Add the missing ps.proto and remove ps_pb2.py (#50040)

* add missing proto file

* fix windows ci

* fix ci compile error

paddle/fluid/distributed/CMakeLists.txt

@@ -2,18 +2,26 @@ add_subdirectory(auto_parallel)
 add_subdirectory(collective)
 add_subdirectory(fleet_executor)
 if(WITH_PYTHON)
+  py_proto_compile(pslib_py_proto SRCS ps.proto)
   py_proto_compile(ps_py_proto SRCS the_one_ps.proto)
   add_custom_target(
     ps_py_proto_init ALL
     COMMAND ${CMAKE_COMMAND} -E make_directory
             ${PADDLE_BINARY_DIR}/python/paddle/distributed/fleet/proto)
   add_dependencies(ps_py_proto ps_py_proto_init)
+  set(PSLIB_PROTO_DSTPATH
+      "${PADDLE_SOURCE_DIR}/python/paddle/fluid/incubate/fleet/parameter_server/pslib/"
+  )
   if(NOT WIN32)
     add_custom_command(
       TARGET ps_py_proto
       POST_BUILD
       COMMAND mv the_one_ps_pb2.py
               ${PADDLE_BINARY_DIR}/python/paddle/distributed/fleet/proto/)
+    add_custom_command(
+      TARGET pslib_py_proto
+      POST_BUILD
+      COMMAND mv ps_pb2.py "${PSLIB_PROTO_DSTPATH}")
   else()
     string(
       REPLACE "/" "\\" fleet_proto_dstpath
@@ -25,7 +33,15 @@ if(WITH_PYTHON)
       COMMENT
         "Copy generated python the_one_ps_pb2 into directory ${fleet_proto_dstpath}."
     )
+    string(REPLACE "/" "\\" PSLIB_PROTO_DSTPATH "${PSLIB_PROTO_DSTPATH}")
+    add_custom_command(
+      TARGET pslib_py_proto
+      POST_BUILD
+      COMMAND copy /Y ps_pb2.py ${PSLIB_PROTO_DSTPATH})
   endif()
+  message(
+    STATUS
+      "Copy generated python ps_pb2.py into directory ${PSLIB_PROTO_DSTPATH}")
 endif()
 
 if(WITH_RPC)

paddle/fluid/distributed/ps.proto

+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+package paddle;
+option cc_generic_services = true;
+option cc_enable_arenas = true;
+
+enum TableType {
+  PS_SPARSE_TABLE = 0;
+  PS_DENSE_TABLE = 1;
+}
+
+enum PsCmdID {
+  PS_PULL_DENSE_TABLE = 0;
+  PS_PUSH_DENSE_TABLE = 1;
+  PS_PULL_SPARSE_TABLE = 2;
+  PS_PUSH_SPARSE_TABLE = 3;
+  PS_SHRINK_TABLE = 4;
+  PS_SAVE_ONE_TABLE = 5;
+  PS_SAVE_ALL_TABLE = 6;
+  PS_LOAD_ONE_TABLE = 7;
+  PS_LOAD_ALL_TABLE = 8;
+  PS_CLEAR_ONE_TABLE = 9;
+  PS_CLEAR_ALL_TABLE = 10;
+  PS_PUSH_DENSE_PARAM = 11;
+  PS_STOP_SERVER = 12;
+  PS_SAVE_ONE_CACHE_TABLE = 13;
+  PS_GET_CACHE_THRESHOLD = 14;
+  PS_CACHE_SHUFFLE = 15;
+  PS_COPY_TABLE = 16;
+  PS_COPY_TABLE_BY_FEASIGN = 17;
+  PS_PULL_SPARSE_TABLE_WITH_DEPENDENCY = 18;
+  PS_PUSH_SPARSE_TABLE_WITH_DEPENDENCY = 19;
+  PS_PRINT_TABLE_STAT = 20;
+  PS_S2S_MSG = 101;
+}
+
+message PSParameter {
+  optional string worker_class = 1;
+  optional string server_class = 2;
+  optional string instance_class = 3;
+  optional string init_gflags = 4;
+  optional WorkerParameter worker_param = 101;
+  optional ServerParameter server_param = 102;
+  repeated DownpourTrainerParameter trainer_param = 301;
+  optional FsClientParameter fs_client_param = 501;
+}
+
+message WorkerParameter {
+  optional DownpourWorkerParameter downpour_worker_param = 1;
+}
+
+message ServerParameter {
+  optional DownpourServerParameter downpour_server_param = 1;
+}
+
+message DownpourWorkerParameter {
+  repeated TableParameter downpour_table_param = 1;
+}
+
+message DownpourTrainerParameter {
+  repeated DenseTableParameter dense_table = 1;
+  repeated SparseTableParameter sparse_table = 2;
+  optional int32 push_sparse_per_batch = 3;
+  optional int32 push_dense_per_batch = 4;
+  repeated string skip_op = 5;
+  repeated ProgramConfig program_config = 6;
+}
+
+message ProgramConfig {
+  required string program_id = 1;
+  repeated int32 push_sparse_table_id = 2;
+  repeated int32 push_dense_table_id = 3;
+  repeated int32 pull_sparse_table_id = 4;
+  repeated int32 pull_dense_table_id = 5;
+}
+
+message DenseTableParameter {
+  optional int32 table_id = 1;
+  repeated string dense_variable_name = 2;
+  repeated string dense_gradient_variable_name = 3;
+  optional int32 fea_dim = 4;
+}
+
+message SparseTableParameter {
+  optional int32 table_id = 1;
+  optional int32 feature_dim = 2;
+  repeated string slot_key = 3;
+  repeated string slot_value = 4;
+  repeated string slot_gradient = 5;
+}
+
+message DownpourServerParameter {
+  repeated TableParameter downpour_table_param = 1;
+  optional ServerServiceParameter service_param = 2;
+}
+
+message ServerServiceParameter {
+  optional string server_class = 1 [ default = "DownpourBrpcPsServer" ];
+  optional string client_class = 2 [ default = "DownpourBrpcPsClient" ];
+  optional string service_class = 3 [ default = "DownpourPsService" ];
+  optional uint32 start_server_port = 4 [ default = 0 ];
+  optional uint32 server_thread_num = 5 [ default = 12 ];
+}
+
+message TableParameter {
+  optional uint64 table_id = 1;
+  optional string table_class = 2;
+  optional uint64 shard_num = 3 [ default = 1000 ];
+  optional TableAccessorParameter accessor = 4;
+  optional TableType type = 5;
+  optional bool compress_in_save = 6 [ default = false ];
+  optional bool enable_sparse_table_cache = 7 [ default = true ];
+  optional double sparse_table_cache_rate = 8 [ default = 0.00055 ];
+  optional uint32 sparse_table_cache_file_num = 9 [ default = 16 ];
+}
+
+message TableAccessorParameter {
+  optional string accessor_class = 1;
+  optional SparseSGDRuleParameter sparse_sgd_param = 2;
+  optional DenseSGDRuleParameter dense_sgd_param = 3;
+  optional uint32 fea_dim = 4 [ default = 11 ];
+  optional uint32 embedx_dim = 5 [ default = 8 ];
+  optional uint32 embedx_threshold = 6 [ default = 10 ];
+  optional DownpourTableAccessorParameter downpour_accessor_param = 7;
+  repeated TableAccessorSaveParameter table_accessor_save_param = 8;
+  optional SparseCommonSGDRuleParameter sparse_commonsgd_param = 9;
+  optional SparseCommonSGDRuleParameter embed_sgd_param = 10;
+  optional SparseCommonSGDRuleParameter embedx_sgd_param = 11;
+}
+
+message DownpourTableAccessorParameter {
+  optional float nonclk_coeff = 1 [ default = 0.1 ];
+  optional float click_coeff = 2 [ default = 1 ];
+  optional float base_threshold = 3 [ default = 1.5 ];
+  optional float delta_threshold = 4 [ default = 0.25 ];
+  optional float delta_keep_days = 5 [ default = 16 ];
+  optional float show_click_decay_rate = 6 [ default = 0.98 ];
+  optional float delete_threshold = 7 [ default = 0.8 ];
+  optional float delete_after_unseen_days = 8 [ default = 30 ];
+  optional int32 ssd_unseenday_threshold = 9 [ default = 1 ];
+}
+
+message TableAccessorSaveParameter {
+  optional uint32 param = 1;
+  optional string converter = 2;
+  optional string deconverter = 3;
+}
+
+message PsRequestMessage {
+  required uint32 cmd_id = 1;
+  optional uint32 table_id = 2;
+  repeated bytes params = 3;
+  optional int32 client_id = 4;
+  optional bytes data = 5;
+}
+
+message SparseSGDRuleParameter {
+  optional double learning_rate = 1 [ default = 0.05 ];
+  optional double initial_g2sum = 2 [ default = 3 ];
+  optional double initial_range = 3 [ default = 0.0001 ];
+  repeated float weight_bounds = 4;
+}
+
+message SparseCommonSGDRuleParameter {
+  optional string name = 1;
+  optional SparseNaiveSGDRuleParameter naive = 2;
+  optional SparseAdagradSGDRuleParameter adagrad = 3;
+  optional SparseAdamSGDParameter adam = 4;
+}
+
+message SparseNaiveSGDRuleParameter {
+  optional double learning_rate = 1 [ default = 0.05 ];
+  optional double initial_range = 2 [ default = 0.0001 ];
+  repeated float weight_bounds = 3;
+}
+
+message SparseAdagradSGDRuleParameter {
+  optional double learning_rate = 1 [ default = 0.05 ];
+  optional double initial_g2sum = 2 [ default = 3 ];
+  optional double initial_range = 3 [ default = 0.0001 ];
+  repeated float weight_bounds = 4;
+}
+
+message SparseAdamSGDParameter {
+  optional double learning_rate = 1 [ default = 0.001 ];
+  optional double initial_range = 2 [ default = 0.0001 ];
+  optional double beta1_decay_rate = 3 [ default = 0.9 ];
+  optional double beta2_decay_rate = 4 [ default = 0.999 ];
+  optional double ada_epsilon = 5 [ default = 1e-08 ];
+  repeated float weight_bounds = 6;
+}
+
+message DenseSGDRuleParameter {
+  optional string name = 1;
+  optional AdamSGDParameter adam = 2;
+  optional NaiveSGDParameter naive = 3;
+  optional SummarySGDParameter summary = 4;
+  optional MovingAverageRuleParameter moving_average = 5;
+}
+
+message AdamSGDParameter {
+  optional double learning_rate = 1 [ default = 5e-06 ];
+  optional double avg_decay_rate = 2 [ default = 0.999993 ];
+  optional double ada_decay_rate = 3 [ default = 0.9999 ];
+  optional double ada_epsilon = 4 [ default = 1e-08 ];
+  optional double mom_decay_rate = 5 [ default = 0.99 ];
+}
+
+message NaiveSGDParameter {
+  optional double learning_rate = 1 [ default = 0.0002 ];
+  optional double avg_decay_rate = 2;
+}
+
+message SummarySGDParameter {
+  optional double summary_decay_rate = 1 [ default = 0.999999 ];
+}
+
+message MovingAverageRuleParameter {
+  optional double momentum = 1;
+}
+
+message PsResponseMessage {
+  required int32 err_code = 1 [ default = 0 ];
+  required string err_msg = 2;
+  optional bytes data = 3;
+}
+
+message FsClientParameter {
+  enum FsApiType {
+    HDFS = 0;
+    AFS = 1;
+  }
+
+  optional FsApiType fs_type = 1 [ default = HDFS ];
+  optional string uri = 2;
+  optional string user = 3;
+  optional string passwd = 4;
+  optional int32 buffer_size = 5;
+  optional string hadoop_bin = 51;
+  optional string afs_conf = 101;
+}
+
+service PsService {
+  rpc service(PsRequestMessage) returns (PsResponseMessage);
+}

paddle/fluid/framework/CMakeLists.txt

@@ -618,7 +618,8 @@ if(WITH_PYTHON)
     fleet_proto_init
     pass_desc_py_proto
     ps_py_proto
-    ps_py_proto_init)
+    ps_py_proto_init
+    pslib_py_proto)
   if(NOT WIN32)
     add_custom_command(
       TARGET framework_py_proto

python/paddle/fluid/incubate/fleet/parameter_server/pslib/.gitignore

+ps_pb2.py