diff --git a/paddle/fluid/distributed/ps.proto b/paddle/fluid/distributed/ps.proto index 8299fd32ee4800e4b12fc59d68b5c419b2dd121c..5d4ab954bbdaec0b18192802787e272ecf2e6b24 100644 --- a/paddle/fluid/distributed/ps.proto +++ b/paddle/fluid/distributed/ps.proto @@ -15,244 +15,251 @@ syntax = "proto2"; package paddle; option cc_generic_services = true; -option cc_enable_arenas = true; - -enum TableType { - PS_SPARSE_TABLE = 0; - PS_DENSE_TABLE = 1; -} - -enum PsCmdID { - PS_PULL_DENSE_TABLE = 0; - PS_PUSH_DENSE_TABLE = 1; - PS_PULL_SPARSE_TABLE = 2; - PS_PUSH_SPARSE_TABLE = 3; - PS_SHRINK_TABLE = 4; - PS_SAVE_ONE_TABLE = 5; - PS_SAVE_ALL_TABLE = 6; - PS_LOAD_ONE_TABLE = 7; - PS_LOAD_ALL_TABLE = 8; - PS_CLEAR_ONE_TABLE = 9; - PS_CLEAR_ALL_TABLE = 10; - PS_PUSH_DENSE_PARAM = 11; - PS_STOP_SERVER = 12; - PS_SAVE_ONE_CACHE_TABLE = 13; - PS_GET_CACHE_THRESHOLD = 14; - PS_CACHE_SHUFFLE = 15; - PS_COPY_TABLE = 16; - PS_COPY_TABLE_BY_FEASIGN = 17; - PS_PULL_SPARSE_TABLE_WITH_DEPENDENCY = 18; - PS_PUSH_SPARSE_TABLE_WITH_DEPENDENCY = 19; - PS_PRINT_TABLE_STAT = 20; - PS_S2S_MSG = 101; -} +option cc_enable_arenas=true; message PSParameter { - optional string worker_class = 1; - optional string server_class = 2; - optional string instance_class = 3; - optional string init_gflags = 4; - optional WorkerParameter worker_param = 101; - optional ServerParameter server_param = 102; - repeated DownpourTrainerParameter trainer_param = 301; - optional FsClientParameter fs_client_param = 501; + optional string worker_class = 1; + optional string server_class = 2; + optional string instance_class = 3; + optional string init_gflags = 4 [ default = "" ]; + optional WorkerParameter worker_param = 101; + optional ServerParameter server_param = 102; + repeated DownpourTrainerParameter trainer_param = 301; + optional FsClientParameter fs_client_param = 501; } message WorkerParameter { - optional DownpourWorkerParameter downpour_worker_param = 1; + optional DownpourWorkerParameter downpour_worker_param = 1; } message ServerParameter { - optional DownpourServerParameter downpour_server_param = 1; + optional DownpourServerParameter downpour_server_param = 1; } message DownpourWorkerParameter { - repeated TableParameter downpour_table_param = 1; + repeated TableParameter downpour_table_param = 1; } message DownpourTrainerParameter { - repeated DenseTableParameter dense_table = 1; - repeated SparseTableParameter sparse_table = 2; - optional int32 push_sparse_per_batch = 3; - optional int32 push_dense_per_batch = 4; - repeated string skip_op = 5; - repeated ProgramConfig program_config = 6; + repeated DenseTableParameter dense_table = 1; + repeated SparseTableParameter sparse_table = 2; + optional int32 push_sparse_per_batch = 3; + optional int32 push_dense_per_batch = 4; + repeated string skip_op = 5; + repeated ProgramConfig program_config = 6; } message ProgramConfig { - required string program_id = 1; - repeated int32 push_sparse_table_id = 2; - repeated int32 push_dense_table_id = 3; - repeated int32 pull_sparse_table_id = 4; - repeated int32 pull_dense_table_id = 5; + required string program_id = 1; + repeated int32 push_sparse_table_id = 2; + repeated int32 push_dense_table_id = 3; + repeated int32 pull_sparse_table_id = 4; + repeated int32 pull_dense_table_id = 5; } message DenseTableParameter { - optional int32 table_id = 1; - repeated string dense_variable_name = 2; - repeated string dense_gradient_variable_name = 3; - optional int32 fea_dim = 4; + optional int32 table_id = 1; + repeated string dense_variable_name = 2; + repeated string dense_gradient_variable_name = 3; + optional int32 fea_dim = 4; } message SparseTableParameter { - optional int32 table_id = 1; - optional int32 feature_dim = 2; - repeated string slot_key = 3; - repeated string slot_value = 4; - repeated string slot_gradient = 5; + optional int32 table_id = 1; + optional int32 feature_dim = 2; + repeated string slot_key = 3; + repeated string slot_value = 4; + repeated string slot_gradient = 5; } message DownpourServerParameter { - repeated TableParameter downpour_table_param = 1; - optional ServerServiceParameter service_param = 2; + repeated TableParameter downpour_table_param = 1; + optional ServerServiceParameter service_param = 2; } message ServerServiceParameter { - optional string server_class = 1 [ default = "DownpourBrpcPsServer" ]; - optional string client_class = 2 [ default = "DownpourBrpcPsClient" ]; - optional string service_class = 3 [ default = "DownpourPsService" ]; - optional uint32 start_server_port = 4 [ default = 0 ]; - optional uint32 server_thread_num = 5 [ default = 12 ]; + optional string server_class = 1 [ default = "DownpourBrpcPsServer" ]; + optional string client_class = 2 [ default = "DownpourBrpcPsClient" ]; + optional string service_class = 3 [ default = "DownpourPsService"]; + optional uint32 start_server_port = 4 [ default = 0 ]; //will find a avaliable port from it + optional uint32 server_thread_num = 5 [ default = 12 ]; +} + +enum TableType { + PS_SPARSE_TABLE = 0; + PS_DENSE_TABLE = 1; } message TableParameter { - optional uint64 table_id = 1; - optional string table_class = 2; - optional uint64 shard_num = 3 [ default = 1000 ]; - optional TableAccessorParameter accessor = 4; - optional TableType type = 5; - optional bool compress_in_save = 6 [ default = false ]; - optional bool enable_sparse_table_cache = 7 [ default = true ]; - optional double sparse_table_cache_rate = 8 [ default = 0.00055 ]; - optional uint32 sparse_table_cache_file_num = 9 [ default = 16 ]; + optional uint64 table_id = 1; + optional string table_class = 2; + optional uint64 shard_num = 3 [ default = 1000 ]; + optional TableAccessorParameter accessor = 4; + optional TableType type = 5; + optional bool compress_in_save = 6 [default = false]; + //for cache model + optional bool enable_sparse_table_cache = 7 [default = true]; + optional double sparse_table_cache_rate = 8 [default = 0.00055]; + optional uint32 sparse_table_cache_file_num = 9 [default = 16]; + optional double sparse_table_mem_cache_rate = 10 [default = 0.5]; + } message TableAccessorParameter { - optional string accessor_class = 1; - optional SparseSGDRuleParameter sparse_sgd_param = 2; - optional DenseSGDRuleParameter dense_sgd_param = 3; - optional uint32 fea_dim = 4 [ default = 11 ]; - optional uint32 embedx_dim = 5 [ default = 8 ]; - optional uint32 embedx_threshold = 6 [ default = 10 ]; - optional DownpourTableAccessorParameter downpour_accessor_param = 7; - repeated TableAccessorSaveParameter table_accessor_save_param = 8; - optional SparseCommonSGDRuleParameter sparse_commonsgd_param = 9; - optional SparseCommonSGDRuleParameter embed_sgd_param = 10; - optional SparseCommonSGDRuleParameter embedx_sgd_param = 11; + optional string accessor_class = 1; + optional SparseSGDRuleParameter sparse_sgd_param = 2; + optional DenseSGDRuleParameter dense_sgd_param = 3; + optional uint32 fea_dim = 4 [default = 11]; + optional uint32 embedx_dim = 5 [default = 8]; + optional uint32 embedx_threshold = 6 [default = 10]; + optional DownpourTableAccessorParameter downpour_accessor_param = 7; + repeated TableAccessorSaveParameter table_accessor_save_param = 8; + optional SparseCommonSGDRuleParameter sparse_commonsgd_param = 9; + optional SparseCommonSGDRuleParameter embed_sgd_param = 10; + optional SparseCommonSGDRuleParameter embedx_sgd_param = 11; } message DownpourTableAccessorParameter { - optional float nonclk_coeff = 1 [ default = 0.1 ]; - optional float click_coeff = 2 [ default = 1 ]; - optional float base_threshold = 3 [ default = 1.5 ]; - optional float delta_threshold = 4 [ default = 0.25 ]; - optional float delta_keep_days = 5 [ default = 16 ]; - optional float show_click_decay_rate = 6 [ default = 0.98 ]; - optional float delete_threshold = 7 [ default = 0.8 ]; - optional float delete_after_unseen_days = 8 [ default = 30 ]; - optional int32 ssd_unseenday_threshold = 9 [ default = 1 ]; + optional float nonclk_coeff = 1 [default = 0.1]; // to calculate show_click_score + optional float click_coeff = 2 [default = 1]; // to calculate show_click_score + optional float base_threshold = 3 [default = 1.5]; // show_click_score > base_threshold, this feature can be saved + optional float delta_threshold = 4 [default = 0.25]; // delta_score > delta_threshold, this feature can be saved + optional float delta_keep_days = 5 [default = 16]; // unseen_day < delta_keep_days, this feature can be saved + optional float show_click_decay_rate = 6 [default = 0.98]; // show/click will update to show/click * show_click_decay_rate after a day + optional float delete_threshold = 7 [default = 0.8]; // threshold to shrink a feasign + optional float delete_after_unseen_days = 8 [default = 30]; // unseen_day > delete_after_unseen_days, this feature will be delete in shrink_model + optional int32 ssd_unseenday_threshold = 9 [default = 1]; // threshold to save ssd } message TableAccessorSaveParameter { - optional uint32 param = 1; - optional string converter = 2; - optional string deconverter = 3; + optional uint32 param = 1; + optional string converter = 2; + optional string deconverter = 3; } -message PsRequestMessage { - required uint32 cmd_id = 1; - optional uint32 table_id = 2; - repeated bytes params = 3; - optional int32 client_id = 4; - optional bytes data = 5; +enum PsCmdID { + PS_PULL_DENSE_TABLE = 0; + PS_PUSH_DENSE_TABLE = 1; + PS_PULL_SPARSE_TABLE = 2; + PS_PUSH_SPARSE_TABLE = 3; + PS_SHRINK_TABLE = 4; + PS_SAVE_ONE_TABLE = 5; + PS_SAVE_ALL_TABLE = 6; + PS_LOAD_ONE_TABLE = 7; + PS_LOAD_ALL_TABLE = 8; + PS_CLEAR_ONE_TABLE = 9; + PS_CLEAR_ALL_TABLE = 10; + PS_PUSH_DENSE_PARAM = 11; + PS_STOP_SERVER = 12; + PS_SAVE_ONE_CACHE_TABLE = 13; + PS_GET_CACHE_THRESHOLD = 14; + PS_CACHE_SHUFFLE = 15; + PS_COPY_TABLE = 16; + PS_COPY_TABLE_BY_FEASIGN = 17; + PS_PULL_SPARSE_TABLE_WITH_DEPENDENCY = 18; + PS_PUSH_SPARSE_TABLE_WITH_DEPENDENCY = 19; + PS_PRINT_TABLE_STAT = 20; + PS_SAVE_ONE_TABLE_PREFIX = 21; + PS_SAVE_MEM_CACHE_TABLE = 22; + //pserver2pserver cmd start from 100 + PS_S2S_MSG = 101; + //local_client2local_client cmd start from 200 + PS_C2C_PULL_SPARSE_TABLE = 201; } +message PsRequestMessage { + required uint32 cmd_id = 1; + optional uint32 table_id = 2; + repeated bytes params = 3; + optional int32 client_id = 4; + optional bytes data = 5; +}; + message SparseSGDRuleParameter { - optional double learning_rate = 1 [ default = 0.05 ]; - optional double initial_g2sum = 2 [ default = 3 ]; - optional double initial_range = 3 [ default = 0.0001 ]; - repeated float weight_bounds = 4; + optional double learning_rate = 1 [default = 0.05]; + optional double initial_g2sum = 2 [default = 3.0]; + optional double initial_range = 3 [default = 0.0001]; + repeated float weight_bounds = 4; } message SparseCommonSGDRuleParameter { - optional string name = 1; - optional SparseNaiveSGDRuleParameter naive = 2; - optional SparseAdagradSGDRuleParameter adagrad = 3; - optional SparseAdamSGDParameter adam = 4; + optional string name = 1; + optional SparseNaiveSGDRuleParameter naive = 2; + optional SparseAdagradSGDRuleParameter adagrad = 3; + optional SparseAdamSGDParameter adam = 4; } message SparseNaiveSGDRuleParameter { - optional double learning_rate = 1 [ default = 0.05 ]; - optional double initial_range = 2 [ default = 0.0001 ]; - repeated float weight_bounds = 3; + optional double learning_rate = 1 [default = 0.05]; + optional double initial_range = 2 [default = 0.0001]; + repeated float weight_bounds = 3; } message SparseAdagradSGDRuleParameter { - optional double learning_rate = 1 [ default = 0.05 ]; - optional double initial_g2sum = 2 [ default = 3 ]; - optional double initial_range = 3 [ default = 0.0001 ]; - repeated float weight_bounds = 4; + optional double learning_rate = 1 [default = 0.05]; + optional double initial_g2sum = 2 [default = 3.0]; + optional double initial_range = 3 [default = 0.0001]; + repeated float weight_bounds = 4; } message SparseAdamSGDParameter { - optional double learning_rate = 1 [ default = 0.001 ]; - optional double initial_range = 2 [ default = 0.0001 ]; - optional double beta1_decay_rate = 3 [ default = 0.9 ]; - optional double beta2_decay_rate = 4 [ default = 0.999 ]; - optional double ada_epsilon = 5 [ default = 1e-08 ]; - repeated float weight_bounds = 6; + optional double learning_rate = 1 [default = 0.001]; + optional double initial_range = 2 [default = 0.0001]; + optional double beta1_decay_rate = 3 [default = 0.9]; + optional double beta2_decay_rate = 4 [default = 0.999]; + optional double ada_epsilon = 5 [default = 1e-08]; + repeated float weight_bounds = 6; } message DenseSGDRuleParameter { - optional string name = 1; - optional AdamSGDParameter adam = 2; - optional NaiveSGDParameter naive = 3; - optional SummarySGDParameter summary = 4; - optional MovingAverageRuleParameter moving_average = 5; + optional string name = 1; + optional AdamSGDParameter adam = 2; + optional NaiveSGDParameter naive = 3; + optional SummarySGDParameter summary = 4; + optional MovingAverageRuleParameter moving_average = 5; } message AdamSGDParameter { - optional double learning_rate = 1 [ default = 5e-06 ]; - optional double avg_decay_rate = 2 [ default = 0.999993 ]; - optional double ada_decay_rate = 3 [ default = 0.9999 ]; - optional double ada_epsilon = 4 [ default = 1e-08 ]; - optional double mom_decay_rate = 5 [ default = 0.99 ]; + optional double learning_rate = 1 [default = 5e-06]; // \u5B66\u4E60\u7387 + optional double avg_decay_rate = 2 [default = 0.999993]; // avg_weight\u7684\u8870\u51CF\u7CFB\u6570 + optional double ada_decay_rate = 3 [default = 0.9999]; + optional double ada_epsilon = 4 [default = 1e-08]; + optional double mom_decay_rate = 5 [default = 0.99]; } message NaiveSGDParameter { - optional double learning_rate = 1 [ default = 0.0002 ]; - optional double avg_decay_rate = 2; + optional double learning_rate = 1 [default = 0.0002]; + optional double avg_decay_rate = 2; } message SummarySGDParameter { - optional double summary_decay_rate = 1 [ default = 0.999999 ]; + optional double summary_decay_rate = 1 [default = 0.999999]; // \u6743\u91CD\u7684\u8870\u51CF\u7CFB\u6570 } message MovingAverageRuleParameter { - optional double momentum = 1; + optional double momentum = 1; } message PsResponseMessage { - required int32 err_code = 1 [ default = 0 ]; - required string err_msg = 2; - optional bytes data = 3; -} - -message FsClientParameter { - enum FsApiType { - HDFS = 0; - AFS = 1; - } - - optional FsApiType fs_type = 1 [ default = HDFS ]; - optional string uri = 2; - optional string user = 3; - optional string passwd = 4; - optional int32 buffer_size = 5; - optional string hadoop_bin = 51; - optional string afs_conf = 101; -} + required int32 err_code = 1 [default = 0]; + required string err_msg = 2 [default = ""]; + optional bytes data = 3; +}; service PsService { - rpc service(PsRequestMessage) returns (PsResponseMessage); + rpc service(PsRequestMessage) returns (PsResponseMessage); +}; + +message FsClientParameter { + enum FsApiType { + HDFS = 0; + AFS = 1; + } + optional FsApiType fs_type = 1 [default = HDFS]; + optional string uri = 2; //such as afs://tianqi.afs.baidu.com:9902 + optional string user = 3; //user_name to access fs + optional string passwd = 4; //password + optional int32 buffer_size = 5; //buffer for read/write + optional string hadoop_bin = 51; + optional string afs_conf = 101; }