Skip to content

P
Paddle

BaiXuePrincess / Paddle
与 Fork 源项目一致

Fork自 PaddlePaddle / Paddle

通知 1
Star 1
Fork 0
P
Paddle
未验证 提交 3da73f8f 编写于 作者: S sneaxiy 提交者： GitHub
浏览文件
操作

update latest ps.proto (#50054)

上级 468f0e91
隐藏空白更改
内联 并排
Showing with 169 addition and 162 deletion
paddle/fluid/distributed/ps.proto
浏览文件 @ 3da73f8f
...@@ -15,244 +15,251 @@ ...@@ -15,244 +15,251 @@
syntax = "proto2"; syntax = "proto2";
package paddle; package paddle;
option cc_generic_services = true; option cc_generic_services = true;
option cc_enable_arenas = true; option cc_enable_arenas=true;
enum TableType {
PS_SPARSE_TABLE = 0;
PS_DENSE_TABLE = 1;
}
enum PsCmdID {
PS_PULL_DENSE_TABLE = 0;
PS_PUSH_DENSE_TABLE = 1;
PS_PULL_SPARSE_TABLE = 2;
PS_PUSH_SPARSE_TABLE = 3;
PS_SHRINK_TABLE = 4;
PS_SAVE_ONE_TABLE = 5;
PS_SAVE_ALL_TABLE = 6;
PS_LOAD_ONE_TABLE = 7;
PS_LOAD_ALL_TABLE = 8;
PS_CLEAR_ONE_TABLE = 9;
PS_CLEAR_ALL_TABLE = 10;
PS_PUSH_DENSE_PARAM = 11;
PS_STOP_SERVER = 12;
PS_SAVE_ONE_CACHE_TABLE = 13;
PS_GET_CACHE_THRESHOLD = 14;
PS_CACHE_SHUFFLE = 15;
PS_COPY_TABLE = 16;
PS_COPY_TABLE_BY_FEASIGN = 17;
PS_PULL_SPARSE_TABLE_WITH_DEPENDENCY = 18;
PS_PUSH_SPARSE_TABLE_WITH_DEPENDENCY = 19;
PS_PRINT_TABLE_STAT = 20;
PS_S2S_MSG = 101;
}
message PSParameter { message PSParameter {
optional string worker_class = 1; optional string worker_class = 1;
optional string server_class = 2; optional string server_class = 2;
optional string instance_class = 3; optional string instance_class = 3;
optional string init_gflags = 4; optional string init_gflags = 4 [ default = "" ];
optional WorkerParameter worker_param = 101; optional WorkerParameter worker_param = 101;
optional ServerParameter server_param = 102; optional ServerParameter server_param = 102;
repeated DownpourTrainerParameter trainer_param = 301; repeated DownpourTrainerParameter trainer_param = 301;
optional FsClientParameter fs_client_param = 501; optional FsClientParameter fs_client_param = 501;
} }
message WorkerParameter { message WorkerParameter {
optional DownpourWorkerParameter downpour_worker_param = 1; optional DownpourWorkerParameter downpour_worker_param = 1;
} }
message ServerParameter { message ServerParameter {
optional DownpourServerParameter downpour_server_param = 1; optional DownpourServerParameter downpour_server_param = 1;
} }
message DownpourWorkerParameter { message DownpourWorkerParameter {
repeated TableParameter downpour_table_param = 1; repeated TableParameter downpour_table_param = 1;
} }
message DownpourTrainerParameter { message DownpourTrainerParameter {
repeated DenseTableParameter dense_table = 1; repeated DenseTableParameter dense_table = 1;
repeated SparseTableParameter sparse_table = 2; repeated SparseTableParameter sparse_table = 2;
optional int32 push_sparse_per_batch = 3; optional int32 push_sparse_per_batch = 3;
optional int32 push_dense_per_batch = 4; optional int32 push_dense_per_batch = 4;
repeated string skip_op = 5; repeated string skip_op = 5;
repeated ProgramConfig program_config = 6; repeated ProgramConfig program_config = 6;
} }
message ProgramConfig { message ProgramConfig {
required string program_id = 1; required string program_id = 1;
repeated int32 push_sparse_table_id = 2; repeated int32 push_sparse_table_id = 2;
repeated int32 push_dense_table_id = 3; repeated int32 push_dense_table_id = 3;
repeated int32 pull_sparse_table_id = 4; repeated int32 pull_sparse_table_id = 4;
repeated int32 pull_dense_table_id = 5; repeated int32 pull_dense_table_id = 5;
} }
message DenseTableParameter { message DenseTableParameter {
optional int32 table_id = 1; optional int32 table_id = 1;
repeated string dense_variable_name = 2; repeated string dense_variable_name = 2;
repeated string dense_gradient_variable_name = 3; repeated string dense_gradient_variable_name = 3;
optional int32 fea_dim = 4; optional int32 fea_dim = 4;
} }
message SparseTableParameter { message SparseTableParameter {
optional int32 table_id = 1; optional int32 table_id = 1;
optional int32 feature_dim = 2; optional int32 feature_dim = 2;
repeated string slot_key = 3; repeated string slot_key = 3;
repeated string slot_value = 4; repeated string slot_value = 4;
repeated string slot_gradient = 5; repeated string slot_gradient = 5;
} }
message DownpourServerParameter { message DownpourServerParameter {
repeated TableParameter downpour_table_param = 1; repeated TableParameter downpour_table_param = 1;
optional ServerServiceParameter service_param = 2; optional ServerServiceParameter service_param = 2;
} }
message ServerServiceParameter { message ServerServiceParameter {
optional string server_class = 1 [ default = "DownpourBrpcPsServer" ]; optional string server_class = 1 [ default = "DownpourBrpcPsServer" ];
optional string client_class = 2 [ default = "DownpourBrpcPsClient" ]; optional string client_class = 2 [ default = "DownpourBrpcPsClient" ];
optional string service_class = 3 [ default = "DownpourPsService" ]; optional string service_class = 3 [ default = "DownpourPsService"];
optional uint32 start_server_port = 4 [ default = 0 ]; optional uint32 start_server_port = 4 [ default = 0 ]; //will find a avaliable port from it
optional uint32 server_thread_num = 5 [ default = 12 ]; optional uint32 server_thread_num = 5 [ default = 12 ];
}
enum TableType {
PS_SPARSE_TABLE = 0;
PS_DENSE_TABLE = 1;
} }
message TableParameter { message TableParameter {
optional uint64 table_id = 1; optional uint64 table_id = 1;
optional string table_class = 2; optional string table_class = 2;
optional uint64 shard_num = 3 [ default = 1000 ]; optional uint64 shard_num = 3 [ default = 1000 ];
optional TableAccessorParameter accessor = 4; optional TableAccessorParameter accessor = 4;
optional TableType type = 5; optional TableType type = 5;
optional bool compress_in_save = 6 [ default = false ]; optional bool compress_in_save = 6 [default = false];
optional bool enable_sparse_table_cache = 7 [ default = true ]; //for cache model
optional double sparse_table_cache_rate = 8 [ default = 0.00055 ]; optional bool enable_sparse_table_cache = 7 [default = true];
optional uint32 sparse_table_cache_file_num = 9 [ default = 16 ]; optional double sparse_table_cache_rate = 8 [default = 0.00055];
optional uint32 sparse_table_cache_file_num = 9 [default = 16];
optional double sparse_table_mem_cache_rate = 10 [default = 0.5];
} }
message TableAccessorParameter { message TableAccessorParameter {
optional string accessor_class = 1; optional string accessor_class = 1;
optional SparseSGDRuleParameter sparse_sgd_param = 2; optional SparseSGDRuleParameter sparse_sgd_param = 2;
optional DenseSGDRuleParameter dense_sgd_param = 3; optional DenseSGDRuleParameter dense_sgd_param = 3;
optional uint32 fea_dim = 4 [ default = 11 ]; optional uint32 fea_dim = 4 [default = 11];
optional uint32 embedx_dim = 5 [ default = 8 ]; optional uint32 embedx_dim = 5 [default = 8];
optional uint32 embedx_threshold = 6 [ default = 10 ]; optional uint32 embedx_threshold = 6 [default = 10];
optional DownpourTableAccessorParameter downpour_accessor_param = 7; optional DownpourTableAccessorParameter downpour_accessor_param = 7;
repeated TableAccessorSaveParameter table_accessor_save_param = 8; repeated TableAccessorSaveParameter table_accessor_save_param = 8;
optional SparseCommonSGDRuleParameter sparse_commonsgd_param = 9; optional SparseCommonSGDRuleParameter sparse_commonsgd_param = 9;
optional SparseCommonSGDRuleParameter embed_sgd_param = 10; optional SparseCommonSGDRuleParameter embed_sgd_param = 10;
optional SparseCommonSGDRuleParameter embedx_sgd_param = 11; optional SparseCommonSGDRuleParameter embedx_sgd_param = 11;
} }
message DownpourTableAccessorParameter { message DownpourTableAccessorParameter {
optional float nonclk_coeff = 1 [ default = 0.1 ]; optional float nonclk_coeff = 1 [default = 0.1]; // to calculate show_click_score
optional float click_coeff = 2 [ default = 1 ]; optional float click_coeff = 2 [default = 1]; // to calculate show_click_score
optional float base_threshold = 3 [ default = 1.5 ]; optional float base_threshold = 3 [default = 1.5]; // show_click_score > base_threshold, this feature can be saved
optional float delta_threshold = 4 [ default = 0.25 ]; optional float delta_threshold = 4 [default = 0.25]; // delta_score > delta_threshold, this feature can be saved
optional float delta_keep_days = 5 [ default = 16 ]; optional float delta_keep_days = 5 [default = 16]; // unseen_day < delta_keep_days, this feature can be saved
optional float show_click_decay_rate = 6 [ default = 0.98 ]; optional float show_click_decay_rate = 6 [default = 0.98]; // show/click will update to show/click * show_click_decay_rate after a day
optional float delete_threshold = 7 [ default = 0.8 ]; optional float delete_threshold = 7 [default = 0.8]; // threshold to shrink a feasign
optional float delete_after_unseen_days = 8 [ default = 30 ]; optional float delete_after_unseen_days = 8 [default = 30]; // unseen_day > delete_after_unseen_days, this feature will be delete in shrink_model
optional int32 ssd_unseenday_threshold = 9 [ default = 1 ]; optional int32 ssd_unseenday_threshold = 9 [default = 1]; // threshold to save ssd
} }
message TableAccessorSaveParameter { message TableAccessorSaveParameter {
optional uint32 param = 1; optional uint32 param = 1;
optional string converter = 2; optional string converter = 2;
optional string deconverter = 3; optional string deconverter = 3;
} }
message PsRequestMessage { enum PsCmdID {
required uint32 cmd_id = 1; PS_PULL_DENSE_TABLE = 0;
optional uint32 table_id = 2; PS_PUSH_DENSE_TABLE = 1;
repeated bytes params = 3; PS_PULL_SPARSE_TABLE = 2;
optional int32 client_id = 4; PS_PUSH_SPARSE_TABLE = 3;
optional bytes data = 5; PS_SHRINK_TABLE = 4;
PS_SAVE_ONE_TABLE = 5;
PS_SAVE_ALL_TABLE = 6;
PS_LOAD_ONE_TABLE = 7;
PS_LOAD_ALL_TABLE = 8;
PS_CLEAR_ONE_TABLE = 9;
PS_CLEAR_ALL_TABLE = 10;
PS_PUSH_DENSE_PARAM = 11;
PS_STOP_SERVER = 12;
PS_SAVE_ONE_CACHE_TABLE = 13;
PS_GET_CACHE_THRESHOLD = 14;
PS_CACHE_SHUFFLE = 15;
PS_COPY_TABLE = 16;
PS_COPY_TABLE_BY_FEASIGN = 17;
PS_PULL_SPARSE_TABLE_WITH_DEPENDENCY = 18;
PS_PUSH_SPARSE_TABLE_WITH_DEPENDENCY = 19;
PS_PRINT_TABLE_STAT = 20;
PS_SAVE_ONE_TABLE_PREFIX = 21;
PS_SAVE_MEM_CACHE_TABLE = 22;
//pserver2pserver cmd start from 100
PS_S2S_MSG = 101;
//local_client2local_client cmd start from 200
PS_C2C_PULL_SPARSE_TABLE = 201;
} }
message PsRequestMessage {
required uint32 cmd_id = 1;
optional uint32 table_id = 2;
repeated bytes params = 3;
optional int32 client_id = 4;
optional bytes data = 5;
};
message SparseSGDRuleParameter { message SparseSGDRuleParameter {
optional double learning_rate = 1 [ default = 0.05 ]; optional double learning_rate = 1 [default = 0.05];
optional double initial_g2sum = 2 [ default = 3 ]; optional double initial_g2sum = 2 [default = 3.0];
optional double initial_range = 3 [ default = 0.0001 ]; optional double initial_range = 3 [default = 0.0001];
repeated float weight_bounds = 4; repeated float weight_bounds = 4;
} }
message SparseCommonSGDRuleParameter { message SparseCommonSGDRuleParameter {
optional string name = 1; optional string name = 1;
optional SparseNaiveSGDRuleParameter naive = 2; optional SparseNaiveSGDRuleParameter naive = 2;
optional SparseAdagradSGDRuleParameter adagrad = 3; optional SparseAdagradSGDRuleParameter adagrad = 3;
optional SparseAdamSGDParameter adam = 4; optional SparseAdamSGDParameter adam = 4;
} }
message SparseNaiveSGDRuleParameter { message SparseNaiveSGDRuleParameter {
optional double learning_rate = 1 [ default = 0.05 ]; optional double learning_rate = 1 [default = 0.05];
optional double initial_range = 2 [ default = 0.0001 ]; optional double initial_range = 2 [default = 0.0001];
repeated float weight_bounds = 3; repeated float weight_bounds = 3;
} }
message SparseAdagradSGDRuleParameter { message SparseAdagradSGDRuleParameter {
optional double learning_rate = 1 [ default = 0.05 ]; optional double learning_rate = 1 [default = 0.05];
optional double initial_g2sum = 2 [ default = 3 ]; optional double initial_g2sum = 2 [default = 3.0];
optional double initial_range = 3 [ default = 0.0001 ]; optional double initial_range = 3 [default = 0.0001];
repeated float weight_bounds = 4; repeated float weight_bounds = 4;
} }
message SparseAdamSGDParameter { message SparseAdamSGDParameter {
optional double learning_rate = 1 [ default = 0.001 ]; optional double learning_rate = 1 [default = 0.001];
optional double initial_range = 2 [ default = 0.0001 ]; optional double initial_range = 2 [default = 0.0001];
optional double beta1_decay_rate = 3 [ default = 0.9 ]; optional double beta1_decay_rate = 3 [default = 0.9];
optional double beta2_decay_rate = 4 [ default = 0.999 ]; optional double beta2_decay_rate = 4 [default = 0.999];
optional double ada_epsilon = 5 [ default = 1e-08 ]; optional double ada_epsilon = 5 [default = 1e-08];
repeated float weight_bounds = 6; repeated float weight_bounds = 6;
} }
message DenseSGDRuleParameter { message DenseSGDRuleParameter {
optional string name = 1; optional string name = 1;
optional AdamSGDParameter adam = 2; optional AdamSGDParameter adam = 2;
optional NaiveSGDParameter naive = 3; optional NaiveSGDParameter naive = 3;
optional SummarySGDParameter summary = 4; optional SummarySGDParameter summary = 4;
optional MovingAverageRuleParameter moving_average = 5; optional MovingAverageRuleParameter moving_average = 5;
} }
message AdamSGDParameter { message AdamSGDParameter {
optional double learning_rate = 1 [ default = 5e-06 ]; optional double learning_rate = 1 [default = 5e-06]; // \u5B66\u4E60\u7387
optional double avg_decay_rate = 2 [ default = 0.999993 ]; optional double avg_decay_rate = 2 [default = 0.999993]; // avg_weight\u7684\u8870\u51CF\u7CFB\u6570
optional double ada_decay_rate = 3 [ default = 0.9999 ]; optional double ada_decay_rate = 3 [default = 0.9999];
optional double ada_epsilon = 4 [ default = 1e-08 ]; optional double ada_epsilon = 4 [default = 1e-08];
optional double mom_decay_rate = 5 [ default = 0.99 ]; optional double mom_decay_rate = 5 [default = 0.99];
} }
message NaiveSGDParameter { message NaiveSGDParameter {
optional double learning_rate = 1 [ default = 0.0002 ]; optional double learning_rate = 1 [default = 0.0002];
optional double avg_decay_rate = 2; optional double avg_decay_rate = 2;
} }
message SummarySGDParameter { message SummarySGDParameter {
optional double summary_decay_rate = 1 [ default = 0.999999 ]; optional double summary_decay_rate = 1 [default = 0.999999]; // \u6743\u91CD\u7684\u8870\u51CF\u7CFB\u6570
} }
message MovingAverageRuleParameter { message MovingAverageRuleParameter {
optional double momentum = 1; optional double momentum = 1;
} }
message PsResponseMessage { message PsResponseMessage {
required int32 err_code = 1 [ default = 0 ]; required int32 err_code = 1 [default = 0];
required string err_msg = 2; required string err_msg = 2 [default = ""];
optional bytes data = 3; optional bytes data = 3;
} };
message FsClientParameter {
enum FsApiType {
HDFS = 0;
AFS = 1;
}
optional FsApiType fs_type = 1 [ default = HDFS ];
optional string uri = 2;
optional string user = 3;
optional string passwd = 4;
optional int32 buffer_size = 5;
optional string hadoop_bin = 51;
optional string afs_conf = 101;
}
service PsService { service PsService {
rpc service(PsRequestMessage) returns (PsResponseMessage); rpc service(PsRequestMessage) returns (PsResponseMessage);
};
message FsClientParameter {
enum FsApiType {
HDFS = 0;
AFS = 1;
}
optional FsApiType fs_type = 1 [default = HDFS];
optional string uri = 2; //such as afs://tianqi.afs.baidu.com:9902
optional string user = 3; //user_name to access fs
optional string passwd = 4; //password
optional int32 buffer_size = 5; //buffer for read/write
optional string hadoop_bin = 51;
optional string afs_conf = 101;
} }
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或