ps.proto

// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
// 
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// 
//     http://www.apache.org/licenses/LICENSE-2.0
// 
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto2";
package paddle;
option cc_generic_services = true;
option cc_enable_arenas = true;

enum TableType {
  PS_SPARSE_TABLE = 0;
  PS_DENSE_TABLE = 1;
}

enum PsCmdID {
  PS_PULL_DENSE_TABLE = 0;
  PS_PUSH_DENSE_TABLE = 1;
  PS_PULL_SPARSE_TABLE = 2;
  PS_PUSH_SPARSE_TABLE = 3;
  PS_SHRINK_TABLE = 4;
  PS_SAVE_ONE_TABLE = 5;
  PS_SAVE_ALL_TABLE = 6;
  PS_LOAD_ONE_TABLE = 7;
  PS_LOAD_ALL_TABLE = 8;
  PS_CLEAR_ONE_TABLE = 9;
  PS_CLEAR_ALL_TABLE = 10;
  PS_PUSH_DENSE_PARAM = 11;
  PS_STOP_SERVER = 12;
  PS_SAVE_ONE_CACHE_TABLE = 13;
  PS_GET_CACHE_THRESHOLD = 14;
  PS_CACHE_SHUFFLE = 15;
  PS_COPY_TABLE = 16;
  PS_COPY_TABLE_BY_FEASIGN = 17;
  PS_PULL_SPARSE_TABLE_WITH_DEPENDENCY = 18;
  PS_PUSH_SPARSE_TABLE_WITH_DEPENDENCY = 19;
  PS_PRINT_TABLE_STAT = 20;
  PS_S2S_MSG = 101;
}

message PSParameter {
  optional string worker_class = 1;
  optional string server_class = 2;
  optional string instance_class = 3;
  optional string init_gflags = 4;
  optional WorkerParameter worker_param = 101;
  optional ServerParameter server_param = 102;
  repeated DownpourTrainerParameter trainer_param = 301;
  optional FsClientParameter fs_client_param = 501;
}

message WorkerParameter {
  optional DownpourWorkerParameter downpour_worker_param = 1;
}

message ServerParameter {
  optional DownpourServerParameter downpour_server_param = 1;
}

message DownpourWorkerParameter {
  repeated TableParameter downpour_table_param = 1;
}

message DownpourTrainerParameter {
  repeated DenseTableParameter dense_table = 1;
  repeated SparseTableParameter sparse_table = 2;
  optional int32 push_sparse_per_batch = 3;
  optional int32 push_dense_per_batch = 4;
  repeated string skip_op = 5;
  repeated ProgramConfig program_config = 6;
}

message ProgramConfig {
  required string program_id = 1;
  repeated int32 push_sparse_table_id = 2;
  repeated int32 push_dense_table_id = 3;
  repeated int32 pull_sparse_table_id = 4;
  repeated int32 pull_dense_table_id = 5;
}

message DenseTableParameter {
  optional int32 table_id = 1;
  repeated string dense_variable_name = 2;
  repeated string dense_gradient_variable_name = 3;
  optional int32 fea_dim = 4;
}

message SparseTableParameter {
  optional int32 table_id = 1;
  optional int32 feature_dim = 2;
  repeated string slot_key = 3;
  repeated string slot_value = 4;
  repeated string slot_gradient = 5;
}

message DownpourServerParameter {
  repeated TableParameter downpour_table_param = 1;
  optional ServerServiceParameter service_param = 2;
}

message ServerServiceParameter {
  optional string server_class = 1 [ default = "DownpourBrpcPsServer" ];
  optional string client_class = 2 [ default = "DownpourBrpcPsClient" ];
  optional string service_class = 3 [ default = "DownpourPsService" ];
  optional uint32 start_server_port = 4 [ default = 0 ];
  optional uint32 server_thread_num = 5 [ default = 12 ];
}

message TableParameter {
  optional uint64 table_id = 1;
  optional string table_class = 2;
  optional uint64 shard_num = 3 [ default = 1000 ];
  optional TableAccessorParameter accessor = 4;
  optional TableType type = 5;
  optional bool compress_in_save = 6 [ default = false ];
  optional bool enable_sparse_table_cache = 7 [ default = true ];
  optional double sparse_table_cache_rate = 8 [ default = 0.00055 ];
  optional uint32 sparse_table_cache_file_num = 9 [ default = 16 ];
}

message TableAccessorParameter {
  optional string accessor_class = 1;
  optional SparseSGDRuleParameter sparse_sgd_param = 2;
  optional DenseSGDRuleParameter dense_sgd_param = 3;
  optional uint32 fea_dim = 4 [ default = 11 ];
  optional uint32 embedx_dim = 5 [ default = 8 ];
  optional uint32 embedx_threshold = 6 [ default = 10 ];
  optional DownpourTableAccessorParameter downpour_accessor_param = 7;
  repeated TableAccessorSaveParameter table_accessor_save_param = 8;
  optional SparseCommonSGDRuleParameter sparse_commonsgd_param = 9;
  optional SparseCommonSGDRuleParameter embed_sgd_param = 10;
  optional SparseCommonSGDRuleParameter embedx_sgd_param = 11;
}

message DownpourTableAccessorParameter {
  optional float nonclk_coeff = 1 [ default = 0.1 ];
  optional float click_coeff = 2 [ default = 1 ];
  optional float base_threshold = 3 [ default = 1.5 ];
  optional float delta_threshold = 4 [ default = 0.25 ];
  optional float delta_keep_days = 5 [ default = 16 ];
  optional float show_click_decay_rate = 6 [ default = 0.98 ];
  optional float delete_threshold = 7 [ default = 0.8 ];
  optional float delete_after_unseen_days = 8 [ default = 30 ];
  optional int32 ssd_unseenday_threshold = 9 [ default = 1 ];
}

message TableAccessorSaveParameter {
  optional uint32 param = 1;
  optional string converter = 2;
  optional string deconverter = 3;
}

message PsRequestMessage {
  required uint32 cmd_id = 1;
  optional uint32 table_id = 2;
  repeated bytes params = 3;
  optional int32 client_id = 4;
  optional bytes data = 5;
}

message SparseSGDRuleParameter {
  optional double learning_rate = 1 [ default = 0.05 ];
  optional double initial_g2sum = 2 [ default = 3 ];
  optional double initial_range = 3 [ default = 0.0001 ];
  repeated float weight_bounds = 4;
}

message SparseCommonSGDRuleParameter {
  optional string name = 1;
  optional SparseNaiveSGDRuleParameter naive = 2;
  optional SparseAdagradSGDRuleParameter adagrad = 3;
  optional SparseAdamSGDParameter adam = 4;
}

message SparseNaiveSGDRuleParameter {
  optional double learning_rate = 1 [ default = 0.05 ];
  optional double initial_range = 2 [ default = 0.0001 ];
  repeated float weight_bounds = 3;
}

message SparseAdagradSGDRuleParameter {
  optional double learning_rate = 1 [ default = 0.05 ];
  optional double initial_g2sum = 2 [ default = 3 ];
  optional double initial_range = 3 [ default = 0.0001 ];
  repeated float weight_bounds = 4;
}

message SparseAdamSGDParameter {
  optional double learning_rate = 1 [ default = 0.001 ];
  optional double initial_range = 2 [ default = 0.0001 ];
  optional double beta1_decay_rate = 3 [ default = 0.9 ];
  optional double beta2_decay_rate = 4 [ default = 0.999 ];
  optional double ada_epsilon = 5 [ default = 1e-08 ];
  repeated float weight_bounds = 6;
}

message DenseSGDRuleParameter {
  optional string name = 1;
  optional AdamSGDParameter adam = 2;
  optional NaiveSGDParameter naive = 3;
  optional SummarySGDParameter summary = 4;
  optional MovingAverageRuleParameter moving_average = 5;
}

message AdamSGDParameter {
  optional double learning_rate = 1 [ default = 5e-06 ];
  optional double avg_decay_rate = 2 [ default = 0.999993 ];
  optional double ada_decay_rate = 3 [ default = 0.9999 ];
  optional double ada_epsilon = 4 [ default = 1e-08 ];
  optional double mom_decay_rate = 5 [ default = 0.99 ];
}

message NaiveSGDParameter {
  optional double learning_rate = 1 [ default = 0.0002 ];
  optional double avg_decay_rate = 2;
}

message SummarySGDParameter {
  optional double summary_decay_rate = 1 [ default = 0.999999 ];
}

message MovingAverageRuleParameter {
  optional double momentum = 1;
}

message PsResponseMessage {
  required int32 err_code = 1 [ default = 0 ];
  required string err_msg = 2;
  optional bytes data = 3;
}

message FsClientParameter {
  enum FsApiType {
    HDFS = 0;
    AFS = 1;
  }

  optional FsApiType fs_type = 1 [ default = HDFS ];
  optional string uri = 2;
  optional string user = 3;
  optional string passwd = 4;
  optional int32 buffer_size = 5;
  optional string hadoop_bin = 51;
  optional string afs_conf = 101;
}

service PsService {
  rpc service(PsRequestMessage) returns (PsResponseMessage);
}