ps.proto

// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
// 
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// 
//     http://www.apache.org/licenses/LICENSE-2.0
// 
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto2";
package paddle;
option cc_generic_services = true;
option cc_enable_arenas=true;

message PSParameter {
    optional string worker_class = 1;
    optional string server_class = 2;
    optional string instance_class = 3;
    optional string init_gflags = 4 [ default = "" ];
    optional WorkerParameter worker_param = 101;
    optional ServerParameter server_param = 102;
    repeated DownpourTrainerParameter trainer_param = 301;
    optional FsClientParameter fs_client_param = 501;
}

message WorkerParameter {
    optional DownpourWorkerParameter downpour_worker_param = 1;
}

message ServerParameter {
    optional DownpourServerParameter downpour_server_param = 1;
}

message DownpourWorkerParameter {
    repeated TableParameter downpour_table_param = 1;
}

message DownpourTrainerParameter {
    repeated DenseTableParameter dense_table = 1;
    repeated SparseTableParameter sparse_table = 2;
    optional int32 push_sparse_per_batch = 3;
    optional int32 push_dense_per_batch = 4;
    repeated string skip_op = 5;
    repeated ProgramConfig program_config = 6;
}

message ProgramConfig {
    required string program_id = 1;
    repeated int32 push_sparse_table_id = 2;
    repeated int32 push_dense_table_id = 3;
    repeated int32 pull_sparse_table_id = 4;
    repeated int32 pull_dense_table_id = 5;
}

message DenseTableParameter {
    optional int32 table_id = 1;
    repeated string dense_variable_name = 2;
    repeated string dense_gradient_variable_name = 3;
    optional int32 fea_dim = 4;
}

message SparseTableParameter {
    optional int32 table_id = 1;
    optional int32 feature_dim = 2;
    repeated string slot_key = 3;
    repeated string slot_value = 4;
    repeated string slot_gradient = 5;
}

message DownpourServerParameter {
    repeated TableParameter downpour_table_param = 1;
    optional ServerServiceParameter service_param = 2;
}

message ServerServiceParameter {
    optional string server_class      = 1 [ default = "DownpourBrpcPsServer" ];
    optional string client_class      = 2 [ default = "DownpourBrpcPsClient" ];
    optional string service_class     = 3 [ default = "DownpourPsService"];
    optional uint32 start_server_port = 4 [ default = 0 ];  //will find a avaliable port from it
    optional uint32 server_thread_num = 5 [ default = 12 ];
}

enum TableType {
    PS_SPARSE_TABLE = 0;
    PS_DENSE_TABLE = 1;
}

message TableParameter {
    optional uint64 table_id = 1;
    optional string table_class = 2;
    optional uint64 shard_num = 3 [ default = 1000 ];
    optional TableAccessorParameter accessor = 4;
    optional TableType type = 5;
    optional bool compress_in_save = 6 [default = false];
    //for cache model
    optional bool enable_sparse_table_cache = 7 [default = true];
    optional double sparse_table_cache_rate = 8 [default = 0.00055];
    optional uint32 sparse_table_cache_file_num = 9 [default = 16];
    optional double sparse_table_mem_cache_rate = 10 [default = 0.5];

}

message TableAccessorParameter {
    optional string accessor_class = 1;
    optional SparseSGDRuleParameter sparse_sgd_param = 2;
    optional DenseSGDRuleParameter dense_sgd_param = 3;
    optional uint32 fea_dim = 4 [default = 11];
    optional uint32 embedx_dim = 5 [default = 8];
    optional uint32 embedx_threshold = 6 [default = 10];
    optional DownpourTableAccessorParameter downpour_accessor_param = 7;
    repeated TableAccessorSaveParameter table_accessor_save_param = 8;
    optional SparseCommonSGDRuleParameter sparse_commonsgd_param = 9;
    optional SparseCommonSGDRuleParameter embed_sgd_param = 10;
    optional SparseCommonSGDRuleParameter embedx_sgd_param = 11;
}

message DownpourTableAccessorParameter {
    optional float nonclk_coeff = 1 [default = 0.1];             // to calculate show_click_score
    optional float click_coeff = 2 [default = 1];                // to calculate show_click_score
    optional float base_threshold = 3 [default = 1.5];           // show_click_score > base_threshold, this feature can be saved
    optional float delta_threshold = 4 [default = 0.25];         // delta_score > delta_threshold, this feature can be saved
    optional float delta_keep_days = 5 [default = 16];           // unseen_day < delta_keep_days, this feature can be saved
    optional float show_click_decay_rate = 6 [default = 0.98];   // show/click will update to show/click * show_click_decay_rate after a day
    optional float delete_threshold = 7 [default = 0.8];         // threshold to shrink a feasign
    optional float delete_after_unseen_days = 8 [default = 30];  // unseen_day > delete_after_unseen_days, this feature will be delete in shrink_model
    optional int32 ssd_unseenday_threshold = 9 [default = 1];    // threshold to save ssd
}

message TableAccessorSaveParameter {
    optional uint32 param = 1;
    optional string converter = 2;
    optional string deconverter = 3;
}

enum PsCmdID {
    PS_PULL_DENSE_TABLE                      = 0;
    PS_PUSH_DENSE_TABLE                      = 1;
    PS_PULL_SPARSE_TABLE                     = 2;
    PS_PUSH_SPARSE_TABLE                     = 3;
    PS_SHRINK_TABLE                          = 4;
    PS_SAVE_ONE_TABLE                        = 5;
    PS_SAVE_ALL_TABLE                        = 6;
    PS_LOAD_ONE_TABLE                        = 7;
    PS_LOAD_ALL_TABLE                        = 8;
    PS_CLEAR_ONE_TABLE                       = 9;
    PS_CLEAR_ALL_TABLE                       = 10;
    PS_PUSH_DENSE_PARAM                      = 11;
    PS_STOP_SERVER                           = 12;
    PS_SAVE_ONE_CACHE_TABLE                  = 13;
    PS_GET_CACHE_THRESHOLD                   = 14;
    PS_CACHE_SHUFFLE                         = 15;
    PS_COPY_TABLE                            = 16;
    PS_COPY_TABLE_BY_FEASIGN                 = 17;
    PS_PULL_SPARSE_TABLE_WITH_DEPENDENCY     = 18;
    PS_PUSH_SPARSE_TABLE_WITH_DEPENDENCY     = 19;
    PS_PRINT_TABLE_STAT                      = 20;
    PS_SAVE_ONE_TABLE_PREFIX                 = 21;
    PS_SAVE_MEM_CACHE_TABLE                  = 22;
    //pserver2pserver cmd start from 100
    PS_S2S_MSG                               = 101;
    //local_client2local_client cmd start from 200
    PS_C2C_PULL_SPARSE_TABLE                 = 201;
}

message PsRequestMessage {
    required uint32 cmd_id = 1;
    optional uint32 table_id = 2;
    repeated bytes params = 3;
    optional int32 client_id = 4;
    optional bytes data = 5;
};

message SparseSGDRuleParameter {
    optional double learning_rate = 1 [default = 0.05];
    optional double initial_g2sum = 2 [default = 3.0];
    optional double initial_range = 3 [default = 0.0001];
    repeated float weight_bounds = 4;
}

message SparseCommonSGDRuleParameter {
    optional string name = 1;
    optional SparseNaiveSGDRuleParameter naive = 2;
    optional SparseAdagradSGDRuleParameter adagrad = 3;
    optional SparseAdamSGDParameter adam = 4;
}

message SparseNaiveSGDRuleParameter {
    optional double learning_rate = 1 [default = 0.05];
    optional double initial_range = 2 [default = 0.0001];
    repeated float weight_bounds = 3;
}

message SparseAdagradSGDRuleParameter {
    optional double learning_rate = 1 [default = 0.05];
    optional double initial_g2sum = 2 [default = 3.0];
    optional double initial_range = 3 [default = 0.0001];
    repeated float weight_bounds = 4;
}

message SparseAdamSGDParameter {
    optional double learning_rate = 1 [default = 0.001];
    optional double initial_range = 2 [default = 0.0001];
    optional double beta1_decay_rate = 3 [default = 0.9];
    optional double beta2_decay_rate = 4 [default = 0.999];
    optional double ada_epsilon = 5 [default = 1e-08];
    repeated float weight_bounds = 6;
}

message DenseSGDRuleParameter {
    optional string name = 1;
    optional AdamSGDParameter adam = 2;
    optional NaiveSGDParameter naive = 3;
    optional SummarySGDParameter summary = 4;
    optional MovingAverageRuleParameter moving_average = 5;
}

message AdamSGDParameter {
    optional double learning_rate = 1 [default = 5e-06];         // \u5B66\u4E60\u7387
    optional double avg_decay_rate = 2 [default = 0.999993];     // avg_weight\u7684\u8870\u51CF\u7CFB\u6570
    optional double ada_decay_rate = 3 [default = 0.9999];
    optional double ada_epsilon = 4 [default = 1e-08];
    optional double mom_decay_rate = 5 [default = 0.99];
}

message NaiveSGDParameter {
    optional double learning_rate = 1 [default = 0.0002];
    optional double avg_decay_rate = 2;
}

message SummarySGDParameter {
    optional double summary_decay_rate = 1 [default = 0.999999];     // \u6743\u91CD\u7684\u8870\u51CF\u7CFB\u6570
}

message MovingAverageRuleParameter {
    optional double momentum = 1;
}

message PsResponseMessage {
    required int32 err_code = 1 [default = 0];
    required string err_msg = 2 [default = ""];
    optional bytes data = 3;
};

service PsService {
    rpc service(PsRequestMessage) returns (PsResponseMessage);
};

message FsClientParameter {
    enum FsApiType {
         HDFS = 0;
         AFS = 1;
    }
    optional FsApiType fs_type      = 1 [default = HDFS];
    optional string    uri          = 2;                  //such as afs://tianqi.afs.baidu.com:9902
    optional string    user         = 3;                  //user_name to access fs
    optional string    passwd       = 4;                  //password
    optional int32     buffer_size  = 5;                  //buffer for read/write
    optional string hadoop_bin = 51;
    optional string afs_conf = 101;
}