ps.proto

// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto2";
package paddle.distributed;
option cc_generic_services = true;
option cc_enable_arenas = true;

message FsClientParameter {
  enum FsApiType {
    HDFS = 0;
    AFS = 1;
  }
  optional FsApiType fs_type = 1 [ default = HDFS ];
  optional string uri = 2;        // such as afs://xxx.afs.com:9902
  optional string user = 3;       // user_name to access fs
  optional string passwd = 4;     // password
  optional int32 buffer_size = 5; // buffer for read/write
  optional string hadoop_bin = 51;
  optional string afs_conf = 101;
}

message PSParameter {
  optional string worker_class = 1;
  optional string server_class = 2;
  optional string instance_class = 3;
  optional string init_gflags = 4 [ default = "" ];
  optional WorkerParameter worker_param = 101;
  optional ServerParameter server_param = 102;
  repeated DownpourTrainerParameter trainer_param = 301;
  optional FsClientParameter fs_client_param = 501;
}

message WorkerParameter {
  optional DownpourWorkerParameter downpour_worker_param = 1;
}

message DownpourWorkerParameter {
  repeated TableParameter downpour_table_param = 1;
}

message DownpourServerParameter {
  repeated TableParameter downpour_table_param = 1;
  optional ServerServiceParameter service_param = 2;
}

message ServerParameter {
  optional DownpourServerParameter downpour_server_param = 1;
}

message DownpourTrainerParameter {
  repeated DenseTableParameter dense_table = 1;
  repeated SparseTableParameter sparse_table = 2;
  optional int32 push_sparse_per_batch = 3;
  optional int32 push_dense_per_batch = 4;
  repeated string skip_op = 5;
  repeated ProgramConfig program_config = 6;
}

message DenseTableParameter {
  optional int32 table_id = 1;
  repeated string dense_variable_name = 2;
  repeated string dense_gradient_variable_name = 3;
  optional int32 fea_dim = 4;
}

message SparseTableParameter {
  optional int32 table_id = 1;
  optional int32 feature_dim = 2;
  repeated string slot_key = 3;
  repeated string slot_value = 4;
  repeated string slot_gradient = 5;
}

message ServerServiceParameter {
  optional string server_class = 1 [ default = "BrpcPsServer" ];
  optional string client_class = 2 [ default = "BrpcPsClient" ];
  optional string service_class = 3 [ default = "PsService" ];
  optional uint32 start_server_port = 4
      [ default = 0 ]; // will find a avaliable port from it
  optional uint32 server_thread_num = 5 [ default = 12 ];
}

message ProgramConfig {
  required string program_id = 1;
  repeated int32 push_sparse_table_id = 2;
  repeated int32 push_dense_table_id = 3;
  repeated int32 pull_sparse_table_id = 4;
  repeated int32 pull_dense_table_id = 5;
}

enum TableType {
  PS_SPARSE_TABLE = 0;
  PS_DENSE_TABLE = 1;
  PS_OTHER_TABLE = 2;
}

message TableParameter {
  optional uint64 table_id = 1;
  optional string table_class = 2;
  optional uint64 shard_num = 3 [ default = 1000 ];
  optional TableAccessorParameter accessor = 4;
  optional TensorAccessorParameter tensor = 5;
  optional CommonAccessorParameter common = 6;
  optional TableType type = 7;
  optional bool compress_in_save = 8 [ default = false ];
}

message TableAccessorParameter {
  optional string accessor_class = 1;
  optional uint32 fea_dim = 4 [ default = 11 ];
  optional uint32 embedx_dim = 5 [ default = 8 ];
  optional uint32 embedx_threshold = 6 [ default = 10 ];
  repeated TableAccessorSaveParameter table_accessor_save_param = 8;
}

message TensorAccessorParameter {
  optional string tensor_class = 1;
  optional uint32 fea_dim = 2;
  optional uint32 emb_dim = 3;
  optional string param = 4;
  optional string grad = 5;
  optional string common_block_map = 6;
}

message CommonAccessorParameter {
  optional string name = 1;
  optional string table_name = 2;
  repeated string attributes = 3;
  repeated string params = 4;
  repeated uint32 dims = 5;
  repeated string initializers = 6;
  optional int32 trainer_num = 7;
  optional bool sync = 8;
}

message TableAccessorSaveParameter {
  optional uint32 param = 1;
  optional string converter = 2;
  optional string deconverter = 3;
}