Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
afeixing77
Paddle
提交
575ae7c6
P
Paddle
项目概览
afeixing77
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
575ae7c6
编写于
12月 07, 2018
作者:
H
heqiaozhi
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
refine pslib inferface & fix some bugs
上级
d3ca359e
变更
8
隐藏空白更改
内联
并排
Showing
8 changed file
with
100 addition
and
65 deletion
+100
-65
CMakeLists.txt
CMakeLists.txt
+2
-2
paddle/fluid/framework/async_executor.cc
paddle/fluid/framework/async_executor.cc
+24
-5
paddle/fluid/framework/async_executor.h
paddle/fluid/framework/async_executor.h
+4
-2
paddle/fluid/framework/executor_thread_worker.cc
paddle/fluid/framework/executor_thread_worker.cc
+17
-14
paddle/fluid/pybind/async_executor_py.cc
paddle/fluid/pybind/async_executor_py.cc
+3
-1
python/paddle/fluid/async_executor.py
python/paddle/fluid/async_executor.py
+10
-1
python/paddle/fluid/distributed/downpour.py
python/paddle/fluid/distributed/downpour.py
+1
-1
python/paddle/fluid/distributed/ps_pb2.py
python/paddle/fluid/distributed/ps_pb2.py
+39
-39
未找到文件。
CMakeLists.txt
浏览文件 @
575ae7c6
...
@@ -217,7 +217,7 @@ include(cupti)
...
@@ -217,7 +217,7 @@ include(cupti)
include
(
external/gzstream
)
include
(
external/gzstream
)
endif
(
NOT WIN32
)
endif
(
NOT WIN32
)
include
(
external/libmct
)
include
(
external/libmct
)
#
include(external/pslib_brpc)
include
(
external/pslib_brpc
)
include
(
external/pslib
)
include
(
external/pslib
)
if
(
WITH_DISTRIBUTE
)
if
(
WITH_DISTRIBUTE
)
...
@@ -280,7 +280,7 @@ set(EXTERNAL_LIBS
...
@@ -280,7 +280,7 @@ set(EXTERNAL_LIBS
zlib
zlib
${
PYTHON_LIBRARIES
}
${
PYTHON_LIBRARIES
}
pslib
pslib
#
pslib_brpc
pslib_brpc
libmct
libmct
)
)
...
...
paddle/fluid/framework/async_executor.cc
浏览文件 @
575ae7c6
...
@@ -65,18 +65,35 @@ void PrepareReaders(std::vector<std::shared_ptr<DataFeed>>& readers, // NOLINT
...
@@ -65,18 +65,35 @@ void PrepareReaders(std::vector<std::shared_ptr<DataFeed>>& readers, // NOLINT
readers
[
0
]
->
SetFileList
(
filelist
);
readers
[
0
]
->
SetFileList
(
filelist
);
}
}
void
AsyncExecutor
::
ConfigPslib
(
const
std
::
string
&
dist_desc
,
std
::
vector
<
uint64_t
>&
host_sign_list
,
int
node_num
,
int
index
)
{
void
AsyncExecutor
::
InitServer
(
const
std
::
string
&
dist_desc
,
int
index
)
{
_pslib_ptr
=
std
::
shared_ptr
<
paddle
::
distributed
::
PSlib
>
(
new
paddle
::
distributed
::
PSlib
());
_pslib_ptr
=
std
::
shared_ptr
<
paddle
::
distributed
::
PSlib
>
(
new
paddle
::
distributed
::
PSlib
());
_pslib_ptr
->
init_and_config
(
dist_desc
,
host_sign_list
,
node_num
,
index
);
//TODO done
_pslib_ptr
->
init_server
(
dist_desc
,
index
);
//TODO done
InitParamConfig
();
}
}
void
AsyncExecutor
::
StartServer
()
{
void
AsyncExecutor
::
InitWorker
(
const
std
::
string
&
dist_desc
,
std
::
vector
<
uint64_t
>&
host_sign_list
,
int
node_num
,
int
index
)
{
_pslib_ptr
=
std
::
shared_ptr
<
paddle
::
distributed
::
PSlib
>
(
new
paddle
::
distributed
::
PSlib
());
_pslib_ptr
->
init_worker
(
dist_desc
,
host_sign_list
.
data
(),
node_num
,
index
);
//TODO done
InitParamConfig
();
InitParamConfig
();
_pslib_ptr
->
run_server
();
}
uint64_t
AsyncExecutor
::
StartServer
()
{
return
_pslib_ptr
->
run_server
();
}
void
AsyncExecutor
::
GatherServers
(
std
::
vector
<
uint64_t
>&
host_sign_list
,
int
node_num
)
{
_pslib_ptr
->
gather_servers
(
host_sign_list
.
data
(),
node_num
);
}
}
void
AsyncExecutor
::
InitParamConfig
()
{
void
AsyncExecutor
::
InitParamConfig
()
{
_param_config
.
fea_dim
=
_pslib_ptr
->
get_param
()
->
trainer_param
().
sparse_table
(
0
).
feature_dim
();
//TODO
for
(
int
i
=
0
;
i
<
_pslib_ptr
->
get_param
()
->
server_param
().
downpour_server_param
().
downpour_table_param_size
();
++
i
)
{
if
(
_pslib_ptr
->
get_param
()
->
server_param
().
downpour_server_param
().
downpour_table_param
(
i
).
table_class
().
find
(
"SparseTable"
)
!=
-
1
)
{
_param_config
.
fea_dim
=
_pslib_ptr
->
get_param
()
->
server_param
().
downpour_server_param
().
downpour_table_param
(
i
).
accessor
().
fea_dim
();
//TODO
break
;
}
}
_param_config
.
slot_dim
=
_param_config
.
fea_dim
-
2
;
//TODO
_param_config
.
slot_dim
=
_param_config
.
fea_dim
-
2
;
//TODO
_param_config
.
tmp_push_dense_wait_times
=
(
int32_t
)(
_pslib_ptr
->
get_param
()
->
trainer_param
().
pull_dense_per_batch
());
_param_config
.
tmp_push_dense_wait_times
=
(
int32_t
)(
_pslib_ptr
->
get_param
()
->
trainer_param
().
pull_dense_per_batch
());
_param_config
.
tmp_push_sparse_wait_times
=
(
int32_t
)(
_pslib_ptr
->
get_param
()
->
trainer_param
().
push_dense_per_batch
());
_param_config
.
tmp_push_sparse_wait_times
=
(
int32_t
)(
_pslib_ptr
->
get_param
()
->
trainer_param
().
push_dense_per_batch
());
...
@@ -176,6 +193,7 @@ void AsyncExecutor::PrepareDenseThread() {
...
@@ -176,6 +193,7 @@ void AsyncExecutor::PrepareDenseThread() {
param
.
dense_params
=
&
_param_config
.
dense_variable_name
;
param
.
dense_params
=
&
_param_config
.
dense_variable_name
;
_pull_dense_thread
=
std
::
shared_ptr
<
DensePullThread
>
(
new
DensePullThread
(
param
));
_pull_dense_thread
=
std
::
shared_ptr
<
DensePullThread
>
(
new
DensePullThread
(
param
));
_pull_dense_thread
->
start
();
}
}
...
@@ -238,6 +256,7 @@ void AsyncExecutor::RunFromFile(const ProgramDesc& main_program,
...
@@ -238,6 +256,7 @@ void AsyncExecutor::RunFromFile(const ProgramDesc& main_program,
fetch_var_names
,
root_scope_
,
thidx
,
debug
);
fetch_var_names
,
root_scope_
,
thidx
,
debug
);
}
}
// start executing ops in multiple threads
// start executing ops in multiple threads
for
(
int
thidx
=
0
;
thidx
<
actual_thread_num
;
++
thidx
)
{
for
(
int
thidx
=
0
;
thidx
<
actual_thread_num
;
++
thidx
)
{
threads
.
push_back
(
threads
.
push_back
(
...
...
paddle/fluid/framework/async_executor.h
浏览文件 @
575ae7c6
...
@@ -63,9 +63,11 @@ class AsyncExecutor {
...
@@ -63,9 +63,11 @@ class AsyncExecutor {
const
std
::
vector
<
std
::
string
>&
fetch_names
,
const
std
::
vector
<
std
::
string
>&
fetch_names
,
const
bool
debug
=
false
);
const
bool
debug
=
false
);
//void ConfigPslib(const char* dist_desc, uint64_t* host_sign_list, int node_num, int index);
//void ConfigPslib(const char* dist_desc, uint64_t* host_sign_list, int node_num, int index);
void
ConfigPslib
(
const
std
::
string
&
dist_desc
,
std
::
vector
<
uint64_t
>&
host_sign_list
,
int
node_num
,
int
index
);
void
InitServer
(
const
std
::
string
&
dist_desc
,
int
index
);
void
InitWorker
(
const
std
::
string
&
dist_desc
,
std
::
vector
<
uint64_t
>&
host_sign_list
,
int
node_num
,
int
index
);
//void ConfigWorker() {}
//void ConfigWorker() {}
void
StartServer
();
uint64_t
StartServer
();
void
GatherServers
(
std
::
vector
<
uint64_t
>&
host_sign_list
,
int
node_num
);
void
InitModel
();
void
InitModel
();
void
SaveModel
(
const
std
::
string
&
path
);
void
SaveModel
(
const
std
::
string
&
path
);
void
InitParamConfig
();
void
InitParamConfig
();
...
...
paddle/fluid/framework/executor_thread_worker.cc
浏览文件 @
575ae7c6
...
@@ -345,9 +345,12 @@ void AsyncExecutorThreadWorker::TrainOneNetwork() {
...
@@ -345,9 +345,12 @@ void AsyncExecutorThreadWorker::TrainOneNetwork() {
if
(
op
->
Type
().
find
(
"sgd"
)
!=
std
::
string
::
npos
)
{
if
(
op
->
Type
().
find
(
"sgd"
)
!=
std
::
string
::
npos
)
{
continue
;
continue
;
}
}
if
(
op
->
Type
().
find
(
"lookup_table"
)
!=
std
::
string
::
npos
||
op
->
Type
().
find
(
"lookup_table_grad"
)
!=
std
::
string
::
npos
)
{
continue
;
}
op
->
Run
(
*
thread_scope_
,
place_
);
op
->
Run
(
*
thread_scope_
,
place_
);
}
}
UpdateParams
();
UpdateParams
();
}
}
...
@@ -416,8 +419,8 @@ void AsyncExecutorThreadWorker::UpdateParams() {
...
@@ -416,8 +419,8 @@ void AsyncExecutorThreadWorker::UpdateParams() {
for
(
auto
i
:
_param_config
->
dense_table_id
)
{
for
(
auto
i
:
_param_config
->
dense_table_id
)
{
PushDense
(
i
);
PushDense
(
i
);
}
}
int32_t
tmp_push_dense_wait_times
=
_param_config
->
tmp_push_dense_wait_times
;
//TODO
int32_t
tmp_push_dense_wait_times
=
-
1
;
//
_param_config->tmp_push_dense_wait_times; //TODO
int32_t
tmp_push_sparse_wait_times
=
_param_config
->
tmp_push_sparse_wait_times
;
//TODO
int32_t
tmp_push_sparse_wait_times
=
-
1
;
//
_param_config->tmp_push_sparse_wait_times; //TODO
static
uint32_t
push_dense_wait_times
=
static_cast
<
uint32_t
>
(
tmp_push_dense_wait_times
);
static
uint32_t
push_dense_wait_times
=
static_cast
<
uint32_t
>
(
tmp_push_dense_wait_times
);
static
uint32_t
push_sparse_wait_times
=
static_cast
<
uint32_t
>
(
tmp_push_sparse_wait_times
);
static
uint32_t
push_sparse_wait_times
=
static_cast
<
uint32_t
>
(
tmp_push_sparse_wait_times
);
...
@@ -430,7 +433,6 @@ void AsyncExecutorThreadWorker::UpdateParams() {
...
@@ -430,7 +433,6 @@ void AsyncExecutorThreadWorker::UpdateParams() {
if
(
tmp_push_dense_wait_times
==
-
1
)
{
if
(
tmp_push_dense_wait_times
==
-
1
)
{
_push_dense_status
.
resize
(
0
);
_push_dense_status
.
resize
(
0
);
}
}
if
(
_push_sparse_status
.
size
()
>=
push_sparse_wait_times
)
{
if
(
_push_sparse_status
.
size
()
>=
push_sparse_wait_times
)
{
for
(
auto
&
t
:
_push_sparse_status
)
{
for
(
auto
&
t
:
_push_sparse_status
)
{
t
.
wait
();
t
.
wait
();
...
@@ -440,7 +442,6 @@ void AsyncExecutorThreadWorker::UpdateParams() {
...
@@ -440,7 +442,6 @@ void AsyncExecutorThreadWorker::UpdateParams() {
if
(
tmp_push_sparse_wait_times
==
-
1
)
{
if
(
tmp_push_sparse_wait_times
==
-
1
)
{
_push_sparse_status
.
resize
(
0
);
_push_sparse_status
.
resize
(
0
);
}
}
//for (auto dense_table_id : GlobalConfig::instance().dense_table_id) {//TODO
//for (auto dense_table_id : GlobalConfig::instance().dense_table_id) {//TODO
for
(
auto
dense_table_id
:
_param_config
->
dense_table_id
)
{
for
(
auto
dense_table_id
:
_param_config
->
dense_table_id
)
{
_pull_dense_thread
->
increase_thread_version
(
thread_id_
,
dense_table_id
);
_pull_dense_thread
->
increase_thread_version
(
thread_id_
,
dense_table_id
);
...
@@ -451,8 +452,8 @@ void AsyncExecutorThreadWorker::UpdateParams() {
...
@@ -451,8 +452,8 @@ void AsyncExecutorThreadWorker::UpdateParams() {
void
AsyncExecutorThreadWorker
::
PushDense
(
int
table_id
)
{
void
AsyncExecutorThreadWorker
::
PushDense
(
int
table_id
)
{
std
::
vector
<
paddle
::
ps
::
Region
>
regions
;
std
::
vector
<
paddle
::
ps
::
Region
>
regions
;
//auto& variables = GlobalConfig::instance().dense_gradient_variable_name[table_id];
//auto& variables = GlobalConfig::instance().dense_gradient_variable_name[table_id];
std
::
vector
<
std
::
string
>
variables
;
//
std::vector<std::string> variables;
for
(
auto
&
t
:
variables
)
{
for
(
auto
&
t
:
_param_config
->
dense_gradient_variable_name
[
table_id
]
)
{
Variable
*
var
=
thread_scope_
->
FindVar
(
t
);
Variable
*
var
=
thread_scope_
->
FindVar
(
t
);
CHECK
(
var
!=
nullptr
)
<<
"var["
<<
t
<<
"] not found"
;
CHECK
(
var
!=
nullptr
)
<<
"var["
<<
t
<<
"] not found"
;
LoDTensor
*
tensor
=
var
->
GetMutable
<
LoDTensor
>
();
LoDTensor
*
tensor
=
var
->
GetMutable
<
LoDTensor
>
();
...
@@ -469,7 +470,6 @@ void AsyncExecutorThreadWorker::PushDense(int table_id) {
...
@@ -469,7 +470,6 @@ void AsyncExecutorThreadWorker::PushDense(int table_id) {
void
AsyncExecutorThreadWorker
::
PullSparse
(
int
table_id
)
{
void
AsyncExecutorThreadWorker
::
PullSparse
(
int
table_id
)
{
auto
&
features
=
_features
[
table_id
];
auto
&
features
=
_features
[
table_id
];
auto
&
feature_value
=
_feature_value
[
table_id
];
auto
&
feature_value
=
_feature_value
[
table_id
];
auto
fea_dim
=
_param_config
->
fea_dim
;
//TODO
auto
fea_dim
=
_param_config
->
fea_dim
;
//TODO
...
@@ -477,7 +477,6 @@ void AsyncExecutorThreadWorker::PullSparse(int table_id) {
...
@@ -477,7 +477,6 @@ void AsyncExecutorThreadWorker::PullSparse(int table_id) {
features
.
clear
();
features
.
clear
();
features
.
resize
(
0
);
features
.
resize
(
0
);
features
.
reserve
(
MAX_FEASIGN_NUM
);
features
.
reserve
(
MAX_FEASIGN_NUM
);
const
std
::
vector
<
std
::
string
>&
feed_vec
=
thread_reader_
->
GetUseSlotAlias
();
const
std
::
vector
<
std
::
string
>&
feed_vec
=
thread_reader_
->
GetUseSlotAlias
();
// slot_idx = 0 is label TODO
// slot_idx = 0 is label TODO
for
(
auto
slot_idx
=
1u
;
slot_idx
<
feed_vec
.
size
();
++
slot_idx
)
{
for
(
auto
slot_idx
=
1u
;
slot_idx
<
feed_vec
.
size
();
++
slot_idx
)
{
...
@@ -493,14 +492,14 @@ void AsyncExecutorThreadWorker::PullSparse(int table_id) {
...
@@ -493,14 +492,14 @@ void AsyncExecutorThreadWorker::PullSparse(int table_id) {
features
.
push_back
(
static_cast
<
uint64_t
>
(
ids
[
i
]));
features
.
push_back
(
static_cast
<
uint64_t
>
(
ids
[
i
]));
}
}
}
}
check_pull_push_memory
(
features
,
feature_value
,
fea_dim
);
check_pull_push_memory
(
features
,
feature_value
,
fea_dim
);
std
::
vector
<
float
*>
pull_feature_value
;
std
::
vector
<
float
*>
pull_feature_value
;
for
(
auto
i
=
0u
;
i
<
features
.
size
();
++
i
)
{
for
(
auto
i
=
0u
;
i
<
features
.
size
();
++
i
)
{
pull_feature_value
.
push_back
(
feature_value
[
i
].
data
());
pull_feature_value
.
push_back
(
feature_value
[
i
].
data
());
}
}
for
(
int
i
=
0
;
i
<
features
.
size
();
++
i
)
{
}
auto
status
=
_pslib_ptr
->
_worker_ptr
->
pull_sparse
(
auto
status
=
_pslib_ptr
->
_worker_ptr
->
pull_sparse
(
pull_feature_value
.
data
(),
table_id
,
features
.
data
(),
features
.
size
());
pull_feature_value
.
data
(),
table_id
,
features
.
data
(),
features
.
size
());
_pull_sparse_status
.
push_back
(
std
::
move
(
status
));
_pull_sparse_status
.
push_back
(
std
::
move
(
status
));
...
@@ -532,10 +531,15 @@ void AsyncExecutorThreadWorker::FillSparse(int table_id) {
...
@@ -532,10 +531,15 @@ void AsyncExecutorThreadWorker::FillSparse(int table_id) {
LoDTensor
*
tensor
=
var
->
GetMutable
<
LoDTensor
>
();
LoDTensor
*
tensor
=
var
->
GetMutable
<
LoDTensor
>
();
int64_t
*
ids
=
tensor
->
data
<
int64_t
>
();
int64_t
*
ids
=
tensor
->
data
<
int64_t
>
();
int
len
=
tensor
->
numel
();
int
len
=
tensor
->
numel
();
Variable
*
var_emb
=
thread_scope_
->
FindVar
(
_param_config
->
slot_input_vec
[
table_id
][
slot_idx
-
1
]);
Variable
*
var_emb
=
thread_scope_
->
FindVar
(
_param_config
->
slot_input_vec
[
table_id
][
slot_idx
-
1
]);
LoDTensor
*
tensor_emb
=
var_emb
->
GetMutable
<
LoDTensor
>
();
LoDTensor
*
tensor_emb
=
var_emb
->
GetMutable
<
LoDTensor
>
();
float
*
ptr
=
tensor_emb
->
data
<
float
>
();
float
*
ptr
=
tensor_emb
->
mutable_data
<
float
>
({
len
,
slot_dim
},
platform
::
CPUPlace
());
memset
(
ptr
,
0
,
sizeof
(
float
)
*
len
*
slot_dim
);
auto
&
tensor_lod
=
tensor
->
lod
()[
0
];
LoD
data_lod
{
tensor_lod
};
tensor_emb
->
set_lod
(
data_lod
);
//float* ptr = tensor_emb->data<float>();
for
(
auto
index
=
0u
;
index
<
len
;
++
index
){
for
(
auto
index
=
0u
;
index
<
len
;
++
index
){
//if (_current_train_job.use_cvm_feature()) {
//if (_current_train_job.use_cvm_feature()) {
...
@@ -576,7 +580,6 @@ void AsyncExecutorThreadWorker::PushSparse(int table_id) {
...
@@ -576,7 +580,6 @@ void AsyncExecutorThreadWorker::PushSparse(int table_id) {
//}
//}
const
std
::
vector
<
std
::
string
>&
feed_vec
=
thread_reader_
->
GetUseSlotAlias
();
const
std
::
vector
<
std
::
string
>&
feed_vec
=
thread_reader_
->
GetUseSlotAlias
();
// slot_idx = 0 is label TODO
// slot_idx = 0 is label TODO
for
(
auto
slot_idx
=
1u
;
slot_idx
<
feed_vec
.
size
();
++
slot_idx
)
{
for
(
auto
slot_idx
=
1u
;
slot_idx
<
feed_vec
.
size
();
++
slot_idx
)
{
if
(
_param_config
->
slot_alias_to_table
[
feed_vec
[
slot_idx
]]
!=
table_id
)
{
if
(
_param_config
->
slot_alias_to_table
[
feed_vec
[
slot_idx
]]
!=
table_id
)
{
...
...
paddle/fluid/pybind/async_executor_py.cc
浏览文件 @
575ae7c6
...
@@ -48,8 +48,10 @@ void BindAsyncExecutor(py::module* m) {
...
@@ -48,8 +48,10 @@ void BindAsyncExecutor(py::module* m) {
new
framework
::
AsyncExecutor
(
scope
,
place
));
new
framework
::
AsyncExecutor
(
scope
,
place
));
}))
}))
.
def
(
"run_from_files"
,
&
framework
::
AsyncExecutor
::
RunFromFile
)
.
def
(
"run_from_files"
,
&
framework
::
AsyncExecutor
::
RunFromFile
)
.
def
(
"config_pslib"
,
&
framework
::
AsyncExecutor
::
ConfigPslib
)
.
def
(
"init_server"
,
&
framework
::
AsyncExecutor
::
InitServer
)
.
def
(
"init_worker"
,
&
framework
::
AsyncExecutor
::
InitWorker
)
.
def
(
"start_server"
,
&
framework
::
AsyncExecutor
::
StartServer
)
.
def
(
"start_server"
,
&
framework
::
AsyncExecutor
::
StartServer
)
.
def
(
"gather_servers"
,
&
framework
::
AsyncExecutor
::
GatherServers
)
.
def
(
"init_model"
,
&
framework
::
AsyncExecutor
::
InitModel
)
.
def
(
"init_model"
,
&
framework
::
AsyncExecutor
::
InitModel
)
.
def
(
"save_model"
,
&
framework
::
AsyncExecutor
::
SaveModel
);
.
def
(
"save_model"
,
&
framework
::
AsyncExecutor
::
SaveModel
);
}
// end BindAsyncExecutor
}
// end BindAsyncExecutor
...
...
python/paddle/fluid/async_executor.py
浏览文件 @
575ae7c6
...
@@ -158,8 +158,17 @@ class AsyncExecutor(object):
...
@@ -158,8 +158,17 @@ class AsyncExecutor(object):
return
return
def
init_server
(
self
,
filename
,
index
):
self
.
executor
.
init_server
(
filename
,
index
)
def
init_worker
(
self
,
filename
,
ips
,
nodes_cnt
,
index
):
self
.
executor
.
init_worker
(
filename
,
ips
,
nodes_cnt
,
index
)
def
start_server
(
self
):
def
start_server
(
self
):
self
.
executor
.
start_server
()
return
self
.
executor
.
start_server
()
def
gather_servers
(
self
,
ips
,
nodes_cnt
):
self
.
executor
.
gather_servers
(
ips
,
nodes_cnt
)
def
init_model
(
self
):
def
init_model
(
self
):
self
.
executor
.
init_model
()
self
.
executor
.
init_model
()
...
...
python/paddle/fluid/distributed/downpour.py
浏览文件 @
575ae7c6
...
@@ -56,7 +56,7 @@ class DownpourSGD(object):
...
@@ -56,7 +56,7 @@ class DownpourSGD(object):
params_grads
[
0
],
params_grads
[
1
])
params_grads
[
0
],
params_grads
[
1
])
ps_param
=
pslib
.
PSParameter
()
ps_param
=
pslib
.
PSParameter
()
ps_param
.
server_param
.
CopyFrom
(
server
.
get_desc
())
ps_param
.
server_param
.
CopyFrom
(
server
.
get_desc
())
ps_param
.
work
er_param
.
CopyFrom
(
worker
.
get_desc
())
ps_param
.
train
er_param
.
CopyFrom
(
worker
.
get_desc
())
# Todo(guru4elephant): figure out how to support more sparse parameters
# Todo(guru4elephant): figure out how to support more sparse parameters
# currently only support lookup_table
# currently only support lookup_table
worker_skipped_ops
=
[
"lookup_table"
,
"lookup_table_grad"
]
worker_skipped_ops
=
[
"lookup_table"
,
"lookup_table_grad"
]
...
...
python/paddle/fluid/distributed/ps_pb2.py
浏览文件 @
575ae7c6
...
@@ -20,7 +20,7 @@ DESCRIPTOR = _descriptor.FileDescriptor(
...
@@ -20,7 +20,7 @@ DESCRIPTOR = _descriptor.FileDescriptor(
name
=
'ps.proto'
,
name
=
'ps.proto'
,
package
=
'paddle'
,
package
=
'paddle'
,
syntax
=
'proto2'
,
syntax
=
'proto2'
,
serialized_pb
=
_b
(
'
\n\x08
ps.proto
\x12\x06
paddle
\"\x9e\x02\n\x0b
PSParameter
\x12\x14\n\x0c
worker_class
\x18\x01
\x01
(
\t\x12\x14\n\x0c
server_class
\x18\x02
\x01
(
\t\x12\x16\n\x0e
instance_class
\x18\x03
\x01
(
\t\x12
-
\n\x0c
worker_param
\x18\x65
\x01
(
\x0b\x32\x17
.paddle.WorkerParameter
\x12
-
\n\x0c
server_param
\x18\x66
\x01
(
\x0b\x32\x17
.paddle.ServerParameter
\x12\x38\n\r
trainer_param
\x18\xad\x02
\x01
(
\x0b\x32
.paddle.DownpourTrainerParameter
\x12\x33\n\x0f\x66
s_client_param
\x18\xf5\x03
\x01
(
\x0b\x32\x19
.paddle.FsClientParameter
\"
Q
\n\x0f
WorkerParameter
\x12
>
\n\x15\x64
ownpour_worker_param
\x18\x01
\x01
(
\x0b\x32\x1f
.paddle.DownpourWorkerParameter
\"
Q
\n\x0f
ServerParameter
\x12
>
\n\x15\x64
ownpour_server_param
\x18\x01
\x01
(
\x0b\x32\x1f
.paddle.DownpourServerParameter
\"
O
\n\x17\x44
ownpourWorkerParameter
\x12\x34\n\x14\x64
ownpour_table_param
\x18\x01
\x03
(
\x0b\x32\x16
.paddle.TableParameter
\"\xbc\x01\n\x18\x44
ownpourTrainerParameter
\x12\x30\n\x0b\x64\x65
nse_table
\x18\x01
\x03
(
\x0b\x32\x1b
.paddle.DenseTableParameter
\x12\x32\n\x0c
sparse_table
\x18\x02
\x03
(
\x0b\x32\x1c
.paddle.SparseTableParameter
\x12\x1c\n\x14
pull_dense_per_batch
\x18\x03
\x01
(
\x05\x12\x1c\n\x14
push_dense_per_batch
\x18\x04
\x01
(
\x05\"
{
\n\x13\x44\x65
nseTableParameter
\x12\x10\n\x08
table_id
\x18\x01
\x01
(
\x05\x12\x1b\n\x13\x64\x65
nse_variable_name
\x18\x02
\x03
(
\t\x12
$
\n\x1c\x64\x65
nse_gradient_variable_name
\x18\x03
\x03
(
\t\x12\x0f\n\x07\x66\x65\x61
_dim
\x18\x04
\x01
(
\x05\"
z
\n\x14
SparseTableParameter
\x12\x10\n\x08
table_id
\x18\x01
\x01
(
\x05\x12\x13\n\x0b\x66\x65\x61
ture_dim
\x18\x02
\x01
(
\x05\x12\x10\n\x08
slot_key
\x18\x03
\x03
(
\t\x12\x12\n\n
slot_value
\x18\x04
\x03
(
\t\x12\x15\n\r
slot_gradient
\x18\x05
\x03
(
\t\"\x86\x01\n\x17\x44
ownpourServerParameter
\x12\x34\n\x14\x64
ownpour_table_param
\x18\x01
\x03
(
\x0b\x32\x16
.paddle.TableParameter
\x12\x35\n\r
service_param
\x18\x02
\x01
(
\x0b\x32\x1e
.paddle.ServerServiceParameter
\"\x
91\x01\n\x16
ServerServiceParameter
\x12\x14\n\x0c
server_class
\x18\x01
\x01
(
\t\x12\x14\n\x0c\x63
lient_class
\x18\x02
\x01
(
\t\x12\x15\n\r
service_class
\x18\x03
\x01
(
\t\x12\x19\n\x11
start_server_port
\x18\x04
\x01
(
\r\x12\x19\n\x11
server_thread_num
\x18\x05
\x01
(
\r
\"\xbf\x01\n\x0e
TableParameter
\x12\x10\n\x08
table_id
\x18\x01
\x01
(
\x04\x12\x13\n\x0b
table_class
\x18\x02
\x01
(
\t\x12\x12\n\n
shared_num
\x18\x03
\x01
(
\x04\x12\x30\n\x08\x61\x63\x63\x65
ssor
\x18\x04
\x01
(
\x0b\x32\x1e
.paddle.TableAccessorParameter
\x12\x1f\n\x04
type
\x18\x05
\x01
(
\x0e\x32\x11
.paddle.TableType
\x12\x1f\n\x10\x63
ompress_in_save
\x18\x06
\x01
(
\x08
:
\x05\x66\x61
lse
\"\xf1\x02\n\x16
TableAccessorParameter
\x12\x16\n\x0e\x61\x63\x63\x65
ssor_class
\x18\x01
\x01
(
\t\x12\x38\n\x10
sparse_sgd_param
\x18\x02
\x01
(
\x0b\x32\x1e
.paddle.SparseSGDRuleParameter
\x12\x36\n\x0f\x64\x65
nse_sgd_param
\x18\x03
\x01
(
\x0b\x32\x1d
.paddle.DenseSGDRuleParameter
\x12\x0f\n\x07\x66\x65\x61
_dim
\x18\x04
\x01
(
\r\x12\x12\n\n
embedx_dim
\x18\x05
\x01
(
\r\x12\x18\n\x10\x65
mbedx_threshold
\x18\x06
\x01
(
\r\x12
G
\n\x17\x64
ownpour_accessor_param
\x18\x07
\x01
(
\x0b\x32
&.paddle.DownpourTableAccessorParameter
\x12\x45\n\x19
table_accessor_save_param
\x18\x08
\x03
(
\x0b\x32\"
.paddle.TableAccessorSaveParameter
\"\xce\x01\n\x1e\x44
ownpourTableAccessorParameter
\x12\x14\n\x0c
nonclk_coeff
\x18\x01
\x01
(
\x02\x12\x13\n\x0b\x63
lick_coeff
\x18\x02
\x01
(
\x02\x12\x16\n\x0e\x62\x61
se_threshold
\x18\x03
\x01
(
\x02\x12\x17\n\x0f\x64\x65
lta_threshold
\x18\x04
\x01
(
\x02\x12\x17\n\x0f\x64\x65
lta_keep_days
\x18\x05
\x01
(
\x02\x12\x1d\n\x15
show_click_decay_rate
\x18\x06
\x01
(
\x02\x12\x18\n\x10\x64\x65
lete_threshold
\x18\x07
\x01
(
\x02\"
S
\n\x1a
TableAccessorSaveParameter
\x12\r\n\x05
param
\x18\x01
\x01
(
\r\x12\x11\n\t
converter
\x18\x02
\x01
(
\t\x12\x13\n\x0b\x64\x65\x63
onverter
\x18\x03
\x01
(
\t\"
e
\n\x10
PsRequestMessage
\x12\x0e\n\x06\x63
md_id
\x18\x01
\x02
(
\r\x12\x10\n\x08
table_id
\x18\x02
\x01
(
\r\x12\x0e\n\x06
params
\x18\x03
\x03
(
\x0c\x12\x11\n\t
client_id
\x18\x04
\x01
(
\x05\x12\x0c\n\x04\x64\x61
ta
\x18\x05
\x01
(
\x0c\"
w
\n\x16
SparseSGDRuleParameter
\x12\x15\n\r
learning_rate
\x18\x01
\x01
(
\x01\x12\x15\n\r
initial_g2sum
\x18\x02
\x01
(
\x01\x12\x18\n\r
initial_range
\x18\x03
\x01
(
\x01
:
\x01\x30\x12\x15\n\r
weight_bounds
\x18\x04
\x03
(
\x02\"\xe1\x01\n\x15\x44\x65
nseSGDRuleParameter
\x12\x0c\n\x04
name
\x18\x01
\x01
(
\t\x12
&
\n\x04\x61\x64\x61
m
\x18\x02
\x01
(
\x0b\x32\x18
.paddle.AdamSGDParameter
\x12
(
\n\x05
naive
\x18\x03
\x01
(
\x0b\x32\x19
.paddle.NaiveSGDParameter
\x12
,
\n\x07
summary
\x18\x04
\x01
(
\x0b\x32\x1b
.paddle.SummarySGDParameter
\x12
:
\n\x0e
moving_average
\x18\x05
\x01
(
\x0b\x32\"
.paddle.MovingAverageRuleParameter
\"\x86\x01\n\x10\x41\x64\x61
mSGDParameter
\x12\x15\n\r
learning_rate
\x18\x01
\x01
(
\x01\x12\x16\n\x0e\x61
vg_decay_rate
\x18\x02
\x01
(
\x01\x12\x16\n\x0e\x61\x64\x61
_decay_rate
\x18\x03
\x01
(
\x01\x12\x13\n\x0b\x61\x64\x61
_epsilon
\x18\x04
\x01
(
\x01\x12\x16\n\x0e
mom_decay_rate
\x18\x05
\x01
(
\x01\"
B
\n\x11
NaiveSGDParameter
\x12\x15\n\r
learning_rate
\x18\x01
\x01
(
\x01\x12\x16\n\x0e\x61
vg_decay_rate
\x18\x02
\x01
(
\x01\"
;
\n\x13
SummarySGDParameter
\x12
$
\n\x12
summary_decay_rate
\x18\x01
\x01
(
\x01
:
\x08\x30
.999999
\"
.
\n\x1a
MovingAverageRuleParameter
\x12\x10\n\x08
momentum
\x18\x01
\x01
(
\x01\"
I
\n\x11
PsResponseMessage
\x12\x13\n\x08\x65
rr_code
\x18\x01
\x02
(
\x05
:
\x01\x30\x12\x11\n\x07\x65
rr_msg
\x18\x02
\x02
(
\t
:
\x00\x12\x0c\n\x04\x64\x61
ta
\x18\x03
\x01
(
\x0c\"\xd5\x01\n\x11\x46
sClientParameter
\x12
:
\n\x07\x66
s_type
\x18\x01
\x01
(
\x0e\x32
#.paddle.FsClientParameter.FsApiType:
\x04
HDFS
\x12\x0b\n\x03
uri
\x18\x02
\x01
(
\t\x12\x0c\n\x04
user
\x18\x03
\x01
(
\t\x12\x0e\n\x06
passwd
\x18\x04
\x01
(
\t\x12\x13\n\x0b\x62
uffer_size
\x18\x05
\x01
(
\x05\x12\x12\n\n
hadoop_bin
\x18\x33
\x01
(
\t\x12\x10\n\x08\x61\x66
s_conf
\x18\x65
\x01
(
\t\"\x1e\n\t
FsApiType
\x12\x08\n\x04
HDFS
\x10\x00\x12\x07\n\x03\x41\x46
S
\x10\x01
*4
\n\t
TableType
\x12\x13\n\x0f
PS_SPARSE_TABLE
\x10\x00\x12\x12\n\x0e
PS_DENSE_TABLE
\x10\x01
*
\xbd\x02\n\x07
PsCmdID
\x12\x17\n\x13
PS_PULL_DENSE_TABLE
\x10\x00\x12\x17\n\x13
PS_PUSH_DENSE_TABLE
\x10\x01\x12\x18\n\x14
PS_PULL_SPARSE_TABLE
\x10\x02\x12\x18\n\x14
PS_PUSH_SPARSE_TABLE
\x10\x03\x12\x13\n\x0f
PS_SHRINK_TABLE
\x10\x04\x12\x15\n\x11
PS_SAVE_ONE_TABLE
\x10\x05\x12\x15\n\x11
PS_SAVE_ALL_TABLE
\x10\x06\x12\x15\n\x11
PS_LOAD_ONE_TABLE
\x10\x07\x12\x15\n\x11
PS_LOAD_ALL_TABLE
\x10\x08\x12\x16\n\x12
PS_CLEAR_ONE_TABLE
\x10\t\x12\x16\n\x12
PS_CLEAR_ALL_TABLE
\x10\n\x12\x17\n\x13
PS_PUSH_DENSE_PARAM
\x10\x0b\x12\x12\n\x0e
PS_STOP_SERVER
\x10\x0c\x32
K
\n\t
PsService
\x12
>
\n\x07
service
\x12\x18
.paddle.PsRequestMessage
\x1a\x19
.paddle.PsResponseMessageB
\x03\x80\x01\x01
'
)
serialized_pb
=
_b
(
'
\n\x08
ps.proto
\x12\x06
paddle
\"\x9e\x02\n\x0b
PSParameter
\x12\x14\n\x0c
worker_class
\x18\x01
\x01
(
\t\x12\x14\n\x0c
server_class
\x18\x02
\x01
(
\t\x12\x16\n\x0e
instance_class
\x18\x03
\x01
(
\t\x12
-
\n\x0c
worker_param
\x18\x65
\x01
(
\x0b\x32\x17
.paddle.WorkerParameter
\x12
-
\n\x0c
server_param
\x18\x66
\x01
(
\x0b\x32\x17
.paddle.ServerParameter
\x12\x38\n\r
trainer_param
\x18\xad\x02
\x01
(
\x0b\x32
.paddle.DownpourTrainerParameter
\x12\x33\n\x0f\x66
s_client_param
\x18\xf5\x03
\x01
(
\x0b\x32\x19
.paddle.FsClientParameter
\"
Q
\n\x0f
WorkerParameter
\x12
>
\n\x15\x64
ownpour_worker_param
\x18\x01
\x01
(
\x0b\x32\x1f
.paddle.DownpourWorkerParameter
\"
Q
\n\x0f
ServerParameter
\x12
>
\n\x15\x64
ownpour_server_param
\x18\x01
\x01
(
\x0b\x32\x1f
.paddle.DownpourServerParameter
\"
O
\n\x17\x44
ownpourWorkerParameter
\x12\x34\n\x14\x64
ownpour_table_param
\x18\x01
\x03
(
\x0b\x32\x16
.paddle.TableParameter
\"\xbc\x01\n\x18\x44
ownpourTrainerParameter
\x12\x30\n\x0b\x64\x65
nse_table
\x18\x01
\x03
(
\x0b\x32\x1b
.paddle.DenseTableParameter
\x12\x32\n\x0c
sparse_table
\x18\x02
\x03
(
\x0b\x32\x1c
.paddle.SparseTableParameter
\x12\x1c\n\x14
pull_dense_per_batch
\x18\x03
\x01
(
\x05\x12\x1c\n\x14
push_dense_per_batch
\x18\x04
\x01
(
\x05\"
{
\n\x13\x44\x65
nseTableParameter
\x12\x10\n\x08
table_id
\x18\x01
\x01
(
\x05\x12\x1b\n\x13\x64\x65
nse_variable_name
\x18\x02
\x03
(
\t\x12
$
\n\x1c\x64\x65
nse_gradient_variable_name
\x18\x03
\x03
(
\t\x12\x0f\n\x07\x66\x65\x61
_dim
\x18\x04
\x01
(
\x05\"
z
\n\x14
SparseTableParameter
\x12\x10\n\x08
table_id
\x18\x01
\x01
(
\x05\x12\x13\n\x0b\x66\x65\x61
ture_dim
\x18\x02
\x01
(
\x05\x12\x10\n\x08
slot_key
\x18\x03
\x03
(
\t\x12\x12\n\n
slot_value
\x18\x04
\x03
(
\t\x12\x15\n\r
slot_gradient
\x18\x05
\x03
(
\t\"\x86\x01\n\x17\x44
ownpourServerParameter
\x12\x34\n\x14\x64
ownpour_table_param
\x18\x01
\x03
(
\x0b\x32\x16
.paddle.TableParameter
\x12\x35\n\r
service_param
\x18\x02
\x01
(
\x0b\x32\x1e
.paddle.ServerServiceParameter
\"\x
d1\x01\n\x16
ServerServiceParameter
\x12
(
\n\x0c
server_class
\x18\x01
\x01
(
\t
:
\x12\x41\x62\x61\x63
usBrpcPsServer
\x12
(
\n\x0c\x63
lient_class
\x18\x02
\x01
(
\t
:
\x12\x41\x62\x61\x63
usBrpcPsClient
\x12
&
\n\r
service_class
\x18\x03
\x01
(
\t
:
\x0f\x41\x62\x61\x63
usPsService
\x12\x1c\n\x11
start_server_port
\x18\x04
\x01
(
\r
:
\x01\x30\x12\x1d\n\x11
server_thread_num
\x18\x05
\x01
(
\r
:
\x02\x31\x32
\"\xbf\x01\n\x0e
TableParameter
\x12\x10\n\x08
table_id
\x18\x01
\x01
(
\x04\x12\x13\n\x0b
table_class
\x18\x02
\x01
(
\t\x12\x12\n\n
shared_num
\x18\x03
\x01
(
\x04\x12\x30\n\x08\x61\x63\x63\x65
ssor
\x18\x04
\x01
(
\x0b\x32\x1e
.paddle.TableAccessorParameter
\x12\x1f\n\x04
type
\x18\x05
\x01
(
\x0e\x32\x11
.paddle.TableType
\x12\x1f\n\x10\x63
ompress_in_save
\x18\x06
\x01
(
\x08
:
\x05\x66\x61
lse
\"\xf1\x02\n\x16
TableAccessorParameter
\x12\x16\n\x0e\x61\x63\x63\x65
ssor_class
\x18\x01
\x01
(
\t\x12\x38\n\x10
sparse_sgd_param
\x18\x02
\x01
(
\x0b\x32\x1e
.paddle.SparseSGDRuleParameter
\x12\x36\n\x0f\x64\x65
nse_sgd_param
\x18\x03
\x01
(
\x0b\x32\x1d
.paddle.DenseSGDRuleParameter
\x12\x0f\n\x07\x66\x65\x61
_dim
\x18\x04
\x01
(
\r\x12\x12\n\n
embedx_dim
\x18\x05
\x01
(
\r\x12\x18\n\x10\x65
mbedx_threshold
\x18\x06
\x01
(
\r\x12
G
\n\x17\x64
ownpour_accessor_param
\x18\x07
\x01
(
\x0b\x32
&.paddle.DownpourTableAccessorParameter
\x12\x45\n\x19
table_accessor_save_param
\x18\x08
\x03
(
\x0b\x32\"
.paddle.TableAccessorSaveParameter
\"\xce\x01\n\x1e\x44
ownpourTableAccessorParameter
\x12\x14\n\x0c
nonclk_coeff
\x18\x01
\x01
(
\x02\x12\x13\n\x0b\x63
lick_coeff
\x18\x02
\x01
(
\x02\x12\x16\n\x0e\x62\x61
se_threshold
\x18\x03
\x01
(
\x02\x12\x17\n\x0f\x64\x65
lta_threshold
\x18\x04
\x01
(
\x02\x12\x17\n\x0f\x64\x65
lta_keep_days
\x18\x05
\x01
(
\x02\x12\x1d\n\x15
show_click_decay_rate
\x18\x06
\x01
(
\x02\x12\x18\n\x10\x64\x65
lete_threshold
\x18\x07
\x01
(
\x02\"
S
\n\x1a
TableAccessorSaveParameter
\x12\r\n\x05
param
\x18\x01
\x01
(
\r\x12\x11\n\t
converter
\x18\x02
\x01
(
\t\x12\x13\n\x0b\x64\x65\x63
onverter
\x18\x03
\x01
(
\t\"
e
\n\x10
PsRequestMessage
\x12\x0e\n\x06\x63
md_id
\x18\x01
\x02
(
\r\x12\x10\n\x08
table_id
\x18\x02
\x01
(
\r\x12\x0e\n\x06
params
\x18\x03
\x03
(
\x0c\x12\x11\n\t
client_id
\x18\x04
\x01
(
\x05\x12\x0c\n\x04\x64\x61
ta
\x18\x05
\x01
(
\x0c\"
w
\n\x16
SparseSGDRuleParameter
\x12\x15\n\r
learning_rate
\x18\x01
\x01
(
\x01\x12\x15\n\r
initial_g2sum
\x18\x02
\x01
(
\x01\x12\x18\n\r
initial_range
\x18\x03
\x01
(
\x01
:
\x01\x30\x12\x15\n\r
weight_bounds
\x18\x04
\x03
(
\x02\"\xe1\x01\n\x15\x44\x65
nseSGDRuleParameter
\x12\x0c\n\x04
name
\x18\x01
\x01
(
\t\x12
&
\n\x04\x61\x64\x61
m
\x18\x02
\x01
(
\x0b\x32\x18
.paddle.AdamSGDParameter
\x12
(
\n\x05
naive
\x18\x03
\x01
(
\x0b\x32\x19
.paddle.NaiveSGDParameter
\x12
,
\n\x07
summary
\x18\x04
\x01
(
\x0b\x32\x1b
.paddle.SummarySGDParameter
\x12
:
\n\x0e
moving_average
\x18\x05
\x01
(
\x0b\x32\"
.paddle.MovingAverageRuleParameter
\"\x86\x01\n\x10\x41\x64\x61
mSGDParameter
\x12\x15\n\r
learning_rate
\x18\x01
\x01
(
\x01\x12\x16\n\x0e\x61
vg_decay_rate
\x18\x02
\x01
(
\x01\x12\x16\n\x0e\x61\x64\x61
_decay_rate
\x18\x03
\x01
(
\x01\x12\x13\n\x0b\x61\x64\x61
_epsilon
\x18\x04
\x01
(
\x01\x12\x16\n\x0e
mom_decay_rate
\x18\x05
\x01
(
\x01\"
B
\n\x11
NaiveSGDParameter
\x12\x15\n\r
learning_rate
\x18\x01
\x01
(
\x01\x12\x16\n\x0e\x61
vg_decay_rate
\x18\x02
\x01
(
\x01\"
;
\n\x13
SummarySGDParameter
\x12
$
\n\x12
summary_decay_rate
\x18\x01
\x01
(
\x01
:
\x08\x30
.999999
\"
.
\n\x1a
MovingAverageRuleParameter
\x12\x10\n\x08
momentum
\x18\x01
\x01
(
\x01\"
I
\n\x11
PsResponseMessage
\x12\x13\n\x08\x65
rr_code
\x18\x01
\x02
(
\x05
:
\x01\x30\x12\x11\n\x07\x65
rr_msg
\x18\x02
\x02
(
\t
:
\x00\x12\x0c\n\x04\x64\x61
ta
\x18\x03
\x01
(
\x0c\"\xd5\x01\n\x11\x46
sClientParameter
\x12
:
\n\x07\x66
s_type
\x18\x01
\x01
(
\x0e\x32
#.paddle.FsClientParameter.FsApiType:
\x04
HDFS
\x12\x0b\n\x03
uri
\x18\x02
\x01
(
\t\x12\x0c\n\x04
user
\x18\x03
\x01
(
\t\x12\x0e\n\x06
passwd
\x18\x04
\x01
(
\t\x12\x13\n\x0b\x62
uffer_size
\x18\x05
\x01
(
\x05\x12\x12\n\n
hadoop_bin
\x18\x33
\x01
(
\t\x12\x10\n\x08\x61\x66
s_conf
\x18\x65
\x01
(
\t\"\x1e\n\t
FsApiType
\x12\x08\n\x04
HDFS
\x10\x00\x12\x07\n\x03\x41\x46
S
\x10\x01
*4
\n\t
TableType
\x12\x13\n\x0f
PS_SPARSE_TABLE
\x10\x00\x12\x12\n\x0e
PS_DENSE_TABLE
\x10\x01
*
\xbd\x02\n\x07
PsCmdID
\x12\x17\n\x13
PS_PULL_DENSE_TABLE
\x10\x00\x12\x17\n\x13
PS_PUSH_DENSE_TABLE
\x10\x01\x12\x18\n\x14
PS_PULL_SPARSE_TABLE
\x10\x02\x12\x18\n\x14
PS_PUSH_SPARSE_TABLE
\x10\x03\x12\x13\n\x0f
PS_SHRINK_TABLE
\x10\x04\x12\x15\n\x11
PS_SAVE_ONE_TABLE
\x10\x05\x12\x15\n\x11
PS_SAVE_ALL_TABLE
\x10\x06\x12\x15\n\x11
PS_LOAD_ONE_TABLE
\x10\x07\x12\x15\n\x11
PS_LOAD_ALL_TABLE
\x10\x08\x12\x16\n\x12
PS_CLEAR_ONE_TABLE
\x10\t\x12\x16\n\x12
PS_CLEAR_ALL_TABLE
\x10\n\x12\x17\n\x13
PS_PUSH_DENSE_PARAM
\x10\x0b\x12\x12\n\x0e
PS_STOP_SERVER
\x10\x0c\x32
K
\n\t
PsService
\x12
>
\n\x07
service
\x12\x18
.paddle.PsRequestMessage
\x1a\x19
.paddle.PsResponseMessageB
\x03\x80\x01\x01
'
)
)
)
_sym_db
.
RegisterFileDescriptor
(
DESCRIPTOR
)
_sym_db
.
RegisterFileDescriptor
(
DESCRIPTOR
)
...
@@ -41,8 +41,8 @@ _TABLETYPE = _descriptor.EnumDescriptor(
...
@@ -41,8 +41,8 @@ _TABLETYPE = _descriptor.EnumDescriptor(
],
],
containing_type
=
None
,
containing_type
=
None
,
options
=
None
,
options
=
None
,
serialized_start
=
3
198
,
serialized_start
=
3
262
,
serialized_end
=
3
250
,
serialized_end
=
3
314
,
)
)
_sym_db
.
RegisterEnumDescriptor
(
_TABLETYPE
)
_sym_db
.
RegisterEnumDescriptor
(
_TABLETYPE
)
...
@@ -108,8 +108,8 @@ _PSCMDID = _descriptor.EnumDescriptor(
...
@@ -108,8 +108,8 @@ _PSCMDID = _descriptor.EnumDescriptor(
],
],
containing_type
=
None
,
containing_type
=
None
,
options
=
None
,
options
=
None
,
serialized_start
=
3
253
,
serialized_start
=
3
317
,
serialized_end
=
3
570
,
serialized_end
=
3
634
,
)
)
_sym_db
.
RegisterEnumDescriptor
(
_PSCMDID
)
_sym_db
.
RegisterEnumDescriptor
(
_PSCMDID
)
...
@@ -148,8 +148,8 @@ _FSCLIENTPARAMETER_FSAPITYPE = _descriptor.EnumDescriptor(
...
@@ -148,8 +148,8 @@ _FSCLIENTPARAMETER_FSAPITYPE = _descriptor.EnumDescriptor(
],
],
containing_type
=
None
,
containing_type
=
None
,
options
=
None
,
options
=
None
,
serialized_start
=
3
166
,
serialized_start
=
3
230
,
serialized_end
=
3
196
,
serialized_end
=
3
260
,
)
)
_sym_db
.
RegisterEnumDescriptor
(
_FSCLIENTPARAMETER_FSAPITYPE
)
_sym_db
.
RegisterEnumDescriptor
(
_FSCLIENTPARAMETER_FSAPITYPE
)
...
@@ -531,35 +531,35 @@ _SERVERSERVICEPARAMETER = _descriptor.Descriptor(
...
@@ -531,35 +531,35 @@ _SERVERSERVICEPARAMETER = _descriptor.Descriptor(
_descriptor
.
FieldDescriptor
(
_descriptor
.
FieldDescriptor
(
name
=
'server_class'
,
full_name
=
'paddle.ServerServiceParameter.server_class'
,
index
=
0
,
name
=
'server_class'
,
full_name
=
'paddle.ServerServiceParameter.server_class'
,
index
=
0
,
number
=
1
,
type
=
9
,
cpp_type
=
9
,
label
=
1
,
number
=
1
,
type
=
9
,
cpp_type
=
9
,
label
=
1
,
has_default_value
=
False
,
default_value
=
_b
(
"
"
).
decode
(
'utf-8'
),
has_default_value
=
True
,
default_value
=
_b
(
"AbacusBrpcPsServer
"
).
decode
(
'utf-8'
),
message_type
=
None
,
enum_type
=
None
,
containing_type
=
None
,
message_type
=
None
,
enum_type
=
None
,
containing_type
=
None
,
is_extension
=
False
,
extension_scope
=
None
,
is_extension
=
False
,
extension_scope
=
None
,
options
=
None
),
options
=
None
),
_descriptor
.
FieldDescriptor
(
_descriptor
.
FieldDescriptor
(
name
=
'client_class'
,
full_name
=
'paddle.ServerServiceParameter.client_class'
,
index
=
1
,
name
=
'client_class'
,
full_name
=
'paddle.ServerServiceParameter.client_class'
,
index
=
1
,
number
=
2
,
type
=
9
,
cpp_type
=
9
,
label
=
1
,
number
=
2
,
type
=
9
,
cpp_type
=
9
,
label
=
1
,
has_default_value
=
False
,
default_value
=
_b
(
"
"
).
decode
(
'utf-8'
),
has_default_value
=
True
,
default_value
=
_b
(
"AbacusBrpcPsClient
"
).
decode
(
'utf-8'
),
message_type
=
None
,
enum_type
=
None
,
containing_type
=
None
,
message_type
=
None
,
enum_type
=
None
,
containing_type
=
None
,
is_extension
=
False
,
extension_scope
=
None
,
is_extension
=
False
,
extension_scope
=
None
,
options
=
None
),
options
=
None
),
_descriptor
.
FieldDescriptor
(
_descriptor
.
FieldDescriptor
(
name
=
'service_class'
,
full_name
=
'paddle.ServerServiceParameter.service_class'
,
index
=
2
,
name
=
'service_class'
,
full_name
=
'paddle.ServerServiceParameter.service_class'
,
index
=
2
,
number
=
3
,
type
=
9
,
cpp_type
=
9
,
label
=
1
,
number
=
3
,
type
=
9
,
cpp_type
=
9
,
label
=
1
,
has_default_value
=
False
,
default_value
=
_b
(
"
"
).
decode
(
'utf-8'
),
has_default_value
=
True
,
default_value
=
_b
(
"AbacusPsService
"
).
decode
(
'utf-8'
),
message_type
=
None
,
enum_type
=
None
,
containing_type
=
None
,
message_type
=
None
,
enum_type
=
None
,
containing_type
=
None
,
is_extension
=
False
,
extension_scope
=
None
,
is_extension
=
False
,
extension_scope
=
None
,
options
=
None
),
options
=
None
),
_descriptor
.
FieldDescriptor
(
_descriptor
.
FieldDescriptor
(
name
=
'start_server_port'
,
full_name
=
'paddle.ServerServiceParameter.start_server_port'
,
index
=
3
,
name
=
'start_server_port'
,
full_name
=
'paddle.ServerServiceParameter.start_server_port'
,
index
=
3
,
number
=
4
,
type
=
13
,
cpp_type
=
3
,
label
=
1
,
number
=
4
,
type
=
13
,
cpp_type
=
3
,
label
=
1
,
has_default_value
=
Fals
e
,
default_value
=
0
,
has_default_value
=
Tru
e
,
default_value
=
0
,
message_type
=
None
,
enum_type
=
None
,
containing_type
=
None
,
message_type
=
None
,
enum_type
=
None
,
containing_type
=
None
,
is_extension
=
False
,
extension_scope
=
None
,
is_extension
=
False
,
extension_scope
=
None
,
options
=
None
),
options
=
None
),
_descriptor
.
FieldDescriptor
(
_descriptor
.
FieldDescriptor
(
name
=
'server_thread_num'
,
full_name
=
'paddle.ServerServiceParameter.server_thread_num'
,
index
=
4
,
name
=
'server_thread_num'
,
full_name
=
'paddle.ServerServiceParameter.server_thread_num'
,
index
=
4
,
number
=
5
,
type
=
13
,
cpp_type
=
3
,
label
=
1
,
number
=
5
,
type
=
13
,
cpp_type
=
3
,
label
=
1
,
has_default_value
=
False
,
default_value
=
0
,
has_default_value
=
True
,
default_value
=
12
,
message_type
=
None
,
enum_type
=
None
,
containing_type
=
None
,
message_type
=
None
,
enum_type
=
None
,
containing_type
=
None
,
is_extension
=
False
,
extension_scope
=
None
,
is_extension
=
False
,
extension_scope
=
None
,
options
=
None
),
options
=
None
),
...
@@ -576,7 +576,7 @@ _SERVERSERVICEPARAMETER = _descriptor.Descriptor(
...
@@ -576,7 +576,7 @@ _SERVERSERVICEPARAMETER = _descriptor.Descriptor(
oneofs
=
[
oneofs
=
[
],
],
serialized_start
=
1134
,
serialized_start
=
1134
,
serialized_end
=
1
279
,
serialized_end
=
1
343
,
)
)
...
@@ -641,8 +641,8 @@ _TABLEPARAMETER = _descriptor.Descriptor(
...
@@ -641,8 +641,8 @@ _TABLEPARAMETER = _descriptor.Descriptor(
extension_ranges
=
[],
extension_ranges
=
[],
oneofs
=
[
oneofs
=
[
],
],
serialized_start
=
1
282
,
serialized_start
=
1
346
,
serialized_end
=
1
473
,
serialized_end
=
1
537
,
)
)
...
@@ -721,8 +721,8 @@ _TABLEACCESSORPARAMETER = _descriptor.Descriptor(
...
@@ -721,8 +721,8 @@ _TABLEACCESSORPARAMETER = _descriptor.Descriptor(
extension_ranges
=
[],
extension_ranges
=
[],
oneofs
=
[
oneofs
=
[
],
],
serialized_start
=
1
476
,
serialized_start
=
1
540
,
serialized_end
=
1
845
,
serialized_end
=
1
909
,
)
)
...
@@ -794,8 +794,8 @@ _DOWNPOURTABLEACCESSORPARAMETER = _descriptor.Descriptor(
...
@@ -794,8 +794,8 @@ _DOWNPOURTABLEACCESSORPARAMETER = _descriptor.Descriptor(
extension_ranges
=
[],
extension_ranges
=
[],
oneofs
=
[
oneofs
=
[
],
],
serialized_start
=
1
848
,
serialized_start
=
1
912
,
serialized_end
=
2
054
,
serialized_end
=
2
118
,
)
)
...
@@ -839,8 +839,8 @@ _TABLEACCESSORSAVEPARAMETER = _descriptor.Descriptor(
...
@@ -839,8 +839,8 @@ _TABLEACCESSORSAVEPARAMETER = _descriptor.Descriptor(
extension_ranges
=
[],
extension_ranges
=
[],
oneofs
=
[
oneofs
=
[
],
],
serialized_start
=
2
056
,
serialized_start
=
2
120
,
serialized_end
=
2
139
,
serialized_end
=
2
203
,
)
)
...
@@ -898,8 +898,8 @@ _PSREQUESTMESSAGE = _descriptor.Descriptor(
...
@@ -898,8 +898,8 @@ _PSREQUESTMESSAGE = _descriptor.Descriptor(
extension_ranges
=
[],
extension_ranges
=
[],
oneofs
=
[
oneofs
=
[
],
],
serialized_start
=
2
141
,
serialized_start
=
2
205
,
serialized_end
=
2
242
,
serialized_end
=
2
306
,
)
)
...
@@ -950,8 +950,8 @@ _SPARSESGDRULEPARAMETER = _descriptor.Descriptor(
...
@@ -950,8 +950,8 @@ _SPARSESGDRULEPARAMETER = _descriptor.Descriptor(
extension_ranges
=
[],
extension_ranges
=
[],
oneofs
=
[
oneofs
=
[
],
],
serialized_start
=
2
244
,
serialized_start
=
2
308
,
serialized_end
=
2
363
,
serialized_end
=
2
427
,
)
)
...
@@ -1009,8 +1009,8 @@ _DENSESGDRULEPARAMETER = _descriptor.Descriptor(
...
@@ -1009,8 +1009,8 @@ _DENSESGDRULEPARAMETER = _descriptor.Descriptor(
extension_ranges
=
[],
extension_ranges
=
[],
oneofs
=
[
oneofs
=
[
],
],
serialized_start
=
2
366
,
serialized_start
=
2
430
,
serialized_end
=
2
591
,
serialized_end
=
2
655
,
)
)
...
@@ -1068,8 +1068,8 @@ _ADAMSGDPARAMETER = _descriptor.Descriptor(
...
@@ -1068,8 +1068,8 @@ _ADAMSGDPARAMETER = _descriptor.Descriptor(
extension_ranges
=
[],
extension_ranges
=
[],
oneofs
=
[
oneofs
=
[
],
],
serialized_start
=
2
594
,
serialized_start
=
2
658
,
serialized_end
=
27
28
,
serialized_end
=
27
92
,
)
)
...
@@ -1106,8 +1106,8 @@ _NAIVESGDPARAMETER = _descriptor.Descriptor(
...
@@ -1106,8 +1106,8 @@ _NAIVESGDPARAMETER = _descriptor.Descriptor(
extension_ranges
=
[],
extension_ranges
=
[],
oneofs
=
[
oneofs
=
[
],
],
serialized_start
=
27
30
,
serialized_start
=
27
94
,
serialized_end
=
2
796
,
serialized_end
=
2
860
,
)
)
...
@@ -1137,8 +1137,8 @@ _SUMMARYSGDPARAMETER = _descriptor.Descriptor(
...
@@ -1137,8 +1137,8 @@ _SUMMARYSGDPARAMETER = _descriptor.Descriptor(
extension_ranges
=
[],
extension_ranges
=
[],
oneofs
=
[
oneofs
=
[
],
],
serialized_start
=
2
798
,
serialized_start
=
2
862
,
serialized_end
=
2
857
,
serialized_end
=
2
921
,
)
)
...
@@ -1168,8 +1168,8 @@ _MOVINGAVERAGERULEPARAMETER = _descriptor.Descriptor(
...
@@ -1168,8 +1168,8 @@ _MOVINGAVERAGERULEPARAMETER = _descriptor.Descriptor(
extension_ranges
=
[],
extension_ranges
=
[],
oneofs
=
[
oneofs
=
[
],
],
serialized_start
=
2
859
,
serialized_start
=
2
923
,
serialized_end
=
29
05
,
serialized_end
=
29
69
,
)
)
...
@@ -1213,8 +1213,8 @@ _PSRESPONSEMESSAGE = _descriptor.Descriptor(
...
@@ -1213,8 +1213,8 @@ _PSRESPONSEMESSAGE = _descriptor.Descriptor(
extension_ranges
=
[],
extension_ranges
=
[],
oneofs
=
[
oneofs
=
[
],
],
serialized_start
=
29
07
,
serialized_start
=
29
71
,
serialized_end
=
2980
,
serialized_end
=
3044
,
)
)
...
@@ -1287,8 +1287,8 @@ _FSCLIENTPARAMETER = _descriptor.Descriptor(
...
@@ -1287,8 +1287,8 @@ _FSCLIENTPARAMETER = _descriptor.Descriptor(
extension_ranges
=
[],
extension_ranges
=
[],
oneofs
=
[
oneofs
=
[
],
],
serialized_start
=
2983
,
serialized_start
=
3047
,
serialized_end
=
3
196
,
serialized_end
=
3
260
,
)
)
_PSPARAMETER
.
fields_by_name
[
'worker_param'
].
message_type
=
_WORKERPARAMETER
_PSPARAMETER
.
fields_by_name
[
'worker_param'
].
message_type
=
_WORKERPARAMETER
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录