Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
a18792752712
Paddle
提交
98dda08a
P
Paddle
项目概览
a18792752712
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
98dda08a
编写于
3月 29, 2019
作者:
D
dongdaxiang
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
fix pull sparse slow problem
test=develop
上级
93c3c7f9
变更
5
隐藏空白更改
内联
并排
Showing
5 changed file
with
30 addition
and
6 deletion
+30
-6
paddle/fluid/framework/async_executor.cc
paddle/fluid/framework/async_executor.cc
+3
-0
paddle/fluid/framework/downpour_worker.cc
paddle/fluid/framework/downpour_worker.cc
+11
-0
paddle/fluid/framework/fleet/fleet_wrapper.cc
paddle/fluid/framework/fleet/fleet_wrapper.cc
+14
-4
paddle/fluid/framework/fleet/fleet_wrapper.h
paddle/fluid/framework/fleet/fleet_wrapper.h
+1
-1
paddle/fluid/platform/lodtensor_printer.cc
paddle/fluid/platform/lodtensor_printer.cc
+1
-1
未找到文件。
paddle/fluid/framework/async_executor.cc
浏览文件 @
98dda08a
...
...
@@ -153,11 +153,14 @@ void AsyncExecutor::RunFromFile(const ProgramDesc& main_program,
for
(
auto
&
th
:
threads
)
{
th
.
join
();
}
// TODO(guru4elephant): we don't need this
/*
#ifdef PADDLE_WITH_PSLIB
if (mode == "mpi") {
_pull_dense_thread->stop();
}
#endif
*/
VLOG
(
3
)
<<
"start to run from files in async_executor"
;
VLOG
(
3
)
<<
"Drop current scope kids"
;
root_scope_
->
DropKids
();
...
...
paddle/fluid/framework/downpour_worker.cc
浏览文件 @
98dda08a
...
...
@@ -210,6 +210,7 @@ void DownpourWorker::TrainFilesWithProfiler() {
timeline
.
Pause
();
pull_sparse_time
+=
timeline
.
ElapsedSec
();
total_time
+=
timeline
.
ElapsedSec
();
timeline
.
Start
();
CollectLabelInfo
(
i
);
timeline
.
Pause
();
collect_label_time
+=
timeline
.
ElapsedSec
();
...
...
@@ -336,6 +337,16 @@ void DownpourWorker::TrainFilesWithProfiler() {
}
fprintf
(
stderr
,
"mean read time: %fs
\n
"
,
read_time
/
batch_cnt
);
fprintf
(
stderr
,
"IO percent: %f
\n
"
,
read_time
/
total_time
*
100
);
fprintf
(
stderr
,
"pull sparse time percent: %f
\n
"
,
pull_sparse_time
/
total_time
*
100
);
fprintf
(
stderr
,
"collect label time percent: %f
\n
"
,
collect_label_time
/
total_time
*
100
);
fprintf
(
stderr
,
"fill sparse time percent: %f
\n
"
,
fill_sparse_time
/
total_time
*
100
);
fprintf
(
stderr
,
"push sparse time percent: %f
\n
"
,
push_sparse_time
/
total_time
*
100
);
fprintf
(
stderr
,
"push dense time percent: %f
\n
"
,
push_dense_time
/
total_time
*
100
);
fprintf
(
stderr
,
"%6.2f instances/s
\n
"
,
total_inst
/
total_time
);
}
}
...
...
paddle/fluid/framework/fleet/fleet_wrapper.cc
浏览文件 @
98dda08a
...
...
@@ -142,6 +142,7 @@ void FleetWrapper::PullSparseVarsSync(
}
fea_keys
->
push_back
(
static_cast
<
uint64_t
>
(
ids
[
i
]));
}
/*
fea_values->resize(fea_keys->size() + 1);
for (auto& t : *fea_values) {
t.resize(fea_value_dim);
...
...
@@ -150,10 +151,19 @@ void FleetWrapper::PullSparseVarsSync(
for (auto& t : *fea_values) {
pull_result_ptr.push_back(t.data());
}
auto
status
=
pslib_ptr_
->
_worker_ptr
->
pull_sparse
(
pull_result_ptr
.
data
(),
table_id
,
fea_keys
->
data
(),
fea_keys
->
size
());
pull_sparse_status
.
push_back
(
std
::
move
(
status
));
*/
}
fea_values
->
resize
(
fea_keys
->
size
()
+
1
);
for
(
auto
&
t
:
*
fea_values
)
{
t
.
resize
(
fea_value_dim
);
}
std
::
vector
<
float
*>
pull_result_ptr
;
for
(
auto
&
t
:
*
fea_values
)
{
pull_result_ptr
.
push_back
(
t
.
data
());
}
auto
status
=
pslib_ptr_
->
_worker_ptr
->
pull_sparse
(
pull_result_ptr
.
data
(),
table_id
,
fea_keys
->
data
(),
fea_keys
->
size
());
pull_sparse_status
.
push_back
(
std
::
move
(
status
));
for
(
auto
&
t
:
pull_sparse_status
)
{
t
.
wait
();
auto
status
=
t
.
get
();
...
...
@@ -207,7 +217,7 @@ void FleetWrapper::PullDenseVarsSync(
}
void
FleetWrapper
::
PushDenseParamSync
(
const
ProgramDesc
&
program
,
const
uint64_t
table_id
,
const
Scope
&
scope
,
const
uint64_t
table_id
,
const
std
::
vector
<
std
::
string
>&
var_names
)
{
#ifdef PADDLE_WITH_PSLIB
auto
place
=
platform
::
CPUPlace
();
...
...
paddle/fluid/framework/fleet/fleet_wrapper.h
浏览文件 @
98dda08a
...
...
@@ -73,7 +73,7 @@ class FleetWrapper {
const
std
::
vector
<
std
::
string
>&
var_names
,
std
::
vector
<::
std
::
future
<
int32_t
>>*
pull_dense_status
);
void
PushDenseParamSync
(
const
ProgramDesc
&
program
,
const
uint64_t
table_id
,
void
PushDenseParamSync
(
const
Scope
&
scope
,
const
uint64_t
table_id
,
const
std
::
vector
<
std
::
string
>&
var_names
);
// Push dense variables to server in async mode
...
...
paddle/fluid/platform/lodtensor_printer.cc
浏览文件 @
98dda08a
...
...
@@ -41,7 +41,7 @@ void print_lod_tensor(const std::string& var_name,
void
PrintVar
(
framework
::
Scope
*
scope
,
const
std
::
string
&
var_name
,
const
std
::
string
&
print_info
)
{
framework
::
Variable
*
var
=
scope
->
FindVar
(
var_name
);
if
(
tenso
r
==
nullptr
)
{
if
(
va
r
==
nullptr
)
{
VLOG
(
1
)
<<
"Variable Name "
<<
var_name
<<
" does not exist in your scope"
;
return
;
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录