Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
TonyTonyFun
Paddle
提交
0a7cb901
P
Paddle
项目概览
TonyTonyFun
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
0a7cb901
编写于
1月 10, 2022
作者:
Y
yaoxuefeng
提交者:
GitHub
1月 10, 2022
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
add retry on pull dense sync (#38793)
上级
1f8fe035
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
53 addition
and
4 deletion
+53
-4
paddle/fluid/framework/fleet/fleet_wrapper.cc
paddle/fluid/framework/fleet/fleet_wrapper.cc
+53
-3
paddle/fluid/framework/fleet/heter_context.h
paddle/fluid/framework/fleet/heter_context.h
+0
-1
未找到文件。
paddle/fluid/framework/fleet/fleet_wrapper.cc
浏览文件 @
0a7cb901
...
...
@@ -632,6 +632,7 @@ void FleetWrapper::PullSparseToTensorSync(const uint64_t table_id, int fea_dim,
if
(
ret
!=
0
)
{
LOG
(
ERROR
)
<<
"fleet pull sparse failed, status["
<<
ret
<<
"]"
;
sleep
(
sleep_seconds_before_fail_exit_
);
exit
(
-
1
);
}
#else
for
(
size_t
index
=
0
;
index
<
inputs
->
size
();
++
index
)
{
...
...
@@ -685,9 +686,36 @@ void FleetWrapper::PullDenseVarsSync(
paddle
::
ps
::
Region
reg
(
w
,
tensor
->
numel
());
regions
.
emplace_back
(
std
::
move
(
reg
));
}
auto
status
=
pslib_ptr_
->
_worker_ptr
->
pull_dense
(
regions
.
data
(),
regions
.
size
(),
tid
);
status
.
wait
();
int32_t
status
=
-
1
;
int32_t
cnt
=
0
;
while
(
true
)
{
auto
tt
=
pslib_ptr_
->
_worker_ptr
->
pull_dense
(
regions
.
data
(),
regions
.
size
(),
tid
);
bool
flag
=
true
;
tt
.
wait
();
try
{
status
=
tt
.
get
();
}
catch
(
const
std
::
future_error
&
e
)
{
VLOG
(
0
)
<<
"Caught a future_error with code"
<<
e
.
code
()
<<
", Message:"
<<
e
.
what
();
}
if
(
status
!=
0
)
{
VLOG
(
0
)
<<
"fleet pull dense sync failed, status["
<<
status
<<
"]"
;
sleep
(
sleep_seconds_before_fail_exit_
);
flag
=
false
;
cnt
++
;
}
if
(
cnt
>
3
)
{
VLOG
(
0
)
<<
"fleet pull dense sync failed, retry 3 times"
;
exit
(
-
1
);
}
if
(
flag
)
{
break
;
}
}
#endif
}
...
...
@@ -1248,6 +1276,7 @@ void FleetWrapper::LoadModelOneTable(const uint64_t table_id,
if
(
ret
.
get
()
!=
0
)
{
LOG
(
ERROR
)
<<
"load model of table id: "
<<
table_id
<<
", from path: "
<<
path
<<
" failed"
;
exit
(
-
1
);
}
#else
VLOG
(
0
)
<<
"FleetWrapper::LoadModel does nothing when no pslib"
;
...
...
@@ -1263,6 +1292,7 @@ void FleetWrapper::LoadWithWhitelist(const uint64_t table_id,
if
(
ret
.
get
()
!=
0
)
{
LOG
(
ERROR
)
<<
"load model of table id: "
<<
table_id
<<
", from path: "
<<
path
<<
" failed"
;
exit
(
-
1
);
}
#else
VLOG
(
0
)
<<
"FleetWrapper::LoadWhitelist does nothing when no pslib"
;
...
...
@@ -1311,6 +1341,7 @@ void FleetWrapper::SaveModelOneTable(const uint64_t table_id,
if
(
ret
.
get
()
!=
0
)
{
LOG
(
ERROR
)
<<
"save model of table id: "
<<
table_id
<<
", to path: "
<<
path
<<
" failed"
;
exit
(
-
1
);
}
#else
VLOG
(
0
)
<<
"FleetWrapper::SaveModelOneTable does nothing when no pslib"
;
...
...
@@ -1328,6 +1359,7 @@ void FleetWrapper::SaveModelOneTablePrefix(const uint64_t table_id,
if
(
ret
.
get
()
!=
0
)
{
LOG
(
ERROR
)
<<
"save model (with prefix) of table id: "
<<
table_id
<<
", to path: "
<<
path
<<
" failed"
;
exit
(
-
1
);
}
#else
VLOG
(
0
)
<<
"FleetWrapper::SaveModelOneTablePrefix does nothing when no pslib"
;
...
...
@@ -1351,6 +1383,7 @@ void FleetWrapper::SetDate(const uint64_t table_id, const std::string& date) {
ret
.
wait
();
if
(
ret
.
get
()
!=
0
)
{
LOG
(
ERROR
)
<<
"setdate : "
<<
date
<<
" failed"
;
exit
(
-
1
);
}
#else
VLOG
(
0
)
<<
"FleetWrapper::SetDate does nothing when no pslib-gpu"
;
...
...
@@ -1463,6 +1496,11 @@ void FleetWrapper::ShrinkSparseTable(int table_id) {
#ifdef PADDLE_WITH_PSLIB
auto
ret
=
pslib_ptr_
->
_worker_ptr
->
shrink
(
table_id
);
ret
.
wait
();
int32_t
err_code
=
ret
.
get
();
if
(
err_code
==
-
1
)
{
LOG
(
ERROR
)
<<
"Shrink Sparse Table failed"
;
exit
(
-
1
);
}
#else
VLOG
(
0
)
<<
"FleetWrapper::ShrinkSparseTable does nothing when no pslib"
;
#endif
...
...
@@ -1472,6 +1510,10 @@ void FleetWrapper::ClearModel() {
#ifdef PADDLE_WITH_PSLIB
auto
ret
=
pslib_ptr_
->
_worker_ptr
->
clear
();
ret
.
wait
();
int32_t
err_code
=
ret
.
get
();
if
(
err_code
==
-
1
)
{
LOG
(
ERROR
)
<<
"Clear Model failed"
;
}
#else
VLOG
(
0
)
<<
"FleetWrapper::ClearModel does nothing when no pslib"
;
#endif
...
...
@@ -1481,6 +1523,10 @@ void FleetWrapper::ClearOneTable(const uint64_t table_id) {
#ifdef PADDLE_WITH_PSLIB
auto
ret
=
pslib_ptr_
->
_worker_ptr
->
clear
(
table_id
);
ret
.
wait
();
int32_t
err_code
=
ret
.
get
();
if
(
err_code
==
-
1
)
{
LOG
(
ERROR
)
<<
"Clear One Table failed table_id: "
<<
table_id
;
}
#else
VLOG
(
0
)
<<
"FleetWrapper::ClearOneTable does nothing when no pslib"
;
#endif
...
...
@@ -1541,6 +1587,10 @@ void FleetWrapper::ClientFlush() {
#ifdef PADDLE_WITH_PSLIB
auto
ret
=
pslib_ptr_
->
_worker_ptr
->
flush
();
ret
.
wait
();
int32_t
err_code
=
ret
.
get
();
if
(
err_code
==
-
1
)
{
LOG
(
ERROR
)
<<
"Client Flush failed"
;
}
#else
VLOG
(
0
)
<<
"FleetWrapper::ServerFlush does nothing when no pslib"
;
#endif
...
...
paddle/fluid/framework/fleet/heter_context.h
浏览文件 @
0a7cb901
...
...
@@ -235,7 +235,6 @@ class HeterContext {
}
VLOG
(
3
)
<<
"heter_context unique keys with dynamic mf dimention"
;
}
for
(
std
::
thread
&
t
:
threads
)
{
t
.
join
();
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录