Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
兽拳
Paddle
提交
d98084e7
P
Paddle
项目概览
兽拳
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
d98084e7
编写于
4月 11, 2020
作者:
X
xujiaqi01
提交者:
GitHub
4月 11, 2020
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
add save with prefix (#23449)
* add save with prefix * test=develop
上级
588eb8e2
变更
8
显示空白变更内容
内联
并排
Showing
8 changed file
with
173 addition
and
3 deletion
+173
-3
paddle/fluid/framework/fleet/fleet_wrapper.cc
paddle/fluid/framework/fleet/fleet_wrapper.cc
+32
-0
paddle/fluid/framework/fleet/fleet_wrapper.h
paddle/fluid/framework/fleet/fleet_wrapper.h
+9
-2
paddle/fluid/framework/fleet/test_fleet.cc
paddle/fluid/framework/fleet/test_fleet.cc
+2
-0
paddle/fluid/pybind/fleet_wrapper_py.cc
paddle/fluid/pybind/fleet_wrapper_py.cc
+3
-0
python/paddle/fluid/incubate/fleet/parameter_server/pslib/__init__.py
...e/fluid/incubate/fleet/parameter_server/pslib/__init__.py
+82
-0
python/paddle/fluid/tests/unittests/test_fleet.py
python/paddle/fluid/tests/unittests/test_fleet.py
+21
-1
python/paddle/fluid/tests/unittests/test_fleet_1.py
python/paddle/fluid/tests/unittests/test_fleet_1.py
+5
-0
python/paddle/fluid/tests/unittests/test_fleet_nocvm_1.py
python/paddle/fluid/tests/unittests/test_fleet_nocvm_1.py
+19
-0
未找到文件。
paddle/fluid/framework/fleet/fleet_wrapper.cc
浏览文件 @
d98084e7
...
@@ -917,6 +917,38 @@ void FleetWrapper::SaveModel(const std::string& path, const int mode) {
...
@@ -917,6 +917,38 @@ void FleetWrapper::SaveModel(const std::string& path, const int mode) {
#endif
#endif
}
}
void
FleetWrapper
::
SaveModelOneTable
(
const
uint64_t
table_id
,
const
std
::
string
&
path
,
const
int
mode
)
{
#ifdef PADDLE_WITH_PSLIB
auto
ret
=
pslib_ptr_
->
_worker_ptr
->
save
(
table_id
,
path
,
std
::
to_string
(
mode
));
ret
.
wait
();
if
(
ret
.
get
()
!=
0
)
{
LOG
(
ERROR
)
<<
"save model of table id: "
<<
table_id
<<
", to path: "
<<
path
<<
" failed"
;
}
#else
VLOG
(
0
)
<<
"FleetWrapper::SaveModelOneTable does nothing when no pslib"
;
#endif
}
void
FleetWrapper
::
SaveModelOneTablePrefix
(
const
uint64_t
table_id
,
const
std
::
string
&
path
,
const
int
mode
,
const
std
::
string
&
prefix
)
{
#ifdef PADDLE_WITH_PSLIB
auto
ret
=
pslib_ptr_
->
_worker_ptr
->
save
(
table_id
,
path
,
std
::
to_string
(
mode
),
prefix
);
ret
.
wait
();
if
(
ret
.
get
()
!=
0
)
{
LOG
(
ERROR
)
<<
"save model (with prefix) of table id: "
<<
table_id
<<
", to path: "
<<
path
<<
" failed"
;
}
#else
VLOG
(
0
)
<<
"FleetWrapper::SaveModelOneTablePrefix does nothing when no pslib"
;
#endif
}
void
FleetWrapper
::
PrintTableStat
(
const
uint64_t
table_id
)
{
void
FleetWrapper
::
PrintTableStat
(
const
uint64_t
table_id
)
{
#ifdef PADDLE_WITH_PSLIB
#ifdef PADDLE_WITH_PSLIB
auto
ret
=
pslib_ptr_
->
_worker_ptr
->
print_table_stat
(
table_id
);
auto
ret
=
pslib_ptr_
->
_worker_ptr
->
print_table_stat
(
table_id
);
...
...
paddle/fluid/framework/fleet/fleet_wrapper.h
浏览文件 @
d98084e7
...
@@ -221,15 +221,22 @@ class FleetWrapper {
...
@@ -221,15 +221,22 @@ class FleetWrapper {
void
PrintTableStat
(
const
uint64_t
table_id
);
void
PrintTableStat
(
const
uint64_t
table_id
);
// mode = 0, load all feature
// mode = 0, load all feature
// mode = 1, l
ao
d delta feature, which means load diff
// mode = 1, l
oa
d delta feature, which means load diff
void
LoadModel
(
const
std
::
string
&
path
,
const
int
mode
);
void
LoadModel
(
const
std
::
string
&
path
,
const
int
mode
);
// mode = 0, load all feature
// mode = 0, load all feature
// mode = 1, l
ao
d delta feature, which means load diff
// mode = 1, l
oa
d delta feature, which means load diff
void
LoadModelOneTable
(
const
uint64_t
table_id
,
const
std
::
string
&
path
,
void
LoadModelOneTable
(
const
uint64_t
table_id
,
const
std
::
string
&
path
,
const
int
mode
);
const
int
mode
);
// mode = 0, save all feature
// mode = 0, save all feature
// mode = 1, save delta feature, which means save diff
// mode = 1, save delta feature, which means save diff
void
SaveModel
(
const
std
::
string
&
path
,
const
int
mode
);
void
SaveModel
(
const
std
::
string
&
path
,
const
int
mode
);
// mode = 0, save all feature
// mode = 1, save delta feature, which means save diff
void
SaveModelOneTable
(
const
uint64_t
table_id
,
const
std
::
string
&
path
,
const
int
mode
);
// save model with prefix
void
SaveModelOneTablePrefix
(
const
uint64_t
table_id
,
const
std
::
string
&
path
,
const
int
mode
,
const
std
::
string
&
prefix
);
// get save cache threshold
// get save cache threshold
double
GetCacheThreshold
(
int
table_id
);
double
GetCacheThreshold
(
int
table_id
);
// shuffle cache model between servers
// shuffle cache model between servers
...
...
paddle/fluid/framework/fleet/test_fleet.cc
浏览文件 @
d98084e7
...
@@ -61,5 +61,7 @@ TEST(TEST_FLEET, fleet_1) {
...
@@ -61,5 +61,7 @@ TEST(TEST_FLEET, fleet_1) {
#ifdef PADDLE_WITH_PSLIB
#ifdef PADDLE_WITH_PSLIB
#else
#else
fleet
->
RunServer
(
""
,
0
);
fleet
->
RunServer
(
""
,
0
);
fleet
->
SaveModelOneTable
(
0
,
""
,
0
);
fleet
->
SaveModelOneTablePrefix
(
0
,
""
,
0
,
""
);
#endif
#endif
}
}
paddle/fluid/pybind/fleet_wrapper_py.cc
浏览文件 @
d98084e7
...
@@ -78,6 +78,9 @@ void BindFleetWrapper(py::module* m) {
...
@@ -78,6 +78,9 @@ void BindFleetWrapper(py::module* m) {
&
framework
::
FleetWrapper
::
SetClient2ClientConfig
)
&
framework
::
FleetWrapper
::
SetClient2ClientConfig
)
.
def
(
"set_pull_local_thread_num"
,
.
def
(
"set_pull_local_thread_num"
,
&
framework
::
FleetWrapper
::
SetPullLocalThreadNum
)
&
framework
::
FleetWrapper
::
SetPullLocalThreadNum
)
.
def
(
"save_model_one_table"
,
&
framework
::
FleetWrapper
::
SaveModelOneTable
)
.
def
(
"save_model_one_table_with_prefix"
,
&
framework
::
FleetWrapper
::
SaveModelOneTablePrefix
)
.
def
(
"copy_table"
,
&
framework
::
FleetWrapper
::
CopyTable
)
.
def
(
"copy_table"
,
&
framework
::
FleetWrapper
::
CopyTable
)
.
def
(
"copy_table_by_feasign"
,
.
def
(
"copy_table_by_feasign"
,
&
framework
::
FleetWrapper
::
CopyTableByFeasign
);
&
framework
::
FleetWrapper
::
CopyTableByFeasign
);
...
...
python/paddle/fluid/incubate/fleet/parameter_server/pslib/__init__.py
浏览文件 @
d98084e7
...
@@ -567,6 +567,88 @@ class PSLib(Fleet):
...
@@ -567,6 +567,88 @@ class PSLib(Fleet):
model_proto_file
,
table_var_names
,
load_combine
)
model_proto_file
,
table_var_names
,
load_combine
)
self
.
_role_maker
.
_barrier_worker
()
self
.
_role_maker
.
_barrier_worker
()
def
load_model
(
self
,
model_dir
=
None
,
**
kwargs
):
"""
load pslib model, there are at least 4 modes, these modes are the same
in load one table/save model/save one table:
0: load checkpoint model
1: load delta model (delta means diff, it's usually for online predict)
2: load base model (base model filters some feasigns in checkpoint, it's
usually for online predict)
3: load batch model (do some statistic works in checkpoint, such as
calculate unseen days of each feasign)
Args:
model_dir(str): if you use hdfs, model_dir should starts with
'hdfs:', otherwise means local dir
kwargs(dict): user-defined properties.
mode(int): the modes illustrated above, default 0
Examples:
.. code-block:: python
fleet.load_model("afs:/user/path/")
"""
mode
=
kwargs
.
get
(
"mode"
,
0
)
self
.
_role_maker
.
_barrier_worker
()
if
self
.
_role_maker
.
is_first_worker
():
self
.
_fleet_ptr
.
load_model
(
model_dir
,
mode
)
self
.
_role_maker
.
_barrier_worker
()
def
save_model
(
self
,
model_dir
=
None
,
**
kwargs
):
"""
save pslib model, the modes are same with load model.
Args:
model_dir(str): if you use hdfs, model_dir should starts with
'hdfs:', otherwise means local dir
kwargs(dict): user-defined properties.
mode(int): the modes illustrated above, default 0
Examples:
.. code-block:: python
fleet.save_model("afs:/user/path/")
"""
mode
=
kwargs
.
get
(
"mode"
,
0
)
prefix
=
kwargs
.
get
(
"prefix"
,
None
)
self
.
_role_maker
.
_barrier_worker
()
if
self
.
_role_maker
.
is_first_worker
():
self
.
_fleet_ptr
.
save_model
(
model_dir
,
mode
)
self
.
_role_maker
.
_barrier_worker
()
def
save_one_table
(
self
,
table_id
,
model_dir
,
**
kwargs
):
"""
save pslib model's one table, the modes are same with load model.
Args:
table_id(int): table id
model_dir(str): if you use hdfs, model_dir should starts with
'hdfs:', otherwise means local dir
kwargs(dict): user-defined properties.
mode(int): the modes illustrated above, default 0
prefix(str): the parts to save can have prefix,
for example, part-prefix-000-00000
Examples:
.. code-block:: python
fleet.save_one_table("afs:/user/path/")
"""
mode
=
kwargs
.
get
(
"mode"
,
0
)
prefix
=
kwargs
.
get
(
"prefix"
,
None
)
self
.
_role_maker
.
_barrier_worker
()
if
self
.
_role_maker
.
is_first_worker
():
if
prefix
is
not
None
:
self
.
_fleet_ptr
.
save_model_one_table_with_prefix
(
table_id
,
model_dir
,
mode
,
prefix
)
else
:
self
.
_fleet_ptr
.
save_model_one_table
(
table_id
,
model_dir
,
mode
)
self
.
_role_maker
.
_barrier_worker
()
def
_set_opt_info
(
self
,
opt_info
):
def
_set_opt_info
(
self
,
opt_info
):
"""
"""
this function saves the result from DistributedOptimizer.minimize()
this function saves the result from DistributedOptimizer.minimize()
...
...
python/paddle/fluid/tests/unittests/test_fleet.py
浏览文件 @
d98084e7
...
@@ -21,7 +21,8 @@ import paddle.fluid.incubate.fleet.base.role_maker as role_maker
...
@@ -21,7 +21,8 @@ import paddle.fluid.incubate.fleet.base.role_maker as role_maker
class
TestFleet1
(
unittest
.
TestCase
):
class
TestFleet1
(
unittest
.
TestCase
):
"""
"""
Test cases for fleet minimize.
Test cases for fleet minimize,
and some other fleet apu tests.
"""
"""
def
setUp
(
self
):
def
setUp
(
self
):
...
@@ -80,6 +81,25 @@ class TestFleet1(unittest.TestCase):
...
@@ -80,6 +81,25 @@ class TestFleet1(unittest.TestCase):
except
:
except
:
print
(
"do not support pslib test, skip"
)
print
(
"do not support pslib test, skip"
)
return
return
try
:
# worker should call these methods instead of server
# the following is only for test when with_pslib=off
def
test_func
():
"""
it is only a test function
"""
return
True
fleet
.
_role_maker
.
is_first_worker
=
test_func
fleet
.
_role_maker
.
_barrier_worker
=
test_func
fleet
.
save_model
(
"./model_000"
)
fleet
.
save_one_table
(
0
,
"./model_001"
)
fleet
.
save_one_table
(
0
,
"./model_002"
,
prefix
=
"hahaha"
)
fleet
.
load_model
(
"./model_0003"
)
fleet
.
load_one_table
(
0
,
"./model_004"
)
except
:
print
(
"do not support pslib test, skip"
)
return
if
__name__
==
"__main__"
:
if
__name__
==
"__main__"
:
...
...
python/paddle/fluid/tests/unittests/test_fleet_1.py
浏览文件 @
d98084e7
...
@@ -221,6 +221,11 @@ class TestFleet2(unittest.TestCase):
...
@@ -221,6 +221,11 @@ class TestFleet2(unittest.TestCase):
emb1
=
fluid
.
embedding
(
input
=
show
,
size
=
[
1
,
1
],
\
emb1
=
fluid
.
embedding
(
input
=
show
,
size
=
[
1
,
1
],
\
is_sparse
=
True
,
is_distributed
=
True
,
\
is_sparse
=
True
,
is_distributed
=
True
,
\
param_attr
=
fluid
.
ParamAttr
(
name
=
"embedding"
))
param_attr
=
fluid
.
ParamAttr
(
name
=
"embedding"
))
fleet
.
save_model
(
"./tmodel_000"
)
fleet
.
save_one_table
(
0
,
"./tmodel_001"
)
fleet
.
save_one_table
(
0
,
"./tmodel_002"
,
prefix
=
"thahaha"
)
fleet
.
load_model
(
"./tmodel_0003"
)
fleet
.
load_one_table
(
0
,
"./tmodel_004"
)
if
__name__
==
"__main__"
:
if
__name__
==
"__main__"
:
...
...
python/paddle/fluid/tests/unittests/test_fleet_nocvm_1.py
浏览文件 @
d98084e7
...
@@ -80,6 +80,25 @@ class TestFleet1(unittest.TestCase):
...
@@ -80,6 +80,25 @@ class TestFleet1(unittest.TestCase):
except
:
except
:
print
(
"do not support pslib test, skip"
)
print
(
"do not support pslib test, skip"
)
return
return
try
:
# worker should call these methods instead of server
# the following is only for test when with_pslib=off
def
test_func
():
"""
it is only a test function
"""
return
True
fleet
.
_role_maker
.
is_first_worker
=
test_func
fleet
.
_role_maker
.
_barrier_worker
=
test_func
fleet
.
save_model
(
"./model_000"
)
fleet
.
save_one_table
(
0
,
"./model_001"
)
fleet
.
save_one_table
(
0
,
"./model_002"
,
prefix
=
"hahaha"
)
fleet
.
load_model
(
"./model_0003"
)
fleet
.
load_one_table
(
0
,
"./model_004"
)
except
:
print
(
"do not support pslib test, skip"
)
return
if
__name__
==
"__main__"
:
if
__name__
==
"__main__"
:
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录