Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
urhero
Paddle
提交
995a6376
P
Paddle
项目概览
urhero
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
995a6376
编写于
3月 20, 2020
作者:
Y
yaoxuefeng
提交者:
GitHub
3月 20, 2020
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
add pslib SparseDoubleTable test=develop (#23053)
上级
3af47711
变更
3
展开全部
隐藏空白更改
内联
并排
Showing
3 changed file
with
155 addition
and
74 deletion
+155
-74
python/paddle/fluid/incubate/fleet/parameter_server/pslib/node.py
...addle/fluid/incubate/fleet/parameter_server/pslib/node.py
+11
-7
python/paddle/fluid/incubate/fleet/parameter_server/pslib/ps_pb2.py
...dle/fluid/incubate/fleet/parameter_server/pslib/ps_pb2.py
+141
-66
python/paddle/fluid/tests/unittests/test_downpoursgd.py
python/paddle/fluid/tests/unittests/test_downpoursgd.py
+3
-1
未找到文件。
python/paddle/fluid/incubate/fleet/parameter_server/pslib/node.py
浏览文件 @
995a6376
...
@@ -81,25 +81,25 @@ class DownpourServer(Server):
...
@@ -81,25 +81,25 @@ class DownpourServer(Server):
'sparse_delete_after_unseen_days'
,
'sparse_show_click_decay_rate'
,
'sparse_delete_threshold'
,
\
'sparse_delete_after_unseen_days'
,
'sparse_show_click_decay_rate'
,
'sparse_delete_threshold'
,
\
'sparse_converter'
,
'sparse_deconverter'
,
'sparse_enable_cache'
,
'sparse_cache_rate'
,
\
'sparse_converter'
,
'sparse_deconverter'
,
'sparse_enable_cache'
,
'sparse_cache_rate'
,
\
'sparse_cache_file_num'
,
'sparse_beta1_decay_rate'
,
'sparse_beta2_decay_rate'
,
\
'sparse_cache_file_num'
,
'sparse_beta1_decay_rate'
,
'sparse_beta2_decay_rate'
,
\
'sparse_ada_epsilon'
,
'sparse_optimizer'
]
'sparse_ada_epsilon'
,
'sparse_optimizer'
,
'sparse_ssd_unseenday_threshold'
]
for
key
in
strategy
:
for
key
in
strategy
:
if
key
not
in
support_sparse_key_list
:
if
key
not
in
support_sparse_key_list
:
raise
ValueError
(
"strategy key '%s' not support"
%
(
key
))
raise
ValueError
(
"strategy key '%s' not support"
%
(
key
))
support_table_calss
=
[
'DownpourSparseTable'
]
support_table_calss
=
[
'DownpourSparseTable'
,
'DownpourSparseSSDTable'
]
if
strategy
.
get
(
'sparse_table_class'
)
is
not
None
:
if
strategy
.
get
(
'sparse_table_class'
)
is
not
None
:
table_class
=
strategy
.
get
(
'sparse_table_class'
)
table_class
=
strategy
.
get
(
'sparse_table_class'
)
if
table_class
not
in
support_table_calss
:
if
table_class
not
in
support_table_calss
:
raise
ValueError
(
raise
ValueError
(
"support sparse_table_class: [ 'DownpourSparseTable'
],
\
"support sparse_table_class: [ 'DownpourSparseTable'
, 'DownpourSparseSSDTable'
],
\
but actual %s"
%
(
table_class
))
but actual %s"
%
(
table_class
))
else
:
else
:
table_class
=
'DownpourSparseTable'
table_class
=
'DownpourSparseTable'
table
.
table_class
=
table_class
table
.
table_class
=
table_class
if
table_class
==
'DownpourSparseTable'
:
if
table_class
==
'DownpourSparseTable'
or
table_class
==
'DownpourSparseSSDTable'
:
table
.
enable_sparse_table_cache
=
strategy
.
get
(
table
.
enable_sparse_table_cache
=
strategy
.
get
(
'sparse_enable_cache'
,
True
)
'sparse_enable_cache'
,
True
)
table
.
sparse_table_cache_rate
=
strategy
.
get
(
'sparse_cache_rate'
,
table
.
sparse_table_cache_rate
=
strategy
.
get
(
'sparse_cache_rate'
,
...
@@ -112,23 +112,25 @@ class DownpourServer(Server):
...
@@ -112,23 +112,25 @@ class DownpourServer(Server):
# DownpourFeatureValueAccessor: for ctr task, has cvm, embedding and sgd info
# DownpourFeatureValueAccessor: for ctr task, has cvm, embedding and sgd info
# DownpourCtrAccessor : for ctr task, has cvm, slot, embedding and sgd info
# DownpourCtrAccessor : for ctr task, has cvm, slot, embedding and sgd info
# DownpourSparseValueAccessor : for general task, has embedding and sgd info
# DownpourSparseValueAccessor : for general task, has embedding and sgd info
# DownpourCtrDoubleAccessor : for ctr task, which show clk are in double
support_accessor_class
=
[
support_accessor_class
=
[
'DownpourFeatureValueAccessor'
,
'DownpourCtrAccessor'
,
'DownpourFeatureValueAccessor'
,
'DownpourCtrAccessor'
,
'DownpourSparseValueAccessor'
'DownpourSparseValueAccessor'
,
'DownpourCtrDoubleAccessor'
]
]
if
strategy
.
get
(
'sparse_accessor_class'
)
is
not
None
:
if
strategy
.
get
(
'sparse_accessor_class'
)
is
not
None
:
accessor_class
=
strategy
.
get
(
'sparse_accessor_class'
)
accessor_class
=
strategy
.
get
(
'sparse_accessor_class'
)
if
accessor_class
not
in
support_accessor_class
:
if
accessor_class
not
in
support_accessor_class
:
raise
ValueError
(
raise
ValueError
(
"support sparse_accessor_class: ['DownpourFeatureValueAccessor', 'DownpourCtrAccessor'],
\
"support sparse_accessor_class: ['DownpourFeatureValueAccessor', 'DownpourCtrAccessor',
\
'DownpourSparseValueAccessor', 'DownpourCtrDoubleAccessor'],
\
but actual %s"
%
(
accessor_class
))
but actual %s"
%
(
accessor_class
))
else
:
else
:
accessor_class
=
'DownpourCtrAccessor'
accessor_class
=
'DownpourCtrAccessor'
table
.
accessor
.
accessor_class
=
accessor_class
table
.
accessor
.
accessor_class
=
accessor_class
if
accessor_class
==
'DownpourFeatureValueAccessor'
or
accessor_class
==
'DownpourCtrAccessor'
:
if
accessor_class
==
'DownpourFeatureValueAccessor'
or
accessor_class
==
'DownpourCtrAccessor'
or
accessor_class
==
'DownpourCtrDoubleAccessor'
:
table
.
accessor
.
sparse_sgd_param
.
learning_rate
=
strategy
.
get
(
table
.
accessor
.
sparse_sgd_param
.
learning_rate
=
strategy
.
get
(
'sparse_learning_rate'
,
0.05
)
'sparse_learning_rate'
,
0.05
)
table
.
accessor
.
sparse_sgd_param
.
initial_g2sum
=
strategy
.
get
(
table
.
accessor
.
sparse_sgd_param
.
initial_g2sum
=
strategy
.
get
(
...
@@ -157,6 +159,8 @@ class DownpourServer(Server):
...
@@ -157,6 +159,8 @@ class DownpourServer(Server):
'sparse_delta_keep_days'
,
16
)
'sparse_delta_keep_days'
,
16
)
table
.
accessor
.
downpour_accessor_param
.
delete_after_unseen_days
=
strategy
.
get
(
table
.
accessor
.
downpour_accessor_param
.
delete_after_unseen_days
=
strategy
.
get
(
'sparse_delete_after_unseen_days'
,
30
)
'sparse_delete_after_unseen_days'
,
30
)
table
.
accessor
.
downpour_accessor_param
.
ssd_unseenday_threshold
=
strategy
.
get
(
'sparse_ssd_unseenday_threshold'
,
1
)
table
.
accessor
.
downpour_accessor_param
.
show_click_decay_rate
=
strategy
.
get
(
table
.
accessor
.
downpour_accessor_param
.
show_click_decay_rate
=
strategy
.
get
(
'sparse_show_click_decay_rate'
,
0.98
)
'sparse_show_click_decay_rate'
,
0.98
)
table
.
accessor
.
downpour_accessor_param
.
delete_threshold
=
strategy
.
get
(
table
.
accessor
.
downpour_accessor_param
.
delete_threshold
=
strategy
.
get
(
...
...
python/paddle/fluid/incubate/fleet/parameter_server/pslib/ps_pb2.py
浏览文件 @
995a6376
此差异已折叠。
点击以展开。
python/paddle/fluid/tests/unittests/test_downpoursgd.py
浏览文件 @
995a6376
...
@@ -26,7 +26,7 @@ import sys
...
@@ -26,7 +26,7 @@ import sys
from
op_test
import
OpTest
from
op_test
import
OpTest
from
paddle.fluid.trainer_desc
import
DistMultiTrainer
from
paddle.fluid.trainer_desc
import
DistMultiTrainer
from
paddle.fluid.device_worker
import
DownpourSGD
,
DownpourSGDOPT
from
paddle.fluid.device_worker
import
DownpourSGD
,
DownpourSGDOPT
from
paddle.fluid.incubate.fleet.parameter_server.pslib.node
import
DownpourWorker
from
paddle.fluid.incubate.fleet.parameter_server.pslib.node
import
DownpourWorker
,
DownpourServer
from
google.protobuf
import
text_format
from
google.protobuf
import
text_format
import
paddle.fluid.incubate.fleet.parameter_server.pslib.ps_pb2
as
pslib
import
paddle.fluid.incubate.fleet.parameter_server.pslib.ps_pb2
as
pslib
from
paddle.fluid.trainer_factory
import
TrainerFactory
from
paddle.fluid.trainer_factory
import
TrainerFactory
...
@@ -91,6 +91,8 @@ class TestListenAndServOp(unittest.TestCase):
...
@@ -91,6 +91,8 @@ class TestListenAndServOp(unittest.TestCase):
opt_info
[
"dump_slot"
]
=
False
opt_info
[
"dump_slot"
]
=
False
opt_info
[
"stat_var_names"
]
=
[]
opt_info
[
"stat_var_names"
]
=
[]
worker
=
DownpourWorker
(
None
)
worker
=
DownpourWorker
(
None
)
server
=
DownpourServer
()
server
.
add_sparse_table
(
0
,
{})
worker
.
get_desc
().
CopyFrom
(
ps_param
.
trainer_param
[
0
])
worker
.
get_desc
().
CopyFrom
(
ps_param
.
trainer_param
[
0
])
opt_info
[
"program_id_to_worker"
]
=
{
program_id
:
worker
}
opt_info
[
"program_id_to_worker"
]
=
{
program_id
:
worker
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录