Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
Crayon鑫
Paddle
提交
52329f6f
P
Paddle
项目概览
Crayon鑫
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
未验证
提交
52329f6f
编写于
12月 24, 2021
作者:
Z
zmxdream
提交者:
GitHub
12月 24, 2021
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
[heterps]move pre-init id logic from common_sparse_table to sparse_geo_table (#38173)
* remove pre-init id in common_sparse_tabl.cc
上级
33185000
变更
4
显示空白变更内容
内联
并排
Showing
4 changed file
with
49 addition
and
29 deletion
+49
-29
paddle/fluid/distributed/service/communicator.h
paddle/fluid/distributed/service/communicator.h
+4
-0
paddle/fluid/distributed/table/common_sparse_table.cc
paddle/fluid/distributed/table/common_sparse_table.cc
+0
-27
paddle/fluid/distributed/table/sparse_geo_table.cc
paddle/fluid/distributed/table/sparse_geo_table.cc
+41
-0
paddle/fluid/distributed/table/sparse_geo_table.h
paddle/fluid/distributed/table/sparse_geo_table.h
+4
-2
未找到文件。
paddle/fluid/distributed/service/communicator.h
浏览文件 @
52329f6f
...
@@ -300,6 +300,10 @@ class Communicator {
...
@@ -300,6 +300,10 @@ class Communicator {
virtual
void
BarrierWithTable
(
uint32_t
barrier_type
)
{
virtual
void
BarrierWithTable
(
uint32_t
barrier_type
)
{
auto
rets
=
_worker_ptr
->
barrier
(
barrier_table_id_
,
barrier_type
);
auto
rets
=
_worker_ptr
->
barrier
(
barrier_table_id_
,
barrier_type
);
rets
.
wait
();
rets
.
wait
();
int
status
=
rets
.
get
();
PADDLE_ENFORCE_EQ
(
status
,
0
,
platform
::
errors
::
InvalidArgument
(
"The ret status must be 0 when barrier with table"
));
}
}
virtual
void
CreateC2CConnection
(
int
pserver_timeout_ms
,
virtual
void
CreateC2CConnection
(
int
pserver_timeout_ms
,
...
...
paddle/fluid/distributed/table/common_sparse_table.cc
浏览文件 @
52329f6f
...
@@ -220,33 +220,6 @@ int32_t CommonSparseTable::initialize_value() {
...
@@ -220,33 +220,6 @@ int32_t CommonSparseTable::initialize_value() {
shard_values_
.
emplace_back
(
shard
);
shard_values_
.
emplace_back
(
shard
);
}
}
auto
accessor
=
_config
.
accessor
();
std
::
vector
<
uint64_t
>
feasigns
;
for
(
size_t
x
=
0
;
x
<
accessor
.
fea_dim
();
++
x
)
{
if
(
x
%
_shard_num
==
_shard_idx
)
{
feasigns
.
push_back
(
x
);
}
}
VLOG
(
3
)
<<
"has "
<<
feasigns
.
size
()
<<
" ids need to be pre inited"
;
auto
buckets
=
bucket
(
feasigns
.
size
(),
10
);
for
(
int
x
=
0
;
x
<
10
;
++
x
)
{
auto
bucket_feasigns
=
buckets
[
x
+
1
]
-
buckets
[
x
];
std
::
vector
<
uint64_t
>
ids
(
bucket_feasigns
);
std
::
copy
(
feasigns
.
begin
()
+
buckets
[
x
],
feasigns
.
begin
()
+
buckets
[
x
+
1
],
ids
.
begin
());
std
::
vector
<
uint32_t
>
fres
;
fres
.
resize
(
ids
.
size
(),
1
);
auto
pull_value
=
PullSparseValue
(
ids
,
fres
,
param_dim_
);
std
::
vector
<
float
>
pulls
;
pulls
.
resize
(
bucket_feasigns
*
param_dim_
);
pull_sparse
(
pulls
.
data
(),
pull_value
);
}
return
0
;
return
0
;
}
}
...
...
paddle/fluid/distributed/table/sparse_geo_table.cc
浏览文件 @
52329f6f
...
@@ -46,5 +46,46 @@ int32_t SparseGeoTable::push_sparse(const uint64_t* keys, const float* values,
...
@@ -46,5 +46,46 @@ int32_t SparseGeoTable::push_sparse(const uint64_t* keys, const float* values,
return
0
;
return
0
;
}
}
int32_t
SparseGeoTable
::
initialize_value
()
{
auto
common
=
_config
.
common
();
shard_values_
.
reserve
(
task_pool_size_
);
for
(
int
x
=
0
;
x
<
task_pool_size_
;
++
x
)
{
auto
shard
=
std
::
make_shared
<
ValueBlock
>
(
value_names_
,
value_dims_
,
value_offsets_
,
value_idx_
,
initializer_attrs_
,
common
.
entry
());
shard_values_
.
emplace_back
(
shard
);
}
auto
accessor
=
_config
.
accessor
();
std
::
vector
<
uint64_t
>
feasigns
;
for
(
size_t
x
=
0
;
x
<
accessor
.
fea_dim
();
++
x
)
{
if
(
x
%
_shard_num
==
_shard_idx
)
{
feasigns
.
push_back
(
x
);
}
}
VLOG
(
3
)
<<
"has "
<<
feasigns
.
size
()
<<
" ids need to be pre inited"
;
auto
buckets
=
bucket
(
feasigns
.
size
(),
10
);
for
(
int
x
=
0
;
x
<
10
;
++
x
)
{
auto
bucket_feasigns
=
buckets
[
x
+
1
]
-
buckets
[
x
];
std
::
vector
<
uint64_t
>
ids
(
bucket_feasigns
);
std
::
copy
(
feasigns
.
begin
()
+
buckets
[
x
],
feasigns
.
begin
()
+
buckets
[
x
+
1
],
ids
.
begin
());
std
::
vector
<
uint32_t
>
fres
;
fres
.
resize
(
ids
.
size
(),
1
);
auto
pull_value
=
PullSparseValue
(
ids
,
fres
,
param_dim_
);
std
::
vector
<
float
>
pulls
;
pulls
.
resize
(
bucket_feasigns
*
param_dim_
);
pull_sparse
(
pulls
.
data
(),
pull_value
);
}
return
0
;
}
}
// namespace distributed
}
// namespace distributed
}
// namespace paddle
}
// namespace paddle
paddle/fluid/distributed/table/sparse_geo_table.h
浏览文件 @
52329f6f
...
@@ -44,10 +44,12 @@ class SparseGeoTable : public CommonSparseTable {
...
@@ -44,10 +44,12 @@ class SparseGeoTable : public CommonSparseTable {
explicit
SparseGeoTable
()
:
CommonSparseTable
()
{
geo_recorder
=
nullptr
;
}
explicit
SparseGeoTable
()
:
CommonSparseTable
()
{
geo_recorder
=
nullptr
;
}
virtual
~
SparseGeoTable
()
{}
virtual
~
SparseGeoTable
()
{}
virtual
int32_t
initialize_value
();
int32_t
pull_geo_param
(
const
uint32_t
trainer_id
,
std
::
vector
<
float
>*
values
,
int32_t
pull_geo_param
(
const
uint32_t
trainer_id
,
std
::
vector
<
float
>*
values
,
std
::
vector
<
uint64_t
>*
keys
);
std
::
vector
<
uint64_t
>*
keys
);
virtual
int32_t
push_sparse
(
const
uint64_t
*
keys
,
const
float
*
values
,
int32_t
push_sparse
(
const
uint64_t
*
keys
,
const
float
*
values
,
size_t
num
)
override
;
size_t
num
)
override
;
virtual
int32_t
initialize_recorder
()
{
virtual
int32_t
initialize_recorder
()
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录