Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
PaddlePaddle
Paddle
提交
2bef8a48
P
Paddle
项目概览
PaddlePaddle
/
Paddle
大约 1 年 前同步成功
通知
2298
Star
20931
Fork
5422
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1423
列表
看板
标记
里程碑
合并请求
543
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1,423
Issue
1,423
列表
看板
标记
里程碑
合并请求
543
合并请求
543
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
未验证
提交
2bef8a48
编写于
6月 24, 2022
作者:
Z
zhangchunle
提交者:
GitHub
6月 24, 2022
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
fix unused-variable warning (#43791)
上级
b2704837
变更
14
隐藏空白更改
内联
并排
Showing
14 changed file
with
1 addition
and
44 deletion
+1
-44
paddle/fluid/distributed/ps/service/brpc_ps_client.cc
paddle/fluid/distributed/ps/service/brpc_ps_client.cc
+0
-5
paddle/fluid/distributed/ps/service/brpc_ps_server.cc
paddle/fluid/distributed/ps/service/brpc_ps_server.cc
+0
-2
paddle/fluid/distributed/ps/service/communicator/communicator.cc
...fluid/distributed/ps/service/communicator/communicator.cc
+0
-2
paddle/fluid/distributed/ps/service/graph_brpc_client.cc
paddle/fluid/distributed/ps/service/graph_brpc_client.cc
+0
-1
paddle/fluid/distributed/ps/service/graph_brpc_server.cc
paddle/fluid/distributed/ps/service/graph_brpc_server.cc
+1
-3
paddle/fluid/distributed/ps/service/ps_local_client.cc
paddle/fluid/distributed/ps/service/ps_local_client.cc
+0
-2
paddle/fluid/distributed/ps/table/ctr_accessor.cc
paddle/fluid/distributed/ps/table/ctr_accessor.cc
+0
-6
paddle/fluid/distributed/ps/table/ctr_double_accessor.cc
paddle/fluid/distributed/ps/table/ctr_double_accessor.cc
+0
-4
paddle/fluid/distributed/ps/table/ctr_dymf_accessor.cc
paddle/fluid/distributed/ps/table/ctr_dymf_accessor.cc
+0
-3
paddle/fluid/distributed/ps/table/memory_dense_table.cc
paddle/fluid/distributed/ps/table/memory_dense_table.cc
+0
-2
paddle/fluid/distributed/ps/table/memory_sparse_table.cc
paddle/fluid/distributed/ps/table/memory_sparse_table.cc
+0
-2
paddle/fluid/distributed/ps/table/sparse_accessor.cc
paddle/fluid/distributed/ps/table/sparse_accessor.cc
+0
-6
paddle/fluid/distributed/ps/table/ssd_sparse_table.cc
paddle/fluid/distributed/ps/table/ssd_sparse_table.cc
+0
-5
paddle/fluid/distributed/test/barrier_table_test.cc
paddle/fluid/distributed/test/barrier_table_test.cc
+0
-1
未找到文件。
paddle/fluid/distributed/ps/service/brpc_ps_client.cc
浏览文件 @
2bef8a48
...
@@ -1494,8 +1494,6 @@ void BrpcPsClient::PushSparseTaskConsume() {
...
@@ -1494,8 +1494,6 @@ void BrpcPsClient::PushSparseTaskConsume() {
merge_status
.
clear
();
merge_status
.
clear
();
std
::
vector
<
std
::
future
<
int
>>
().
swap
(
merge_status
);
std
::
vector
<
std
::
future
<
int
>>
().
swap
(
merge_status
);
_push_sparse_merge_count_map
[
table_id
]
=
0
;
_push_sparse_merge_count_map
[
table_id
]
=
0
;
auto
queue_size
=
task_queue
->
Size
();
}
else
{
// 未达到阈值 只做多路归并
}
else
{
// 未达到阈值 只做多路归并
std
::
vector
<
std
::
future
<
int
>>
merge_status
(
request_call_num
);
std
::
vector
<
std
::
future
<
int
>>
merge_status
(
request_call_num
);
for
(
size_t
shard_idx
=
0
;
shard_idx
<
request_call_num
;
++
shard_idx
)
{
for
(
size_t
shard_idx
=
0
;
shard_idx
<
request_call_num
;
++
shard_idx
)
{
...
@@ -1542,7 +1540,6 @@ int BrpcPsClient::PushSparseAsyncShardMerge(
...
@@ -1542,7 +1540,6 @@ int BrpcPsClient::PushSparseAsyncShardMerge(
std
::
vector
<
int
>
&
request_kv_num
,
int
table_id
,
int
shard_idx
,
std
::
vector
<
int
>
&
request_kv_num
,
int
table_id
,
int
shard_idx
,
ValueAccessor
*
accessor
)
{
ValueAccessor
*
accessor
)
{
size_t
merged_kv_count
=
0
;
size_t
merged_kv_count
=
0
;
uint64_t
min_key
=
UINT64_MAX
;
uint32_t
value_size
=
accessor
->
GetAccessorInfo
().
update_size
;
uint32_t
value_size
=
accessor
->
GetAccessorInfo
().
update_size
;
thread_local
std
::
vector
<
std
::
pair
<
uint64_t
,
const
float
*>>
sorted_kv_list
;
thread_local
std
::
vector
<
std
::
pair
<
uint64_t
,
const
float
*>>
sorted_kv_list
;
...
@@ -1771,8 +1768,6 @@ void BrpcPsClient::PushDenseTaskConsume() {
...
@@ -1771,8 +1768,6 @@ void BrpcPsClient::PushDenseTaskConsume() {
accessor
->
Merge
(
&
total_send_data
,
&
merge_data
,
accessor
->
Merge
(
&
total_send_data
,
&
merge_data
,
total_send_data_size
);
total_send_data_size
);
#pragma optimize("", off)
#pragma optimize("", off)
auto
*
debug_closure
=
closure
;
auto
*
debug_task
=
async_task
;
delete
async_task
;
delete
async_task
;
#pragma optimize("", on)
#pragma optimize("", on)
return
0
;
return
0
;
...
...
paddle/fluid/distributed/ps/service/brpc_ps_server.cc
浏览文件 @
2bef8a48
...
@@ -648,7 +648,6 @@ int32_t BrpcPsService::SaveAllTable(Table *table,
...
@@ -648,7 +648,6 @@ int32_t BrpcPsService::SaveAllTable(Table *table,
PsResponseMessage
&
response
,
PsResponseMessage
&
response
,
brpc
::
Controller
*
cntl
)
{
brpc
::
Controller
*
cntl
)
{
auto
&
table_map
=
*
(
_server
->
GetTable
());
auto
&
table_map
=
*
(
_server
->
GetTable
());
int32_t
all_feasign_size
=
0
;
int32_t
feasign_size
=
0
;
int32_t
feasign_size
=
0
;
for
(
auto
&
itr
:
table_map
)
{
for
(
auto
&
itr
:
table_map
)
{
...
@@ -827,7 +826,6 @@ int32_t BrpcPsService::PushGlobalStep(Table *table,
...
@@ -827,7 +826,6 @@ int32_t BrpcPsService::PushGlobalStep(Table *table,
set_response_code
(
response
,
0
,
"run_program data is empty"
);
set_response_code
(
response
,
0
,
"run_program data is empty"
);
return
0
;
return
0
;
}
}
uint32_t
num
=
*
(
const
uint32_t
*
)(
request
.
data
().
data
());
const
int64_t
*
values
=
const
int64_t
*
values
=
(
const
int64_t
*
)(
request
.
data
().
data
()
+
sizeof
(
uint32_t
));
(
const
int64_t
*
)(
request
.
data
().
data
()
+
sizeof
(
uint32_t
));
auto
trainer_id
=
request
.
client_id
();
auto
trainer_id
=
request
.
client_id
();
...
...
paddle/fluid/distributed/ps/service/communicator/communicator.cc
浏览文件 @
2bef8a48
...
@@ -658,8 +658,6 @@ void AsyncCommunicator::PushSparseFromTensorAsync(
...
@@ -658,8 +658,6 @@ void AsyncCommunicator::PushSparseFromTensorAsync(
// TODO(zhaocaibei123): check type of show/clk is int? float? uint64?
// TODO(zhaocaibei123): check type of show/clk is int? float? uint64?
// const long int* show_tensor = shows->data<int64_t>();
// const long int* show_tensor = shows->data<int64_t>();
// const long int* clk_tensor = clks->data<int64_t>();
// const long int* clk_tensor = clks->data<int64_t>();
const
int64_t
*
show_tensor
=
shows
->
data
<
int64_t
>
();
const
int64_t
*
clk_tensor
=
clks
->
data
<
int64_t
>
();
for
(
size_t
index
=
0
;
index
<
inputs
->
size
();
++
index
)
{
for
(
size_t
index
=
0
;
index
<
inputs
->
size
();
++
index
)
{
framework
::
LoDTensor
*
g_tensor
=
outputs
->
at
(
index
);
framework
::
LoDTensor
*
g_tensor
=
outputs
->
at
(
index
);
...
...
paddle/fluid/distributed/ps/service/graph_brpc_client.cc
浏览文件 @
2bef8a48
...
@@ -491,7 +491,6 @@ std::future<int32_t> GraphBrpcClient::random_sample_nodes(
...
@@ -491,7 +491,6 @@ std::future<int32_t> GraphBrpcClient::random_sample_nodes(
butil
::
IOBufBytesIterator
io_buffer_itr
(
res_io_buffer
);
butil
::
IOBufBytesIterator
io_buffer_itr
(
res_io_buffer
);
size_t
bytes_size
=
io_buffer_itr
.
bytes_left
();
size_t
bytes_size
=
io_buffer_itr
.
bytes_left
();
char
*
buffer
=
new
char
[
bytes_size
];
char
*
buffer
=
new
char
[
bytes_size
];
auto
size
=
io_buffer_itr
.
copy_and_forward
((
void
*
)(
buffer
),
bytes_size
);
size_t
index
=
0
;
size_t
index
=
0
;
while
(
index
<
bytes_size
)
{
while
(
index
<
bytes_size
)
{
ids
.
push_back
(
*
(
int64_t
*
)(
buffer
+
index
));
ids
.
push_back
(
*
(
int64_t
*
)(
buffer
+
index
));
...
...
paddle/fluid/distributed/ps/service/graph_brpc_server.cc
浏览文件 @
2bef8a48
...
@@ -498,8 +498,7 @@ int32_t GraphBrpcService::sample_neighbors_across_multi_servers(
...
@@ -498,8 +498,7 @@ int32_t GraphBrpcService::sample_neighbors_across_multi_servers(
}
}
int
idx_
=
*
(
int
*
)(
request
.
params
(
0
).
c_str
());
int
idx_
=
*
(
int
*
)(
request
.
params
(
0
).
c_str
());
size_t
node_num
=
request
.
params
(
1
).
size
()
/
sizeof
(
int64_t
),
size_t
node_num
=
request
.
params
(
1
).
size
()
/
sizeof
(
int64_t
);
size_of_size_t
=
sizeof
(
size_t
);
int64_t
*
node_data
=
(
int64_t
*
)(
request
.
params
(
1
).
c_str
());
int64_t
*
node_data
=
(
int64_t
*
)(
request
.
params
(
1
).
c_str
());
int
sample_size
=
*
(
int64_t
*
)(
request
.
params
(
2
).
c_str
());
int
sample_size
=
*
(
int64_t
*
)(
request
.
params
(
2
).
c_str
());
bool
need_weight
=
*
(
int64_t
*
)(
request
.
params
(
3
).
c_str
());
bool
need_weight
=
*
(
int64_t
*
)(
request
.
params
(
3
).
c_str
());
...
@@ -572,7 +571,6 @@ int32_t GraphBrpcService::sample_neighbors_across_multi_servers(
...
@@ -572,7 +571,6 @@ int32_t GraphBrpcService::sample_neighbors_across_multi_servers(
failed
[
request2server
[
request_idx
]]
=
true
;
failed
[
request2server
[
request_idx
]]
=
true
;
}
else
{
}
else
{
auto
&
res_io_buffer
=
closure
->
cntl
(
request_idx
)
->
response_attachment
();
auto
&
res_io_buffer
=
closure
->
cntl
(
request_idx
)
->
response_attachment
();
size_t
node_size
;
res
[
request_idx
].
reset
(
new
butil
::
IOBufBytesIterator
(
res_io_buffer
));
res
[
request_idx
].
reset
(
new
butil
::
IOBufBytesIterator
(
res_io_buffer
));
size_t
num
;
size_t
num
;
res
[
request_idx
]
->
copy_and_forward
(
&
num
,
sizeof
(
size_t
));
res
[
request_idx
]
->
copy_and_forward
(
&
num
,
sizeof
(
size_t
));
...
...
paddle/fluid/distributed/ps/service/ps_local_client.cc
浏览文件 @
2bef8a48
...
@@ -283,7 +283,6 @@ int32_t PsLocalClient::Initialize() {
...
@@ -283,7 +283,6 @@ int32_t PsLocalClient::Initialize() {
size_t
table_id
,
const
uint64_t
*
keys
,
const
float
**
update_values
,
size_t
table_id
,
const
uint64_t
*
keys
,
const
float
**
update_values
,
size_t
num
,
void
*
callback
)
{
size_t
num
,
void
*
callback
)
{
PSClientClosure
*
closure
=
reinterpret_cast
<
PSClientClosure
*>
(
callback
);
PSClientClosure
*
closure
=
reinterpret_cast
<
PSClientClosure
*>
(
callback
);
auto
*
accessor
=
GetTableAccessor
(
table_id
);
auto
*
table_ptr
=
GetTable
(
table_id
);
auto
*
table_ptr
=
GetTable
(
table_id
);
TableContext
table_context
;
TableContext
table_context
;
...
@@ -303,7 +302,6 @@ int32_t PsLocalClient::Initialize() {
...
@@ -303,7 +302,6 @@ int32_t PsLocalClient::Initialize() {
const
uint64_t
*
keys
,
const
uint64_t
*
keys
,
const
float
**
update_values
,
const
float
**
update_values
,
size_t
num
)
{
size_t
num
)
{
auto
*
accessor
=
GetTableAccessor
(
table_id
);
auto
*
table_ptr
=
GetTable
(
table_id
);
auto
*
table_ptr
=
GetTable
(
table_id
);
TableContext
table_context
;
TableContext
table_context
;
...
...
paddle/fluid/distributed/ps/table/ctr_accessor.cc
浏览文件 @
2bef8a48
...
@@ -61,8 +61,6 @@ void CtrCommonAccessor::InitAccessorInfo() {
...
@@ -61,8 +61,6 @@ void CtrCommonAccessor::InitAccessorInfo() {
}
}
bool
CtrCommonAccessor
::
Shrink
(
float
*
value
)
{
bool
CtrCommonAccessor
::
Shrink
(
float
*
value
)
{
auto
base_threshold
=
_config
.
ctr_accessor_param
().
base_threshold
();
auto
delta_threshold
=
_config
.
ctr_accessor_param
().
delta_threshold
();
auto
delete_after_unseen_days
=
auto
delete_after_unseen_days
=
_config
.
ctr_accessor_param
().
delete_after_unseen_days
();
_config
.
ctr_accessor_param
().
delete_after_unseen_days
();
auto
delete_threshold
=
_config
.
ctr_accessor_param
().
delete_threshold
();
auto
delete_threshold
=
_config
.
ctr_accessor_param
().
delete_threshold
();
...
@@ -171,7 +169,6 @@ void CtrCommonAccessor::UpdateStatAfterSave(float* value, int param) {
...
@@ -171,7 +169,6 @@ void CtrCommonAccessor::UpdateStatAfterSave(float* value, int param) {
}
}
int32_t
CtrCommonAccessor
::
Create
(
float
**
values
,
size_t
num
)
{
int32_t
CtrCommonAccessor
::
Create
(
float
**
values
,
size_t
num
)
{
auto
embedx_dim
=
_config
.
embedx_dim
();
for
(
size_t
value_item
=
0
;
value_item
<
num
;
++
value_item
)
{
for
(
size_t
value_item
=
0
;
value_item
<
num
;
++
value_item
)
{
float
*
value
=
values
[
value_item
];
float
*
value
=
values
[
value_item
];
value
[
common_feature_value
.
UnseenDaysIndex
()]
=
0
;
value
[
common_feature_value
.
UnseenDaysIndex
()]
=
0
;
...
@@ -245,7 +242,6 @@ int32_t CtrCommonAccessor::Merge(float** update_values,
...
@@ -245,7 +242,6 @@ int32_t CtrCommonAccessor::Merge(float** update_values,
// second dim: field num
// second dim: field num
int32_t
CtrCommonAccessor
::
Update
(
float
**
update_values
,
int32_t
CtrCommonAccessor
::
Update
(
float
**
update_values
,
const
float
**
push_values
,
size_t
num
)
{
const
float
**
push_values
,
size_t
num
)
{
auto
embedx_dim
=
_config
.
embedx_dim
();
for
(
size_t
value_item
=
0
;
value_item
<
num
;
++
value_item
)
{
for
(
size_t
value_item
=
0
;
value_item
<
num
;
++
value_item
)
{
float
*
update_value
=
update_values
[
value_item
];
float
*
update_value
=
update_values
[
value_item
];
const
float
*
push_value
=
push_values
[
value_item
];
const
float
*
push_value
=
push_values
[
value_item
];
...
@@ -330,8 +326,6 @@ std::string CtrCommonAccessor::ParseToString(const float* v, int param) {
...
@@ -330,8 +326,6 @@ std::string CtrCommonAccessor::ParseToString(const float* v, int param) {
}
}
int
CtrCommonAccessor
::
ParseFromString
(
const
std
::
string
&
str
,
float
*
value
)
{
int
CtrCommonAccessor
::
ParseFromString
(
const
std
::
string
&
str
,
float
*
value
)
{
int
embedx_dim
=
_config
.
embedx_dim
();
_embedx_sgd_rule
->
InitValue
(
value
+
common_feature_value
.
EmbedxWIndex
(),
_embedx_sgd_rule
->
InitValue
(
value
+
common_feature_value
.
EmbedxWIndex
(),
value
+
common_feature_value
.
EmbedxG2SumIndex
());
value
+
common_feature_value
.
EmbedxG2SumIndex
());
auto
ret
=
paddle
::
string
::
str_to_float
(
str
.
data
(),
value
);
auto
ret
=
paddle
::
string
::
str_to_float
(
str
.
data
(),
value
);
...
...
paddle/fluid/distributed/ps/table/ctr_double_accessor.cc
浏览文件 @
2bef8a48
...
@@ -59,8 +59,6 @@ bool CtrDoubleAccessor::Shrink(float* value) {
...
@@ -59,8 +59,6 @@ bool CtrDoubleAccessor::Shrink(float* value) {
// auto base_threshold = _config.ctr_accessor_param().base_threshold();
// auto base_threshold = _config.ctr_accessor_param().base_threshold();
// auto delta_threshold = _config.ctr_accessor_param().delta_threshold();
// auto delta_threshold = _config.ctr_accessor_param().delta_threshold();
// auto delete_threshold = _config.ctr_accessor_param().delete_threshold();
// auto delete_threshold = _config.ctr_accessor_param().delete_threshold();
auto
base_threshold
=
_config
.
ctr_accessor_param
().
base_threshold
();
auto
delta_threshold
=
_config
.
ctr_accessor_param
().
delta_threshold
();
auto
delete_after_unseen_days
=
auto
delete_after_unseen_days
=
_config
.
ctr_accessor_param
().
delete_after_unseen_days
();
_config
.
ctr_accessor_param
().
delete_after_unseen_days
();
auto
delete_threshold
=
_config
.
ctr_accessor_param
().
delete_threshold
();
auto
delete_threshold
=
_config
.
ctr_accessor_param
().
delete_threshold
();
...
@@ -170,7 +168,6 @@ void CtrDoubleAccessor::UpdateStatAfterSave(float* value, int param) {
...
@@ -170,7 +168,6 @@ void CtrDoubleAccessor::UpdateStatAfterSave(float* value, int param) {
}
}
int32_t
CtrDoubleAccessor
::
Create
(
float
**
values
,
size_t
num
)
{
int32_t
CtrDoubleAccessor
::
Create
(
float
**
values
,
size_t
num
)
{
auto
embedx_dim
=
_config
.
embedx_dim
();
for
(
size_t
value_item
=
0
;
value_item
<
num
;
++
value_item
)
{
for
(
size_t
value_item
=
0
;
value_item
<
num
;
++
value_item
)
{
float
*
value
=
values
[
value_item
];
float
*
value
=
values
[
value_item
];
value
[
CtrDoubleFeatureValue
::
UnseenDaysIndex
()]
=
0
;
value
[
CtrDoubleFeatureValue
::
UnseenDaysIndex
()]
=
0
;
...
@@ -246,7 +243,6 @@ int32_t CtrDoubleAccessor::Merge(float** update_values,
...
@@ -246,7 +243,6 @@ int32_t CtrDoubleAccessor::Merge(float** update_values,
// second dim: field num
// second dim: field num
int32_t
CtrDoubleAccessor
::
Update
(
float
**
update_values
,
int32_t
CtrDoubleAccessor
::
Update
(
float
**
update_values
,
const
float
**
push_values
,
size_t
num
)
{
const
float
**
push_values
,
size_t
num
)
{
auto
embedx_dim
=
_config
.
embedx_dim
();
for
(
size_t
value_item
=
0
;
value_item
<
num
;
++
value_item
)
{
for
(
size_t
value_item
=
0
;
value_item
<
num
;
++
value_item
)
{
float
*
update_value
=
update_values
[
value_item
];
float
*
update_value
=
update_values
[
value_item
];
const
float
*
push_value
=
push_values
[
value_item
];
const
float
*
push_value
=
push_values
[
value_item
];
...
...
paddle/fluid/distributed/ps/table/ctr_dymf_accessor.cc
浏览文件 @
2bef8a48
...
@@ -62,8 +62,6 @@ void CtrDymfAccessor::InitAccessorInfo() {
...
@@ -62,8 +62,6 @@ void CtrDymfAccessor::InitAccessorInfo() {
}
}
bool
CtrDymfAccessor
::
Shrink
(
float
*
value
)
{
bool
CtrDymfAccessor
::
Shrink
(
float
*
value
)
{
auto
base_threshold
=
_config
.
ctr_accessor_param
().
base_threshold
();
auto
delta_threshold
=
_config
.
ctr_accessor_param
().
delta_threshold
();
auto
delete_after_unseen_days
=
auto
delete_after_unseen_days
=
_config
.
ctr_accessor_param
().
delete_after_unseen_days
();
_config
.
ctr_accessor_param
().
delete_after_unseen_days
();
auto
delete_threshold
=
_config
.
ctr_accessor_param
().
delete_threshold
();
auto
delete_threshold
=
_config
.
ctr_accessor_param
().
delete_threshold
();
...
@@ -172,7 +170,6 @@ void CtrDymfAccessor::UpdateStatAfterSave(float* value, int param) {
...
@@ -172,7 +170,6 @@ void CtrDymfAccessor::UpdateStatAfterSave(float* value, int param) {
}
}
int32_t
CtrDymfAccessor
::
Create
(
float
**
values
,
size_t
num
)
{
int32_t
CtrDymfAccessor
::
Create
(
float
**
values
,
size_t
num
)
{
auto
embedx_dim
=
_config
.
embedx_dim
();
for
(
size_t
value_item
=
0
;
value_item
<
num
;
++
value_item
)
{
for
(
size_t
value_item
=
0
;
value_item
<
num
;
++
value_item
)
{
float
*
value
=
values
[
value_item
];
float
*
value
=
values
[
value_item
];
value
[
common_feature_value
.
UnseenDaysIndex
()]
=
0
;
value
[
common_feature_value
.
UnseenDaysIndex
()]
=
0
;
...
...
paddle/fluid/distributed/ps/table/memory_dense_table.cc
浏览文件 @
2bef8a48
...
@@ -249,7 +249,6 @@ int32_t MemoryDenseTable::Load(const std::string& path,
...
@@ -249,7 +249,6 @@ int32_t MemoryDenseTable::Load(const std::string& path,
float
data_buffer
[
5
];
float
data_buffer
[
5
];
float
*
data_buff_ptr
=
data_buffer
;
float
*
data_buff_ptr
=
data_buffer
;
std
::
string
line_data
;
std
::
string
line_data
;
int
size
=
static_cast
<
int
>
(
values_
.
size
());
auto
common
=
_config
.
common
();
auto
common
=
_config
.
common
();
for
(
size_t
i
=
start_file_idx
;
i
<
end_file_idx
+
1
;
++
i
)
{
for
(
size_t
i
=
start_file_idx
;
i
<
end_file_idx
+
1
;
++
i
)
{
...
@@ -354,7 +353,6 @@ int32_t MemoryDenseTable::Save(const std::string& path,
...
@@ -354,7 +353,6 @@ int32_t MemoryDenseTable::Save(const std::string& path,
}
else
{
}
else
{
std
::
ostringstream
os
;
std
::
ostringstream
os
;
for
(
int
x
=
0
;
x
<
size
;
++
x
)
{
for
(
int
x
=
0
;
x
<
size
;
++
x
)
{
auto
&
varname
=
common
.
params
()[
x
];
int
dim
=
common
.
dims
()[
x
];
int
dim
=
common
.
dims
()[
x
];
VLOG
(
3
)
<<
"MemoryDenseTable::save dim "
<<
x
<<
" size: "
<<
dim
;
VLOG
(
3
)
<<
"MemoryDenseTable::save dim "
<<
x
<<
" size: "
<<
dim
;
for
(
int
y
=
0
;
y
<
dim
;
++
y
)
{
for
(
int
y
=
0
;
y
<
dim
;
++
y
)
{
...
...
paddle/fluid/distributed/ps/table/memory_sparse_table.cc
浏览文件 @
2bef8a48
...
@@ -163,8 +163,6 @@ int32_t MemorySparseTable::LoadLocalFS(const std::string& path,
...
@@ -163,8 +163,6 @@ int32_t MemorySparseTable::LoadLocalFS(const std::string& path,
const
std
::
string
&
param
)
{
const
std
::
string
&
param
)
{
std
::
string
table_path
=
TableDir
(
path
);
std
::
string
table_path
=
TableDir
(
path
);
auto
file_list
=
paddle
::
framework
::
localfs_list
(
table_path
);
auto
file_list
=
paddle
::
framework
::
localfs_list
(
table_path
);
int
load_param
=
atoi
(
param
.
c_str
());
size_t
expect_shard_num
=
_sparse_table_shard_num
;
size_t
expect_shard_num
=
_sparse_table_shard_num
;
if
(
file_list
.
size
()
!=
expect_shard_num
)
{
if
(
file_list
.
size
()
!=
expect_shard_num
)
{
LOG
(
WARNING
)
<<
"MemorySparseTable file_size:"
<<
file_list
.
size
()
LOG
(
WARNING
)
<<
"MemorySparseTable file_size:"
<<
file_list
.
size
()
...
...
paddle/fluid/distributed/ps/table/sparse_accessor.cc
浏览文件 @
2bef8a48
...
@@ -55,8 +55,6 @@ void SparseAccessor::InitAccessorInfo() {
...
@@ -55,8 +55,6 @@ void SparseAccessor::InitAccessorInfo() {
}
}
bool
SparseAccessor
::
Shrink
(
float
*
value
)
{
bool
SparseAccessor
::
Shrink
(
float
*
value
)
{
auto
base_threshold
=
_config
.
ctr_accessor_param
().
base_threshold
();
auto
delta_threshold
=
_config
.
ctr_accessor_param
().
delta_threshold
();
auto
delete_after_unseen_days
=
auto
delete_after_unseen_days
=
_config
.
ctr_accessor_param
().
delete_after_unseen_days
();
_config
.
ctr_accessor_param
().
delete_after_unseen_days
();
auto
delete_threshold
=
_config
.
ctr_accessor_param
().
delete_threshold
();
auto
delete_threshold
=
_config
.
ctr_accessor_param
().
delete_threshold
();
...
@@ -146,7 +144,6 @@ void SparseAccessor::UpdateStatAfterSave(float* value, int param) {
...
@@ -146,7 +144,6 @@ void SparseAccessor::UpdateStatAfterSave(float* value, int param) {
}
}
int32_t
SparseAccessor
::
Create
(
float
**
values
,
size_t
num
)
{
int32_t
SparseAccessor
::
Create
(
float
**
values
,
size_t
num
)
{
auto
embedx_dim
=
_config
.
embedx_dim
();
for
(
size_t
value_item
=
0
;
value_item
<
num
;
++
value_item
)
{
for
(
size_t
value_item
=
0
;
value_item
<
num
;
++
value_item
)
{
float
*
value
=
values
[
value_item
];
float
*
value
=
values
[
value_item
];
value
[
sparse_feature_value
.
UnseenDaysIndex
()]
=
0
;
value
[
sparse_feature_value
.
UnseenDaysIndex
()]
=
0
;
...
@@ -215,7 +212,6 @@ int32_t SparseAccessor::Merge(float** update_values,
...
@@ -215,7 +212,6 @@ int32_t SparseAccessor::Merge(float** update_values,
// second dim: field num
// second dim: field num
int32_t
SparseAccessor
::
Update
(
float
**
update_values
,
const
float
**
push_values
,
int32_t
SparseAccessor
::
Update
(
float
**
update_values
,
const
float
**
push_values
,
size_t
num
)
{
size_t
num
)
{
auto
embedx_dim
=
_config
.
embedx_dim
();
for
(
size_t
value_item
=
0
;
value_item
<
num
;
++
value_item
)
{
for
(
size_t
value_item
=
0
;
value_item
<
num
;
++
value_item
)
{
float
*
update_value
=
update_values
[
value_item
];
float
*
update_value
=
update_values
[
value_item
];
const
float
*
push_value
=
push_values
[
value_item
];
const
float
*
push_value
=
push_values
[
value_item
];
...
@@ -294,8 +290,6 @@ std::string SparseAccessor::ParseToString(const float* v, int param) {
...
@@ -294,8 +290,6 @@ std::string SparseAccessor::ParseToString(const float* v, int param) {
}
}
int
SparseAccessor
::
ParseFromString
(
const
std
::
string
&
str
,
float
*
value
)
{
int
SparseAccessor
::
ParseFromString
(
const
std
::
string
&
str
,
float
*
value
)
{
int
embedx_dim
=
_config
.
embedx_dim
();
_embedx_sgd_rule
->
InitValue
(
value
+
sparse_feature_value
.
EmbedxWIndex
(),
_embedx_sgd_rule
->
InitValue
(
value
+
sparse_feature_value
.
EmbedxWIndex
(),
value
+
sparse_feature_value
.
EmbedxG2SumIndex
());
value
+
sparse_feature_value
.
EmbedxG2SumIndex
());
auto
ret
=
paddle
::
string
::
str_to_float
(
str
.
data
(),
value
);
auto
ret
=
paddle
::
string
::
str_to_float
(
str
.
data
(),
value
);
...
...
paddle/fluid/distributed/ps/table/ssd_sparse_table.cc
浏览文件 @
2bef8a48
...
@@ -290,9 +290,6 @@ int64_t SSDSparseTable::LocalSize() {
...
@@ -290,9 +290,6 @@ int64_t SSDSparseTable::LocalSize() {
local_size
+=
_local_shards
[
i
].
size
();
local_size
+=
_local_shards
[
i
].
size
();
}
}
// TODO rocksdb size
// TODO rocksdb size
uint64_t
ssd_size
=
0
;
// _db->get_estimate_key_num(ssd_size);
// return local_size + ssd_size;
return
local_size
;
return
local_size
;
}
}
...
@@ -473,7 +470,6 @@ int64_t SSDSparseTable::CacheShuffle(
...
@@ -473,7 +470,6 @@ int64_t SSDSparseTable::CacheShuffle(
}
}
int
shuffle_node_num
=
_config
.
sparse_table_cache_file_num
();
int
shuffle_node_num
=
_config
.
sparse_table_cache_file_num
();
LOG
(
INFO
)
<<
"Table>> shuffle node num is: "
<<
shuffle_node_num
;
LOG
(
INFO
)
<<
"Table>> shuffle node num is: "
<<
shuffle_node_num
;
size_t
file_start_idx
=
_avg_local_shard_num
*
_shard_idx
;
int
thread_num
=
_real_local_shard_num
<
20
?
_real_local_shard_num
:
20
;
int
thread_num
=
_real_local_shard_num
<
20
?
_real_local_shard_num
:
20
;
std
::
vector
<
std
::
vector
<
...
@@ -578,7 +574,6 @@ int32_t SSDSparseTable::SaveCache(
...
@@ -578,7 +574,6 @@ int32_t SSDSparseTable::SaveCache(
return
0
;
return
0
;
}
}
int
save_param
=
atoi
(
param
.
c_str
());
// batch_model:0 xbox:1
int
save_param
=
atoi
(
param
.
c_str
());
// batch_model:0 xbox:1
size_t
file_start_idx
=
_avg_local_shard_num
*
_shard_idx
;
std
::
string
table_path
=
paddle
::
string
::
format_string
(
std
::
string
table_path
=
paddle
::
string
::
format_string
(
"%s/%03d_cache/"
,
path
.
c_str
(),
_config
.
table_id
());
"%s/%03d_cache/"
,
path
.
c_str
(),
_config
.
table_id
());
_afs_client
.
remove
(
paddle
::
string
::
format_string
(
_afs_client
.
remove
(
paddle
::
string
::
format_string
(
...
...
paddle/fluid/distributed/test/barrier_table_test.cc
浏览文件 @
2bef8a48
...
@@ -26,7 +26,6 @@ namespace paddle {
...
@@ -26,7 +26,6 @@ namespace paddle {
namespace
distributed
{
namespace
distributed
{
TEST
(
BarrierTable
,
Barrier
)
{
TEST
(
BarrierTable
,
Barrier
)
{
int
emb_dim
=
10
;
int
trainers
=
2
;
int
trainers
=
2
;
bool
sync
=
true
;
bool
sync
=
true
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录