Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
wmsofts
Paddle
提交
d4514949
P
Paddle
项目概览
wmsofts
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
d4514949
编写于
3月 26, 2019
作者:
D
dongdaxiang
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
remove local random engine in fleet with rand_r()
test=develop
上级
e82969ee
变更
7
隐藏空白更改
内联
并排
Showing
7 changed file
with
11 addition
and
58 deletion
+11
-58
paddle/fluid/framework/data_feed.cc
paddle/fluid/framework/data_feed.cc
+1
-1
paddle/fluid/framework/data_feed.h
paddle/fluid/framework/data_feed.h
+1
-0
paddle/fluid/framework/data_set.cc
paddle/fluid/framework/data_set.cc
+1
-1
paddle/fluid/framework/data_set.h
paddle/fluid/framework/data_set.h
+1
-0
paddle/fluid/framework/fleet/fleet_wrapper.cc
paddle/fluid/framework/fleet/fleet_wrapper.cc
+6
-54
paddle/fluid/framework/fleet/fleet_wrapper.h
paddle/fluid/framework/fleet/fleet_wrapper.h
+0
-1
paddle/fluid/string/string_helper.cc
paddle/fluid/string/string_helper.cc
+1
-1
未找到文件。
paddle/fluid/framework/data_feed.cc
浏览文件 @
d4514949
...
...
@@ -349,7 +349,7 @@ void InMemoryDataFeed<T>::GlobalShuffle() {
for
(
int64_t
i
=
interval
.
first
;
i
<
interval
.
second
;
++
i
)
{
// if get ins id, can also use hash
// std::string ins_id = memory_data_[i].ins_id;
int64_t
random_num
=
fleet_ptr
->
LocalRandomEngine
()(
);
int64_t
random_num
=
rand_r
(
&
rand_seed
);
int64_t
node_id
=
random_num
%
trainer_num_
;
send_vec
[
node_id
].
push_back
(
&
((
*
memory_data_
)[
i
]));
if
(
i
%
fleet_send_batch_size_
==
0
&&
i
!=
0
)
{
...
...
paddle/fluid/framework/data_feed.h
浏览文件 @
d4514949
...
...
@@ -232,6 +232,7 @@ class InMemoryDataFeed : public PrivateQueueDataFeed<T> {
int
thread_id_
;
int
thread_num_
;
int
trainer_num_
;
uint32_t
rand_seed
;
std
::
vector
<
T
>*
memory_data_
;
std
::
mutex
*
mutex_for_update_memory_data_
;
// when read ins, we put ins from one channel to the other,
...
...
paddle/fluid/framework/data_set.cc
浏览文件 @
d4514949
...
...
@@ -250,7 +250,7 @@ int DatasetImpl<T>::ReceiveFromClient(int msg_type, int client_id,
VLOG
(
3
)
<<
"ReceiveFromClient msg_type="
<<
msg_type
<<
", client_id="
<<
client_id
<<
", msg length="
<<
msg
.
length
();
auto
fleet_ptr
=
FleetWrapper
::
GetInstance
();
int64_t
index
=
fleet_ptr
->
LocalRandomEngine
()(
)
%
thread_num_
;
int64_t
index
=
rand_r
(
&
rand_seed
)
%
thread_num_
;
VLOG
(
3
)
<<
"ramdom index="
<<
index
;
readers_
[
index
]
->
PutInsToChannel
(
msg
);
return
0
;
...
...
paddle/fluid/framework/data_set.h
浏览文件 @
d4514949
...
...
@@ -136,6 +136,7 @@ class DatasetImpl : public Dataset {
std
::
mutex
mutex_for_pick_file_
;
std
::
string
fs_name_
;
std
::
string
fs_ugi_
;
unsigned
int
rand_seed
;
};
// use std::vector<MultiSlotType> as data type
...
...
paddle/fluid/framework/fleet/fleet_wrapper.cc
浏览文件 @
d4514949
...
...
@@ -210,52 +210,20 @@ void FleetWrapper::PushDenseParamSync(
const
ProgramDesc
&
program
,
const
uint64_t
table_id
,
const
std
::
vector
<
std
::
string
>&
var_names
)
{
#ifdef PADDLE_WITH_PSLIB
paddle
::
framework
::
Scope
scope
;
auto
&
block
=
program
.
Block
(
0
);
for
(
auto
&
var
:
block
.
AllVars
())
{
if
(
var
->
Persistable
())
{
auto
*
ptr
=
scope
.
Var
(
var
->
Name
());
InitializeVariable
(
ptr
,
var
->
GetType
());
}
else
{
auto
*
ptr
=
scope
.
Var
(
var
->
Name
());
InitializeVariable
(
ptr
,
var
->
GetType
());
}
}
auto
place
=
platform
::
CPUPlace
();
std
::
vector
<
paddle
::
ps
::
Region
>
regions
;
for
(
auto
&
t
:
var_names
)
{
Variable
*
var
=
scope
.
FindVar
(
t
);
CHECK
(
var
!=
nullptr
)
<<
"var["
<<
t
<<
"] not found"
;
LoDTensor
*
tensor
=
var
->
GetMutable
<
LoDTensor
>
();
std
::
vector
<
int64_t
>
dim
;
for
(
auto
&
var
:
block
.
AllVars
())
{
if
(
var
->
Name
()
==
t
)
{
dim
=
var
->
GetShape
();
break
;
}
}
int
cnt
=
1
;
for
(
auto
&
i
:
dim
)
{
cnt
*=
i
;
}
DDim
d
(
std
::
vector
<
int64_t
>
{
cnt
}.
data
(),
1
);
float
*
g
=
tensor
->
mutable_data
<
float
>
(
d
,
place
);
CHECK
(
g
!=
nullptr
)
<<
"var["
<<
t
<<
"] value not initialized"
;
float
init_range
=
0.2
;
int
rown
=
tensor
->
dims
()[
0
];
init_range
/=
sqrt
(
rown
);
std
::
normal_distribution
<
float
>
ndistr
(
0.0
,
1.0
);
for
(
auto
i
=
0u
;
i
<
tensor
->
numel
();
++
i
)
{
g
[
i
]
=
ndistr
(
LocalRandomEngine
())
*
init_range
;
}
float
*
g
=
tensor
->
mutable_data
<
float
>
(
place
);
paddle
::
ps
::
Region
reg
(
g
,
tensor
->
numel
());
regions
.
emplace_back
(
std
::
move
(
reg
));
auto
push_status
=
pslib_ptr_
->
_worker_ptr
->
push_dense_param
(
regions
.
data
(),
regions
.
size
(),
table_id
);
push_status
.
wait
();
auto
status
=
push_status
.
get
();
CHECK
(
status
==
0
)
<<
"push dense param failed, status["
<<
status
<<
"]"
;
}
auto
push_status
=
pslib_ptr_
->
_worker_ptr
->
push_dense_param
(
regions
.
data
(),
regions
.
size
(),
table_id
);
push_status
.
wait
();
auto
status
=
push_status
.
get
();
CHECK
(
status
==
0
)
<<
"push dense param failed, status["
<<
status
<<
"]"
;
#endif
}
...
...
@@ -372,22 +340,6 @@ std::future<int32_t> FleetWrapper::SendClientToClientMsg(
return
std
::
future
<
int32_t
>
();
}
std
::
default_random_engine
&
FleetWrapper
::
LocalRandomEngine
()
{
struct
engine_wrapper_t
{
std
::
default_random_engine
engine
;
engine_wrapper_t
()
{
struct
timespec
tp
;
clock_gettime
(
CLOCK_REALTIME
,
&
tp
);
double
cur_time
=
tp
.
tv_sec
+
tp
.
tv_nsec
*
1e-9
;
static
std
::
atomic
<
uint64_t
>
x
(
0
);
std
::
seed_seq
sseq
=
{
x
++
,
x
++
,
x
++
,
(
uint64_t
)(
cur_time
*
1000
)};
engine
.
seed
(
sseq
);
}
};
thread_local
engine_wrapper_t
r
;
return
r
.
engine
;
}
template
<
typename
T
>
void
FleetWrapper
::
Serialize
(
const
std
::
vector
<
T
*>&
t
,
std
::
string
*
str
)
{
#ifdef PADDLE_WITH_PSLIB
...
...
paddle/fluid/framework/fleet/fleet_wrapper.h
浏览文件 @
d4514949
...
...
@@ -127,7 +127,6 @@ class FleetWrapper {
std
::
future
<
int32_t
>
SendClientToClientMsg
(
int
msg_type
,
int
to_client_id
,
const
std
::
string
&
msg
);
std
::
default_random_engine
&
LocalRandomEngine
();
template
<
typename
T
>
void
Serialize
(
const
std
::
vector
<
T
*>&
t
,
std
::
string
*
str
);
template
<
typename
T
>
...
...
paddle/fluid/string/string_helper.cc
浏览文件 @
d4514949
...
...
@@ -79,7 +79,7 @@ inline int str_to_float(const char* str, float* v) {
// A line buffer is maintained. It
// doesn't need to know the maximum possible length of a line.
char
*
LineFileReader
::
getdelim
(
FILE
*
f
,
char
delim
)
{
#ifndef _
_
WIN32
#ifndef _WIN32
int32_t
ret
=
::
getdelim
(
&
_buffer
,
&
_buf_size
,
delim
,
f
);
if
(
ret
>=
0
)
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录