Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
OPTHREE
Paddle
提交
ee4e567d
P
Paddle
项目概览
OPTHREE
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
ee4e567d
编写于
6月 04, 2018
作者:
F
fengjiayi
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Creating readers before training begining
上级
e0a8c584
变更
1
隐藏空白更改
内联
并排
Showing
1 changed file
with
28 addition
and
24 deletion
+28
-24
paddle/fluid/operators/reader/open_files_op.cc
paddle/fluid/operators/reader/open_files_op.cc
+28
-24
未找到文件。
paddle/fluid/operators/reader/open_files_op.cc
浏览文件 @
ee4e567d
...
...
@@ -26,7 +26,11 @@ class MultiFileReader : public framework::ReaderBase {
MultiFileReader
(
const
std
::
vector
<
std
::
string
>&
file_names
,
const
std
::
vector
<
framework
::
DDim
>&
dims
,
size_t
thread_num
,
size_t
buffer_size
)
:
file_names_
(
file_names
),
dims_
(
dims
),
buffer_size_
(
buffer_size
)
{
:
buffer_size_
(
buffer_size
)
{
readers_
.
resize
(
file_names
.
size
());
for
(
const
std
::
string
&
f_name
:
file_names
)
{
readers_
.
emplace_back
(
CreateReaderByFileName
(
f_name
,
dims
));
}
prefetchers_
.
resize
(
thread_num
);
StartNewScheduler
();
}
...
...
@@ -40,14 +44,13 @@ class MultiFileReader : public framework::ReaderBase {
void
StartNewScheduler
();
void
EndScheduler
();
void
ScheduleThreadFunc
();
void
PrefetchThreadFunc
(
s
td
::
string
file_name
,
size_t
thread_idx
);
void
PrefetchThreadFunc
(
s
ize_t
reader_idx
,
size_t
thread_idx
);
std
::
vector
<
std
::
string
>
file_names_
;
std
::
vector
<
framework
::
DDim
>
dims_
;
std
::
vector
<
std
::
unique_ptr
<
framework
::
ReaderBase
>>
readers_
;
std
::
thread
scheduler_
;
std
::
vector
<
std
::
thread
>
prefetchers_
;
size_t
buffer_size_
;
reader
::
BlockingQueue
<
size_t
>*
waiting_
file
_idx_
;
reader
::
BlockingQueue
<
size_t
>*
waiting_
reader
_idx_
;
reader
::
BlockingQueue
<
size_t
>*
available_thread_idx_
;
reader
::
BlockingQueue
<
std
::
vector
<
framework
::
LoDTensor
>>*
buffer_
;
};
...
...
@@ -60,20 +63,23 @@ void MultiFileReader::ReadNext(std::vector<framework::LoDTensor>* out) {
void
MultiFileReader
::
ReInit
()
{
EndScheduler
();
for
(
auto
&
reader
:
readers_
)
{
reader
->
ReInit
();
}
StartNewScheduler
();
}
void
MultiFileReader
::
StartNewScheduler
()
{
size_t
thread_num
=
prefetchers_
.
size
();
waiting_
file_idx_
=
new
reader
::
BlockingQueue
<
size_t
>
(
file_name
s_
.
size
());
waiting_
reader_idx_
=
new
reader
::
BlockingQueue
<
size_t
>
(
reader
s_
.
size
());
available_thread_idx_
=
new
reader
::
BlockingQueue
<
size_t
>
(
thread_num
);
buffer_
=
new
reader
::
BlockingQueue
<
std
::
vector
<
framework
::
LoDTensor
>>
(
buffer_size_
);
for
(
size_t
i
=
0
;
i
<
file_name
s_
.
size
();
++
i
)
{
waiting_
file
_idx_
->
Send
(
i
);
for
(
size_t
i
=
0
;
i
<
reader
s_
.
size
();
++
i
)
{
waiting_
reader
_idx_
->
Send
(
i
);
}
waiting_
file
_idx_
->
Close
();
waiting_
reader
_idx_
->
Close
();
for
(
size_t
i
=
0
;
i
<
thread_num
;
++
i
)
{
available_thread_idx_
->
Send
(
i
);
}
...
...
@@ -84,13 +90,13 @@ void MultiFileReader::StartNewScheduler() {
void
MultiFileReader
::
EndScheduler
()
{
available_thread_idx_
->
Close
();
buffer_
->
Close
();
waiting_
file
_idx_
->
Close
();
waiting_
reader
_idx_
->
Close
();
if
(
scheduler_
.
joinable
())
{
scheduler_
.
join
();
}
delete
buffer_
;
delete
available_thread_idx_
;
delete
waiting_
file
_idx_
;
delete
waiting_
reader
_idx_
;
}
void
MultiFileReader
::
ScheduleThreadFunc
()
{
...
...
@@ -102,12 +108,11 @@ void MultiFileReader::ScheduleThreadFunc() {
if
(
prefetcher
.
joinable
())
{
prefetcher
.
join
();
}
size_t
file
_idx
;
if
(
waiting_
file_idx_
->
Receive
(
&
file
_idx
))
{
size_t
reader
_idx
;
if
(
waiting_
reader_idx_
->
Receive
(
&
reader
_idx
))
{
// Still have files to read. Start a new prefetch thread.
std
::
string
file_name
=
file_names_
[
file_idx
];
prefetcher
=
std
::
thread
([
this
,
file_name
,
thread_idx
]
{
PrefetchThreadFunc
(
file_name
,
thread_idx
);
prefetcher
=
std
::
thread
([
this
,
reader_idx
,
thread_idx
]
{
PrefetchThreadFunc
(
reader_idx
,
thread_idx
);
});
}
else
{
// No more file to read.
...
...
@@ -129,11 +134,9 @@ void MultiFileReader::ScheduleThreadFunc() {
VLOG
(
5
)
<<
"MultiFileReader schedule thread terminates."
;
}
void
MultiFileReader
::
PrefetchThreadFunc
(
std
::
string
file_name
,
size_t
thread_idx
)
{
VLOG
(
5
)
<<
"The prefetch thread of file '"
<<
file_name
<<
"' starts."
;
std
::
unique_ptr
<
framework
::
ReaderBase
>
reader
=
CreateReaderByFileName
(
file_name
,
dims_
);
void
MultiFileReader
::
PrefetchThreadFunc
(
size_t
reader_idx
,
size_t
thread_idx
)
{
VLOG
(
5
)
<<
"The prefetch thread of file idx '"
<<
reader_idx
<<
"' starts."
;
std
::
unique_ptr
<
framework
::
ReaderBase
>&
reader
=
readers_
[
reader_idx
];
while
(
true
)
{
std
::
vector
<
framework
::
LoDTensor
>
ins
;
reader
->
ReadNext
(
&
ins
);
...
...
@@ -144,8 +147,8 @@ void MultiFileReader::PrefetchThreadFunc(std::string file_name,
buffer_
->
Send
(
std
::
move
(
ins
));
}
catch
(
paddle
::
platform
::
EnforceNotMet
e
)
{
VLOG
(
5
)
<<
"WARNING: The buffer channel has been closed. The prefetch "
"thread of file '"
<<
file_name
<<
"' will terminate."
;
"thread of file
idx
'"
<<
reader_idx
<<
"' will terminate."
;
break
;
}
}
...
...
@@ -154,7 +157,8 @@ void MultiFileReader::PrefetchThreadFunc(std::string file_name,
VLOG
(
5
)
<<
"WARNING: The available_thread_idx_ channel has been closed. "
"Fail to send thread_idx."
;
}
VLOG
(
5
)
<<
"The prefetch thread of file '"
<<
file_name
<<
"' terminates."
;
VLOG
(
5
)
<<
"The prefetch thread of file idx '"
<<
reader_idx
<<
"' terminates."
;
}
class
OpenFilesOp
:
public
framework
::
OperatorBase
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录