Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
PaddlePaddle
PaddleDetection
提交
87a5590b
P
PaddleDetection
项目概览
PaddlePaddle
/
PaddleDetection
大约 1 年 前同步成功
通知
694
Star
11112
Fork
2696
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
184
列表
看板
标记
里程碑
合并请求
40
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
PaddleDetection
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
184
Issue
184
列表
看板
标记
里程碑
合并请求
40
合并请求
40
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
87a5590b
编写于
6月 04, 2018
作者:
F
fengjiayi
提交者:
GitHub
6月 04, 2018
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #11151 from JiayiFeng/dev_update_open_files_op
Update open files op
上级
2a5cb2ec
3526ac11
变更
1
隐藏空白更改
内联
并排
Showing
1 changed file
with
26 addition
and
24 deletion
+26
-24
paddle/fluid/operators/reader/open_files_op.cc
paddle/fluid/operators/reader/open_files_op.cc
+26
-24
未找到文件。
paddle/fluid/operators/reader/open_files_op.cc
浏览文件 @
87a5590b
...
@@ -26,7 +26,11 @@ class MultiFileReader : public framework::ReaderBase {
...
@@ -26,7 +26,11 @@ class MultiFileReader : public framework::ReaderBase {
MultiFileReader
(
const
std
::
vector
<
std
::
string
>&
file_names
,
MultiFileReader
(
const
std
::
vector
<
std
::
string
>&
file_names
,
const
std
::
vector
<
framework
::
DDim
>&
dims
,
size_t
thread_num
,
const
std
::
vector
<
framework
::
DDim
>&
dims
,
size_t
thread_num
,
size_t
buffer_size
)
size_t
buffer_size
)
:
file_names_
(
file_names
),
dims_
(
dims
),
buffer_size_
(
buffer_size
)
{
:
buffer_size_
(
buffer_size
)
{
readers_
.
reserve
(
file_names
.
size
());
for
(
const
std
::
string
&
f_name
:
file_names
)
{
readers_
.
emplace_back
(
CreateReaderByFileName
(
f_name
,
dims
));
}
prefetchers_
.
resize
(
thread_num
);
prefetchers_
.
resize
(
thread_num
);
StartNewScheduler
();
StartNewScheduler
();
}
}
...
@@ -40,14 +44,13 @@ class MultiFileReader : public framework::ReaderBase {
...
@@ -40,14 +44,13 @@ class MultiFileReader : public framework::ReaderBase {
void
StartNewScheduler
();
void
StartNewScheduler
();
void
EndScheduler
();
void
EndScheduler
();
void
ScheduleThreadFunc
();
void
ScheduleThreadFunc
();
void
PrefetchThreadFunc
(
s
td
::
string
file_name
,
size_t
thread_idx
);
void
PrefetchThreadFunc
(
s
ize_t
reader_idx
,
size_t
thread_idx
);
std
::
vector
<
std
::
string
>
file_names_
;
std
::
vector
<
std
::
unique_ptr
<
framework
::
ReaderBase
>>
readers_
;
std
::
vector
<
framework
::
DDim
>
dims_
;
std
::
thread
scheduler_
;
std
::
thread
scheduler_
;
std
::
vector
<
std
::
thread
>
prefetchers_
;
std
::
vector
<
std
::
thread
>
prefetchers_
;
size_t
buffer_size_
;
size_t
buffer_size_
;
reader
::
BlockingQueue
<
size_t
>*
waiting_
file
_idx_
;
reader
::
BlockingQueue
<
size_t
>*
waiting_
reader
_idx_
;
reader
::
BlockingQueue
<
size_t
>*
available_thread_idx_
;
reader
::
BlockingQueue
<
size_t
>*
available_thread_idx_
;
reader
::
BlockingQueue
<
std
::
vector
<
framework
::
LoDTensor
>>*
buffer_
;
reader
::
BlockingQueue
<
std
::
vector
<
framework
::
LoDTensor
>>*
buffer_
;
};
};
...
@@ -65,15 +68,15 @@ void MultiFileReader::ReInit() {
...
@@ -65,15 +68,15 @@ void MultiFileReader::ReInit() {
void
MultiFileReader
::
StartNewScheduler
()
{
void
MultiFileReader
::
StartNewScheduler
()
{
size_t
thread_num
=
prefetchers_
.
size
();
size_t
thread_num
=
prefetchers_
.
size
();
waiting_
file_idx_
=
new
reader
::
BlockingQueue
<
size_t
>
(
file_name
s_
.
size
());
waiting_
reader_idx_
=
new
reader
::
BlockingQueue
<
size_t
>
(
reader
s_
.
size
());
available_thread_idx_
=
new
reader
::
BlockingQueue
<
size_t
>
(
thread_num
);
available_thread_idx_
=
new
reader
::
BlockingQueue
<
size_t
>
(
thread_num
);
buffer_
=
new
reader
::
BlockingQueue
<
std
::
vector
<
framework
::
LoDTensor
>>
(
buffer_
=
new
reader
::
BlockingQueue
<
std
::
vector
<
framework
::
LoDTensor
>>
(
buffer_size_
);
buffer_size_
);
for
(
size_t
i
=
0
;
i
<
file_name
s_
.
size
();
++
i
)
{
for
(
size_t
i
=
0
;
i
<
reader
s_
.
size
();
++
i
)
{
waiting_
file
_idx_
->
Send
(
i
);
waiting_
reader
_idx_
->
Send
(
i
);
}
}
waiting_
file
_idx_
->
Close
();
waiting_
reader
_idx_
->
Close
();
for
(
size_t
i
=
0
;
i
<
thread_num
;
++
i
)
{
for
(
size_t
i
=
0
;
i
<
thread_num
;
++
i
)
{
available_thread_idx_
->
Send
(
i
);
available_thread_idx_
->
Send
(
i
);
}
}
...
@@ -84,13 +87,13 @@ void MultiFileReader::StartNewScheduler() {
...
@@ -84,13 +87,13 @@ void MultiFileReader::StartNewScheduler() {
void
MultiFileReader
::
EndScheduler
()
{
void
MultiFileReader
::
EndScheduler
()
{
available_thread_idx_
->
Close
();
available_thread_idx_
->
Close
();
buffer_
->
Close
();
buffer_
->
Close
();
waiting_
file
_idx_
->
Close
();
waiting_
reader
_idx_
->
Close
();
if
(
scheduler_
.
joinable
())
{
if
(
scheduler_
.
joinable
())
{
scheduler_
.
join
();
scheduler_
.
join
();
}
}
delete
buffer_
;
delete
buffer_
;
delete
available_thread_idx_
;
delete
available_thread_idx_
;
delete
waiting_
file
_idx_
;
delete
waiting_
reader
_idx_
;
}
}
void
MultiFileReader
::
ScheduleThreadFunc
()
{
void
MultiFileReader
::
ScheduleThreadFunc
()
{
...
@@ -102,12 +105,11 @@ void MultiFileReader::ScheduleThreadFunc() {
...
@@ -102,12 +105,11 @@ void MultiFileReader::ScheduleThreadFunc() {
if
(
prefetcher
.
joinable
())
{
if
(
prefetcher
.
joinable
())
{
prefetcher
.
join
();
prefetcher
.
join
();
}
}
size_t
file
_idx
;
size_t
reader
_idx
;
if
(
waiting_
file_idx_
->
Receive
(
&
file
_idx
))
{
if
(
waiting_
reader_idx_
->
Receive
(
&
reader
_idx
))
{
// Still have files to read. Start a new prefetch thread.
// Still have files to read. Start a new prefetch thread.
std
::
string
file_name
=
file_names_
[
file_idx
];
prefetcher
=
std
::
thread
([
this
,
reader_idx
,
thread_idx
]
{
prefetcher
=
std
::
thread
([
this
,
file_name
,
thread_idx
]
{
PrefetchThreadFunc
(
reader_idx
,
thread_idx
);
PrefetchThreadFunc
(
file_name
,
thread_idx
);
});
});
}
else
{
}
else
{
// No more file to read.
// No more file to read.
...
@@ -129,23 +131,22 @@ void MultiFileReader::ScheduleThreadFunc() {
...
@@ -129,23 +131,22 @@ void MultiFileReader::ScheduleThreadFunc() {
VLOG
(
5
)
<<
"MultiFileReader schedule thread terminates."
;
VLOG
(
5
)
<<
"MultiFileReader schedule thread terminates."
;
}
}
void
MultiFileReader
::
PrefetchThreadFunc
(
std
::
string
file_name
,
void
MultiFileReader
::
PrefetchThreadFunc
(
size_t
reader_idx
,
size_t
thread_idx
)
{
size_t
thread_idx
)
{
VLOG
(
5
)
<<
"The prefetch thread of file idx '"
<<
reader_idx
<<
"' starts."
;
VLOG
(
5
)
<<
"The prefetch thread of file '"
<<
file_name
<<
"' starts."
;
std
::
unique_ptr
<
framework
::
ReaderBase
>&
reader
=
readers_
[
reader_idx
];
std
::
unique_ptr
<
framework
::
ReaderBase
>
reader
=
CreateReaderByFileName
(
file_name
,
dims_
);
while
(
true
)
{
while
(
true
)
{
std
::
vector
<
framework
::
LoDTensor
>
ins
;
std
::
vector
<
framework
::
LoDTensor
>
ins
;
reader
->
ReadNext
(
&
ins
);
reader
->
ReadNext
(
&
ins
);
if
(
ins
.
empty
())
{
if
(
ins
.
empty
())
{
reader
->
ReInit
();
break
;
break
;
}
}
try
{
try
{
buffer_
->
Send
(
std
::
move
(
ins
));
buffer_
->
Send
(
std
::
move
(
ins
));
}
catch
(
paddle
::
platform
::
EnforceNotMet
e
)
{
}
catch
(
paddle
::
platform
::
EnforceNotMet
e
)
{
VLOG
(
5
)
<<
"WARNING: The buffer channel has been closed. The prefetch "
VLOG
(
5
)
<<
"WARNING: The buffer channel has been closed. The prefetch "
"thread of file '"
"thread of file
idx
'"
<<
file_name
<<
"' will terminate."
;
<<
reader_idx
<<
"' will terminate."
;
break
;
break
;
}
}
}
}
...
@@ -154,7 +155,8 @@ void MultiFileReader::PrefetchThreadFunc(std::string file_name,
...
@@ -154,7 +155,8 @@ void MultiFileReader::PrefetchThreadFunc(std::string file_name,
VLOG
(
5
)
<<
"WARNING: The available_thread_idx_ channel has been closed. "
VLOG
(
5
)
<<
"WARNING: The available_thread_idx_ channel has been closed. "
"Fail to send thread_idx."
;
"Fail to send thread_idx."
;
}
}
VLOG
(
5
)
<<
"The prefetch thread of file '"
<<
file_name
<<
"' terminates."
;
VLOG
(
5
)
<<
"The prefetch thread of file idx '"
<<
reader_idx
<<
"' terminates."
;
}
}
class
OpenFilesOp
:
public
framework
::
OperatorBase
{
class
OpenFilesOp
:
public
framework
::
OperatorBase
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录