Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
橘蓝
Paddle
提交
3fcd16ed
P
Paddle
项目概览
橘蓝
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
3fcd16ed
编写于
3月 06, 2018
作者:
F
fengjiayi
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
init double buffer
上级
86263b2f
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
66 addition
and
0 deletion
+66
-0
paddle/fluid/framework/reader.cc
paddle/fluid/framework/reader.cc
+41
-0
paddle/fluid/framework/reader.h
paddle/fluid/framework/reader.h
+25
-0
未找到文件。
paddle/fluid/framework/reader.cc
浏览文件 @
3fcd16ed
...
...
@@ -112,5 +112,46 @@ void BatchReader::ReadNext(std::vector<LoDTensor>* out) {
out
->
push_back
(
out_tensor
);
}
}
void
DoubleBufferReader
::
ReadNext
(
std
::
vector
<
LoDTensor
>*
out
)
{
std
::
unique_lock
<
std
::
mutex
>
lck
(
mtx_
);
while
(
write_pos_
==
read_pos_
)
{
buffer_not_empty_
.
wait
(
lck
);
}
out
->
clear
();
out
->
resize
(
buffer_
[
read_pos_
].
size
());
// TODO(fengjiayi): This copy shall be reduced.
for
(
size_t
i
=
0
;
i
<
buffer_
[
read_pos_
].
size
();
++
i
)
{
TensorCopy
(
buffer_
[
read_pos_
][
i
],
platform
::
CPUPlace
(),
&
out
[
i
]);
out
[
i
].
set_lod
(
buffer_
[
read_pos_
][
i
].
lod
());
}
++
read_pos_
;
if
(
read_pos_
>=
kDoubleBufferSize
)
{
read_pos_
=
0
;
}
buffer_not_full_
.
notify_all
();
}
bool
DoubleBufferReader
::
HasNext
()
{
return
reader_
->
HasNext
()
||
!
buffer_
.
empty
();
}
void
DoubleBufferReader
::
ProducerThreadFunc
()
{
while
(
reader_
->
HasNext
())
{
std
::
unique_lock
<
std
::
mutex
>
lck
(
mtx
);
while
(((
write_pos_
+
1
)
%
kDoubleBufferSize
)
==
read_pos_
)
{
buffer_not_full_
.
wait
(
lck
);
}
reader_
->
ReadNext
(
&
buffer_
[
write_pos_
]);
++
write_pos_
;
if
(
write_pos_
>=
kDoubleBufferSize
)
{
write_pos_
=
0
;
}
buffer_not_empty_
.
notify_all
();
}
}
}
// namespace framework
}
// namespace paddle
paddle/fluid/framework/reader.h
浏览文件 @
3fcd16ed
...
...
@@ -16,10 +16,13 @@
#include "paddle/fluid/framework/ddim.h"
#include "paddle/fluid/framework/lod_tensor_array.h"
#include "paddle/fluid/framework/threadpool.h"
namespace
paddle
{
namespace
framework
{
static
constexpr
size_t
kDoubleBufferSize
=
3
;
class
ReaderBase
{
public:
explicit
ReaderBase
(
const
std
::
vector
<
DDim
>&
shapes
)
:
shapes_
(
shapes
)
{
...
...
@@ -135,6 +138,28 @@ class BatchReader : public DecoratedReader {
std
::
vector
<
std
::
vector
<
LoDTensor
>>
buffer_
;
};
class
DoubleBufferReader
:
public
DecoratedReader
{
public:
DoubleBufferReader
(
ReaderBase
*
reader
)
:
DecoratedReader
(
reader
),
buffer_
(
kDoubleBufferSize
)
{
framework
::
Async
(
std
::
bind
(
&
DoubleBufferReader
::
ProducerThreadFunc
,
this
));
}
void
ReadNext
(
std
::
vector
<
LoDTensor
>*
out
)
override
;
bool
HasNext
()
const
override
;
private:
void
ProducerThreadFunc
();
std
::
vector
<
std
::
vector
<
LoDTensor
>>
buffer_
;
size_t
write_pos_
;
size_t
read_pos_
;
std
::
mutex
mtx_
;
std
::
condition_variable
buffer_not_full_
;
std
::
condition_variable
buffer_not_empty_
;
};
// The ReaderHolder is used as readers' unified wrapper,
// making it easier to access different type readers in Variables.
class
ReaderHolder
{
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录