Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
afeixing77
Paddle
提交
eec412b2
P
Paddle
项目概览
afeixing77
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
eec412b2
编写于
7月 23, 2018
作者:
F
fengjiayi
提交者:
GitHub
7月 23, 2018
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #12273 from JiayiFeng/update_py_reader
Some enhancement on readers
上级
9e0a94f0
ea8a375f
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
26 addition
and
10 deletion
+26
-10
paddle/fluid/operators/reader/buffered_reader.cc
paddle/fluid/operators/reader/buffered_reader.cc
+13
-1
python/paddle/fluid/layers/io.py
python/paddle/fluid/layers/io.py
+5
-4
python/paddle/fluid/tests/demo/pyreader.py
python/paddle/fluid/tests/demo/pyreader.py
+8
-5
未找到文件。
paddle/fluid/operators/reader/buffered_reader.cc
浏览文件 @
eec412b2
...
...
@@ -18,7 +18,14 @@
namespace
paddle
{
namespace
operators
{
namespace
reader
{
BufferedReader
::~
BufferedReader
()
{
reader_
->
Shutdown
();
}
BufferedReader
::~
BufferedReader
()
{
reader_
->
Shutdown
();
while
(
!
position_
.
empty
())
{
position_
.
front
().
wait
();
position_
.
pop
();
}
}
BufferedReader
::
BufferedReader
(
const
std
::
shared_ptr
<
framework
::
ReaderBase
>
&
reader
,
const
platform
::
Place
&
place
,
size_t
buffer_size
)
...
...
@@ -30,12 +37,14 @@ BufferedReader::BufferedReader(
gpu_buffer_
.
resize
(
buffer_size
);
ReadTillBufferFullAsync
();
}
void
BufferedReader
::
ReadTillBufferFullAsync
()
{
PADDLE_ENFORCE_EQ
(
position_
.
size
(),
0U
);
for
(
size_t
i
=
0
;
i
<
buffer_size_
;
++
i
)
{
ReadAsync
(
i
);
}
}
void
BufferedReader
::
ReadAsync
(
size_t
i
)
{
position_
.
emplace
(
thread_pool_
.
enqueue
([
this
,
i
]()
->
size_t
{
TensorVec
&
cpu
=
cpu_buffer_
[
i
];
...
...
@@ -56,6 +65,7 @@ void BufferedReader::ReadAsync(size_t i) {
return
i
;
}));
}
void
BufferedReader
::
ShutdownImpl
()
{
reader_
->
Shutdown
();
while
(
!
position_
.
empty
())
{
...
...
@@ -63,10 +73,12 @@ void BufferedReader::ShutdownImpl() {
}
prev_pos_
=
-
1UL
;
}
void
BufferedReader
::
StartImpl
()
{
reader_
->
Start
();
ReadTillBufferFullAsync
();
}
void
BufferedReader
::
ReadNextImpl
(
std
::
vector
<
framework
::
LoDTensor
>
*
out
)
{
if
(
position_
.
empty
())
{
out
->
clear
();
...
...
python/paddle/fluid/layers/io.py
浏览文件 @
eec412b2
...
...
@@ -457,7 +457,7 @@ def py_reader(capacity,
use_double_buffer
=
True
):
"""
Create a reader and blocking queue for data feeding in Python
This layer returns a Reader Variable and a BlockingQueue.
The BlockingQueue provides `push()` method to push a `LoDTensorArray`
object into the queue in Python side. In C++ side, the Reader
...
...
@@ -478,7 +478,7 @@ def py_reader(capacity,
Returns:
tuple(Variable, BlockingQueue):
A Reader Variable from which we can get feeding data.
A BlockingQueue object for data feeding.
Examples:
...
...
@@ -491,7 +491,7 @@ def py_reader(capacity,
dtypes=['float32', 'int64'])
# Via the reader, we can use 'read_file' layer to get data:
image, label = fluid.layers.read_file(reader)
# Via the blocking queue, we can feed data using threads
def feed_data(queue, feed_images, feed_labels):
for feed_image, feed_label in zip(feed_images, feed_labels):
...
...
@@ -499,7 +499,7 @@ def py_reader(capacity,
data.append(feed_image)
data.append(feed_label)
queue.push(data)
thread = threading.Thread(target=feed_data, args=(queue, feed_images, feed_labels))
thread.start()
"""
...
...
@@ -579,6 +579,7 @@ def py_reader(capacity,
feed_queue
.
close
()
reader
.
thread
=
threading
.
Thread
(
target
=
__provider_thread__
)
reader
.
thread
.
daemon
=
True
reader
.
thread
.
start
()
def
__set_tensor_provider__
(
func
):
...
...
python/paddle/fluid/tests/demo/pyreader.py
浏览文件 @
eec412b2
...
...
@@ -25,7 +25,8 @@ def network(is_train):
capacity
=
10
,
shapes
=
((
-
1
,
784
),
(
-
1
,
1
)),
dtypes
=
(
'float32'
,
'int64'
),
name
=
"train_reader"
if
is_train
else
"test_reader"
)
name
=
"train_reader"
if
is_train
else
"test_reader"
,
use_double_buffer
=
True
)
img
,
label
=
fluid
.
layers
.
read_file
(
reader
)
hidden
=
img
...
...
@@ -56,14 +57,16 @@ def main():
with
fluid
.
unique_name
.
guard
():
test_loss
,
test_reader
=
network
(
False
)
fluid
.
Executor
(
fluid
.
CUDAPlace
(
0
)).
run
(
startup_prog
)
fluid
.
Executor
(
fluid
.
CUDAPlace
(
0
)).
run
(
test_startup
)
use_cuda
=
fluid
.
core
.
is_compiled_with_cuda
()
place
=
fluid
.
CUDAPlace
(
0
)
if
use_cuda
else
fluid
.
CPUPlace
()
fluid
.
Executor
(
place
).
run
(
startup_prog
)
fluid
.
Executor
(
place
).
run
(
test_startup
)
trainer
=
fluid
.
ParallelExecutor
(
use_cuda
=
True
,
loss_name
=
loss
.
name
,
main_program
=
train_prog
)
use_cuda
=
use_cuda
,
loss_name
=
loss
.
name
,
main_program
=
train_prog
)
tester
=
fluid
.
ParallelExecutor
(
use_cuda
=
True
,
share_vars_from
=
trainer
,
main_program
=
test_prog
)
use_cuda
=
use_cuda
,
share_vars_from
=
trainer
,
main_program
=
test_prog
)
train_reader
.
decorate_paddle_reader
(
paddle
.
v2
.
reader
.
shuffle
(
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录