Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
兽拳
Paddle
提交
61b3a597
P
Paddle
项目概览
兽拳
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
61b3a597
编写于
7月 14, 2018
作者:
Y
yuyang18
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Refine Python Reader
上级
c204f0cc
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
23 addition
and
11 deletion
+23
-11
paddle/fluid/operators/reader/create_py_reader_op.cc
paddle/fluid/operators/reader/create_py_reader_op.cc
+2
-0
python/paddle/fluid/layers/io.py
python/paddle/fluid/layers/io.py
+17
-9
python/paddle/fluid/tests/demo/pyreader.py
python/paddle/fluid/tests/demo/pyreader.py
+4
-2
未找到文件。
paddle/fluid/operators/reader/create_py_reader_op.cc
浏览文件 @
61b3a597
...
...
@@ -33,6 +33,8 @@ class PyReader : public framework::FileReader {
if
(
!
success
)
out
->
clear
();
}
~
PyReader
()
{
queue_
->
Close
();
}
void
Shutdown
()
override
{
queue_
->
Close
();
}
void
Start
()
override
{
queue_
->
ReOpen
();
}
...
...
python/paddle/fluid/layers/io.py
浏览文件 @
61b3a597
...
...
@@ -558,6 +558,7 @@ def py_reader(capacity,
current_reset_method
=
reader
.
reset
reader
.
thread
=
None
reader
.
tensor_provider
=
None
reader
.
exited
=
False
def
start_provide_thread
(
func
):
def
__provider_thread__
():
...
...
@@ -571,17 +572,20 @@ def py_reader(capacity,
array
.
append
(
item
)
if
reader
.
exited
:
break
feed_queue
.
push
(
array
)
if
reader
.
exited
:
break
feed_queue
.
close
()
reader
.
thread
=
threading
.
Thread
(
target
=
__provider_thread__
)
reader
.
thread
.
start
()
def
__set_tensor_provider__
(
func
):
reader
.
_tensor_provider
=
func
start_provide_thread
(
reader
.
_tensor_provider
)
reader
.
tensor_provider
=
func
def
__set_paddle_reader__
(
reader
):
def
__set_paddle_reader__
(
paddle_
reader
):
with
program_guard
(
Program
(),
Program
()):
feed_list
=
[]
counter
=
0
...
...
@@ -596,25 +600,29 @@ def py_reader(capacity,
counter
+=
1
feeder
=
DataFeeder
(
feed_list
=
feed_list
,
place
=
core
.
CPUPlace
())
reader
=
feeder
.
decorate_reader
(
reader
,
multi_devices
=
False
)
paddle_reader
=
feeder
.
decorate_reader
(
paddle_
reader
,
multi_devices
=
False
)
def
__tensor_provider__
():
for
data
in
reader
():
yield
[
data
[
str
(
idx
)]
for
idx
in
xrange
(
counter
)]
for
slots
in
paddle_
reader
():
yield
[
slots
[
str
(
idx
)]
for
idx
in
xrange
(
counter
)]
__set_tensor_provider__
(
__tensor_provider__
)
def
__reset__
():
current_reset_method
()
if
reader
.
thread
is
not
None
and
reader
.
tensor_provider
is
not
None
:
reader
.
exited
=
True
reader
.
thread
.
join
()
# restart provider thread.
start_provide_thread
(
reader
.
tensor_provider
)
reader
.
exited
=
False
def
__start__
():
start_provide_thread
(
reader
.
tensor_provider
)
reader
.
reset
=
__reset__
reader
.
decorate_tensor_provider
=
__set_tensor_provider__
reader
.
decorate_paddle_reader
=
__set_paddle_reader__
reader
.
start
=
__start__
return
reader
...
...
python/paddle/fluid/tests/demo/pyreader.py
浏览文件 @
61b3a597
...
...
@@ -67,11 +67,12 @@ def main():
train_reader
.
decorate_paddle_reader
(
paddle
.
v2
.
reader
.
shuffle
(
paddle
.
batch
(
mnist
.
train
(),
256
),
buf_size
=
8192
))
paddle
.
batch
(
mnist
.
train
(),
512
),
buf_size
=
8192
))
test_reader
.
decorate_paddle_reader
(
paddle
.
batch
(
mnist
.
test
(),
256
))
test_reader
.
decorate_paddle_reader
(
paddle
.
batch
(
mnist
.
test
(),
512
))
for
epoch_id
in
xrange
(
10
):
train_reader
.
start
()
try
:
while
True
:
print
'train_loss'
,
numpy
.
array
(
...
...
@@ -80,6 +81,7 @@ def main():
print
'End of epoch'
,
epoch_id
train_reader
.
reset
()
test_reader
.
start
()
try
:
while
True
:
print
'test loss'
,
numpy
.
array
(
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录