Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
Crayon鑫
Paddle
提交
cadea35a
P
Paddle
项目概览
Crayon鑫
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
提交
cadea35a
编写于
6月 20, 2017
作者:
W
wanghaoshuang
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
format code
上级
09cc4408
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
15 addition
and
10 deletion
+15
-10
python/paddle/v2/reader/decorator.py
python/paddle/v2/reader/decorator.py
+9
-8
python/paddle/v2/reader/tests/decorator_test.py
python/paddle/v2/reader/tests/decorator_test.py
+6
-2
未找到文件。
python/paddle/v2/reader/decorator.py
浏览文件 @
cadea35a
...
...
@@ -251,18 +251,19 @@ def xmap_readers(mapper, reader, process_num, buffer_size, order=False):
in_queue
=
Queue
(
buffer_size
)
out_queue
=
Queue
(
buffer_size
)
out_order
=
[
0
]
# define a worker to read samples from reader to in_queue
def
read_worker
(
reader
,
in_queue
):
for
i
in
reader
():
in_queue
.
put
(
i
)
in_queue
.
put
(
end
)
# define a worker to read samples from reader to in_queue with order flag
def
order_read_worker
(
reader
,
in_queue
):
in_order
=
0
for
i
in
reader
():
in_queue
.
put
((
in_order
,
i
))
in_order
+=
1
in_queue
.
put
((
in_order
,
i
))
in_order
+=
1
in_queue
.
put
(
end
)
# start a read worker in a thread
...
...
@@ -281,7 +282,7 @@ def xmap_readers(mapper, reader, process_num, buffer_size, order=False):
sample
=
in_queue
.
get
()
in_queue
.
put
(
end
)
out_queue
.
put
(
end
)
# define a worker to handle samples from in_queue by mapper
# and put mapped samples into out_queue by order
def
order_handle_worker
(
in_queue
,
out_queue
,
mapper
,
out_order
):
...
...
@@ -292,18 +293,18 @@ def xmap_readers(mapper, reader, process_num, buffer_size, order=False):
while
order
!=
out_order
[
0
]:
pass
out_queue
.
put
(
r
)
out_order
[
0
]
+=
1
out_order
[
0
]
+=
1
ins
=
in_queue
.
get
()
in_queue
.
put
(
end
)
out_queue
.
put
(
end
)
# start several handle_workers
target
=
order_handle_worker
if
order
else
handle_worker
args
=
(
in_queue
,
out_queue
,
mapper
,
out_order
)
if
order
else
(
in_queue
,
out_queue
,
mapper
)
args
=
(
in_queue
,
out_queue
,
mapper
,
out_order
)
if
order
else
(
in_queue
,
out_queue
,
mapper
)
workers
=
[]
for
i
in
xrange
(
process_num
):
worker
=
Thread
(
target
=
target
,
args
=
args
)
worker
=
Thread
(
target
=
target
,
args
=
args
)
worker
.
daemon
=
True
workers
.
append
(
worker
)
for
w
in
workers
:
...
...
python/paddle/v2/reader/tests/decorator_test.py
浏览文件 @
cadea35a
...
...
@@ -120,10 +120,12 @@ class TestShuffle(unittest.TestCase):
total
+=
1
self
.
assertEqual
(
total
,
10
)
class
TestXmap
(
unittest
.
TestCase
):
def
test_xmap
(
self
):
def
mapper
(
x
):
return
(
x
+
1
)
orders
=
(
True
,
False
)
thread_nums
=
(
1
,
2
,
4
,
8
,
16
)
buffered_size
=
(
1
,
2
,
4
,
8
,
16
)
...
...
@@ -131,13 +133,15 @@ class TestXmap(unittest.TestCase):
for
tNum
in
thread_nums
:
for
size
in
buffered_size
:
result
=
[]
for
i
in
paddle
.
v2
.
reader
.
xmap_readers
(
mapper
,
reader_creator_10
(),
tNum
,
size
,
order
)():
for
i
in
paddle
.
v2
.
reader
.
xmap_readers
(
mapper
,
reader_creator_10
(),
tNum
,
size
,
order
)():
result
.
append
(
i
)
if
not
order
:
result
.
sort
()
for
idx
,
e
in
enumerate
(
result
):
self
.
assertEqual
(
e
,
mapper
(
idx
))
if
__name__
==
'__main__'
:
unittest
.
main
()
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录