Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
SummerGao.
Paddle
提交
2e67f8ae
P
Paddle
项目概览
SummerGao.
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
2e67f8ae
编写于
2月 26, 2019
作者:
S
sneaxiy
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
add doc
test=develop
上级
c545f1ed
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
152 addition
and
2 deletion
+152
-2
python/paddle/fluid/framework.py
python/paddle/fluid/framework.py
+49
-0
python/paddle/fluid/layers/io.py
python/paddle/fluid/layers/io.py
+1
-2
python/paddle/fluid/reader.py
python/paddle/fluid/reader.py
+102
-0
未找到文件。
python/paddle/fluid/framework.py
浏览文件 @
2e67f8ae
...
...
@@ -96,6 +96,27 @@ def _cpu_num():
def
cuda_places
(
device_ids
=
None
):
'''
Create a list of :code:`fluid.CUDAPlace` objects.
If :code:`device_ids` is None, environment variable of
:code:`FLAGS_selected_gpus` would be checked first. If
:code:`FLAGS_selected_gpus=0,1,2`, the returned list would
be [fluid.CUDAPlace(0), fluid.CUDAPlace(1), fluid.CUDAPlace(2)].
If :code:`FLAGS_selected_gpus` is not set, all visible
gpu places would be returned.
If :code:`device_ids` is not None, it should be the device
ids of gpus. For example, if :code:`device_ids=[0,1,2]`,
the returned list would be
[fluid.CUDAPlace(0), fluid.CUDAPlace(1), fluid.CUDAPlace(2)].
Args:
device_ids (None|list(int)|tuple(int)): gpu device id list.
Returns:
out (list(fluid.CUDAPlace)): gpu place list.
'''
assert
core
.
is_compiled_with_cuda
(),
\
"Not compiled with CUDA"
if
device_ids
is
None
:
...
...
@@ -110,12 +131,40 @@ def cuda_places(device_ids=None):
def
cpu_places
(
device_count
=
None
):
'''
Create a list of :code:`fluid.CPUPlace` objects.
If :code:`device_count` is None, the device count would
be determined by environment variable :code:`CPU_NUM`.
If :code:`CPU_NUM` is not set, the device count would
be determined by :code:`multiprocessing.cpu_count()`.
Args:
device_count (None|int): device number.
Returns:
out (list(fluid.CPUPlace)): cpu place list.
'''
if
device_count
is
None
:
device_count
=
_cpu_num
()
return
[
core
.
CPUPlace
()]
*
device_count
def
cuda_pinned_places
(
device_count
=
None
):
'''
Create a list of :code:`fluid.CUDAPinnedPlace` objects.
If :code:`device_count` is None, the device count would
be determined by environment variable :code:`CPU_NUM`.
If :code:`CPU_NUM` is not set, the device count would
be determined by :code:`multiprocessing.cpu_count()`.
Args:
device_count (None|int): device number.
Returns:
out (list(fluid.CUDAPinnedPlace)): cuda pinned place list.
'''
assert
core
.
is_compiled_with_cuda
(),
\
"Not compiled with CUDA"
if
device_count
is
None
:
...
...
python/paddle/fluid/layers/io.py
浏览文件 @
2e67f8ae
...
...
@@ -531,8 +531,7 @@ def _py_reader(capacity,
startup_blk
=
default_startup_program
().
current_block
()
startup_var
=
startup_blk
.
create_var
(
name
=
reader_name
)
startup_blk
.
append_op
(
type
=
'create_py_reader'
if
not
lock_free
else
'create_lock_free_py_reader'
,
type
=
'create_py_reader'
,
inputs
=
{
'blocking_queue'
:
[
queue_name
]},
outputs
=
{
'Out'
:
[
startup_var
]},
attrs
=
{
...
...
python/paddle/fluid/reader.py
浏览文件 @
2e67f8ae
...
...
@@ -47,6 +47,76 @@ class PyReader(object):
capacity
,
use_double_buffer
=
True
,
iterable
=
True
):
"""
Create a reader object for data feeding in Python.
Data would be prefetched using Python thread and be pushed
into a queue asynchronously. Data in the queue would be extracted
automatically when `Executor.run(...)` is called.
Args:
feed_list (list(Variable)|tuple(Variable)): feed variable list.
The variables should be created by :code:`fluid.layers.data()`.
capacity (int): capacity of the queue maintained in PyReader object.
use_double_buffer (bool): whether to use double_buffer_reader to
speed up data feeding.
iterable (bool): whether the created reader object is iterable.
Returns:
reader (Reader): the created reader object.
Examples:
1. If iterable = False, the created PyReader object is almost the
same as :code:`fluid.layers.py_reader()`. Operators would be
inserted into the program. User should call :code:`start()`
before each epoch and catch :code:`fluid.core.EOFException`
thrown by :code:`Executor.run()` when epoch ends. Once the
exception is caught, user should call :code:`reset()` to reset
the reader manually.
.. code-block:: python
image = fluid.layers.data(
name='image', shape=[784], dtype='float32')
label = fluid.layers.data(
name='label', shape=[1], dtype='int64')
reader = fluid.io.PyReader(feed_list=[image, label],
capacity=4, iterable=False)
reader.decorate_paddle_reader(user_defined_reader)
... # definition of network is omitted
executor.run(fluid.default_main_program())
for _ in range(EPOCH_NUM):
reader.start()
while True:
try:
executor.run(feed=None, ...)
except fluid.core.EOFException:
reader.reset()
break
2. If iterable=True, the created PyReader object is decoupled with
the program. No operator would be inserted into the program.
In this case, the created reader is a Python generator, which
is iterable. User should feed the data yielded from PyReader
object into :code:`Executor.run(feed=...)`.
.. code-block:: python
image = fluid.layers.data(
name='image', shape=[784], dtype='float32')
label = fluid.layers.data(
name='label', shape=[1], dtype='int64')
reader = fluid.io.PyReader(feed_list=[image, label],
capacity=4, iterable=True)
reader.decorate_paddle_reader(user_defined_reader,
places=fluid.cuda_places())
... # definition of network is omitted
executor.run(fluid.default_main_program())
for _ in range(EPOCH_NUM):
for data in reader():
executor.run(feed=data, ...)
"""
self
.
_tensor_reader
=
None
self
.
_thread
=
None
self
.
_iterable
=
iterable
...
...
@@ -161,10 +231,18 @@ class PyReader(object):
self
.
_thread
.
join
()
def
start
(
self
):
'''
Start the data feeding thread.
Can only call when the reader object is not iterable.
'''
assert
not
self
.
_iterable
,
"start() cannot be called when PyReader is iterable"
self
.
_start
()
def
reset
(
self
):
'''
Reset the reader object when :code:`fluid.core.EOFException` raises.
Can only call when the reader object is not iterable.
'''
assert
not
self
.
_iterable
,
"reset() cannot be called when PyReader is iterable"
self
.
_reset
()
...
...
@@ -190,6 +268,18 @@ class PyReader(object):
self
.
_thread
.
start
()
def
decorate_paddle_reader
(
self
,
reader
,
places
=
None
):
'''
Set the data source of the PyReader object.
The provided :code:`reader` should be a Python generator,
which yields numpy-typed batched data.
:code:`places` must be set when the PyReader object is iterable.
Args:
reader (generator): Python generator that yields numpy-typed
batched data.
'''
assert
self
.
_tensor_reader
is
None
,
\
"Cannot reset the data source of PyReader"
with
program_guard
(
Program
(),
Program
()):
...
...
@@ -204,6 +294,18 @@ class PyReader(object):
self
.
decorate_tensor_provider
(
__tensor_reader_impl__
,
places
)
def
decorate_tensor_provider
(
self
,
reader
,
places
=
None
):
'''
Set the data source of the PyReader object.
The provided :code:`reader` should be a Python generator,
which yields LoDTensor-typed batched data.
:code:`places` must be set when the PyReader object is iterable.
Args:
reader (generator): Python generator that yields LoDTensor-typed
batched data.
'''
assert
self
.
_tensor_reader
is
None
,
\
"Cannot reset the data source of PyReader"
self
.
_tensor_reader
=
reader
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录