Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
PaddlePaddle
Paddle
提交
9b85f401
P
Paddle
项目概览
PaddlePaddle
/
Paddle
1 年多 前同步成功
通知
2302
Star
20931
Fork
5422
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1423
列表
看板
标记
里程碑
合并请求
543
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1,423
Issue
1,423
列表
看板
标记
里程碑
合并请求
543
合并请求
543
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
提交
9b85f401
编写于
10月 12, 2019
作者:
G
guofei
提交者:
Huihuang Zheng
10月 12, 2019
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Modify English documents (#20452)
上级
9dc83dda
变更
6
显示空白变更内容
内联
并排
Showing
6 changed file
with
302 addition
and
175 deletion
+302
-175
paddle/fluid/API.spec
paddle/fluid/API.spec
+13
-13
python/paddle/batch.py
python/paddle/batch.py
+33
-10
python/paddle/fluid/framework.py
python/paddle/fluid/framework.py
+10
-7
python/paddle/fluid/io.py
python/paddle/fluid/io.py
+116
-72
python/paddle/fluid/layers/io.py
python/paddle/fluid/layers/io.py
+53
-32
python/paddle/fluid/reader.py
python/paddle/fluid/reader.py
+77
-41
未找到文件。
paddle/fluid/API.spec
浏览文件 @
9b85f401
...
@@ -9,7 +9,7 @@ paddle.fluid.Program.parse_from_string (ArgSpec(args=['binary_str'], varargs=Non
...
@@ -9,7 +9,7 @@ paddle.fluid.Program.parse_from_string (ArgSpec(args=['binary_str'], varargs=Non
paddle.fluid.Program.to_string (ArgSpec(args=['self', 'throw_on_error', 'with_details'], varargs=None, keywords=None, defaults=(False,)), ('document', '7dde33f16b63aa50d474870a9cebb539'))
paddle.fluid.Program.to_string (ArgSpec(args=['self', 'throw_on_error', 'with_details'], varargs=None, keywords=None, defaults=(False,)), ('document', '7dde33f16b63aa50d474870a9cebb539'))
paddle.fluid.default_startup_program (ArgSpec(args=[], varargs=None, keywords=None, defaults=None), ('document', 'f53890b2fb8c0642b6047e4fee2d6d58'))
paddle.fluid.default_startup_program (ArgSpec(args=[], varargs=None, keywords=None, defaults=None), ('document', 'f53890b2fb8c0642b6047e4fee2d6d58'))
paddle.fluid.default_main_program (ArgSpec(args=[], varargs=None, keywords=None, defaults=None), ('document', '082aa471d247bd8d7c87814105439e1a'))
paddle.fluid.default_main_program (ArgSpec(args=[], varargs=None, keywords=None, defaults=None), ('document', '082aa471d247bd8d7c87814105439e1a'))
paddle.fluid.program_guard (ArgSpec(args=['main_program', 'startup_program'], varargs=None, keywords=None, defaults=(None,)), ('document', '
78fb5c7f70ef76bcf4a1862c3f6b8191
'))
paddle.fluid.program_guard (ArgSpec(args=['main_program', 'startup_program'], varargs=None, keywords=None, defaults=(None,)), ('document', '
eb4eabc13405a8c6dc2f14308ddf5ed8
'))
paddle.fluid.name_scope (ArgSpec(args=['prefix'], varargs=None, keywords=None, defaults=(None,)), ('document', '907a5f877206079d8e67ae69b06bb3ba'))
paddle.fluid.name_scope (ArgSpec(args=['prefix'], varargs=None, keywords=None, defaults=(None,)), ('document', '907a5f877206079d8e67ae69b06bb3ba'))
paddle.fluid.cuda_places (ArgSpec(args=['device_ids'], varargs=None, keywords=None, defaults=(None,)), ('document', 'ab9bd2079536114aa7c1488a489ee87f'))
paddle.fluid.cuda_places (ArgSpec(args=['device_ids'], varargs=None, keywords=None, defaults=(None,)), ('document', 'ab9bd2079536114aa7c1488a489ee87f'))
paddle.fluid.cpu_places (ArgSpec(args=['device_count'], varargs=None, keywords=None, defaults=(None,)), ('document', 'a7352a3dd39308fde4fbbf6421a4193d'))
paddle.fluid.cpu_places (ArgSpec(args=['device_count'], varargs=None, keywords=None, defaults=(None,)), ('document', 'a7352a3dd39308fde4fbbf6421a4193d'))
...
@@ -72,24 +72,24 @@ paddle.fluid.BuildStrategy.ReduceStrategy.__init__ __init__(self: paddle.fluid.c
...
@@ -72,24 +72,24 @@ paddle.fluid.BuildStrategy.ReduceStrategy.__init__ __init__(self: paddle.fluid.c
paddle.fluid.BuildStrategy.__init__ __init__(self: paddle.fluid.core_avx.ParallelExecutor.BuildStrategy) -> None
paddle.fluid.BuildStrategy.__init__ __init__(self: paddle.fluid.core_avx.ParallelExecutor.BuildStrategy) -> None
paddle.fluid.gradients (ArgSpec(args=['targets', 'inputs', 'target_gradients', 'no_grad_set'], varargs=None, keywords=None, defaults=(None, None)), ('document', 'e2097e1e0ed84ae44951437bfe269a1b'))
paddle.fluid.gradients (ArgSpec(args=['targets', 'inputs', 'target_gradients', 'no_grad_set'], varargs=None, keywords=None, defaults=(None, None)), ('document', 'e2097e1e0ed84ae44951437bfe269a1b'))
paddle.fluid.io.save_vars (ArgSpec(args=['executor', 'dirname', 'main_program', 'vars', 'predicate', 'filename'], varargs=None, keywords=None, defaults=(None, None, None, None)), ('document', '9ff7159eef501e9dfaf520073e681c10'))
paddle.fluid.io.save_vars (ArgSpec(args=['executor', 'dirname', 'main_program', 'vars', 'predicate', 'filename'], varargs=None, keywords=None, defaults=(None, None, None, None)), ('document', '9ff7159eef501e9dfaf520073e681c10'))
paddle.fluid.io.save_params (ArgSpec(args=['executor', 'dirname', 'main_program', 'filename'], varargs=None, keywords=None, defaults=(None, None)), ('document', '
046d7c43d67e08c2660bb3bd7e081015
'))
paddle.fluid.io.save_params (ArgSpec(args=['executor', 'dirname', 'main_program', 'filename'], varargs=None, keywords=None, defaults=(None, None)), ('document', '
a03d0de7594d311103671b7275f1b464
'))
paddle.fluid.io.save_persistables (ArgSpec(args=['executor', 'dirname', 'main_program', 'filename'], varargs=None, keywords=None, defaults=(None, None)), ('document', '
ffcee38044975c29f2ab2fec0576f963
'))
paddle.fluid.io.save_persistables (ArgSpec(args=['executor', 'dirname', 'main_program', 'filename'], varargs=None, keywords=None, defaults=(None, None)), ('document', '
3cd8facbe536a09310e95453914ca322
'))
paddle.fluid.io.load_vars (ArgSpec(args=['executor', 'dirname', 'main_program', 'vars', 'predicate', 'filename'], varargs=None, keywords=None, defaults=(None, None, None, None)), ('document', '12dd2c3f29d63f7a920bb1e0a0e8caff'))
paddle.fluid.io.load_vars (ArgSpec(args=['executor', 'dirname', 'main_program', 'vars', 'predicate', 'filename'], varargs=None, keywords=None, defaults=(None, None, None, None)), ('document', '12dd2c3f29d63f7a920bb1e0a0e8caff'))
paddle.fluid.io.load_params (ArgSpec(args=['executor', 'dirname', 'main_program', 'filename'], varargs=None, keywords=None, defaults=(None, None)), ('document', 'f3f16db75ae076d46608c7e976650cfc'))
paddle.fluid.io.load_params (ArgSpec(args=['executor', 'dirname', 'main_program', 'filename'], varargs=None, keywords=None, defaults=(None, None)), ('document', 'f3f16db75ae076d46608c7e976650cfc'))
paddle.fluid.io.load_persistables (ArgSpec(args=['executor', 'dirname', 'main_program', 'filename'], varargs=None, keywords=None, defaults=(None, None)), ('document', '1e039084ad3781eb43966581eed48688'))
paddle.fluid.io.load_persistables (ArgSpec(args=['executor', 'dirname', 'main_program', 'filename'], varargs=None, keywords=None, defaults=(None, None)), ('document', '1e039084ad3781eb43966581eed48688'))
paddle.fluid.io.save_inference_model (ArgSpec(args=['dirname', 'feeded_var_names', 'target_vars', 'executor', 'main_program', 'model_filename', 'params_filename', 'export_for_deployment', 'program_only'], varargs=None, keywords=None, defaults=(None, None, None, True, False)), ('document', '
fc82bfd137a9b1ab8ebd1651bd35b6e5
'))
paddle.fluid.io.save_inference_model (ArgSpec(args=['dirname', 'feeded_var_names', 'target_vars', 'executor', 'main_program', 'model_filename', 'params_filename', 'export_for_deployment', 'program_only'], varargs=None, keywords=None, defaults=(None, None, None, True, False)), ('document', '
827797614e194d31ceb5a3a68c46efab
'))
paddle.fluid.io.load_inference_model (ArgSpec(args=['dirname', 'executor', 'model_filename', 'params_filename', 'pserver_endpoints'], varargs=None, keywords=None, defaults=(None, None, None)), ('document', '7a863032bf7613dec1c8dd99efbd82e5'))
paddle.fluid.io.load_inference_model (ArgSpec(args=['dirname', 'executor', 'model_filename', 'params_filename', 'pserver_endpoints'], varargs=None, keywords=None, defaults=(None, None, None)), ('document', '7a863032bf7613dec1c8dd99efbd82e5'))
paddle.fluid.io.batch (ArgSpec(args=['reader', 'batch_size', 'drop_last'], varargs=None, keywords=None, defaults=(False,)), ('document', '
cf2869b408b39cadadd95206b4e03b39
'))
paddle.fluid.io.batch (ArgSpec(args=['reader', 'batch_size', 'drop_last'], varargs=None, keywords=None, defaults=(False,)), ('document', '
16acb4e1215d5fc4386add454e717440
'))
paddle.fluid.io.save (ArgSpec(args=['program', 'model_path'], varargs=None, keywords=None, defaults=None), ('document', 'cef7d50c36b93c02b6d12bcea7d025ce'))
paddle.fluid.io.save (ArgSpec(args=['program', 'model_path'], varargs=None, keywords=None, defaults=None), ('document', 'cef7d50c36b93c02b6d12bcea7d025ce'))
paddle.fluid.io.load (ArgSpec(args=['program', 'model_path'], varargs=None, keywords=None, defaults=None), ('document', '8d0f200c20f8a4581e1843967230ad45'))
paddle.fluid.io.load (ArgSpec(args=['program', 'model_path'], varargs=None, keywords=None, defaults=None), ('document', '8d0f200c20f8a4581e1843967230ad45'))
paddle.fluid.io.PyReader ('paddle.fluid.reader.PyReader', ('document', '
b03399246f69cd6fc03b43e87af8bd4e
'))
paddle.fluid.io.PyReader ('paddle.fluid.reader.PyReader', ('document', '
f5875acee86f9f4432933bab40873722
'))
paddle.fluid.io.PyReader.__init__ (ArgSpec(args=['self', 'feed_list', 'capacity', 'use_double_buffer', 'iterable', 'return_list'], varargs=None, keywords=None, defaults=(None, None, True, True, False)), ('document', '6adf97f83acf6453d4a6a4b1070f3754'))
paddle.fluid.io.PyReader.__init__ (ArgSpec(args=['self', 'feed_list', 'capacity', 'use_double_buffer', 'iterable', 'return_list'], varargs=None, keywords=None, defaults=(None, None, True, True, False)), ('document', '6adf97f83acf6453d4a6a4b1070f3754'))
paddle.fluid.io.PyReader.decorate_batch_generator (ArgSpec(args=['self', 'reader', 'places'], varargs=None, keywords=None, defaults=(None,)), ('document', '
4364e836e3cb8ab5e68e411b763c50c7
'))
paddle.fluid.io.PyReader.decorate_batch_generator (ArgSpec(args=['self', 'reader', 'places'], varargs=None, keywords=None, defaults=(None,)), ('document', '
386f969058852594b916be7cc15b8066
'))
paddle.fluid.io.PyReader.decorate_sample_generator (ArgSpec(args=['self', 'sample_generator', 'batch_size', 'drop_last', 'places'], varargs=None, keywords=None, defaults=(True, None)), ('document', '
efa4c8b90fe6d99dcbda637b70351bb1
'))
paddle.fluid.io.PyReader.decorate_sample_generator (ArgSpec(args=['self', 'sample_generator', 'batch_size', 'drop_last', 'places'], varargs=None, keywords=None, defaults=(True, None)), ('document', '
c3ab4fd82a4560e369adcd23a36c2a7b
'))
paddle.fluid.io.PyReader.decorate_sample_list_generator (ArgSpec(args=['self', 'reader', 'places'], varargs=None, keywords=None, defaults=(None,)), ('document', '
6c11980092720de304863de98074a64a
'))
paddle.fluid.io.PyReader.decorate_sample_list_generator (ArgSpec(args=['self', 'reader', 'places'], varargs=None, keywords=None, defaults=(None,)), ('document', '
182443165b1b6a607ca821052b1b9085
'))
paddle.fluid.io.PyReader.next (ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None), ('document', '08b2fd1463f3ea99d79d17303988349b'))
paddle.fluid.io.PyReader.next (ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None), ('document', '08b2fd1463f3ea99d79d17303988349b'))
paddle.fluid.io.PyReader.reset (ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None), ('document', '
7432197701fdaab1848063860dc0b97e
'))
paddle.fluid.io.PyReader.reset (ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None), ('document', '
259e025143f0bb7cfc6d7163bc333679
'))
paddle.fluid.io.PyReader.start (ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None), ('document', '
a0983fb21a0a51e6a31716009fe9a9c1
'))
paddle.fluid.io.PyReader.start (ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None), ('document', '
c91176b919e19be11cbc11ab3d47318e
'))
paddle.fluid.io.DataLoader ('paddle.fluid.reader.DataLoader', ('document', '6adf97f83acf6453d4a6a4b1070f3754'))
paddle.fluid.io.DataLoader ('paddle.fluid.reader.DataLoader', ('document', '6adf97f83acf6453d4a6a4b1070f3754'))
paddle.fluid.io.DataLoader.__init__
paddle.fluid.io.DataLoader.__init__
paddle.fluid.io.DataLoader.from_dataset (ArgSpec(args=['dataset', 'places', 'drop_last'], varargs=None, keywords=None, defaults=(True,)), ('document', 'eb8b6d31e1c2ec2ca8ebbb62fcf46557'))
paddle.fluid.io.DataLoader.from_dataset (ArgSpec(args=['dataset', 'places', 'drop_last'], varargs=None, keywords=None, defaults=(True,)), ('document', 'eb8b6d31e1c2ec2ca8ebbb62fcf46557'))
...
@@ -310,10 +310,10 @@ paddle.fluid.layers.hard_swish (ArgSpec(args=['x', 'threshold', 'scale', 'offset
...
@@ -310,10 +310,10 @@ paddle.fluid.layers.hard_swish (ArgSpec(args=['x', 'threshold', 'scale', 'offset
paddle.fluid.layers.gather_tree (ArgSpec(args=['ids', 'parents'], varargs=None, keywords=None, defaults=None), ('document', '201b54fa7512305078c70a6610beaead'))
paddle.fluid.layers.gather_tree (ArgSpec(args=['ids', 'parents'], varargs=None, keywords=None, defaults=None), ('document', '201b54fa7512305078c70a6610beaead'))
paddle.fluid.layers.mse_loss (ArgSpec(args=['input', 'label'], varargs=None, keywords=None, defaults=None), ('document', '88b967ef5132567396062d5d654b3064'))
paddle.fluid.layers.mse_loss (ArgSpec(args=['input', 'label'], varargs=None, keywords=None, defaults=None), ('document', '88b967ef5132567396062d5d654b3064'))
paddle.fluid.layers.uniform_random (ArgSpec(args=['shape', 'dtype', 'min', 'max', 'seed'], varargs=None, keywords=None, defaults=('float32', -1.0, 1.0, 0)), ('document', '34e7c1ff0263baf9551000b6bb3bc47e'))
paddle.fluid.layers.uniform_random (ArgSpec(args=['shape', 'dtype', 'min', 'max', 'seed'], varargs=None, keywords=None, defaults=('float32', -1.0, 1.0, 0)), ('document', '34e7c1ff0263baf9551000b6bb3bc47e'))
paddle.fluid.layers.data (ArgSpec(args=['name', 'shape', 'append_batch_size', 'dtype', 'lod_level', 'type', 'stop_gradient'], varargs=None, keywords=None, defaults=(True, 'float32', 0, VarType.LOD_TENSOR, True)), ('document', '
9d7806e31bdf727c1a23b8782a09b545
'))
paddle.fluid.layers.data (ArgSpec(args=['name', 'shape', 'append_batch_size', 'dtype', 'lod_level', 'type', 'stop_gradient'], varargs=None, keywords=None, defaults=(True, 'float32', 0, VarType.LOD_TENSOR, True)), ('document', '
a43c597ac4e1cec20cf193c083d946be
'))
paddle.fluid.layers.read_file (ArgSpec(args=['reader'], varargs=None, keywords=None, defaults=None), ('document', 'd5b41c7b2df1b064fbd42dcf435268cd'))
paddle.fluid.layers.read_file (ArgSpec(args=['reader'], varargs=None, keywords=None, defaults=None), ('document', 'd5b41c7b2df1b064fbd42dcf435268cd'))
paddle.fluid.layers.double_buffer (ArgSpec(args=['reader', 'place', 'name'], varargs=None, keywords=None, defaults=(None, None)), ('document', '556fa82daf62cbb0fb393f4125daba77'))
paddle.fluid.layers.double_buffer (ArgSpec(args=['reader', 'place', 'name'], varargs=None, keywords=None, defaults=(None, None)), ('document', '556fa82daf62cbb0fb393f4125daba77'))
paddle.fluid.layers.py_reader (ArgSpec(args=['capacity', 'shapes', 'dtypes', 'lod_levels', 'name', 'use_double_buffer'], varargs=None, keywords=None, defaults=(None, None, True)), ('document', '
d78a1c7344955c5caed8dc13adb7beb6
'))
paddle.fluid.layers.py_reader (ArgSpec(args=['capacity', 'shapes', 'dtypes', 'lod_levels', 'name', 'use_double_buffer'], varargs=None, keywords=None, defaults=(None, None, True)), ('document', '
c9c52b0a57e541d751e7a839ad26ee1a
'))
paddle.fluid.layers.create_py_reader_by_data (ArgSpec(args=['capacity', 'feed_list', 'name', 'use_double_buffer'], varargs=None, keywords=None, defaults=(None, True)), ('document', '1321d4ce89d82f96fcfd5601f816b0f3'))
paddle.fluid.layers.create_py_reader_by_data (ArgSpec(args=['capacity', 'feed_list', 'name', 'use_double_buffer'], varargs=None, keywords=None, defaults=(None, True)), ('document', '1321d4ce89d82f96fcfd5601f816b0f3'))
paddle.fluid.layers.load (ArgSpec(args=['out', 'file_path', 'load_as_fp16'], varargs=None, keywords=None, defaults=(None,)), ('document', '309f9e5249463e1b207a7347b2a91134'))
paddle.fluid.layers.load (ArgSpec(args=['out', 'file_path', 'load_as_fp16'], varargs=None, keywords=None, defaults=(None,)), ('document', '309f9e5249463e1b207a7347b2a91134'))
paddle.fluid.layers.create_tensor (ArgSpec(args=['dtype', 'name', 'persistable'], varargs=None, keywords=None, defaults=(None, False)), ('document', 'fdc2d964488e99fb0743887454c34e36'))
paddle.fluid.layers.create_tensor (ArgSpec(args=['dtype', 'name', 'persistable'], varargs=None, keywords=None, defaults=(None, False)), ('document', 'fdc2d964488e99fb0743887454c34e36'))
...
...
python/paddle/batch.py
浏览文件 @
9b85f401
...
@@ -17,16 +17,39 @@ __all__ = ['batch']
...
@@ -17,16 +17,39 @@ __all__ = ['batch']
def
batch
(
reader
,
batch_size
,
drop_last
=
False
):
def
batch
(
reader
,
batch_size
,
drop_last
=
False
):
"""
"""
Create a batched reader.
This operator creates a batched reader which combines the data from the
input reader to batched data.
:param reader: the data reader to read from.
:type reader: callable
Args:
:param batch_size: size of each mini-batch
reader(generator): the data reader to read from.
:type batch_size: int
batch_size(int): size of each mini-batch.
:param drop_last: drop the last batch, if the size of last batch is not equal to batch_size.
drop_last(bool, optional): If set to True, the last batch is dropped when
:type drop_last: bool
the size of last batch is not equal to batch_size, if set to False,
:return: the batched reader.
it will not. Default: False.
:rtype: callable
Returns:
The batched reader.
Return Type:
generator
Examples:
.. code-block:: python
import paddle.fluid as fluid
def reader():
for i in range(10):
yield i
batch_reader = fluid.io.batch(reader, batch_size=2)
for data in batch_reader():
print(data)
# Output is
# [0, 1]
# [2, 3]
# [4, 5]
# [6, 7]
# [8, 9]
"""
"""
def
batch_reader
():
def
batch_reader
():
...
...
python/paddle/fluid/framework.py
浏览文件 @
9b85f401
...
@@ -4626,6 +4626,13 @@ def program_guard(main_program, startup_program=None):
...
@@ -4626,6 +4626,13 @@ def program_guard(main_program, startup_program=None):
Layer functions in the Python `"with"` block will append operators and
Layer functions in the Python `"with"` block will append operators and
variables to the new main programs.
variables to the new main programs.
Args:
main_program(Program): New main program inside `"with"` statement.
startup_program(Program, optional): New startup program inside `"with"`
statement. :code:`None` means not changing startup program,
default_startup_program is still used.
Default: None.
Examples:
Examples:
.. code-block:: python
.. code-block:: python
...
@@ -4634,7 +4641,7 @@ def program_guard(main_program, startup_program=None):
...
@@ -4634,7 +4641,7 @@ def program_guard(main_program, startup_program=None):
main_program = fluid.Program()
main_program = fluid.Program()
startup_program = fluid.Program()
startup_program = fluid.Program()
with fluid.program_guard(main_program, startup_program):
with fluid.program_guard(main_program, startup_program):
data = fluid.
layers.data(name='image', shape=[
784, 784], dtype='float32')
data = fluid.
data(name='image', shape=[None,
784, 784], dtype='float32')
hidden = fluid.layers.fc(input=data, size=10, act='relu')
hidden = fluid.layers.fc(input=data, size=10, act='relu')
Notes: The temporary :code:`Program` can be used if the user does not need
Notes: The temporary :code:`Program` can be used if the user does not need
...
@@ -4648,12 +4655,8 @@ def program_guard(main_program, startup_program=None):
...
@@ -4648,12 +4655,8 @@ def program_guard(main_program, startup_program=None):
main_program = fluid.Program()
main_program = fluid.Program()
# does not care about startup program. Just pass a temporary value.
# does not care about startup program. Just pass a temporary value.
with fluid.program_guard(main_program, fluid.Program()):
with fluid.program_guard(main_program, fluid.Program()):
data = fluid.
layers.data(name='image', shape=[
784, 784], dtype='float32')
data = fluid.
data(name='image', shape=[None,
784, 784], dtype='float32')
Args:
main_program(Program): New main program inside `"with"` statement.
startup_program(Program): New startup program inside `"with"` statement.
None means not changing startup program.
"""
"""
if
not
isinstance
(
main_program
,
Program
):
if
not
isinstance
(
main_program
,
Program
):
raise
TypeError
(
"main_program should be Program"
)
raise
TypeError
(
"main_program should be Program"
)
...
...
python/paddle/fluid/io.py
浏览文件 @
9b85f401
...
@@ -258,30 +258,38 @@ def save_vars(executor,
...
@@ -258,30 +258,38 @@ def save_vars(executor,
def
save_params
(
executor
,
dirname
,
main_program
=
None
,
filename
=
None
):
def
save_params
(
executor
,
dirname
,
main_program
=
None
,
filename
=
None
):
"""
"""
This function filters out all parameters from the give `main_program`
This operator saves all parameters from the :code:`main_program` to
and then save them to the folder `dirname` or the file `filename`.
the folder :code:`dirname` or file :code:`filename`. You can refer to
:ref:`api_guide_model_save_reader_en` for more details.
Use the `dirname` to specify the saving folder. If you would like to
Use the
:code:
`dirname` to specify the saving folder. If you would like to
save parameters in separate files, set `filename` None; if you would
save parameters in separate files, set
:code:
`filename` None; if you would
like to save all parameters in a single file, use `filename` to specify
like to save all parameters in a single file, use
:code:
`filename` to specify
the file name.
the file name.
NOTICE: Some variables are not Parameter while they are necessary for
Note:
training. So you can NOT save and continue your training just by
Some variables are not Parameter while they are necessary for
`save_params()` and `load_params()`. Please use `save_persistables()`
training, such as learning rate, global step, etc. So you can NOT save
and `load_persistables()` instead. If you want to save your model for
and continue your training just by :ref:`api_fluid_io_save_params`
the inference, please use the `save_inference_model` API. You can refer
and :ref:`api_fluid_io_load_params`. Please use :ref:`api_fluid_io_save_persistables`
to :ref:`api_guide_model_save_reader_en` for more details.
and :ref:`api_fluid_io_load_persistables` instead.
If you want to save your model for the inference, please use the
:ref:`api_fluid_io_save_inference_model`. You can refer to
:ref:`api_guide_model_save_reader_en` for more details.
Args:
Args:
executor(Executor): The executor to run for saving parameters.
executor(Executor): The executor to run for saving parameters, You can
refer to :ref:`api_guide_executor_en`.
dirname(str): The saving directory path.
dirname(str): The saving directory path.
main_program(Program|None): The program whose parameters will be
main_program(Program, optional): The program whose parameters will be
saved. If it is None, the default
saved. You can refer to
main program will be used automatically.
:ref:`api_guide_Program_en` for more
details. If it is None, the default main
program will be used.
Default: None
Default: None
filename(str
|None
): The file to save all parameters. If you prefer
filename(str
, optional
): The file to save all parameters. If you prefer
to save parameters in differne
t files, set it
to save parameters in differen
t files, set it
to None.
to None.
Default: None
Default: None
...
@@ -293,11 +301,20 @@ def save_params(executor, dirname, main_program=None, filename=None):
...
@@ -293,11 +301,20 @@ def save_params(executor, dirname, main_program=None, filename=None):
import paddle.fluid as fluid
import paddle.fluid as fluid
params_path = "./my_paddle_model"
image = fluid.data(name='img', shape=[None, 28, 28], dtype='float32')
label = fluid.data(name='label', shape=[None, 1], dtype='int64')
feeder = fluid.DataFeeder(feed_list=[image, label], place=fluid.CPUPlace())
predict = fluid.layers.fc(input=image, size=10, act='softmax')
loss = fluid.layers.cross_entropy(input=predict, label=label)
avg_loss = fluid.layers.mean(loss)
exe = fluid.Executor(fluid.CPUPlace())
exe = fluid.Executor(fluid.CPUPlace())
param_path = "./my_paddle_model"
exe.run(fluid.default_startup_program())
prog = fluid.default_main_program(
)
fluid.io.save_params(executor=exe, dirname=params_path
)
fluid.io.save_params(executor=exe, dirname=param_path,
# The parameters weights and bias of the fc layer in the network are going to
main_program=None)
# be saved in different files in the path "./my_paddle_model"
"""
"""
save_vars
(
save_vars
(
executor
,
executor
,
...
@@ -491,25 +508,31 @@ def _save_distributed_persistables(executor, dirname, main_program):
...
@@ -491,25 +508,31 @@ def _save_distributed_persistables(executor, dirname, main_program):
def
save_persistables
(
executor
,
dirname
,
main_program
=
None
,
filename
=
None
):
def
save_persistables
(
executor
,
dirname
,
main_program
=
None
,
filename
=
None
):
"""
"""
This function filters out all variables with `persistable==True` from the
This operator saves all persistable variables from :code:`main_program` to
give `main_program` and then saves these variables to the folder `dirname`
the folder :code:`dirname` or file :code:`filename`. You can refer to
or file `filename`.
:ref:`api_guide_model_save_reader_en` for more details. And then
saves these persistables variables to the folder :code:`dirname` or file
:code:`filename`.
The `dirname` is used to specify the folder where persistable variables
The
:code:
`dirname` is used to specify the folder where persistable variables
are going to be saved. If you would like to save variables in separate
are going to be saved. If you would like to save variables in separate
files, set `filename` None; if you would like to save all variables in a
files, set
:code:
`filename` None; if you would like to save all variables in a
single file, use `filename` to specify the file name.
single file, use
:code:
`filename` to specify the file name.
Args:
Args:
executor(Executor): The executor to run for saving persistable variables.
executor(Executor): The executor to run for saving persistable variables.
dirname(str): The directory path.
You can refer to :ref:`api_guide_executor_en` for
main_program(Program|None): The program whose persistbale variables will
more details.
be saved. If it is None, the default main
dirname(str): The saving directory path.
program will be used automatically.
main_program(Program, optional): The program whose persistbale variables will
Default: None
be saved. You can refer to
filename(str|None): The file to saved all variables. If you prefer to
:ref:`api_guide_Program_en` for more details.
save variables in differnet files, set it to None.
If it is None, the default main program will
Default: None
be used.
Default: None.
filename(str, optional): The file to save all variables. If you prefer to
save variables in different files, set it to None.
Default: None.
Returns:
Returns:
None
None
...
@@ -519,12 +542,21 @@ def save_persistables(executor, dirname, main_program=None, filename=None):
...
@@ -519,12 +542,21 @@ def save_persistables(executor, dirname, main_program=None, filename=None):
import paddle.fluid as fluid
import paddle.fluid as fluid
dir_path = "./my_paddle_model"
file_name = "persistables"
image = fluid.data(name='img', shape=[None, 28, 28], dtype='float32')
label = fluid.data(name='label', shape=[None, 1], dtype='int64')
feeder = fluid.DataFeeder(feed_list=[image, label], place=fluid.CPUPlace())
predict = fluid.layers.fc(input=image, size=10, act='softmax')
loss = fluid.layers.cross_entropy(input=predict, label=label)
avg_loss = fluid.layers.mean(loss)
exe = fluid.Executor(fluid.CPUPlace())
exe = fluid.Executor(fluid.CPUPlace())
param_path = "./my_paddle_model"
exe.run(fluid.default_startup_program())
# `prog` can be a program defined by the user
fluid.io.save_persistables(executor=exe, dirname=dir_path, filename=file_name)
prog = fluid.default_main_program()
# The persistables variables weights and bias in the fc layer of the network
fluid.io.save_persistables(executor=exe, dirname=param_path,
# are going to be saved in the same file named "persistables" in the path
main_program=prog)
# "./my_paddle_model"
"""
"""
if
main_program
and
main_program
.
_is_distributed
:
if
main_program
and
main_program
.
_is_distributed
:
_save_distributed_persistables
(
_save_distributed_persistables
(
...
@@ -973,27 +1005,33 @@ def save_inference_model(dirname,
...
@@ -973,27 +1005,33 @@ def save_inference_model(dirname,
program_only
=
False
):
program_only
=
False
):
"""
"""
Prune the given `main_program` to build a new program especially for inference,
Prune the given `main_program` to build a new program especially for inference,
and then save it and all related parameters to given `dirname`
by the `executor`
.
and then save it and all related parameters to given `dirname` .
If you just want to save parameters of your trained model, please use the
If you just want to save parameters of your trained model, please use the
`save_params` API. You can refer to :ref:`api_guide_model_save_reader_en` for
:ref:`api_fluid_io_save_params` . You can refer to :ref:`api_guide_model_save_reader_en`
more details.
for
more details.
Note:
The :code:`dirname` is used to specify the folder where inference model
structure and parameters are going to be saved. If you would like to save params of
Program in separate files, set `params_filename` None; if you would like to save all
params of Program in a single file, use `params_filename` to specify the file name.
Args:
Args:
dirname(str): The directory path to save the inference model.
dirname(str): The directory path to save the inference model.
feeded_var_names(list[str]): Names of variables that need to be feeded data
feeded_var_names(list[str]): list of string. Names of variables that need to be feeded
during inference.
data during inference.
target_vars(list[Variable]): Variables from which we can get inference
target_vars(list[Variable]): list of Variable. Variables from which we can get
results.
inference results.
executor(Executor): The executor that saves the inference model.
executor(Executor): The executor that saves the inference model. You can refer
main_program(Program|None): The original program, which will be pruned to
to :ref:`api_guide_executor_en` for more details.
main_program(Program, optional): The original program, which will be pruned to
build the inference model. If is setted None,
build the inference model. If is setted None,
the default main program
will be used.
the global default :code:`_main_program_`
will be used.
Default: None.
Default: None.
model_filename(str
|None
): The name of file to save the inference program
model_filename(str
, optional
): The name of file to save the inference program
itself. If is setted None, a default filename
itself. If is setted None, a default filename
`__model__` will be used.
:code:
`__model__` will be used.
params_filename(str
|None
): The name of file to save all related parameters.
params_filename(str
, optional
): The name of file to save all related parameters.
If it is setted None, parameters will be saved
If it is setted None, parameters will be saved
in separate files .
in separate files .
export_for_deployment(bool): If True, programs are modified to only support
export_for_deployment(bool): If True, programs are modified to only support
...
@@ -1001,14 +1039,20 @@ def save_inference_model(dirname,
...
@@ -1001,14 +1039,20 @@ def save_inference_model(dirname,
more information will be stored for flexible
more information will be stored for flexible
optimization and re-training. Currently, only
optimization and re-training. Currently, only
True is supported.
True is supported.
program_only(bool): If True, It will save inference program only, and do not save params of Program.
Default: True.
program_only(bool, optional): If True, It will save inference program only, and do not
save params of Program.
Default: False.
Returns:
Returns:
target_var_name_list(list): The fetch variables' name list
The fetch variables' name list
Return Type:
list
Raises:
Raises:
ValueError: If `feed_var_names` is not a list of basestring.
ValueError: If `feed_var_names` is not a list of basestring
, an exception is thrown
.
ValueError: If `target_vars` is not a list of Variable.
ValueError: If `target_vars` is not a list of Variable
, an exception is thrown
.
Examples:
Examples:
.. code-block:: python
.. code-block:: python
...
@@ -1018,8 +1062,8 @@ def save_inference_model(dirname,
...
@@ -1018,8 +1062,8 @@ def save_inference_model(dirname,
path = "./infer_model"
path = "./infer_model"
# User defined network, here a softmax regresssion example
# User defined network, here a softmax regresssion example
image = fluid.
layers.data(name='img', shape=[1
, 28, 28], dtype='float32')
image = fluid.
data(name='img', shape=[None
, 28, 28], dtype='float32')
label = fluid.
layers.data(name='label', shape=[
1], dtype='int64')
label = fluid.
data(name='label', shape=[None,
1], dtype='int64')
feeder = fluid.DataFeeder(feed_list=[image, label], place=fluid.CPUPlace())
feeder = fluid.DataFeeder(feed_list=[image, label], place=fluid.CPUPlace())
predict = fluid.layers.fc(input=image, size=10, act='softmax')
predict = fluid.layers.fc(input=image, size=10, act='softmax')
...
@@ -1037,9 +1081,9 @@ def save_inference_model(dirname,
...
@@ -1037,9 +1081,9 @@ def save_inference_model(dirname,
target_vars=[predict],
target_vars=[predict],
executor=exe)
executor=exe)
# In this example, the
function will prune the default main program
# In this example, the
save_inference_mode inference will prune the default
#
to make it suitable for infering the `predict` var. The pruned
#
main program according to the network's input node (img) and output node(predict).
# inference program is going to be saved in the "./infer_model/__model__"
#
The pruned
inference program is going to be saved in the "./infer_model/__model__"
# and parameters are going to be saved in separate files under folder
# and parameters are going to be saved in separate files under folder
# "./infer_model".
# "./infer_model".
...
...
python/paddle/fluid/layers/io.py
浏览文件 @
9b85f401
...
@@ -47,25 +47,25 @@ def data(name,
...
@@ -47,25 +47,25 @@ def data(name,
"""
"""
**Data Layer**
**Data Layer**
This function takes in the input and based on whether data has
This operator creates the global variable. The global variables can be
to be returned back as a minibatch, it creates the global variable by using
accessed by all the following operators in the graph.
the helper functions. The global variables can be accessed by all the
following operators in the graph.
All the input variables of this function are passed in as local variables
Note:
to the LayerHelper constructor.
:code:`paddle.fluid.layers.data` is deprecated as it will be removed in
a later version. Please use :code:`paddle.fluid.data` .
Notice that paddle would only use :code:`shape` to infer the shapes of
The :code:`paddle.fluid.layers.data` set shape and dtype at compile time
following variables in the network during compile-time. During run-time,
but does NOT check the shape or the dtype of feeded data, this
paddle would not check whether the shape of the feeded data matches the
:code:`paddle.fluid.data` checks the shape and the dtype of data feeded
:code:`shape` settings in this function.
by Executor or ParallelExecutor during run time.
Args:
Args:
name(str): The name/alias of the function
name(str): The name/alias of the variable, see :ref:`api_guide_Name`
for more details.
shape(list): Tuple declaring the shape. If :code:`append_batch_size` is
shape(list): Tuple declaring the shape. If :code:`append_batch_size` is
True and there is no -1 inside :code:`shape`, it should be
True and there is no -1 inside :code:`shape`, it should be
considered as the shape of the each sample. Otherwise, it
considered as the shape of the each sample. Otherwise, it should
should
be considered as the shape of the batched data.
be considered as the shape of the batched data.
append_batch_size(bool):
append_batch_size(bool):
1. If true, it prepends -1 to the shape.
1. If true, it prepends -1 to the shape.
For example if shape=[1], the resulting shape is [-1, 1]. This will
For example if shape=[1], the resulting shape is [-1, 1]. This will
...
@@ -74,13 +74,20 @@ def data(name,
...
@@ -74,13 +74,20 @@ def data(name,
append_batch_size will be enforced to be be False (ineffective)
append_batch_size will be enforced to be be False (ineffective)
because PaddlePaddle cannot set more than 1 unknown number on the
because PaddlePaddle cannot set more than 1 unknown number on the
shape.
shape.
dtype(np.dtype|VarType|str): The type of data : float32, float16, int etc
dtype(np.dtype|VarType|str): The type of the data. Supported dtype: bool,
type(VarType): The output type. By default it is LOD_TENSOR.
float16, float32, float64, int8, int16, int32, int64, uint8.
type(VarType): The output type. Supported dtype: VarType.LOD_TENSOR,
VarType.SELECTED_ROWS, VarType.NCCL_ID. Default: VarType.LOD_TENSOR.
lod_level(int): The LoD Level. 0 means the input data is not a sequence.
lod_level(int): The LoD Level. 0 means the input data is not a sequence.
Default: 0.
stop_gradient(bool): A boolean that mentions whether gradient should flow.
stop_gradient(bool): A boolean that mentions whether gradient should flow.
Default: True.
Returns:
Returns:
Variable: The global variable that gives access to the data.
The global variable that gives access to the data.
Return Type:
Variable
Examples:
Examples:
.. code-block:: python
.. code-block:: python
...
@@ -531,29 +538,43 @@ def py_reader(capacity,
...
@@ -531,29 +538,43 @@ def py_reader(capacity,
"""
"""
Create a Python reader for data feeding in Python
Create a Python reader for data feeding in Python
This
laye
r returns a Reader Variable.
This
operato
r returns a Reader Variable.
The Reader provides :code:`decorate_paddle_reader()` and
The Reader provides :code:`decorate_paddle_reader()` and
:code:`decorate_tensor_provider()` to set a Python generator as the data
:code:`decorate_tensor_provider()` to set a Python generator as the data
source. More details :ref:`user_guide_use_py_reader_en` . When
source and feed the data from the data source to the Reader Variable.
:code:`Executor::Run()` is invoked in C++ side, the data from the generator
When :code:`Executor::Run()` is invoked in C++ side, the data from the
would be read automatically. Unlike :code:`DataFeeder.feed()`, the data
generator would be read automatically. Unlike :code:`DataFeeder.feed()`,
reading process and :code:`Executor::Run()` process can run in parallel
the data reading process and :code:`Executor::Run()` process can run in
using :code:`py_reader`. The :code:`start()` method of the Reader should be
parallel using :code:`py_reader`. The :code:`start()` method of the Reader
called when each pass begins, while the :code:`reset()` method should be
should be called when each pass begins, while the :code:`reset()` method
called when the pass ends and :code:`fluid.core.EOFException` raises.
should be called when the pass ends and :code:`fluid.core.EOFException` raises.
Note that :code:`Program.clone()` method cannot clone :code:`py_reader`.
Note:
:code:`Program.clone()` method cannot clone :code:`py_reader`. You can
refer to :ref:`api_fluid_Program` for more details.
The :code:`read_file` call needs to be in the program block of :code:`py_reader`.
You can refer to :ref:`api_fluid_layers_read_file` for more details.
Args:
Args:
capacity(int): The buffer capacity maintained by :code:`py_reader`.
capacity(int): The buffer capacity maintained by :code:`py_reader`.
shapes(list|tuple): List of tuples which declaring data shapes.
shapes(list|tuple): List of tuples which declaring data shapes. shapes[i]
dtypes(list|tuple): List of strs which declaring data type.
represents the i-th data shape.
dtypes(list|tuple): List of strings which declaring data type. Supported dtype:
bool, float16, float32, float64, int8, int16, int32, int64, uint8.
lod_levels(list|tuple): List of ints which declaring data lod_level.
lod_levels(list|tuple): List of ints which declaring data lod_level.
name(basestring): The prefix Python queue name and Reader name. None will
name(basestring): The default value is None. Normally there is no
be generated automatically.
need for user to set this property. For more information, please
use_double_buffer(bool): Whether use double buffer or not.
refer to :ref:`api_guide_Name`.
use_double_buffer(bool): Whether use double buffer or not. The double buffer is
for pre-reading the data of the next batch and copy the data asynchronously
from CPU to GPU. Default is True.
Returns:
Returns:
Variable: A Reader from which we can get feeding data.
A Reader from which we can get feeding data.
Return Type:
Variable
Examples:
Examples:
1. The basic usage of :code:`py_reader` is as follows:
1. The basic usage of :code:`py_reader` is as follows:
...
...
python/paddle/fluid/reader.py
浏览文件 @
9b85f401
...
@@ -595,7 +595,10 @@ class PyReader(DataLoaderBase):
...
@@ -595,7 +595,10 @@ class PyReader(DataLoaderBase):
use return_list=True in dygraph mode.
use return_list=True in dygraph mode.
Returns:
Returns:
reader (Reader): the created reader object.
the created reader object.
Return type:
reader(Reader)
Examples:
Examples:
1. If iterable = False, the created PyReader object is almost the
1. If iterable = False, the created PyReader object is almost the
...
@@ -616,6 +619,11 @@ class PyReader(DataLoaderBase):
...
@@ -616,6 +619,11 @@ class PyReader(DataLoaderBase):
ITER_NUM = 5
ITER_NUM = 5
BATCH_SIZE = 3
BATCH_SIZE = 3
def network(image, label):
# User-defined network, here is an example of softmax regression.
predict = fluid.layers.fc(input=image, size=10, act='softmax')
return fluid.layers.cross_entropy(input=predict, label=label)
def reader_creator_random_image_and_label(height, width):
def reader_creator_random_image_and_label(height, width):
def reader():
def reader():
for i in range(ITER_NUM):
for i in range(ITER_NUM):
...
@@ -626,8 +634,8 @@ class PyReader(DataLoaderBase):
...
@@ -626,8 +634,8 @@ class PyReader(DataLoaderBase):
yield fake_image, fake_label
yield fake_image, fake_label
return reader
return reader
image = fluid.
layers.data(name='image', shape=[
784, 784], dtype='float32')
image = fluid.
data(name='image', shape=[None,
784, 784], dtype='float32')
label = fluid.
layers.data(name='label', shape=[
1], dtype='int64')
label = fluid.
data(name='label', shape=[None,
1], dtype='int64')
reader = fluid.io.PyReader(feed_list=[image, label],
reader = fluid.io.PyReader(feed_list=[image, label],
capacity=4,
capacity=4,
...
@@ -636,8 +644,8 @@ class PyReader(DataLoaderBase):
...
@@ -636,8 +644,8 @@ class PyReader(DataLoaderBase):
user_defined_reader = reader_creator_random_image_and_label(784, 784)
user_defined_reader = reader_creator_random_image_and_label(784, 784)
reader.decorate_sample_list_generator(
reader.decorate_sample_list_generator(
paddle.batch(user_defined_reader, batch_size=BATCH_SIZE))
paddle.batch(user_defined_reader, batch_size=BATCH_SIZE))
# definition of network is omitted
loss = network(image, label)
executor = fluid.Executor(fluid.C
UDAPlace(0
))
executor = fluid.Executor(fluid.C
PUPlace(
))
executor.run(fluid.default_startup_program())
executor.run(fluid.default_startup_program())
for i in range(EPOCH_NUM):
for i in range(EPOCH_NUM):
reader.start()
reader.start()
...
@@ -665,26 +673,35 @@ class PyReader(DataLoaderBase):
...
@@ -665,26 +673,35 @@ class PyReader(DataLoaderBase):
ITER_NUM = 5
ITER_NUM = 5
BATCH_SIZE = 10
BATCH_SIZE = 10
def network(image, label):
# User-defined network, here is an example of softmax regression.
predict = fluid.layers.fc(input=image, size=10, act='softmax')
return fluid.layers.cross_entropy(input=predict, label=label)
def reader_creator_random_image(height, width):
def reader_creator_random_image(height, width):
def reader():
def reader():
for i in range(ITER_NUM):
for i in range(ITER_NUM):
yield np.random.uniform(low=0, high=255, size=[height, width]),
fake_image = np.random.uniform(low=0, high=255, size=[height, width])
fake_label = np.ones([1])
yield fake_image, fake_label
return reader
return reader
image = fluid.layers.data(name='image', shape=[784, 784], dtype='float32')
image = fluid.data(name='image', shape=[None, 784, 784], dtype='float32')
reader = fluid.io.PyReader(feed_list=[image], capacity=4, iterable=True, return_list=False)
label = fluid.data(name='label', shape=[None, 1], dtype='int64')
reader = fluid.io.PyReader(feed_list=[image, label], capacity=4, iterable=True, return_list=False)
user_defined_reader = reader_creator_random_image(784, 784)
user_defined_reader = reader_creator_random_image(784, 784)
reader.decorate_sample_list_generator(
reader.decorate_sample_list_generator(
paddle.batch(user_defined_reader, batch_size=BATCH_SIZE),
paddle.batch(user_defined_reader, batch_size=BATCH_SIZE),
fluid.core.CUDAPlace(0))
fluid.core.CPUPlace())
# definition of network is omitted
executor = fluid.Executor(fluid.CUDAPlace(0))
loss = network(image, label)
executor.run(fluid.default_main_program())
executor = fluid.Executor(fluid.CPUPlace())
executor.run(fluid.default_startup_program())
for _ in range(EPOCH_NUM):
for _ in range(EPOCH_NUM):
for data in reader():
for data in reader():
executor.run(feed=data)
executor.run(feed=data
, fetch_list=[loss]
)
3. If return_list=True, the return values would be presented as list instead of dict.
3. If return_list=True, the return values would be presented as list instead of dict.
...
@@ -758,12 +775,12 @@ class PyReader(DataLoaderBase):
...
@@ -758,12 +775,12 @@ class PyReader(DataLoaderBase):
for i in range(5):
for i in range(5):
yield np.random.uniform(low=0, high=255, size=[784, 784]),
yield np.random.uniform(low=0, high=255, size=[784, 784]),
image = fluid.
layers.data(name='image', shape=[
784, 784], dtype='float32')
image = fluid.
data(name='image', shape=[None,
784, 784], dtype='float32')
reader = fluid.io.PyReader(feed_list=[image], capacity=4, iterable=False)
reader = fluid.io.PyReader(feed_list=[image], capacity=4, iterable=False)
reader.decorate_sample_list_generator(
reader.decorate_sample_list_generator(
paddle.batch(generator, batch_size=BATCH_SIZE))
paddle.batch(generator, batch_size=BATCH_SIZE))
executor = fluid.Executor(fluid.C
UDAPlace(0
))
executor = fluid.Executor(fluid.C
PUPlace(
))
executor.run(fluid.default_startup_program())
executor.run(fluid.default_startup_program())
for i in range(3):
for i in range(3):
reader.start()
reader.start()
...
@@ -795,12 +812,12 @@ class PyReader(DataLoaderBase):
...
@@ -795,12 +812,12 @@ class PyReader(DataLoaderBase):
for i in range(5):
for i in range(5):
yield np.random.uniform(low=0, high=255, size=[784, 784]),
yield np.random.uniform(low=0, high=255, size=[784, 784]),
image = fluid.
layers.data(name='image', shape=[
784, 784], dtype='float32')
image = fluid.
data(name='image', shape=[None,
784, 784], dtype='float32')
reader = fluid.io.PyReader(feed_list=[image], capacity=4, iterable=False)
reader = fluid.io.PyReader(feed_list=[image], capacity=4, iterable=False)
reader.decorate_sample_list_generator(
reader.decorate_sample_list_generator(
paddle.batch(generator, batch_size=BATCH_SIZE))
paddle.batch(generator, batch_size=BATCH_SIZE))
executor = fluid.Executor(fluid.C
UDAPlace(0
))
executor = fluid.Executor(fluid.C
PUPlace(
))
executor.run(fluid.default_startup_program())
executor.run(fluid.default_startup_program())
for i in range(3):
for i in range(3):
reader.start()
reader.start()
...
@@ -849,6 +866,11 @@ class PyReader(DataLoaderBase):
...
@@ -849,6 +866,11 @@ class PyReader(DataLoaderBase):
ITER_NUM = 15
ITER_NUM = 15
BATCH_SIZE = 3
BATCH_SIZE = 3
def network(image, label):
# User-defined network, here is an example of softmax regression.
predict = fluid.layers.fc(input=image, size=10, act='softmax')
return fluid.layers.cross_entropy(input=predict, label=label)
def random_image_and_label_generator(height, width):
def random_image_and_label_generator(height, width):
def generator():
def generator():
for i in range(ITER_NUM):
for i in range(ITER_NUM):
...
@@ -859,21 +881,21 @@ class PyReader(DataLoaderBase):
...
@@ -859,21 +881,21 @@ class PyReader(DataLoaderBase):
yield fake_image, fake_label
yield fake_image, fake_label
return generator
return generator
image = fluid.
layers.data(name='image', shape=[
784, 784], dtype='float32')
image = fluid.
data(name='image', shape=[None,
784, 784], dtype='float32')
label = fluid.
layers.data(name='label', shape=[1], dtype='int32
')
label = fluid.
data(name='label', shape=[None, 1], dtype='int64
')
reader = fluid.io.PyReader(feed_list=[image, label], capacity=4, iterable=True)
reader = fluid.io.PyReader(feed_list=[image, label], capacity=4, iterable=True)
user_defined_generator = random_image_and_label_generator(784, 784)
user_defined_generator = random_image_and_label_generator(784, 784)
reader.decorate_sample_generator(user_defined_generator,
reader.decorate_sample_generator(user_defined_generator,
batch_size=BATCH_SIZE,
batch_size=BATCH_SIZE,
places=[fluid.C
UDAPlace(0
)])
places=[fluid.C
PUPlace(
)])
# definition of network is omitted
loss = network(image, label)
executor = fluid.Executor(fluid.C
UDAPlace(0
))
executor = fluid.Executor(fluid.C
PUPlace(
))
executor.run(fluid.default_
main
_program())
executor.run(fluid.default_
startup
_program())
for _ in range(EPOCH_NUM):
for _ in range(EPOCH_NUM):
for data in reader():
for data in reader():
executor.run(feed=data)
executor.run(feed=data
, fetch_list=[loss]
)
'''
'''
self
.
_loader
.
set_sample_generator
(
sample_generator
,
batch_size
,
self
.
_loader
.
set_sample_generator
(
sample_generator
,
batch_size
,
...
@@ -905,6 +927,11 @@ class PyReader(DataLoaderBase):
...
@@ -905,6 +927,11 @@ class PyReader(DataLoaderBase):
ITER_NUM = 15
ITER_NUM = 15
BATCH_SIZE = 3
BATCH_SIZE = 3
def network(image, label):
# User-defined network, here is an example of softmax regression.
predict = fluid.layers.fc(input=image, size=10, act='softmax')
return fluid.layers.cross_entropy(input=predict, label=label)
def random_image_and_label_generator(height, width):
def random_image_and_label_generator(height, width):
def generator():
def generator():
for i in range(ITER_NUM):
for i in range(ITER_NUM):
...
@@ -915,21 +942,22 @@ class PyReader(DataLoaderBase):
...
@@ -915,21 +942,22 @@ class PyReader(DataLoaderBase):
yield fake_image, fake_label
yield fake_image, fake_label
return generator
return generator
image = fluid.
layers.data(name='image', shape=[
784, 784], dtype='float32')
image = fluid.
data(name='image', shape=[None,
784, 784], dtype='float32')
label = fluid.
layers.data(name='label', shape=[1], dtype='int32
')
label = fluid.
data(name='label', shape=[None, 1], dtype='int64
')
reader = fluid.io.PyReader(feed_list=[image, label], capacity=4, iterable=True)
reader = fluid.io.PyReader(feed_list=[image, label], capacity=4, iterable=True)
user_defined_generator = random_image_and_label_generator(784, 784)
user_defined_generator = random_image_and_label_generator(784, 784)
reader.decorate_sample_list_generator(
reader.decorate_sample_list_generator(
paddle.batch(user_defined_generator, batch_size=BATCH_SIZE),
paddle.batch(user_defined_generator, batch_size=BATCH_SIZE),
fluid.core.CUDAPlace(0))
fluid.core.CPUPlace())
# definition of network is omitted
executor = fluid.Executor(fluid.core.CUDAPlace(0))
loss = network(image, label)
executor.run(fluid.default_main_program())
executor = fluid.Executor(fluid.core.CPUPlace())
executor.run(fluid.default_startup_program())
for _ in range(EPOCH_NUM):
for _ in range(EPOCH_NUM):
for data in reader():
for data in reader():
executor.run(feed=data)
executor.run(feed=data
, fetch_list=[loss]
)
'''
'''
self
.
_loader
.
set_sample_list_generator
(
reader
,
places
)
self
.
_loader
.
set_sample_list_generator
(
reader
,
places
)
...
@@ -959,6 +987,11 @@ class PyReader(DataLoaderBase):
...
@@ -959,6 +987,11 @@ class PyReader(DataLoaderBase):
ITER_NUM = 15
ITER_NUM = 15
BATCH_SIZE = 3
BATCH_SIZE = 3
def network(image, label):
# User-defined network, here is an example of softmax regression.
predict = fluid.layers.fc(input=image, size=10, act='softmax')
return fluid.layers.cross_entropy(input=predict, label=label)
def random_image_and_label_generator(height, width):
def random_image_and_label_generator(height, width):
def generator():
def generator():
for i in range(ITER_NUM):
for i in range(ITER_NUM):
...
@@ -966,22 +999,25 @@ class PyReader(DataLoaderBase):
...
@@ -966,22 +999,25 @@ class PyReader(DataLoaderBase):
high=255,
high=255,
size=[BATCH_SIZE, height, width])
size=[BATCH_SIZE, height, width])
batch_label = np.ones([BATCH_SIZE, 1])
batch_label = np.ones([BATCH_SIZE, 1])
batch_image = batch_image.astype('float32')
batch_label = batch_label.astype('int64')
yield batch_image, batch_label
yield batch_image, batch_label
return generator
return generator
image = fluid.
layers.data(name='image', shape=[
784, 784], dtype='float32')
image = fluid.
data(name='image', shape=[None,
784, 784], dtype='float32')
label = fluid.
layers.data(name='label', shape=[1], dtype='int32
')
label = fluid.
data(name='label', shape=[None, 1], dtype='int64
')
reader = fluid.io.PyReader(feed_list=[image, label], capacity=4, iterable=True)
reader = fluid.io.PyReader(feed_list=[image, label], capacity=4, iterable=True)
user_defined_generator = random_image_and_label_generator(784, 784)
user_defined_generator = random_image_and_label_generator(784, 784)
reader.decorate_batch_generator(user_defined_generator, fluid.CUDAPlace(0))
reader.decorate_batch_generator(user_defined_generator, fluid.CPUPlace())
# definition of network is omitted
executor = fluid.Executor(fluid.CUDAPlace(0))
loss = network(image, label)
executor.run(fluid.default_main_program())
executor = fluid.Executor(fluid.CPUPlace())
executor.run(fluid.default_startup_program())
for _ in range(EPOCH_NUM):
for _ in range(EPOCH_NUM):
for data in reader():
for data in reader():
executor.run(feed=data)
executor.run(feed=data
, fetch_list=[loss]
)
'''
'''
self
.
_loader
.
set_batch_generator
(
reader
,
places
)
self
.
_loader
.
set_batch_generator
(
reader
,
places
)
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录