From 931698a54ad0e61fc7c811334ea944580d9c6173 Mon Sep 17 00:00:00 2001 From: Huihuang Zheng Date: Mon, 3 Jun 2019 01:11:13 -0700 Subject: [PATCH] Modify doc of program_guard, py_reader, data, and clone (#17727) Note the append_batch_size variable is doing prepend. We should change the name, but due to backward compatibility, I suggest to change at v2.0. Not now. test=develop --- paddle/fluid/API.spec | 8 ++++---- python/paddle/fluid/framework.py | 30 +++++++++++++++++++----------- python/paddle/fluid/layers/io.py | 19 +++++++++++-------- 3 files changed, 34 insertions(+), 23 deletions(-) diff --git a/paddle/fluid/API.spec b/paddle/fluid/API.spec index b698eaf70..c745905aa 100644 --- a/paddle/fluid/API.spec +++ b/paddle/fluid/API.spec @@ -1,6 +1,6 @@ paddle.fluid.Program.__init__ (ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None), ('document', '6adf97f83acf6453d4a6a4b1070f3754')) paddle.fluid.Program.block (ArgSpec(args=['self', 'index'], varargs=None, keywords=None, defaults=None), ('document', '86cd9499e226be661a3d686260ee1150')) -paddle.fluid.Program.clone (ArgSpec(args=['self', 'for_test'], varargs=None, keywords=None, defaults=(False,)), ('document', 'fe23e081126d2d4422b0780c2fea1a6a')) +paddle.fluid.Program.clone (ArgSpec(args=['self', 'for_test'], varargs=None, keywords=None, defaults=(False,)), ('document', '17d059efb24c81dde6166c6b0b93e9d0')) paddle.fluid.Program.current_block (ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None), ('document', 'd601c7719e425e3d9cf862ea4ad194ca')) paddle.fluid.Program.global_block (ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None), ('document', 'd64ea1dc96e9f674499ea3006d470aa4')) paddle.fluid.Program.list_vars (ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None), ('document', '32c14b0f12baae4b352200fa09b5e789')) @@ -8,7 +8,7 @@ paddle.fluid.Program.parse_from_string (ArgSpec(args=['binary_str'], varargs=Non paddle.fluid.Program.to_string (ArgSpec(args=['self', 'throw_on_error', 'with_details'], varargs=None, keywords=None, defaults=(False,)), ('document', '89acca639baf00f3ad08b9d827e81706')) paddle.fluid.default_startup_program (ArgSpec(args=[], varargs=None, keywords=None, defaults=None), ('document', 'ba609cb02e4e55e8d626723567ef1778')) paddle.fluid.default_main_program (ArgSpec(args=[], varargs=None, keywords=None, defaults=None), ('document', '06a5a8f649dfb8496c1f683c909db375')) -paddle.fluid.program_guard (ArgSpec(args=['main_program', 'startup_program'], varargs=None, keywords=None, defaults=(None,)), ('document', 'ae5f806f082cfaeaa5194cacc253a5e4')) +paddle.fluid.program_guard (ArgSpec(args=['main_program', 'startup_program'], varargs=None, keywords=None, defaults=(None,)), ('document', '78fb5c7f70ef76bcf4a1862c3f6b8191')) paddle.fluid.name_scope (ArgSpec(args=['prefix'], varargs=None, keywords=None, defaults=(None,)), ('document', '61660461e1f44e0480ca22fa8a482c41')) paddle.fluid.cuda_places (ArgSpec(args=['device_ids'], varargs=None, keywords=None, defaults=(None,)), ('document', '7f3068b82fc427bfa04b1af953610992')) paddle.fluid.cpu_places (ArgSpec(args=['device_count'], varargs=None, keywords=None, defaults=(None,)), ('document', '8b674e9a7ac7944c27fd853b675c2cb2')) @@ -238,14 +238,14 @@ paddle.fluid.layers.continuous_value_model (ArgSpec(args=['input', 'cvm', 'use_c paddle.fluid.layers.where (ArgSpec(args=['condition'], varargs=None, keywords=None, defaults=None), ('document', '3126e3039e752ce26077f1efaca355c6')) paddle.fluid.layers.sign (ArgSpec(args=['x'], varargs=None, keywords=None, defaults=None), ('document', 'ccf6bb7912afd2818d24bc45461e807a')) paddle.fluid.layers.deformable_conv (ArgSpec(args=['input', 'offset', 'mask', 'num_filters', 'filter_size', 'stride', 'padding', 'dilation', 'groups', 'deformable_groups', 'im2col_step', 'param_attr', 'bias_attr', 'name'], varargs=None, keywords=None, defaults=(1, 0, 1, None, None, None, None, None, None)), ('document', 'c896b66265a60bd3c5510f66e6e02919')) -paddle.fluid.layers.data (ArgSpec(args=['name', 'shape', 'append_batch_size', 'dtype', 'lod_level', 'type', 'stop_gradient'], varargs=None, keywords=None, defaults=(True, 'float32', 0, VarType.LOD_TENSOR, True)), ('document', 'adf285346e23316097f7789b572491e9')) +paddle.fluid.layers.data (ArgSpec(args=['name', 'shape', 'append_batch_size', 'dtype', 'lod_level', 'type', 'stop_gradient'], varargs=None, keywords=None, defaults=(True, 'float32', 0, VarType.LOD_TENSOR, True)), ('document', '6e19128b46936edf9f3fad77860a1da8')) paddle.fluid.layers.open_files (ArgSpec(args=['filenames', 'shapes', 'lod_levels', 'dtypes', 'thread_num', 'buffer_size', 'pass_num', 'is_test'], varargs=None, keywords=None, defaults=(None, None, 1, None)), ('document', 'dce69a78638da8f7ad80b1fc00ed2029')) paddle.fluid.layers.read_file (ArgSpec(args=['reader'], varargs=None, keywords=None, defaults=None), ('document', '32181f6037e387fb6e68a5beaafe33b6')) paddle.fluid.layers.shuffle (ArgSpec(args=['reader', 'buffer_size'], varargs=None, keywords=None, defaults=None), ('document', 'f29d7d159e114f73fc988d9a86805841')) paddle.fluid.layers.batch (ArgSpec(args=['reader', 'batch_size'], varargs=None, keywords=None, defaults=None), ('document', 'fcb24383c6eef2ca040ee824c26e22fd')) paddle.fluid.layers.double_buffer (ArgSpec(args=['reader', 'place', 'name'], varargs=None, keywords=None, defaults=(None, None)), ('document', 'c13b8a8521bea5f8123b925ae2a5d5db')) paddle.fluid.layers.random_data_generator (ArgSpec(args=['low', 'high', 'shapes', 'lod_levels', 'for_parallel'], varargs=None, keywords=None, defaults=(True,)), ('document', '9b7f0f86ec24bbc97643cadcb6499cff')) -paddle.fluid.layers.py_reader (ArgSpec(args=['capacity', 'shapes', 'dtypes', 'lod_levels', 'name', 'use_double_buffer'], varargs=None, keywords=None, defaults=(None, None, True)), ('document', '5c54493d96c7e0760dc6758af1c8dd72')) +paddle.fluid.layers.py_reader (ArgSpec(args=['capacity', 'shapes', 'dtypes', 'lod_levels', 'name', 'use_double_buffer'], varargs=None, keywords=None, defaults=(None, None, True)), ('document', '0a77c56dff556b5ae4c5630d9a0764ef')) paddle.fluid.layers.create_py_reader_by_data (ArgSpec(args=['capacity', 'feed_list', 'name', 'use_double_buffer'], varargs=None, keywords=None, defaults=(None, True)), ('document', 'b42332b894e1e0962c6a43f0151c2640')) paddle.fluid.layers.Preprocessor.__init__ (ArgSpec(args=['self', 'reader', 'name'], varargs=None, keywords=None, defaults=(None,)), ('document', '6adf97f83acf6453d4a6a4b1070f3754')) paddle.fluid.layers.Preprocessor.block (ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None), ('document', '6adf97f83acf6453d4a6a4b1070f3754')) diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py index 44c501ed1..0fe554bfa 100644 --- a/python/paddle/fluid/framework.py +++ b/python/paddle/fluid/framework.py @@ -2969,12 +2969,15 @@ class Program(object): attribute of them to :code:`True` when :code:`for_test=True`. * Set for_test to False when we want to clone the program for training. - * Set for_test to True when we want to clone the program for testing. We will not do any prune - on program here, So if you just want an forward program for testing, please use :code:`clone` - before using :code:`Opimizer.minimize` + * Set for_test to True when we want to clone the program for testing. + We will not do any prune on program here, So if you just want an + forward program for testing, please use :code:`clone` before using + :code:`Opimizer.minimize` - Notes: This API DOES NOT prune any operator. Use - :code:`clone(for_test=True)` before backward and optimization please. e.g. + Notes: + 1. :code:`Program.clone()` method DOES NOT clone :code:`py_reader`. + 2. This API DOES NOT prune any operator. Use + :code:`clone(for_test=True)` before backward and optimization please. E.g. .. code-block:: python @@ -2991,7 +2994,12 @@ class Program(object): Examples: - Notes: The Program Descs' order maybe different after :code:`clone` and this will not affect your training or testing progress. In the following example we give you an simple method :code:`print_prog(program)` to print Program Descs inorder to make sure you have same print result after :code:`clone`: + Notes: The Program Descs' order maybe different after :code:`clone` and + this will not affect your training or testing progress. In the following + example we give you an simple method :code:`print_prog(program)` to + print Program Descs inorder to make sure you have same print result + after :code:`clone`: + .. code-block:: python import paddle.fluid as fluid @@ -3685,8 +3693,8 @@ def switch_startup_program(program): @signature_safe_contextmanager def program_guard(main_program, startup_program=None): """ - Change the global main program and startup program with `with` statement. - Layer functions in the Python `with` block will append operators and + Change the global main program and startup program with `"with"` statement. + Layer functions in the Python `"with"` block will append operators and variables to the new main programs. Examples: @@ -3714,9 +3722,9 @@ def program_guard(main_program, startup_program=None): data = fluid.layers.data(name='image', shape=[784, 784], dtype='float32') Args: - main_program(Program): New main program inside `with` statement. - startup_program(Program): New startup program inside `with` statement. - None means do not change startup program. + main_program(Program): New main program inside `"with"` statement. + startup_program(Program): New startup program inside `"with"` statement. + None means not changing startup program. """ if not isinstance(main_program, Program): raise TypeError("main_program should be Program") diff --git a/python/paddle/fluid/layers/io.py b/python/paddle/fluid/layers/io.py index 3f064b39d..23d5f0dc0 100644 --- a/python/paddle/fluid/layers/io.py +++ b/python/paddle/fluid/layers/io.py @@ -62,9 +62,12 @@ def data(name, should be considered as the shape of the batched data. append_batch_size(bool): 1. If true, it prepends -1 to the shape. - For example if shape=[1], the resulting shape is [-1, 1]. - 2. If shape contains -1, such as shape=[1, -1], - append_batch_size will be enforced to be be False (ineffective). + For example if shape=[1], the resulting shape is [-1, 1]. This will + be useful to set different batch size at run time. + 2. If shape contains -1, such as shape=[1, -1]. + append_batch_size will be enforced to be be False (ineffective) + because PaddlePaddle cannot set more than 1 unknown number on the + shape. dtype(np.dtype|VarType|str): The type of data : float32, float16, int etc type(VarType): The output type. By default it is LOD_TENSOR. lod_level(int): The LoD Level. 0 means the input data is not a sequence. @@ -652,11 +655,11 @@ def py_reader(capacity, This layer returns a Reader Variable. The Reader provides :code:`decorate_paddle_reader()` and :code:`decorate_tensor_provider()` to set a Python generator as the data - source in Python side. When :code:`Executor::Run()` is invoked in C++ - side, the data from the generator would be read automatically. Unlike - :code:`DataFeeder.feed()`, the data reading process and - :code:`Executor::Run()` process can run in parallel using - :code:`py_reader`. The :code:`start()` method of the Reader should be + source. More details :ref:`user_guide_use_py_reader_en` . When + :code:`Executor::Run()` is invoked in C++ side, the data from the generator + would be read automatically. Unlike :code:`DataFeeder.feed()`, the data + reading process and :code:`Executor::Run()` process can run in parallel + using :code:`py_reader`. The :code:`start()` method of the Reader should be called when each pass begins, while the :code:`reset()` method should be called when the pass ends and :code:`fluid.core.EOFException` raises. Note that :code:`Program.clone()` method cannot clone :code:`py_reader`. -- GitLab