From bb166a1e1019ff500f04d3a60c59ca8b5f71ea63 Mon Sep 17 00:00:00 2001 From: sneaxiy Date: Tue, 19 Mar 2019 08:12:43 +0000 Subject: [PATCH] fix API.spec test=develop --- paddle/fluid/API.spec | 8 +- python/paddle/fluid/compiler.py | 10 +-- python/paddle/fluid/reader.py | 143 +++++++++++++++--------------- python/paddle/reader/decorator.py | 3 +- 4 files changed, 81 insertions(+), 83 deletions(-) diff --git a/paddle/fluid/API.spec b/paddle/fluid/API.spec index f168d5608..188f5037c 100644 --- a/paddle/fluid/API.spec +++ b/paddle/fluid/API.spec @@ -47,7 +47,7 @@ paddle.fluid.AsyncExecutor.run (ArgSpec(args=['self', 'program', 'data_feed', 'f paddle.fluid.AsyncExecutor.save_model (ArgSpec(args=['self', 'save_path'], varargs=None, keywords=None, defaults=None), ('document', 'c8ac0dfcb3b187aba25d03af7fea56b2')) paddle.fluid.AsyncExecutor.stop (ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None), ('document', '5f23d043607bb5d55e466ec3f578e093')) paddle.fluid.CompiledProgram.__init__ (ArgSpec(args=['self', 'program_or_graph'], varargs=None, keywords=None, defaults=None), ('document', '6adf97f83acf6453d4a6a4b1070f3754')) -paddle.fluid.CompiledProgram.with_data_parallel (ArgSpec(args=['self', 'loss_name', 'build_strategy', 'exec_strategy', 'share_vars_from', 'places'], varargs=None, keywords=None, defaults=(None, None, None, None, None)), ('document', 'dbf542d1384741650a1238ddb05daa37')) +paddle.fluid.CompiledProgram.with_data_parallel (ArgSpec(args=['self', 'loss_name', 'build_strategy', 'exec_strategy', 'share_vars_from', 'places'], varargs=None, keywords=None, defaults=(None, None, None, None, None)), ('document', '5e8cca4619a5d7c3280fb3cae7021b14')) paddle.fluid.CompiledProgram.with_inference_optimize (ArgSpec(args=['self', 'config'], varargs=None, keywords=None, defaults=None), ('document', '9e5b009d850191a010e859189c127fd8')) paddle.fluid.ExecutionStrategy.__init__ __init__(self: paddle.fluid.core.ParallelExecutor.ExecutionStrategy) -> None paddle.fluid.BuildStrategy.GradientScaleStrategy.__init__ __init__(self: paddle.fluid.core.ParallelExecutor.BuildStrategy.GradientScaleStrategy, arg0: int) -> None @@ -61,8 +61,8 @@ paddle.fluid.io.load_params (ArgSpec(args=['executor', 'dirname', 'main_program' paddle.fluid.io.load_persistables (ArgSpec(args=['executor', 'dirname', 'main_program', 'filename'], varargs=None, keywords=None, defaults=(None, None)), ('document', '28df5bfe26ca7a077f91156abb0fe6d2')) paddle.fluid.io.save_inference_model (ArgSpec(args=['dirname', 'feeded_var_names', 'target_vars', 'executor', 'main_program', 'model_filename', 'params_filename', 'export_for_deployment'], varargs=None, keywords=None, defaults=(None, None, None, True)), ('document', '582d87b8df75a5a639a107db8ff86f9c')) paddle.fluid.io.load_inference_model (ArgSpec(args=['dirname', 'executor', 'model_filename', 'params_filename', 'pserver_endpoints'], varargs=None, keywords=None, defaults=(None, None, None)), ('document', '7a5255386075dac3c75b7058254fcdcb')) -paddle.fluid.io.PyReader.__init__ (ArgSpec(args=['self', 'feed_list', 'capacity', 'use_double_buffer', 'iterable'], varargs=None, keywords=None, defaults=(True, False)), ('document', 'b3d72958b2568aae3f90f72abdcb7d1a')) -paddle.fluid.io.PyReader.decorate_batch_generator (ArgSpec(args=['self', 'reader', 'places'], varargs=None, keywords=None, defaults=(None,)), ('document', 'd10224fef1095247063b6976da793021')) +paddle.fluid.io.PyReader.__init__ (ArgSpec(args=['self', 'feed_list', 'capacity', 'use_double_buffer', 'iterable'], varargs=None, keywords=None, defaults=(True, False)), ('document', '6adf97f83acf6453d4a6a4b1070f3754')) +paddle.fluid.io.PyReader.decorate_batch_generator (ArgSpec(args=['self', 'reader', 'places'], varargs=None, keywords=None, defaults=(None,)), ('document', 'a3fefec8bacd6ce83f49906a9d05e779')) paddle.fluid.io.PyReader.decorate_sample_generator (ArgSpec(args=['self', 'sample_generator', 'batch_size', 'drop_last', 'places'], varargs=None, keywords=None, defaults=(True, None)), ('document', '7abd9cf7d695bab5bb6cf7ded5903cb2')) paddle.fluid.io.PyReader.decorate_sample_list_generator (ArgSpec(args=['self', 'reader', 'places'], varargs=None, keywords=None, defaults=(None,)), ('document', 'faef298f73e91aedcfaf5d184f3109b7')) paddle.fluid.io.PyReader.reset (ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None), ('document', 'ff1cc1e2beb8824d453656c72c28ddfb')) @@ -521,7 +521,7 @@ paddle.fluid.unique_name.guard (ArgSpec(args=['new_generator'], varargs=None, ke paddle.fluid.recordio_writer.convert_reader_to_recordio_file (ArgSpec(args=['filename', 'reader_creator', 'feeder', 'compressor', 'max_num_records', 'feed_order'], varargs=None, keywords=None, defaults=(Compressor.Snappy, 1000, None)), ('document', '65c7523e86f0c50bb729b01667f36310')) paddle.fluid.recordio_writer.convert_reader_to_recordio_files (ArgSpec(args=['filename', 'batch_per_file', 'reader_creator', 'feeder', 'compressor', 'max_num_records', 'feed_order'], varargs=None, keywords=None, defaults=(Compressor.Snappy, 1000, None)), ('document', 'bc643f0f5f1b9db57ff0d8a57d379bd7')) paddle.fluid.Scope Scope() -> paddle.fluid.core._Scope -paddle.reader.cache (ArgSpec(args=['reader'], varargs=None, keywords=None, defaults=None), ('document', '83b94750674c6a04b5f96599d4bf3105')) +paddle.reader.cache (ArgSpec(args=['reader'], varargs=None, keywords=None, defaults=None), ('document', '1676886070eb607cb608f7ba47be0d3c')) paddle.reader.map_readers (ArgSpec(args=['func'], varargs='readers', keywords=None, defaults=None), ('document', '77cbadb09df588e21e5cc0819b69c87d')) paddle.reader.buffered (ArgSpec(args=['reader', 'size'], varargs=None, keywords=None, defaults=None), ('document', '0d6186f109feceb99f60ec50a0a624cb')) paddle.reader.compose (ArgSpec(args=[], varargs='readers', keywords='kwargs', defaults=None), ('document', '884291104e1c3f37f33aae44b7deeb0d')) diff --git a/python/paddle/fluid/compiler.py b/python/paddle/fluid/compiler.py index 894220d96..75cbd141d 100644 --- a/python/paddle/fluid/compiler.py +++ b/python/paddle/fluid/compiler.py @@ -123,7 +123,7 @@ class CompiledProgram(object): will share variables from `share_vars_from`. `share_vars_from` must be run by the executor before this CompiledProgram so that vars are ready. - places(list(CUDAPlace)|list(CPUPlace)|None): If provide, only compile + places(list(CUDAPlace)|list(CPUPlace)|None): If provided, only compile program in the given places. Otherwise, the places used when compiled is determined by the Executor, and the places used are controlled by environment variables: FLAGS_selected_gpus or CUDA_VISIBLE_DEVICES @@ -148,7 +148,7 @@ class CompiledProgram(object): if places is not None: if not isinstance(places, (list, tuple)): places = [places] - self._places = [_place_obj(p) for p in places] + self._places = places else: self._places = None self._build_strategy.is_distribution = _is_pserver_mode(self._program) @@ -195,14 +195,12 @@ class CompiledProgram(object): self._exec_strategy.use_cuda = use_cuda has_set_place = (self._places is not None) if has_set_place: - desire_place = _place_obj(self._place) for p in self._places: - assert p._type() == desire_place._type(), \ + assert p._type() == self._place._type(), \ "Place type not match. You may set the wrong type of places" else: - places = cuda_places( + self._places = cuda_places( ) if self._exec_strategy.use_cuda else cpu_places() - self._places = [_place_obj(p) for p in places] assert self._places, "no place for execution" if self._exec_strategy.num_threads == 0: diff --git a/python/paddle/fluid/reader.py b/python/paddle/fluid/reader.py index 589379806..74ee2828d 100644 --- a/python/paddle/fluid/reader.py +++ b/python/paddle/fluid/reader.py @@ -40,6 +40,77 @@ def _convert_places(places): class PyReader(object): + """ + Create a reader object for data feeding in Python. + Data would be prefetched using Python thread and be pushed + into a queue asynchronously. Data in the queue would be extracted + automatically when `Executor.run(...)` is called. + + Args: + feed_list (list(Variable)|tuple(Variable)): feed variable list. + The variables should be created by :code:`fluid.layers.data()`. + capacity (int): capacity of the queue maintained in PyReader object. + use_double_buffer (bool): whether to use double_buffer_reader to + speed up data feeding. + iterable (bool): whether the created reader object is iterable. + + Returns: + reader (Reader): the created reader object. + + Examples: + 1. If iterable = False, the created PyReader object is almost the + same as :code:`fluid.layers.py_reader()`. Operators would be + inserted into the program. User should call :code:`start()` + before each epoch and catch :code:`fluid.core.EOFException` + thrown by :code:`Executor.run()` when epoch ends. Once the + exception is caught, user should call :code:`reset()` to reset + the reader manually. + + .. code-block:: python + + image = fluid.layers.data( + name='image', shape=[784], dtype='float32') + label = fluid.layers.data( + name='label', shape=[1], dtype='int64') + + reader = fluid.io.PyReader(feed_list=[image, label], + capacity=4, iterable=False) + reader.decorate_sample_list_generator(user_defined_reader) + ... # definition of network is omitted + executor.run(fluid.default_main_program()) + for _ in range(EPOCH_NUM): + reader.start() + while True: + try: + executor.run(feed=None, ...) + except fluid.core.EOFException: + reader.reset() + break + + 2. If iterable=True, the created PyReader object is decoupled with + the program. No operator would be inserted into the program. + In this case, the created reader is a Python generator, which + is iterable. User should feed the data yielded from PyReader + object into :code:`Executor.run(feed=...)`. + + .. code-block:: python + + image = fluid.layers.data( + name='image', shape=[784], dtype='float32') + label = fluid.layers.data( + name='label', shape=[1], dtype='int64') + + reader = fluid.io.PyReader(feed_list=[image, label], + capacity=4, iterable=True) + reader.decorate_sample_list_generator(user_defined_reader, + places=fluid.cuda_places()) + ... # definition of network is omitted + executor.run(fluid.default_main_program()) + for _ in range(EPOCH_NUM): + for data in reader(): + executor.run(feed=data, ...) + """ + unique_name_generator = UniqueNameGenerator() def __init__(self, @@ -47,76 +118,6 @@ class PyReader(object): capacity, use_double_buffer=True, iterable=False): - """ - Create a reader object for data feeding in Python. - Data would be prefetched using Python thread and be pushed - into a queue asynchronously. Data in the queue would be extracted - automatically when `Executor.run(...)` is called. - - Args: - feed_list (list(Variable)|tuple(Variable)): feed variable list. - The variables should be created by :code:`fluid.layers.data()`. - capacity (int): capacity of the queue maintained in PyReader object. - use_double_buffer (bool): whether to use double_buffer_reader to - speed up data feeding. - iterable (bool): whether the created reader object is iterable. - - Returns: - reader (Reader): the created reader object. - - Examples: - 1. If iterable = False, the created PyReader object is almost the - same as :code:`fluid.layers.py_reader()`. Operators would be - inserted into the program. User should call :code:`start()` - before each epoch and catch :code:`fluid.core.EOFException` - thrown by :code:`Executor.run()` when epoch ends. Once the - exception is caught, user should call :code:`reset()` to reset - the reader manually. - - .. code-block:: python - - image = fluid.layers.data( - name='image', shape=[784], dtype='float32') - label = fluid.layers.data( - name='label', shape=[1], dtype='int64') - - reader = fluid.io.PyReader(feed_list=[image, label], - capacity=4, iterable=False) - reader.decorate_sample_list_generator(user_defined_reader) - ... # definition of network is omitted - executor.run(fluid.default_main_program()) - for _ in range(EPOCH_NUM): - reader.start() - while True: - try: - executor.run(feed=None, ...) - except fluid.core.EOFException: - reader.reset() - break - - 2. If iterable=True, the created PyReader object is decoupled with - the program. No operator would be inserted into the program. - In this case, the created reader is a Python generator, which - is iterable. User should feed the data yielded from PyReader - object into :code:`Executor.run(feed=...)`. - - .. code-block:: python - - image = fluid.layers.data( - name='image', shape=[784], dtype='float32') - label = fluid.layers.data( - name='label', shape=[1], dtype='int64') - - reader = fluid.io.PyReader(feed_list=[image, label], - capacity=4, iterable=True) - reader.decorate_sample_list_generator(user_defined_reader, - places=fluid.cuda_places()) - ... # definition of network is omitted - executor.run(fluid.default_main_program()) - for _ in range(EPOCH_NUM): - for data in reader(): - executor.run(feed=data, ...) - """ self._tensor_reader = None self._thread = None self._iterable = iterable @@ -361,7 +362,7 @@ class PyReader(object): Args: reader (generator): Python generator that yields LoDTensor-typed batched data. - places (None|list(CUDAPlace)|list(CPUPlace)): place list. Must + places (None|list(CUDAPlace)|list(CPUPlace)): place list. Must be provided when PyReader is iterable. ''' assert self._tensor_reader is None, \ diff --git a/python/paddle/reader/decorator.py b/python/paddle/reader/decorator.py index 7443a6bb1..168a525b5 100644 --- a/python/paddle/reader/decorator.py +++ b/python/paddle/reader/decorator.py @@ -46,8 +46,7 @@ def cache(reader): data each time. Returns: - reader (generator): a decorated reader object - which yields data from cached memory. + generator: a decorated reader object which yields data from cached memory. """ all_data = tuple(reader()) -- GitLab