From 3a09693f5c73035e762e5d296dbee8a2f2aaff7a Mon Sep 17 00:00:00 2001 From: sneaxiy Date: Mon, 18 Mar 2019 12:39:38 +0000 Subject: [PATCH] change API name test=develop --- paddle/fluid/API.spec | 8 ++++---- python/paddle/fluid/compiler.py | 4 +++- python/paddle/fluid/layers/io.py | 3 +++ python/paddle/fluid/reader.py | 16 ++++++++-------- .../unittests/test_decoupled_py_reader.py | 19 +++++++++++++++++-- .../test_py_reader_sample_generator.py | 2 +- 6 files changed, 36 insertions(+), 16 deletions(-) diff --git a/paddle/fluid/API.spec b/paddle/fluid/API.spec index c14eaa4b8..f168d5608 100644 --- a/paddle/fluid/API.spec +++ b/paddle/fluid/API.spec @@ -61,10 +61,10 @@ paddle.fluid.io.load_params (ArgSpec(args=['executor', 'dirname', 'main_program' paddle.fluid.io.load_persistables (ArgSpec(args=['executor', 'dirname', 'main_program', 'filename'], varargs=None, keywords=None, defaults=(None, None)), ('document', '28df5bfe26ca7a077f91156abb0fe6d2')) paddle.fluid.io.save_inference_model (ArgSpec(args=['dirname', 'feeded_var_names', 'target_vars', 'executor', 'main_program', 'model_filename', 'params_filename', 'export_for_deployment'], varargs=None, keywords=None, defaults=(None, None, None, True)), ('document', '582d87b8df75a5a639a107db8ff86f9c')) paddle.fluid.io.load_inference_model (ArgSpec(args=['dirname', 'executor', 'model_filename', 'params_filename', 'pserver_endpoints'], varargs=None, keywords=None, defaults=(None, None, None)), ('document', '7a5255386075dac3c75b7058254fcdcb')) -paddle.fluid.io.PyReader.__init__ (ArgSpec(args=['self', 'feed_list', 'capacity', 'use_double_buffer', 'iterable'], varargs=None, keywords=None, defaults=(True, False)), ('document', '18211b287474b401bc460d3f73dbc1c7')) -paddle.fluid.io.PyReader.decorate_paddle_reader (ArgSpec(args=['self', 'reader', 'places'], varargs=None, keywords=None, defaults=(None,)), ('document', 'faef298f73e91aedcfaf5d184f3109b7')) -paddle.fluid.io.PyReader.decorate_sample_generator (ArgSpec(args=['self', 'sample_generator', 'batch_size', 'drop_last', 'places'], varargs=None, keywords=None, defaults=(True, None)), ('document', 'd3fe49fc342e7778ed086e965f41bf12')) -paddle.fluid.io.PyReader.decorate_tensor_provider (ArgSpec(args=['self', 'reader', 'places'], varargs=None, keywords=None, defaults=(None,)), ('document', 'd10224fef1095247063b6976da793021')) +paddle.fluid.io.PyReader.__init__ (ArgSpec(args=['self', 'feed_list', 'capacity', 'use_double_buffer', 'iterable'], varargs=None, keywords=None, defaults=(True, False)), ('document', 'b3d72958b2568aae3f90f72abdcb7d1a')) +paddle.fluid.io.PyReader.decorate_batch_generator (ArgSpec(args=['self', 'reader', 'places'], varargs=None, keywords=None, defaults=(None,)), ('document', 'd10224fef1095247063b6976da793021')) +paddle.fluid.io.PyReader.decorate_sample_generator (ArgSpec(args=['self', 'sample_generator', 'batch_size', 'drop_last', 'places'], varargs=None, keywords=None, defaults=(True, None)), ('document', '7abd9cf7d695bab5bb6cf7ded5903cb2')) +paddle.fluid.io.PyReader.decorate_sample_list_generator (ArgSpec(args=['self', 'reader', 'places'], varargs=None, keywords=None, defaults=(None,)), ('document', 'faef298f73e91aedcfaf5d184f3109b7')) paddle.fluid.io.PyReader.reset (ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None), ('document', 'ff1cc1e2beb8824d453656c72c28ddfb')) paddle.fluid.io.PyReader.start (ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None), ('document', 'b7ea0a548991924e4cfe61a577b8e56d')) paddle.fluid.initializer.ConstantInitializer.__init__ (ArgSpec(args=['self', 'value', 'force_cpu'], varargs=None, keywords=None, defaults=(0.0, False)), ('document', '6adf97f83acf6453d4a6a4b1070f3754')) diff --git a/python/paddle/fluid/compiler.py b/python/paddle/fluid/compiler.py index a12fde47d..894220d96 100644 --- a/python/paddle/fluid/compiler.py +++ b/python/paddle/fluid/compiler.py @@ -127,7 +127,9 @@ class CompiledProgram(object): program in the given places. Otherwise, the places used when compiled is determined by the Executor, and the places used are controlled by environment variables: FLAGS_selected_gpus or CUDA_VISIBLE_DEVICES - if using GPU; or CPU_NUM if using CPU. + if using GPU; or CPU_NUM if using CPU. For example, if you want to + run on GPU 0 and 1, set places=[fluid.CUDAPlace(0), fluid.CUDAPlace(1)]. + If you want to run on 2 CPU cores, set places=[fluid.CPUPlace()]*2. Returns: self diff --git a/python/paddle/fluid/layers/io.py b/python/paddle/fluid/layers/io.py index da3ffc9a6..94fd9f3ea 100644 --- a/python/paddle/fluid/layers/io.py +++ b/python/paddle/fluid/layers/io.py @@ -632,6 +632,9 @@ def _py_reader(capacity, reader.reset = __reset__ reader.decorate_tensor_provider = __set_tensor_provider__ reader.decorate_paddle_reader = __set_paddle_reader__ + + reader.decorate_batch_generator = __set_tensor_provider__ + reader.decorate_sample_list_generator = __set_paddle_reader__ reader.start = __start__ return reader diff --git a/python/paddle/fluid/reader.py b/python/paddle/fluid/reader.py index ef212014b..589379806 100644 --- a/python/paddle/fluid/reader.py +++ b/python/paddle/fluid/reader.py @@ -82,7 +82,7 @@ class PyReader(object): reader = fluid.io.PyReader(feed_list=[image, label], capacity=4, iterable=False) - reader.decorate_paddle_reader(user_defined_reader) + reader.decorate_sample_list_generator(user_defined_reader) ... # definition of network is omitted executor.run(fluid.default_main_program()) for _ in range(EPOCH_NUM): @@ -109,7 +109,7 @@ class PyReader(object): reader = fluid.io.PyReader(feed_list=[image, label], capacity=4, iterable=True) - reader.decorate_paddle_reader(user_defined_reader, + reader.decorate_sample_list_generator(user_defined_reader, places=fluid.cuda_places()) ... # definition of network is omitted executor.run(fluid.default_main_program()) @@ -287,7 +287,7 @@ class PyReader(object): :code:`places` must be set when the PyReader object is iterable. If all inputs have no lods, this method is faster than - :code:`decorate_paddle_reader(paddle.batch(sample_generator, ...))` . + :code:`decorate_sample_list_generator(paddle.batch(sample_generator, ...))` . Args: sample_generator (generator): Python generator that yields @@ -306,7 +306,7 @@ class PyReader(object): break if has_lod: - self.decorate_paddle_reader( + self.decorate_sample_list_generator( paddle.batch( sample_generator, batch_size=batch_size, @@ -319,9 +319,9 @@ class PyReader(object): batch_size=batch_size, generator=sample_generator, drop_last=drop_last) - self.decorate_tensor_provider(reader, places=places) + self.decorate_batch_generator(reader, places=places) - def decorate_paddle_reader(self, reader, places=None): + def decorate_sample_list_generator(self, reader, places=None): ''' Set the data source of the PyReader object. @@ -347,9 +347,9 @@ class PyReader(object): for slots in paddle_reader(): yield [slots[var.name] for var in self._feed_list] - self.decorate_tensor_provider(__tensor_reader_impl__, places) + self.decorate_batch_generator(__tensor_reader_impl__, places) - def decorate_tensor_provider(self, reader, places=None): + def decorate_batch_generator(self, reader, places=None): ''' Set the data source of the PyReader object. diff --git a/python/paddle/fluid/tests/unittests/test_decoupled_py_reader.py b/python/paddle/fluid/tests/unittests/test_decoupled_py_reader.py index 7112a5774..377014510 100644 --- a/python/paddle/fluid/tests/unittests/test_decoupled_py_reader.py +++ b/python/paddle/fluid/tests/unittests/test_decoupled_py_reader.py @@ -25,6 +25,7 @@ CLASS_NUM = 10 def random_reader(): + np.random.seed(1) for i in range(BATCH_SIZE * 40): image = np.random.random([784]) label = np.random.random_integers(low=0, high=CLASS_NUM - 1) @@ -79,7 +80,8 @@ class TestBase(unittest.TestCase): reader = paddle.batch(random_reader, batch_size=BATCH_SIZE) ps = places if use_double_buffer else fluid.cpu_places(len(places)) - py_reader.decorate_paddle_reader( + + py_reader.decorate_sample_list_generator( reader, places=ps if py_reader.iterable else None) exe = fluid.Executor(place=places[0]) @@ -92,6 +94,7 @@ class TestBase(unittest.TestCase): step = 0 step_list = [] + loss_list = [] start_t = time.time() if not py_reader.iterable: for _ in six.moves.range(EPOCH_NUM): @@ -102,6 +105,7 @@ class TestBase(unittest.TestCase): L, = exe.run(program=prog, fetch_list=[loss], use_program_cache=True) + loss_list.append(np.mean(L)) step += 1 except fluid.core.EOFException: py_reader.reset() @@ -123,10 +127,15 @@ class TestBase(unittest.TestCase): feed=d, fetch_list=[loss], use_program_cache=True) + loss_list.append(np.mean(L)) step += 1 step_list.append(step) end_t = time.time() - ret = {"time": end_t - start_t, "step": step_list} + ret = { + "time": end_t - start_t, + "step": step_list, + "loss": np.array(loss_list) + } return ret def prepare_places(self, with_data_parallel, with_cpu=True, with_gpu=True): @@ -148,12 +157,18 @@ class TestBase(unittest.TestCase): for with_data_parallel in [True, False]: for p in self.prepare_places(with_data_parallel): for use_double_buffer in [False, True]: + results = [] for use_legacy_py_reader in [False, True]: ret = self.run_main( use_legacy_py_reader=use_legacy_py_reader, with_data_parallel=with_data_parallel, places=p, use_double_buffer=use_double_buffer) + results.append(ret) + if not use_double_buffer: + diff = np.max( + np.abs(results[0]['loss'] - results[1]['loss'])) + self.assertLess(diff, 1e-3) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_py_reader_sample_generator.py b/python/paddle/fluid/tests/unittests/test_py_reader_sample_generator.py index 2f8f0b1b6..4efca5e2a 100644 --- a/python/paddle/fluid/tests/unittests/test_py_reader_sample_generator.py +++ b/python/paddle/fluid/tests/unittests/test_py_reader_sample_generator.py @@ -62,7 +62,7 @@ class TestCaseBase(unittest.TestCase): all_datas = self.generate_all_data(batch_reader) if not use_sample_generator: - py_reader.decorate_paddle_reader( + py_reader.decorate_sample_list_generator( batch_reader, places=fluid.cpu_places()) else: py_reader.decorate_sample_generator( -- GitLab