diff --git a/python/paddle/fluid/layers/io.py b/python/paddle/fluid/layers/io.py index ead57ac370d1bec13c1b21e83dd4be1a7331f87e..34382fb9fecdc256ae8fe3fcdaf1effd6e2597cb 100644 --- a/python/paddle/fluid/layers/io.py +++ b/python/paddle/fluid/layers/io.py @@ -21,7 +21,7 @@ from ..executor import global_scope __all__ = [ 'data', 'BlockGuardServ', 'ListenAndServ', 'Send', 'open_recordio_file', - 'open_files', 'read_file', 'shuffle', 'double_buffer' + 'open_files', 'read_file', 'shuffle', 'batch', 'double_buffer' ] @@ -290,7 +290,7 @@ def open_recordio_file(filename, lod_levels, dtypes, pass_num=1, - for_parallel=False): + for_parallel=True): """ Open a RecordIO file @@ -364,7 +364,7 @@ def open_files(filenames, thread_num, buffer_size=None, pass_num=1, - for_parallel=False): + for_parallel=True): """ Open files @@ -476,6 +476,11 @@ def shuffle(reader, buffer_size): 'create_shuffle_reader', reader, {'buffer_size': int(buffer_size)}) +def batch(reader, batch_size): + return __create_unshared_decorated_reader__( + 'create_batch_reader', reader, {'batch_size': int(batch_size)}) + + def double_buffer(reader, place=None): attrs = dict() if place is not None: diff --git a/python/paddle/fluid/tests/unittests/test_multi_file_reader.py b/python/paddle/fluid/tests/unittests/test_multi_file_reader.py index 5dc41e54d6158787eb966333c894e378b5c706d0..3f940203b9393d266d75b50c9cbf62e89c36cbdf 100644 --- a/python/paddle/fluid/tests/unittests/test_multi_file_reader.py +++ b/python/paddle/fluid/tests/unittests/test_multi_file_reader.py @@ -69,7 +69,6 @@ class TestMultipleReader(unittest.TestCase): break batch_count += 1 self.assertLessEqual(img_val.shape[0], self.batch_size) - data_files.reset() self.assertEqual(batch_count, self.num_batch * 3) def test_main(self): diff --git a/python/paddle/fluid/tests/unittests/test_multi_pass_reader.py b/python/paddle/fluid/tests/unittests/test_multi_pass_reader.py index 1471843ded7a42432a84a9fad76bb97dcf7fb9c2..52e7cc1ffbba40a63ce3cec645c7c0a7a499c1bf 100644 --- a/python/paddle/fluid/tests/unittests/test_multi_pass_reader.py +++ b/python/paddle/fluid/tests/unittests/test_multi_pass_reader.py @@ -43,9 +43,8 @@ class TestMultipleReader(unittest.TestCase): filename='./mnist.recordio', shapes=[(-1, 784), (-1, 1)], lod_levels=[0, 0], - dtypes=['float32', 'int64']) - data_file = fluid.layers.io.multi_pass( - reader=data_file, pass_num=self.pass_num) + dtypes=['float32', 'int64'], + pass_num=self.pass_num) img, label = fluid.layers.read_file(data_file) if fluid.core.is_compiled_with_cuda(): @@ -65,5 +64,4 @@ class TestMultipleReader(unittest.TestCase): break batch_count += 1 self.assertLessEqual(img_val.shape[0], self.batch_size) - data_file.reset() self.assertEqual(batch_count, self.num_batch * self.pass_num) diff --git a/python/paddle/fluid/tests/unittests/test_recordio_reader.py b/python/paddle/fluid/tests/unittests/test_recordio_reader.py index 7c8e7f634fdd3ee3f056a95df774402a7c29e906..f32050014d7ace5aee4aca75a47bfc6a75ff91c2 100644 --- a/python/paddle/fluid/tests/unittests/test_recordio_reader.py +++ b/python/paddle/fluid/tests/unittests/test_recordio_reader.py @@ -74,13 +74,13 @@ class TestRecordIO(unittest.TestCase): avg_loss_np.append(tmp) batch_id += 1 - data_file.reset() self.assertEqual(batch_id, self.num_batches) self.assertLess(avg_loss_np[-1], avg_loss_np[0]) def test_shuffle_reader(self): - self.test_main(decorator_callback=lambda reader: fluid.layers.io.shuffle(reader, buffer_size=200)) + self.test_main(decorator_callback=lambda reader: fluid.layers.io.shuffle( + reader, buffer_size=200)) def test_double_buffer_reader(self): self.test_main(decorator_callback=lambda reader: fluid.layers.io.double_buffer(reader, - place='cuda:0' if fluid.core.is_compiled_with_cuda() else 'cpu')) + place='cuda:0' if fluid.core.is_compiled_with_cuda() else 'cpu'))