From 18f2ea66bd8d4c3f9d9f6c59556851316f421e6d Mon Sep 17 00:00:00 2001 From: Kaipeng Deng Date: Tue, 22 Sep 2020 13:19:18 +0800 Subject: [PATCH] remove paddle.readers API (#26727) * remove paddle.readers API. test=develop --- python/paddle/__init__.py | 1 - python/paddle/dataset/flowers.py | 2 +- python/paddle/fluid/io.py | 6 +++--- .../unittests/test_decoupled_py_reader_data_check.py | 2 +- .../test_imperative_star_gan_with_gradient_penalty.py | 2 +- .../unittests/test_multiprocess_reader_exception.py | 3 ++- .../tests/unittests/test_py_reader_combination.py | 5 +++-- python/paddle/io/__init__.py | 10 ---------- python/paddle/reader/__init__.py | 2 +- python/paddle/tensor/__init__.py | 1 - python/paddle/tensor/random.py | 3 --- 11 files changed, 12 insertions(+), 25 deletions(-) diff --git a/python/paddle/__init__.py b/python/paddle/__init__.py index 1e0dc0e07b4..29e739a0edf 100755 --- a/python/paddle/__init__.py +++ b/python/paddle/__init__.py @@ -204,7 +204,6 @@ from .tensor.math import prod #DEFINE_ALIAS from .tensor.random import standard_normal from .tensor.random import normal from .tensor.random import uniform #DEFINE_ALIAS -from .tensor.random import shuffle #DEFINE_ALIAS from .tensor.random import randn #DEFINE_ALIAS from .tensor.random import rand #DEFINE_ALIAS from .tensor.random import randint #DEFINE_ALIAS diff --git a/python/paddle/dataset/flowers.py b/python/paddle/dataset/flowers.py index 969ad3c922f..bb60c58211c 100644 --- a/python/paddle/dataset/flowers.py +++ b/python/paddle/dataset/flowers.py @@ -37,7 +37,7 @@ from .common import download import tarfile import scipy.io as scio from paddle.dataset.image import * -from paddle.reader import * +from paddle.reader import map_readers, xmap_readers from paddle import compat as cpt import os import numpy as np diff --git a/python/paddle/fluid/io.py b/python/paddle/fluid/io.py index 6e5f7fd035a..fe5b683bdea 100644 --- a/python/paddle/fluid/io.py +++ b/python/paddle/fluid/io.py @@ -26,13 +26,13 @@ from functools import reduce import numpy as np import paddle -import paddle.reader -from paddle.reader import * from paddle.fluid import layers from paddle.fluid.executor import Executor, global_scope from paddle.fluid.evaluator import Evaluator from paddle.fluid.framework import Program, Parameter, default_main_program, default_startup_program, Variable, \ program_guard, dygraph_not_support +from paddle.reader import cache, map_readers, buffered, compose, chain, shuffle, \ + ComposeNotAligned, firstn, xmap_readers, multiprocess_reader from .wrapped_decorator import signature_safe_contextmanager from paddle.fluid.compiler import CompiledProgram from paddle.fluid.log_helper import get_logger @@ -62,7 +62,7 @@ __all__ = [ 'set_program_state', 'get_program_parameter', 'get_program_persistable_vars', -] + reader.__all__ + paddle.reader.__all__ +] + reader.__all__ _logger = get_logger( __name__, logging.INFO, fmt='%(asctime)s-%(levelname)s: %(message)s') diff --git a/python/paddle/fluid/tests/unittests/test_decoupled_py_reader_data_check.py b/python/paddle/fluid/tests/unittests/test_decoupled_py_reader_data_check.py index 4d767709ef5..b2cb3141aad 100644 --- a/python/paddle/fluid/tests/unittests/test_decoupled_py_reader_data_check.py +++ b/python/paddle/fluid/tests/unittests/test_decoupled_py_reader_data_check.py @@ -37,7 +37,7 @@ class TestClass(unittest.TestCase): low=0, high=9, size=label_shape).astype('int64') yield img, label - reader = fluid.io.cache(fake_reader) + reader = paddle.reader.cache(fake_reader) batch_reader = fluid.io.batch(reader, batch_size=batch_size) places = [fluid.CPUPlace()] diff --git a/python/paddle/fluid/tests/unittests/test_imperative_star_gan_with_gradient_penalty.py b/python/paddle/fluid/tests/unittests/test_imperative_star_gan_with_gradient_penalty.py index e94157fa047..1ab37aaed23 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_star_gan_with_gradient_penalty.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_star_gan_with_gradient_penalty.py @@ -592,7 +592,7 @@ class TestStarGANWithGradientPenalty(unittest.TestCase): cfg = Config(place) dataset = create_mnist_dataset(cfg) - dataset = fluid.io.cache(dataset) + dataset = paddle.reader.cache(dataset) static_graph_model = StaticGraphTrainModel(cfg) static_loss = [] diff --git a/python/paddle/fluid/tests/unittests/test_multiprocess_reader_exception.py b/python/paddle/fluid/tests/unittests/test_multiprocess_reader_exception.py index 39cb6651a4b..9634f5af30a 100644 --- a/python/paddle/fluid/tests/unittests/test_multiprocess_reader_exception.py +++ b/python/paddle/fluid/tests/unittests/test_multiprocess_reader_exception.py @@ -12,8 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import paddle import paddle.fluid as fluid -from paddle.fluid.io import multiprocess_reader +from paddle.reader import multiprocess_reader import unittest import numpy as np import six diff --git a/python/paddle/fluid/tests/unittests/test_py_reader_combination.py b/python/paddle/fluid/tests/unittests/test_py_reader_combination.py index 2d977caa033..624927d809f 100644 --- a/python/paddle/fluid/tests/unittests/test_py_reader_combination.py +++ b/python/paddle/fluid/tests/unittests/test_py_reader_combination.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import paddle import paddle.fluid as fluid import unittest import numpy as np @@ -60,8 +61,8 @@ class TestPyReaderCombination(unittest.TestCase): py_reader2 = fluid.io.PyReader( feed_list=[image, label], capacity=16, iterable=True) - reader1 = fluid.io.cache(self.create_reader(self.n1)) - reader2 = fluid.io.cache(self.create_reader(self.n2)) + reader1 = paddle.reader.cache(self.create_reader(self.n1)) + reader2 = paddle.reader.cache(self.create_reader(self.n2)) py_reader1.decorate_batch_generator(reader1, places=place) py_reader2.decorate_batch_generator(reader2, places=place) diff --git a/python/paddle/io/__init__.py b/python/paddle/io/__init__.py index b67779cb2a2..6f0b0f3c9c1 100644 --- a/python/paddle/io/__init__.py +++ b/python/paddle/io/__init__.py @@ -31,15 +31,6 @@ __all__ = [ 'set_program_state', 'load_inference_model', 'save_inference_model', - 'batch', - 'shuffle', - 'buffered', - 'cache', - 'chain', - 'firstn', - 'compose', - 'map_readers', - 'xmap_readers' ] from ..fluid.io import DataLoader @@ -47,4 +38,3 @@ from ..fluid.dataloader import Dataset, IterableDataset, BatchSampler, get_worke TensorDataset, Sampler, SequenceSampler, RandomSampler, DistributedBatchSampler from ..fluid.io import load, save, load_program_state, set_program_state, \ load_inference_model, save_inference_model, batch -from ..reader import shuffle, buffered, cache, chain, firstn, compose, map_readers, xmap_readers diff --git a/python/paddle/reader/__init__.py b/python/paddle/reader/__init__.py index 29337cf0668..881cfd81314 100644 --- a/python/paddle/reader/__init__.py +++ b/python/paddle/reader/__init__.py @@ -66,4 +66,4 @@ An example implementation for multiple item data reader creator: import paddle.reader.decorator from paddle.reader.decorator import * -__all__ = decorator.__all__ +__all__ = [] diff --git a/python/paddle/tensor/__init__.py b/python/paddle/tensor/__init__.py index 2df9473c4b2..cec989fba8b 100755 --- a/python/paddle/tensor/__init__.py +++ b/python/paddle/tensor/__init__.py @@ -171,7 +171,6 @@ from .math import prod #DEFINE_ALIAS from .random import standard_normal from .random import normal from .random import uniform #DEFINE_ALIAS -from .random import shuffle #DEFINE_ALIAS from .random import randn #DEFINE_ALIAS from .random import rand #DEFINE_ALIAS from .random import randint #DEFINE_ALIAS diff --git a/python/paddle/tensor/random.py b/python/paddle/tensor/random.py index b38a1d0f5b7..9ffd81995ed 100644 --- a/python/paddle/tensor/random.py +++ b/python/paddle/tensor/random.py @@ -21,14 +21,11 @@ from ..fluid.data_feeder import check_variable_and_dtype, check_type, check_dtyp from ..fluid.layers import utils import paddle -from ..fluid.io import shuffle #DEFINE_ALIAS - __all__ = [ 'bernoulli', 'standard_normal', 'normal', 'uniform', - 'shuffle', 'randn', 'rand', 'randint', -- GitLab