From df223e87dc89fd531ee9454a82f562dd30a5363f Mon Sep 17 00:00:00 2001 From: Kaipeng Deng Date: Mon, 21 Sep 2020 18:05:16 +0800 Subject: [PATCH] remove paddle.reader APIs. (#2504) * remove paddle.reader APIs. test=develop * remove paddle.reader --- doc/fluid/api/io.rst | 15 +---- doc/fluid/api/io/buffered.rst | 11 ---- doc/fluid/api/io/cache.rst | 11 ---- doc/fluid/api/io/chain.rst | 11 ---- doc/fluid/api/io/compose.rst | 11 ---- doc/fluid/api/io/default_collate_fn.rst | 11 ++++ doc/fluid/api/io/firstn.rst | 11 ---- doc/fluid/api/io/map_readers.rst | 11 ---- doc/fluid/api/io/multiprocess_reader.rst | 11 ---- doc/fluid/api/io/shuffle.rst | 11 ---- doc/fluid/api/io/xmap_readers.rst | 11 ---- doc/paddle/api/gen_doc.py | 3 + .../paddle/reader/ComposeNotAligned_cn.rst | 14 ----- doc/paddle/api/paddle/reader/buffered_cn.rst | 17 ------ doc/paddle/api/paddle/reader/cache_cn.rst | 18 ------ doc/paddle/api/paddle/reader/chain_cn.rst | 47 --------------- doc/paddle/api/paddle/reader/compose_cn.rst | 34 ----------- doc/paddle/api/paddle/reader/firstn_cn.rst | 30 ---------- .../api/paddle/reader/map_readers_cn.rst | 40 ------------- .../paddle/reader/multiprocess_reader_cn.rst | 59 ------------------- .../api/paddle/reader/xmap_readers_cn.rst | 57 ------------------ 21 files changed, 16 insertions(+), 428 deletions(-) delete mode 100644 doc/fluid/api/io/buffered.rst delete mode 100644 doc/fluid/api/io/cache.rst delete mode 100644 doc/fluid/api/io/chain.rst delete mode 100644 doc/fluid/api/io/compose.rst create mode 100644 doc/fluid/api/io/default_collate_fn.rst delete mode 100644 doc/fluid/api/io/firstn.rst delete mode 100644 doc/fluid/api/io/map_readers.rst delete mode 100644 doc/fluid/api/io/multiprocess_reader.rst delete mode 100644 doc/fluid/api/io/shuffle.rst delete mode 100644 doc/fluid/api/io/xmap_readers.rst delete mode 100644 doc/paddle/api/paddle/reader/ComposeNotAligned_cn.rst delete mode 100644 doc/paddle/api/paddle/reader/buffered_cn.rst delete mode 100644 doc/paddle/api/paddle/reader/cache_cn.rst delete mode 100644 doc/paddle/api/paddle/reader/chain_cn.rst delete mode 100644 doc/paddle/api/paddle/reader/compose_cn.rst delete mode 100644 doc/paddle/api/paddle/reader/firstn_cn.rst delete mode 100644 doc/paddle/api/paddle/reader/map_readers_cn.rst delete mode 100644 doc/paddle/api/paddle/reader/multiprocess_reader_cn.rst delete mode 100644 doc/paddle/api/paddle/reader/xmap_readers_cn.rst diff --git a/doc/fluid/api/io.rst b/doc/fluid/api/io.rst index 666d584ca..eb220fe7e 100644 --- a/doc/fluid/api/io.rst +++ b/doc/fluid/api/io.rst @@ -1,5 +1,5 @@ ======== -paddle.io +fluid.io ======== .. toctree:: @@ -7,14 +7,10 @@ paddle.io io/batch.rst io/BatchSampler.rst - io/buffered.rst - io/cache.rst - io/chain.rst - io/compose.rst io/ComposeNotAligned.rst io/DataLoader.rst io/Dataset.rst - io/firstn.rst + io/default_collate_fn.rst io/get_program_parameter.rst io/get_program_persistable_vars.rst io/load.rst @@ -23,17 +19,10 @@ paddle.io io/load_persistables.rst io/load_program_state.rst io/load_vars.rst - io/map_readers.rst - io/multiprocess_reader.rst io/PyReader.rst io/save.rst io/save_inference_model.rst - io/save_inference_model.rst io/save_params.rst io/save_persistables.rst io/save_vars.rst io/set_program_state.rst - io/set_program_state.rst - io/shuffle.rst - io/shuffle.rst - io/xmap_readers.rst diff --git a/doc/fluid/api/io/buffered.rst b/doc/fluid/api/io/buffered.rst deleted file mode 100644 index 125e100dd..000000000 --- a/doc/fluid/api/io/buffered.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. THIS FILE IS GENERATED BY `gen_doc.{py|sh}` - !DO NOT EDIT THIS FILE MANUALLY! - -.. _api_fluid_io_buffered: - -buffered --------- - -.. autofunction:: paddle.fluid.io.buffered - :noindex: - diff --git a/doc/fluid/api/io/cache.rst b/doc/fluid/api/io/cache.rst deleted file mode 100644 index 6da87c92e..000000000 --- a/doc/fluid/api/io/cache.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. THIS FILE IS GENERATED BY `gen_doc.{py|sh}` - !DO NOT EDIT THIS FILE MANUALLY! - -.. _api_fluid_io_cache: - -cache ------ - -.. autofunction:: paddle.fluid.io.cache - :noindex: - diff --git a/doc/fluid/api/io/chain.rst b/doc/fluid/api/io/chain.rst deleted file mode 100644 index efc904b65..000000000 --- a/doc/fluid/api/io/chain.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. THIS FILE IS GENERATED BY `gen_doc.{py|sh}` - !DO NOT EDIT THIS FILE MANUALLY! - -.. _api_fluid_io_chain: - -chain ------ - -.. autofunction:: paddle.fluid.io.chain - :noindex: - diff --git a/doc/fluid/api/io/compose.rst b/doc/fluid/api/io/compose.rst deleted file mode 100644 index bac5affe5..000000000 --- a/doc/fluid/api/io/compose.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. THIS FILE IS GENERATED BY `gen_doc.{py|sh}` - !DO NOT EDIT THIS FILE MANUALLY! - -.. _api_fluid_io_compose: - -compose -------- - -.. autofunction:: paddle.fluid.io.compose - :noindex: - diff --git a/doc/fluid/api/io/default_collate_fn.rst b/doc/fluid/api/io/default_collate_fn.rst new file mode 100644 index 000000000..38f34e1ee --- /dev/null +++ b/doc/fluid/api/io/default_collate_fn.rst @@ -0,0 +1,11 @@ +.. THIS FILE IS GENERATED BY `gen_doc.{py|sh}` + !DO NOT EDIT THIS FILE MANUALLY! + +.. _api_fluid_io_default_collate_fn: + +default_collate_fn +------------------ + +.. autofunction:: paddle.fluid.io.default_collate_fn + :noindex: + diff --git a/doc/fluid/api/io/firstn.rst b/doc/fluid/api/io/firstn.rst deleted file mode 100644 index 512dab717..000000000 --- a/doc/fluid/api/io/firstn.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. THIS FILE IS GENERATED BY `gen_doc.{py|sh}` - !DO NOT EDIT THIS FILE MANUALLY! - -.. _api_fluid_io_firstn: - -firstn ------- - -.. autofunction:: paddle.fluid.io.firstn - :noindex: - diff --git a/doc/fluid/api/io/map_readers.rst b/doc/fluid/api/io/map_readers.rst deleted file mode 100644 index 5e7558baf..000000000 --- a/doc/fluid/api/io/map_readers.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. THIS FILE IS GENERATED BY `gen_doc.{py|sh}` - !DO NOT EDIT THIS FILE MANUALLY! - -.. _api_fluid_io_map_readers: - -map_readers ------------ - -.. autofunction:: paddle.fluid.io.map_readers - :noindex: - diff --git a/doc/fluid/api/io/multiprocess_reader.rst b/doc/fluid/api/io/multiprocess_reader.rst deleted file mode 100644 index 24b2a3bd5..000000000 --- a/doc/fluid/api/io/multiprocess_reader.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. THIS FILE IS GENERATED BY `gen_doc.{py|sh}` - !DO NOT EDIT THIS FILE MANUALLY! - -.. _api_fluid_io_multiprocess_reader: - -multiprocess_reader -------------------- - -.. autofunction:: paddle.fluid.io.multiprocess_reader - :noindex: - diff --git a/doc/fluid/api/io/shuffle.rst b/doc/fluid/api/io/shuffle.rst deleted file mode 100644 index dccf6d5c6..000000000 --- a/doc/fluid/api/io/shuffle.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. THIS FILE IS GENERATED BY `gen_doc.{py|sh}` - !DO NOT EDIT THIS FILE MANUALLY! - -.. _api_fluid_io_shuffle: - -shuffle -------- - -.. autofunction:: paddle.fluid.io.shuffle - :noindex: - diff --git a/doc/fluid/api/io/xmap_readers.rst b/doc/fluid/api/io/xmap_readers.rst deleted file mode 100644 index 77b3e7899..000000000 --- a/doc/fluid/api/io/xmap_readers.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. THIS FILE IS GENERATED BY `gen_doc.{py|sh}` - !DO NOT EDIT THIS FILE MANUALLY! - -.. _api_fluid_io_xmap_readers: - -xmap_readers ------------- - -.. autofunction:: paddle.fluid.io.xmap_readers - :noindex: - diff --git a/doc/paddle/api/gen_doc.py b/doc/paddle/api/gen_doc.py index 93926d56b..211c41cdd 100644 --- a/doc/paddle/api/gen_doc.py +++ b/doc/paddle/api/gen_doc.py @@ -20,6 +20,9 @@ api_set = set() def get_all_api(root_path='paddle'): for filefiner, name, ispkg in pkgutil.walk_packages( path=paddle.__path__, prefix=paddle.__name__ + '.'): + # not show paddle.reader APIs + if name.startswith("paddle.reader"): + continue try: m = eval(name) except AttributeError: diff --git a/doc/paddle/api/paddle/reader/ComposeNotAligned_cn.rst b/doc/paddle/api/paddle/reader/ComposeNotAligned_cn.rst deleted file mode 100644 index 1f4a911c5..000000000 --- a/doc/paddle/api/paddle/reader/ComposeNotAligned_cn.rst +++ /dev/null @@ -1,14 +0,0 @@ -.. THIS FILE IS GENERATED BY `gen_doc.{py|sh}` - !DO NOT EDIT THIS FILE MANUALLY! - -.. _api_fluid_io_ComposeNotAligned: - -ComposeNotAligned ------------------ - -.. autoclass:: paddle.fluid.io.ComposeNotAligned - :members: - :inherited-members: - :noindex: - -表示一种错误状态，说明调用compose API时, reader的输出数据没有对齐。 diff --git a/doc/paddle/api/paddle/reader/buffered_cn.rst b/doc/paddle/api/paddle/reader/buffered_cn.rst deleted file mode 100644 index 027c6346f..000000000 --- a/doc/paddle/api/paddle/reader/buffered_cn.rst +++ /dev/null @@ -1,17 +0,0 @@ -.. _cn_api_fluid_io_buffered: - -buffered -------------------------------- - -.. py:function:: paddle.fluid.io.buffered(reader, size) - - - - -创建一个缓存数据读取器，它读取数据并且存储进缓存区，从缓存区读取数据将会加速，只要缓存不是空的。 - -参数: - - **reader** (callable) – 读取数据的reader - - **size** (int) – 最大buffer的大小 - -返回:缓存的reader（读取器） \ No newline at end of file diff --git a/doc/paddle/api/paddle/reader/cache_cn.rst b/doc/paddle/api/paddle/reader/cache_cn.rst deleted file mode 100644 index e93e4c85d..000000000 --- a/doc/paddle/api/paddle/reader/cache_cn.rst +++ /dev/null @@ -1,18 +0,0 @@ -.. _cn_api_fluid_io_cache: - -cache -------------------------------- - -.. py:function:: paddle.fluid.io.cache(reader) - - - - -缓存reader数据到内存中，小心此方法可能会花长时间来处理数据，并且会占用大量内存。 ``reader()`` 只能被调用一次。 - -参数: - - **reader** (callable) – 读取数据的reader，每次都会yields数据。 - -返回：每次都会从内存中yields数据的一个装饰reader。 - -返回类型：生成器 \ No newline at end of file diff --git a/doc/paddle/api/paddle/reader/chain_cn.rst b/doc/paddle/api/paddle/reader/chain_cn.rst deleted file mode 100644 index 4a4872d26..000000000 --- a/doc/paddle/api/paddle/reader/chain_cn.rst +++ /dev/null @@ -1,47 +0,0 @@ -.. _cn_api_fluid_io_chain: - -chain -------------------------------- - -.. py:function:: paddle.fluid.io.chain(*readers) - - - - -该接口将多个数据读取器组成一个数据读取器，它依次返回多个数据读取器的输出数据，同时不改变输出数据原先的格式。 - -举例来说，如果有3个输入读取器且输出分别为[0，0，0]、[10，10，10]和[20，20，20]，那么调用该接口产生的新数据读取器的输出为[0，0，0], [10，10，10], [20，20，20]。 - -参数： - - **readers(list)** – 输入的数据读取器。 - -返回： 新的数据读取器。 - -返回类型：callable - -**代码示例** - -.. code-block:: python - - import paddle.fluid as fluid - - def reader_creator_3(start): - def reader(): - for i in range(start, start + 3): - yield [i, i, i] - return reader - - c = fluid.io.chain(reader_creator_3(0), reader_creator_3(10), reader_creator_3(20)) - for e in c(): - print(e) - # 输出结果如下： - # [0, 0, 0] - # [1, 1, 1] - # [2, 2, 2] - # [10, 10, 10] - # [11, 11, 11] - # [12, 12, 12] - # [20, 20, 20] - # [21, 21, 21] - # [22, 22, 22] - diff --git a/doc/paddle/api/paddle/reader/compose_cn.rst b/doc/paddle/api/paddle/reader/compose_cn.rst deleted file mode 100644 index b4393bab5..000000000 --- a/doc/paddle/api/paddle/reader/compose_cn.rst +++ /dev/null @@ -1,34 +0,0 @@ -.. _cn_api_fluid_io_compose: - -compose -------------------------------- - -.. py:function:: paddle.fluid.io.compose(*readers, **kwargs) - - - - -该接口将多个数据读取器组合为一个数据读取器，返回读取器的输出包含所有输入读取器的输出。 - -例如：如果输入为三个reader，三个reader的输出分别为：（1，2）、3、（4，5），则组合reader的输出为：（1，2，3，4，5）。 - -参数： - - **readers** - 将被组合的多个数据读取器(Reader)，数据读取器的定义参见 :ref:`cn_api_paddle_data_reader_reader` 。 - - **check_alignment** (bool) - 可选，指明是否对输入reader进行对齐检查，默认值为True。如果为True，将检查输入reader是否正确对齐。如果为False，将不检查对齐并自动丢弃无法对齐的末尾数据。 - -返回：数据读取器(Reader)。 - -**代码示例**: - -.. code-block:: python - - import paddle.fluid as fluid - def reader_creator_10(dur): - def reader(): - for i in range(10): - yield i - return reader - - reader = fluid.io.compose(reader_creator_10(0), reader_creator_10(0)) - -注意： 运行过程可能会抛出异常 ``ComposeNotAligned`` ，原因是输入的readers数据未对齐。 当check_alignment设置为False时，不会检查并触发该异常。 diff --git a/doc/paddle/api/paddle/reader/firstn_cn.rst b/doc/paddle/api/paddle/reader/firstn_cn.rst deleted file mode 100644 index ba9c1d427..000000000 --- a/doc/paddle/api/paddle/reader/firstn_cn.rst +++ /dev/null @@ -1,30 +0,0 @@ -.. _cn_api_fluid_io_firstn: - -firstn -------------------------------- - -.. py:function:: paddle.fluid.io.firstn(reader, n) - - - - -该接口创建一个数据读取器，它可以返回的最大样本数为n。 - -参数： - - **reader** (callable) – 输入的数据读取器。 - - **n** (int) – 可以返回的最大样本数。 - -返回： 新的的数据读取器。 - -返回类型： callable - -.. code-block:: python - - import paddle.fluid as fluid - def reader(): - for i in range(100): - yield i - firstn_reader = fluid.io.firstn(reader, 5) - for e in firstn_reader(): - print(e) - # 输出结果为:0 1 2 3 4 diff --git a/doc/paddle/api/paddle/reader/map_readers_cn.rst b/doc/paddle/api/paddle/reader/map_readers_cn.rst deleted file mode 100644 index cb50e6283..000000000 --- a/doc/paddle/api/paddle/reader/map_readers_cn.rst +++ /dev/null @@ -1,40 +0,0 @@ -.. _cn_api_fluid_io_map_readers: - -map_readers -------------------------------- - -.. py:function:: paddle.fluid.io.map_readers(func, *readers) - - - - -该接口将创建一个数据读取器(Reader)，其中 `func` 函数的输出将直接作为新数据读取器的输出， `readers` 的输出将作为函数 `func` 的输入参数。 - -例如：如果输入的 `readers` 为两个输出分别为：2、3 的 `reader` ，输入的 `func` 为乘法函数 `mul(x, y)` ，则得到的新建 `reader` 的输出为：6。 - -参数： - - **func** - 读取数据并返回数据项的函数， `func` 的输出将直接作为新创建的数据读取器的输出。 - - - **readers** - 输入的一个或多个数据读取器(Reader)，这些数据读取器的输出数据将作为函数 `func` 的输入参数。数据读取器的定义参见 :ref:`cn_api_paddle_data_reader_reader` 。 - -返回： 新创建的数据读取器(Reader) - -**代码示例**: - -.. code-block:: python - - - import paddle.fluid as fluid - d = {"h": 0, "i": 1} - def func(x): - return d[x] - - def reader(): - yield "h" - yield "i" - - map_reader_result = fluid.io.map_readers(func, reader) - - - - diff --git a/doc/paddle/api/paddle/reader/multiprocess_reader_cn.rst b/doc/paddle/api/paddle/reader/multiprocess_reader_cn.rst deleted file mode 100644 index ab600239e..000000000 --- a/doc/paddle/api/paddle/reader/multiprocess_reader_cn.rst +++ /dev/null @@ -1,59 +0,0 @@ -.. _cn_api_fluid_io_multiprocess_reader: - -multiprocess_reader -------------------------------- - -.. py:function:: paddle.fluid.io.multiprocess_reader(readers, use_pipe=True, queue_size=1000) - - - - -使用python多进程从 ``readers`` 中读取数据，然后使用 ``multiprocessing.Pipe`` 或 ``multiprocessing.Queue`` 合并所有数据。 ``readers`` 列表中的每个reader会被创建一个独立的进程来调用，reader之间应该相互独立，互不影响，避免出现多进程读取的冲突问题. - -multiprocess.queue需要/dev/shm的rw访问权限，某些平台不支持。 - -参数： - - **readers** (list(generator)|tuple(generator)) - python生成器list, 用来读取数据 - - **use_pipe** (bool，可选) - use_pipe控制multiprocess_reader内部用 ``pipe`` 还是 ``queue`` 来实现进程间通信，默认为 ``True`` 使用 ``pipe`` 进行通信 - - **queue_size** (int，可选) - 如果使用queue来进行进程间通信 (``use_pipe=False``), 则该参数用来设定队列大小 - -返回：使用多进程封装readers之后的reader - -返回类型：python生成器 - - -**代码示例** - -.. code-block:: python - - import paddle.fluid as fluid - from paddle.fluid.io import multiprocess_reader - import numpy as np - - - def fake_reader(start, end): - def __impl__(): - for i in range(start, end): - yield [np.array([1, 2, 3]) * i], - return __impl__ - - - with fluid.program_guard(fluid.Program(), fluid.Program()): - place = fluid.CPUPlace() - image = fluid.layers.data( - name='image', dtype='int64', shape=[3]) - fluid.layers.Print(image) - reader = fluid.io.PyReader( - feed_list=[image], capacity=2) - image_p_1 = image + 1 - decorated_reader = multiprocess_reader( - [fake_reader(1, 5), fake_reader(6, 10)], False) - - reader.decorate_sample_generator(decorated_reader, batch_size=2, places=[place]) - - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) - - for data in reader(): - exe.run(feed=data, fetch_list=[image_p_1]) - diff --git a/doc/paddle/api/paddle/reader/xmap_readers_cn.rst b/doc/paddle/api/paddle/reader/xmap_readers_cn.rst deleted file mode 100644 index 5f434ecb9..000000000 --- a/doc/paddle/api/paddle/reader/xmap_readers_cn.rst +++ /dev/null @@ -1,57 +0,0 @@ -.. _cn_api_fluid_io_xmap_readers: - -xmap_readers -------------------------------- - -.. py:function:: paddle.fluid.io.xmap_readers(mapper, reader, process_num, buffer_size, order=False) - - - - -多线程下，使用自定义映射器 reader 返回样本到输出队列。 - -参数： - - **mapper** （callable）: 映射 reader 数据的函数。 - - **reader** （callable）: 产生数据的 reader。 - - **process_num** （int）: 处理样本的线程数。 - - **buffer_size** （int）: 数据缓冲队列大小。 - - **order** （bool）: 是否保持原始 reader 数据顺序，默认为 False。 - -返回：一个用户定义的 reader `装饰器 `_ 。 - -返回类型：callable，可调用对象。 - -**代码示例**： - -.. code-block:: python - - import paddle.fluid as fluid - import time - - def reader_creator_10(dur): - def reader(): - for i in range(10): - time.sleep(dur) - yield i - return reader - - def mapper(x): - return (x + 1) - - orders = (True, False) - thread_num = (1, 2, 4, 8, 16) - buffer_size = (1, 2, 4, 8, 16) - for order in orders: - for t_num in thread_num: - for size in buffer_size: - user_reader = fluid.io.xmap_readers(mapper, - reader_creator_10(0), - t_num, size, order) - for n in range(3): - result = list() - for i in user_reader(): - result.append(i) - if not order: - result.sort() - for idx, e in enumerate(result): - assert e == mapper(idx) \ No newline at end of file -- GitLab