diff --git a/doc/fluid/api/io.rst b/doc/fluid/api/io.rst index 666d584ca533027ddeb3f1a16575365804260b81..eb220fe7e911f59bb10869b4a93fcb59d34d475e 100644 --- a/doc/fluid/api/io.rst +++ b/doc/fluid/api/io.rst @@ -1,5 +1,5 @@ ======== -paddle.io +fluid.io ======== .. toctree:: @@ -7,14 +7,10 @@ paddle.io io/batch.rst io/BatchSampler.rst - io/buffered.rst - io/cache.rst - io/chain.rst - io/compose.rst io/ComposeNotAligned.rst io/DataLoader.rst io/Dataset.rst - io/firstn.rst + io/default_collate_fn.rst io/get_program_parameter.rst io/get_program_persistable_vars.rst io/load.rst @@ -23,17 +19,10 @@ paddle.io io/load_persistables.rst io/load_program_state.rst io/load_vars.rst - io/map_readers.rst - io/multiprocess_reader.rst io/PyReader.rst io/save.rst io/save_inference_model.rst - io/save_inference_model.rst io/save_params.rst io/save_persistables.rst io/save_vars.rst io/set_program_state.rst - io/set_program_state.rst - io/shuffle.rst - io/shuffle.rst - io/xmap_readers.rst diff --git a/doc/fluid/api/io/buffered.rst b/doc/fluid/api/io/buffered.rst deleted file mode 100644 index 125e100dd8eaac9e09b40bf04d31756b6cade33a..0000000000000000000000000000000000000000 --- a/doc/fluid/api/io/buffered.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. THIS FILE IS GENERATED BY `gen_doc.{py|sh}` - !DO NOT EDIT THIS FILE MANUALLY! - -.. _api_fluid_io_buffered: - -buffered --------- - -.. autofunction:: paddle.fluid.io.buffered - :noindex: - diff --git a/doc/fluid/api/io/cache.rst b/doc/fluid/api/io/cache.rst deleted file mode 100644 index 6da87c92eb7730d26d45795094688044f490974d..0000000000000000000000000000000000000000 --- a/doc/fluid/api/io/cache.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. THIS FILE IS GENERATED BY `gen_doc.{py|sh}` - !DO NOT EDIT THIS FILE MANUALLY! - -.. _api_fluid_io_cache: - -cache ------ - -.. autofunction:: paddle.fluid.io.cache - :noindex: - diff --git a/doc/fluid/api/io/chain.rst b/doc/fluid/api/io/chain.rst deleted file mode 100644 index efc904b654206f8e107af46df852fafd4472f8c4..0000000000000000000000000000000000000000 --- a/doc/fluid/api/io/chain.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. THIS FILE IS GENERATED BY `gen_doc.{py|sh}` - !DO NOT EDIT THIS FILE MANUALLY! - -.. _api_fluid_io_chain: - -chain ------ - -.. autofunction:: paddle.fluid.io.chain - :noindex: - diff --git a/doc/fluid/api/io/compose.rst b/doc/fluid/api/io/compose.rst deleted file mode 100644 index bac5affe50192ec8fb33c6294f1012d883f8ef15..0000000000000000000000000000000000000000 --- a/doc/fluid/api/io/compose.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. THIS FILE IS GENERATED BY `gen_doc.{py|sh}` - !DO NOT EDIT THIS FILE MANUALLY! - -.. _api_fluid_io_compose: - -compose -------- - -.. autofunction:: paddle.fluid.io.compose - :noindex: - diff --git a/doc/fluid/api/io/default_collate_fn.rst b/doc/fluid/api/io/default_collate_fn.rst new file mode 100644 index 0000000000000000000000000000000000000000..38f34e1eeaa476fb77874ea015dbfd4213a5f9fe --- /dev/null +++ b/doc/fluid/api/io/default_collate_fn.rst @@ -0,0 +1,11 @@ +.. THIS FILE IS GENERATED BY `gen_doc.{py|sh}` + !DO NOT EDIT THIS FILE MANUALLY! + +.. _api_fluid_io_default_collate_fn: + +default_collate_fn +------------------ + +.. autofunction:: paddle.fluid.io.default_collate_fn + :noindex: + diff --git a/doc/fluid/api/io/firstn.rst b/doc/fluid/api/io/firstn.rst deleted file mode 100644 index 512dab717dbba79401450af06f551d4ae0224011..0000000000000000000000000000000000000000 --- a/doc/fluid/api/io/firstn.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. THIS FILE IS GENERATED BY `gen_doc.{py|sh}` - !DO NOT EDIT THIS FILE MANUALLY! - -.. _api_fluid_io_firstn: - -firstn ------- - -.. autofunction:: paddle.fluid.io.firstn - :noindex: - diff --git a/doc/fluid/api/io/map_readers.rst b/doc/fluid/api/io/map_readers.rst deleted file mode 100644 index 5e7558baf1efe603a3415e789439a24d9e8c36bc..0000000000000000000000000000000000000000 --- a/doc/fluid/api/io/map_readers.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. THIS FILE IS GENERATED BY `gen_doc.{py|sh}` - !DO NOT EDIT THIS FILE MANUALLY! - -.. _api_fluid_io_map_readers: - -map_readers ------------ - -.. autofunction:: paddle.fluid.io.map_readers - :noindex: - diff --git a/doc/fluid/api/io/multiprocess_reader.rst b/doc/fluid/api/io/multiprocess_reader.rst deleted file mode 100644 index 24b2a3bd564a114431ffe5e48e73372528b1b443..0000000000000000000000000000000000000000 --- a/doc/fluid/api/io/multiprocess_reader.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. THIS FILE IS GENERATED BY `gen_doc.{py|sh}` - !DO NOT EDIT THIS FILE MANUALLY! - -.. _api_fluid_io_multiprocess_reader: - -multiprocess_reader -------------------- - -.. autofunction:: paddle.fluid.io.multiprocess_reader - :noindex: - diff --git a/doc/fluid/api/io/shuffle.rst b/doc/fluid/api/io/shuffle.rst deleted file mode 100644 index dccf6d5c6b53bf4234a23f37952e341eb9390282..0000000000000000000000000000000000000000 --- a/doc/fluid/api/io/shuffle.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. THIS FILE IS GENERATED BY `gen_doc.{py|sh}` - !DO NOT EDIT THIS FILE MANUALLY! - -.. _api_fluid_io_shuffle: - -shuffle -------- - -.. autofunction:: paddle.fluid.io.shuffle - :noindex: - diff --git a/doc/fluid/api/io/xmap_readers.rst b/doc/fluid/api/io/xmap_readers.rst deleted file mode 100644 index 77b3e78993974c4dd7905c052928307ddc7a6baa..0000000000000000000000000000000000000000 --- a/doc/fluid/api/io/xmap_readers.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. THIS FILE IS GENERATED BY `gen_doc.{py|sh}` - !DO NOT EDIT THIS FILE MANUALLY! - -.. _api_fluid_io_xmap_readers: - -xmap_readers ------------- - -.. autofunction:: paddle.fluid.io.xmap_readers - :noindex: - diff --git a/doc/paddle/api/gen_doc.py b/doc/paddle/api/gen_doc.py index 93926d56b5c44548ee03b80413d976c2adb244e5..211c41cdd2f3ee0da45e2ccba28df29a17a465c9 100644 --- a/doc/paddle/api/gen_doc.py +++ b/doc/paddle/api/gen_doc.py @@ -20,6 +20,9 @@ api_set = set() def get_all_api(root_path='paddle'): for filefiner, name, ispkg in pkgutil.walk_packages( path=paddle.__path__, prefix=paddle.__name__ + '.'): + # not show paddle.reader APIs + if name.startswith("paddle.reader"): + continue try: m = eval(name) except AttributeError: diff --git a/doc/paddle/api/paddle/reader/ComposeNotAligned_cn.rst b/doc/paddle/api/paddle/reader/ComposeNotAligned_cn.rst deleted file mode 100644 index 1f4a911c5cb8b7dbad64807f81a4114e411ebeab..0000000000000000000000000000000000000000 --- a/doc/paddle/api/paddle/reader/ComposeNotAligned_cn.rst +++ /dev/null @@ -1,14 +0,0 @@ -.. THIS FILE IS GENERATED BY `gen_doc.{py|sh}` - !DO NOT EDIT THIS FILE MANUALLY! - -.. _api_fluid_io_ComposeNotAligned: - -ComposeNotAligned ------------------ - -.. autoclass:: paddle.fluid.io.ComposeNotAligned - :members: - :inherited-members: - :noindex: - -表示一种错误状态，说明调用compose API时, reader的输出数据没有对齐。 diff --git a/doc/paddle/api/paddle/reader/buffered_cn.rst b/doc/paddle/api/paddle/reader/buffered_cn.rst deleted file mode 100644 index 027c6346f735874d8c9accc38024c09b5940c429..0000000000000000000000000000000000000000 --- a/doc/paddle/api/paddle/reader/buffered_cn.rst +++ /dev/null @@ -1,17 +0,0 @@ -.. _cn_api_fluid_io_buffered: - -buffered -------------------------------- - -.. py:function:: paddle.fluid.io.buffered(reader, size) - - - - -创建一个缓存数据读取器，它读取数据并且存储进缓存区，从缓存区读取数据将会加速，只要缓存不是空的。 - -参数: - - **reader** (callable) – 读取数据的reader - - **size** (int) – 最大buffer的大小 - -返回:缓存的reader（读取器） \ No newline at end of file diff --git a/doc/paddle/api/paddle/reader/cache_cn.rst b/doc/paddle/api/paddle/reader/cache_cn.rst deleted file mode 100644 index e93e4c85d134c0feb2ff813a6127c89816baed76..0000000000000000000000000000000000000000 --- a/doc/paddle/api/paddle/reader/cache_cn.rst +++ /dev/null @@ -1,18 +0,0 @@ -.. _cn_api_fluid_io_cache: - -cache -------------------------------- - -.. py:function:: paddle.fluid.io.cache(reader) - - - - -缓存reader数据到内存中，小心此方法可能会花长时间来处理数据，并且会占用大量内存。 ``reader()`` 只能被调用一次。 - -参数: - - **reader** (callable) – 读取数据的reader，每次都会yields数据。 - -返回：每次都会从内存中yields数据的一个装饰reader。 - -返回类型：生成器 \ No newline at end of file diff --git a/doc/paddle/api/paddle/reader/chain_cn.rst b/doc/paddle/api/paddle/reader/chain_cn.rst deleted file mode 100644 index 4a4872d268cc2dfbddad0a6d4720be54e5eb41c9..0000000000000000000000000000000000000000 --- a/doc/paddle/api/paddle/reader/chain_cn.rst +++ /dev/null @@ -1,47 +0,0 @@ -.. _cn_api_fluid_io_chain: - -chain -------------------------------- - -.. py:function:: paddle.fluid.io.chain(*readers) - - - - -该接口将多个数据读取器组成一个数据读取器，它依次返回多个数据读取器的输出数据，同时不改变输出数据原先的格式。 - -举例来说，如果有3个输入读取器且输出分别为[0，0，0]、[10，10，10]和[20，20，20]，那么调用该接口产生的新数据读取器的输出为[0，0，0], [10，10，10], [20，20，20]。 - -参数： - - **readers(list)** – 输入的数据读取器。 - -返回： 新的数据读取器。 - -返回类型：callable - -**代码示例** - -.. code-block:: python - - import paddle.fluid as fluid - - def reader_creator_3(start): - def reader(): - for i in range(start, start + 3): - yield [i, i, i] - return reader - - c = fluid.io.chain(reader_creator_3(0), reader_creator_3(10), reader_creator_3(20)) - for e in c(): - print(e) - # 输出结果如下： - # [0, 0, 0] - # [1, 1, 1] - # [2, 2, 2] - # [10, 10, 10] - # [11, 11, 11] - # [12, 12, 12] - # [20, 20, 20] - # [21, 21, 21] - # [22, 22, 22] - diff --git a/doc/paddle/api/paddle/reader/compose_cn.rst b/doc/paddle/api/paddle/reader/compose_cn.rst deleted file mode 100644 index b4393bab5db375413712c27b60fb0699e4d85370..0000000000000000000000000000000000000000 --- a/doc/paddle/api/paddle/reader/compose_cn.rst +++ /dev/null @@ -1,34 +0,0 @@ -.. _cn_api_fluid_io_compose: - -compose -------------------------------- - -.. py:function:: paddle.fluid.io.compose(*readers, **kwargs) - - - - -该接口将多个数据读取器组合为一个数据读取器，返回读取器的输出包含所有输入读取器的输出。 - -例如：如果输入为三个reader，三个reader的输出分别为：（1，2）、3、（4，5），则组合reader的输出为：（1，2，3，4，5）。 - -参数： - - **readers** - 将被组合的多个数据读取器(Reader)，数据读取器的定义参见 :ref:`cn_api_paddle_data_reader_reader` 。 - - **check_alignment** (bool) - 可选，指明是否对输入reader进行对齐检查，默认值为True。如果为True，将检查输入reader是否正确对齐。如果为False，将不检查对齐并自动丢弃无法对齐的末尾数据。 - -返回：数据读取器(Reader)。 - -**代码示例**: - -.. code-block:: python - - import paddle.fluid as fluid - def reader_creator_10(dur): - def reader(): - for i in range(10): - yield i - return reader - - reader = fluid.io.compose(reader_creator_10(0), reader_creator_10(0)) - -注意： 运行过程可能会抛出异常 ``ComposeNotAligned`` ，原因是输入的readers数据未对齐。 当check_alignment设置为False时，不会检查并触发该异常。 diff --git a/doc/paddle/api/paddle/reader/firstn_cn.rst b/doc/paddle/api/paddle/reader/firstn_cn.rst deleted file mode 100644 index ba9c1d427ab3ec1946dab0e78f1a2021a712fe94..0000000000000000000000000000000000000000 --- a/doc/paddle/api/paddle/reader/firstn_cn.rst +++ /dev/null @@ -1,30 +0,0 @@ -.. _cn_api_fluid_io_firstn: - -firstn -------------------------------- - -.. py:function:: paddle.fluid.io.firstn(reader, n) - - - - -该接口创建一个数据读取器，它可以返回的最大样本数为n。 - -参数： - - **reader** (callable) – 输入的数据读取器。 - - **n** (int) – 可以返回的最大样本数。 - -返回： 新的的数据读取器。 - -返回类型： callable - -.. code-block:: python - - import paddle.fluid as fluid - def reader(): - for i in range(100): - yield i - firstn_reader = fluid.io.firstn(reader, 5) - for e in firstn_reader(): - print(e) - # 输出结果为:0 1 2 3 4 diff --git a/doc/paddle/api/paddle/reader/map_readers_cn.rst b/doc/paddle/api/paddle/reader/map_readers_cn.rst deleted file mode 100644 index cb50e62839cd523184fef37d4ff9dc625c52fc6b..0000000000000000000000000000000000000000 --- a/doc/paddle/api/paddle/reader/map_readers_cn.rst +++ /dev/null @@ -1,40 +0,0 @@ -.. _cn_api_fluid_io_map_readers: - -map_readers -------------------------------- - -.. py:function:: paddle.fluid.io.map_readers(func, *readers) - - - - -该接口将创建一个数据读取器(Reader)，其中 `func` 函数的输出将直接作为新数据读取器的输出， `readers` 的输出将作为函数 `func` 的输入参数。 - -例如：如果输入的 `readers` 为两个输出分别为：2、3 的 `reader` ，输入的 `func` 为乘法函数 `mul(x, y)` ，则得到的新建 `reader` 的输出为：6。 - -参数： - - **func** - 读取数据并返回数据项的函数， `func` 的输出将直接作为新创建的数据读取器的输出。 - - - **readers** - 输入的一个或多个数据读取器(Reader)，这些数据读取器的输出数据将作为函数 `func` 的输入参数。数据读取器的定义参见 :ref:`cn_api_paddle_data_reader_reader` 。 - -返回： 新创建的数据读取器(Reader) - -**代码示例**: - -.. code-block:: python - - - import paddle.fluid as fluid - d = {"h": 0, "i": 1} - def func(x): - return d[x] - - def reader(): - yield "h" - yield "i" - - map_reader_result = fluid.io.map_readers(func, reader) - - - - diff --git a/doc/paddle/api/paddle/reader/multiprocess_reader_cn.rst b/doc/paddle/api/paddle/reader/multiprocess_reader_cn.rst deleted file mode 100644 index ab600239eb702ffa2c503f88d5bd9d6ef6bda443..0000000000000000000000000000000000000000 --- a/doc/paddle/api/paddle/reader/multiprocess_reader_cn.rst +++ /dev/null @@ -1,59 +0,0 @@ -.. _cn_api_fluid_io_multiprocess_reader: - -multiprocess_reader -------------------------------- - -.. py:function:: paddle.fluid.io.multiprocess_reader(readers, use_pipe=True, queue_size=1000) - - - - -使用python多进程从 ``readers`` 中读取数据，然后使用 ``multiprocessing.Pipe`` 或 ``multiprocessing.Queue`` 合并所有数据。 ``readers`` 列表中的每个reader会被创建一个独立的进程来调用，reader之间应该相互独立，互不影响，避免出现多进程读取的冲突问题. - -multiprocess.queue需要/dev/shm的rw访问权限，某些平台不支持。 - -参数： - - **readers** (list(generator)|tuple(generator)) - python生成器list, 用来读取数据 - - **use_pipe** (bool，可选) - use_pipe控制multiprocess_reader内部用 ``pipe`` 还是 ``queue`` 来实现进程间通信，默认为 ``True`` 使用 ``pipe`` 进行通信 - - **queue_size** (int，可选) - 如果使用queue来进行进程间通信 (``use_pipe=False``), 则该参数用来设定队列大小 - -返回：使用多进程封装readers之后的reader - -返回类型：python生成器 - - -**代码示例** - -.. code-block:: python - - import paddle.fluid as fluid - from paddle.fluid.io import multiprocess_reader - import numpy as np - - - def fake_reader(start, end): - def __impl__(): - for i in range(start, end): - yield [np.array([1, 2, 3]) * i], - return __impl__ - - - with fluid.program_guard(fluid.Program(), fluid.Program()): - place = fluid.CPUPlace() - image = fluid.layers.data( - name='image', dtype='int64', shape=[3]) - fluid.layers.Print(image) - reader = fluid.io.PyReader( - feed_list=[image], capacity=2) - image_p_1 = image + 1 - decorated_reader = multiprocess_reader( - [fake_reader(1, 5), fake_reader(6, 10)], False) - - reader.decorate_sample_generator(decorated_reader, batch_size=2, places=[place]) - - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) - - for data in reader(): - exe.run(feed=data, fetch_list=[image_p_1]) - diff --git a/doc/paddle/api/paddle/reader/xmap_readers_cn.rst b/doc/paddle/api/paddle/reader/xmap_readers_cn.rst deleted file mode 100644 index 5f434ecb9525c4f8dd746319d2b4baace88c89ae..0000000000000000000000000000000000000000 --- a/doc/paddle/api/paddle/reader/xmap_readers_cn.rst +++ /dev/null @@ -1,57 +0,0 @@ -.. _cn_api_fluid_io_xmap_readers: - -xmap_readers -------------------------------- - -.. py:function:: paddle.fluid.io.xmap_readers(mapper, reader, process_num, buffer_size, order=False) - - - - -多线程下，使用自定义映射器 reader 返回样本到输出队列。 - -参数： - - **mapper** （callable）: 映射 reader 数据的函数。 - - **reader** （callable）: 产生数据的 reader。 - - **process_num** （int）: 处理样本的线程数。 - - **buffer_size** （int）: 数据缓冲队列大小。 - - **order** （bool）: 是否保持原始 reader 数据顺序，默认为 False。 - -返回：一个用户定义的 reader `装饰器 `_ 。 - -返回类型：callable，可调用对象。 - -**代码示例**： - -.. code-block:: python - - import paddle.fluid as fluid - import time - - def reader_creator_10(dur): - def reader(): - for i in range(10): - time.sleep(dur) - yield i - return reader - - def mapper(x): - return (x + 1) - - orders = (True, False) - thread_num = (1, 2, 4, 8, 16) - buffer_size = (1, 2, 4, 8, 16) - for order in orders: - for t_num in thread_num: - for size in buffer_size: - user_reader = fluid.io.xmap_readers(mapper, - reader_creator_10(0), - t_num, size, order) - for n in range(3): - result = list() - for i in user_reader(): - result.append(i) - if not order: - result.sort() - for idx, e in enumerate(result): - assert e == mapper(idx) \ No newline at end of file