Merge branch 'develop' into pool

.pre-commit-hooks/convert_markdown_into_html.py

@@ -81,11 +81,9 @@ def convert_markdown_into_html(argv=None):
     for filename in args.filenames:
         with open(
                 re.sub(r"README", "index", re.sub(r"\.md$", ".html",
-                                                  filename)),
-                "w",
-                encoding="utf-8") as output:
+                                                  filename)), "w") as output:
             output.write(HEAD)
-            with open(filename, encoding="utf-8") as input:
+            with open(filename) as input:
                 for line in input:
                     output.write(line)
             output.write(TAIL)

doc/fluid/advanced_guide/dygraph_to_static/index_cn.rst

+###############
+动态图转静态图
+###############
+
+- `动态图转静态图 <program_translator_cn.html>`_ ：介绍了动态图转静态图的基本使用方法和架构原理
+
+..  toctree::
+    :hidden:
+
+    program_translator_cn.rst

doc/fluid/advanced_guide/dygraph_to_static/program_translator_cn.rst

+动态图转静态图
+================
+
+PaddlePadde的动态图具有接口易用、Python风格的编程体验、友好的debug交互等优点。在动态图模式下，代码是按照我们编写的顺序依次执行。这种机制更符合python程序员的习惯，可以很方便地将大脑中的想法快速地转化为实际代码，也更容易调试。但在部分性能方面，python速度负担太大，无法与C++相提并论。因此在工业界部署很多地方（如大型推荐系统、移动端）都倾向于直接使用C++来提速。
+
+此时，静态图在部署方面更具有性能的优势。静态图程序在编译执行时，先搭建模型的神经网络结构，然后再对神经网络执行计算操作。预先搭建好的神经网络可以脱离python依赖，在C++端被重新解析执行。
+
+动态图代码更易编写和debug，但在部署性能上，静态图更具优势。因此我们新增了动态图转静态图的功能，支持用户依然使用动态图编写组网代码。PaddlePaddle会对用户代码进行分析，自动转换为静态图网络结构，兼顾了动态图易用性和静态图部署性能两方面优势。
+
+基本使用方法
+--------------
+
+PaddlePaddle提供了两种动态图转静态图的方式，基于动态图trace的转换与基于源代码级别的转换的ProgramTranslator。
+
+1. 基于trace的TracedLayer：
+
+trace是指在模型运行时记录下其运行过哪些算子。TracedLayer就是基于这种技术，在一次执行动态图的过程中，记录所有运行的算子，并构建和保存静态图模型。一个使用例子如下：
+
+我们先定义一个简单的Fully Connected网络：
+
+.. code-block:: python
+
+    import numpy as np
+    import paddle
+
+    class SimpleFcLayer(paddle.nn.Layer):
+        def __init__(self, feature_size, batch_size, fc_size):
+            super(SimpleFCLayer, self).__init__()
+            self._linear = paddle.nn.Linear(feature_size, fc_size)
+            self._offset = paddle.to_tensor(
+                np.random.random((batch_size, fc_size)).astype('float32'))
+
+        def forward(self, x):
+            fc = self._linear(x)
+            return fc + self._offset
+
+
+接下来是TracedLayer如何存储模型：
+
+.. code-block:: python
+    import paddle
+    from paddle.jit import TracedLayer
+
+    paddle.disable_static()
+
+    fc_layer = SimpleFcLayer(3, 4, 2)
+    in_np = np.random.random([3, 4]).astype('float32')
+    # 将numpy的ndarray类型的数据转换为Tensor类型
+    input_var = paddle.to_tensor(in_np)
+    # 通过 TracerLayer.trace 接口将命令式模型转换为声明式模型
+    out_dygraph, static_layer = TracedLayer.trace(fc_layer, inputs=[input_var])
+    save_dirname = './saved_infer_model'
+    # 将转换后的模型保存
+    static_layer.save_inference_model(save_dirname, feed=[0], fetch=[0])
+
+
+载入的模型可以使用静态图方式运行
+
+.. code-block:: python
+
+    place = paddle.CPUPlace()
+    exe = paddle.Executor(place)
+    program, feed_vars, fetch_vars = paddle.io.load_inference_model(save_dirname, exe)
+    fetch, = exe.run(program, feed={feed_vars[0]: in_np}, fetch_list=fetch_vars)
+
+
+但是也正如我们阐述的原理，trace只是记录了一次执行涉及算子，若在用户的模型代码中，包含了依赖数据条件（包括输入的值或者shape）的控制流分支，即根据数据条件触发运行不同的算子，则TracedLayer无法正常工作。比如下面
+
+
+.. code-block:: python
+
+    import paddle
+
+    def func(input_var)
+        # if判断与输入input_var的shape有关
+        if input_var.shape[0] > 1:
+            return paddle.cast(input_var, "float64")
+        else:
+            return paddle.cast(input_var, "int64")
+
+    paddle.disable_static()
+    in_np = np.array([-2]).astype('int')
+    input_var = paddle.to_tensor(in_np)
+    out = func(input_var)
+
+
+上例如果在使用TracedLayer.trace(func, inputs=[input_var])，由于trace只能记录if-else其中跑的一次算子，模型就无法按用户想要的根据input_var的形状进行if-else控制流保存。类似的控制流还有while/for循环的情况
+
+2. 基于源代码转写的ProgramTranslator
+
+对于依赖数据的控制流，我们使用基于源代码转写的ProgramTranslator来进行动态图转静态图。其基本原理是通过分析python代码来将动态图代码转写为静态图代码，并在底层自动帮用户使用执行器运行。其基本使用方法十分简便，只需要在要转化的函数（该函数也可以是用户自定义动态图Layer的forward函数）前添加一个装饰器@paddle.jit.to_static，上面的例子转化如下，并且可以依旧使用该函数运行得到结果：
+
+.. code-block:: python
+
+    import paddle
+
+    @paddle.jit.to_static
+    def func(input_var)
+        # if判断与输入input_var的shape有关
+        if input_var.shape[0] > 1:
+            out = paddle.cast(input_var, "float64")
+        else:
+            out = paddle.cast(input_var, "int64")
+
+    paddle.disable_static()
+    in_np = np.array([-2]).astype('int')
+    input_var = paddle.to_tensor(in_np)
+    func(input_var)
+
+
+若要存储转化后的静态图模型，可以调用paddle.jit.save，我们再以SimpleFcLayer为例，需要在SimpleFcLayer的forward函数添加装饰器：
+
+.. code-block:: python
+
+    import numpy as np
+    import paddle
+
+    class SimpleFcLayer(paddle.nn.Layer):
+        def __init__(self, feature_size, batch_size, fc_size):
+            super(SimpleFCLayer, self).__init__()
+            self._linear = paddle.nn.Linear(feature_size, fc_size)
+            self._offset = paddle.to_tensor(
+                np.random.random((batch_size, fc_size)).astype('float32'))
+
+        @paddle.jit.to_static
+        def forward(self, x):
+            fc = self._linear(x)
+            return fc + self._offset
+
+
+存储该模型可以使用paddle.jit.save接口：
+
+.. code-block:: python
+
+    import paddle
+
+    paddle.disable_static()
+
+    fc_layer = SimpleFcLayer(3, 4, 2)
+    in_np = np.random.random([3, 4]).astype('float32')
+    input_var = paddle.to_tensor(in_np)
+    out = fc_layer(input_var)
+
+    paddle.jit.save(mnist, "./mnist_dy2stat", input_spec=[input_var])
+
+内部架构原理
+--------------
+
+TracedLayer的原理就是trace，相对简单，因此我们在这里不展开描述。本节将主要阐述ProgramTranslator基于源代码将动态图代码转化为静态图代码。
+
+
+转化过程发生在用户开始调用被装饰的函数，转换过程在装饰器中实现。我们将内部涉及的过程分为以下几步：
+
+1. 函数与缓存
+
+动态图转静态图的主体是函数（Function）。对于函数内包含的PaddlePaddle接口，如果是仅计算相关算子代码语句，那么因为PaddlePaddle动态图和静态图接口一致，我们不需要额外转换这些代码为静态图代码。但是对于动态图，此类代码接口是直接运行计算和返回结果，而对于静态图此类代码接口其实是组网。那么如果被转化的函数被调用多次，动态图转静态图后会多次组网添加对应算子，这显然会导致问题。为了解决这个问题以及为了加速动转静转化过程，我们维护了被装饰器装饰的函数（Function）与其输入形状（shape），数据类型（dtype）映射到被转化后组网的Program的缓存（Cache）。当要被转化的函数命中缓存，我们直接用对应存储的Program运行静态图得到结果，否则我们才进行语句转化，并且转化成功后的Program存储进缓存。
+
+2. 动态图源码转AST（抽象语法树）
+
+动态图转静态图的最核心部分类似一个编译器，解析动态图代码语句为AST，再对应AST进行改写，最后反转回成静态图代码。从函数转化为代码字符串可以使用Python的inspect.getsource。从字符串Python提供了自带的ast库来解析字符串为 `AST <https://docs.python.org/3/library/ast.html>`_ ，但是由于python2，python3的语法略有不同，为了避免我们需要额外处理这些python2，python3的不同情况，我们使用了统一python2，python3的开源AST处理 `gast库 <https://github.com/serge-sans-paille/gast>`_ 。这些接口使得函数转化为AST没有本质上的困难。
+
+3. AST改写和静态图源码转换
+
+这部分为动转静最核心的部分，我们对支持的各种语法进行ast转写。其中最重要的python控制流，if-else，while，for循环被分别分析转化为PaddlePaddle静态图接口cond，while_loop等接口实现。我们对想转化的每一种主要语法创建一个Transformer（这里的Transformer是python ast转写的概念，而不是自然语言处理NLP领域的Transformer），每个Transformer扫一遍AST并进行对应的改写。最后被转化完成的AST我们使用gast提供的接口转回成源码。
+
+4. 静态图源码作为动态图一部分运行的技术
+
+为了动静转化更加易用和被转化的代码能在动态图中复用，我们在拥有源码后运行生成Program，并将这个Program作为一个大op，包装成动态图的一个op，这样既能把用户的代码转为静态图提速或者保存部署，另一方面如果用户想在python层使用生成的静态图代码作为动态图的一部分继续训练或者别的动态图运算也是可以直接使用。
+
+5. 易用性与Debug功能在动转静过程的实现
+
+正如AST转写类似编译器，而一般编译器都会提供debug断点，报错，输出一些中间代码等功能。我们在进行动转静时，万一用户的动态图代码出错，或者用户想断点调试，或者用户想看看被转化后的静态图代码是否符合其预期，我们也希望能够像编译器一样提供这些易用性功能，使得动转静兼顾性能和部署同时还具有易用性。我们这里将列出这些功能的实现方式
+
+A. 报错对应到动态图代码行。由于被转化后的静态图代码和原动态图代码不同，python运行出错时会报静态图的错误，因此我们在每一次AST转写时添加AST节点对应的原动态图代码行等信息，在python报错栈中将静态图的报错转化成对应的动态图源码报错
+
+B. 设置断点功能。我们保留了被转化后代码的中的pdb.set_trace(), 用户可以使用这种方式进行断点调试
+
+C. 查看最后转化的静态图代码。我们输出为一个StaticLayer class，这个StaticLayer可以直接被调用，但是也存储转化后的代码，可以调用StaticLayer.code来获得转化后的代码。
+
+D. 输出中间转化状态代码，甚至不同语法Transformer转化的代码，比如经过for循环转化后代码是什么样的。我们开放接口设定了log level来让用户可以打印中间状态转化的代码。
+
+

doc/fluid/advanced_guide/index_cn.rst

@@ -10,5 +10,6 @@
 ..  toctree::
     :hidden:
 
+    dygraph_to_static/index_cn.rst
     inference_deployment/index_cn.rst
     flags/flags_cn.rst

doc/fluid/advanced_guide/inference_deployment/inference/windows_cpp_inference.md

@@ -5,13 +5,13 @@
 下载安装包与对应的测试环境
 -------------
 
-| 版本说明      |     预测库(1.8.3版本)     |       编译器        |    构建工具      |  cuDNN  |  CUDA  |
+| 版本说明      |     预测库(1.8.4版本)     |       编译器        |    构建工具      |  cuDNN  |  CUDA  |
 |:---------|:-------------------|:-------------------|:----------------|:--------|:-------|
-|    cpu_avx_mkl | [fluid_inference.zip](https://paddle-wheel.bj.bcebos.com/1.8.3/win-infer/mkl/cpu/fluid_inference_install_dir.zip) | MSVC 2015 update 3|  CMake v3.16.0  |
-|    cpu_avx_openblas | [fluid_inference.zip](https://paddle-wheel.bj.bcebos.com/1.8.3/win-infer/open/cpu/fluid_inference_install_dir.zip) | MSVC 2015 update 3|  CMake v3.16.0  |
-|    cuda9.0_cudnn7_avx_mkl | [fluid_inference.zip](https://paddle-wheel.bj.bcebos.com/1.8.3/win-infer/mkl/post97/fluid_inference_install_dir.zip) |  MSVC 2015 update 3 |  CMake v3.16.0  |  7.3.1  |   9.0    |
-|    cuda9.0_cudnn7_avx_openblas | [fluid_inference.zip](https://paddle-wheel.bj.bcebos.com/1.8.3/win-infer/open/post97/fluid_inference_install_dir.zip) | MSVC 2015 update 3 |  CMake v3.16.0  |  7.3.1  |   9.0    |
-|    cuda10.0_cudnn7_avx_mkl | [fluid_inference.zip](https://paddle-wheel.bj.bcebos.com/1.8.3/win-infer/mkl/post107/fluid_inference_install_dir.zip) | MSVC 2015 update 3 |  CMake v3.16.0  |  7.4.1  |   10.0    |
+|    cpu_avx_mkl | [fluid_inference.zip](https://paddle-wheel.bj.bcebos.com/1.8.4/win-infer/mkl/cpu/fluid_inference_install_dir.zip) | MSVC 2015 update 3|  CMake v3.16.0  |
+|    cpu_avx_openblas | [fluid_inference.zip](https://paddle-wheel.bj.bcebos.com/1.8.4/win-infer/open/cpu/fluid_inference_install_dir.zip) | MSVC 2015 update 3|  CMake v3.16.0  |
+|    cuda9.0_cudnn7_avx_mkl | [fluid_inference.zip](https://paddle-wheel.bj.bcebos.com/1.8.4/win-infer/mkl/post97/fluid_inference_install_dir.zip) |  MSVC 2015 update 3 |  CMake v3.16.0  |  7.3.1  |   9.0    |
+|    cuda9.0_cudnn7_avx_openblas | [fluid_inference.zip](https://paddle-wheel.bj.bcebos.com/1.8.4/win-infer/open/post97/fluid_inference_install_dir.zip) | MSVC 2015 update 3 |  CMake v3.16.0  |  7.3.1  |   9.0    |
+|    cuda10.0_cudnn7_avx_mkl | [fluid_inference.zip](https://paddle-wheel.bj.bcebos.com/1.8.4/win-infer/mkl/post107/fluid_inference_install_dir.zip) | MSVC 2015 update 3 |  CMake v3.16.0  |  7.4.1  |   10.0    |
 
 ### 硬件环境
 

doc/fluid/advanced_guide/inference_deployment/inference/windows_cpp_inference_en.md

@@ -5,13 +5,13 @@ Install and Compile C++ Inference Library on Windows
 Direct Download and Install
 -------------
 
-| Version      |     Inference Libraries(v1.8.3)   | Compiler | Build tools | cuDNN | CUDA |
+| Version      |     Inference Libraries(v1.8.4)   | Compiler | Build tools | cuDNN | CUDA |
 |:---------|:-------------------|:-------------------|:----------------|:--------|:-------|
-|    cpu_avx_mkl | [fluid_inference.zip](https://paddle-wheel.bj.bcebos.com/1.8.3/win-infer/mkl/cpu/fluid_inference_install_dir.zip) | MSVC 2015 update 3|  CMake v3.16.0  |
-|    cpu_avx_openblas | [fluid_inference.zip](https://paddle-wheel.bj.bcebos.com/1.8.3/win-infer/open/cpu/fluid_inference_install_dir.zip) | MSVC 2015 update 3|  CMake v3.16.0  |
-|    cuda9.0_cudnn7_avx_mkl | [fluid_inference.zip](https://paddle-wheel.bj.bcebos.com/1.8.3/win-infer/mkl/post97/fluid_inference_install_dir.zip) |  MSVC 2015 update 3 |  CMake v3.16.0  |  7.3.1  |   9.0    |
-|    cuda9.0_cudnn7_avx_openblas | [fluid_inference.zip](https://paddle-wheel.bj.bcebos.com/1.8.3/win-infer/open/post97/fluid_inference_install_dir.zip) | MSVC 2015 update 3 |  CMake v3.16.0  |  7.3.1  |   9.0    |
-|    cuda10.0_cudnn7_avx_mkl | [fluid_inference.zip](https://paddle-wheel.bj.bcebos.com/1.8.3/win-infer/mkl/post107/fluid_inference_install_dir.zip) | MSVC 2015 update 3 |  CMake v3.16.0  |  7.4.1  |   10.0    |
+|    cpu_avx_mkl | [fluid_inference.zip](https://paddle-wheel.bj.bcebos.com/1.8.4/win-infer/mkl/cpu/fluid_inference_install_dir.zip) | MSVC 2015 update 3|  CMake v3.16.0  |
+|    cpu_avx_openblas | [fluid_inference.zip](https://paddle-wheel.bj.bcebos.com/1.8.4/win-infer/open/cpu/fluid_inference_install_dir.zip) | MSVC 2015 update 3|  CMake v3.16.0  |
+|    cuda9.0_cudnn7_avx_mkl | [fluid_inference.zip](https://paddle-wheel.bj.bcebos.com/1.8.4/win-infer/mkl/post97/fluid_inference_install_dir.zip) |  MSVC 2015 update 3 |  CMake v3.16.0  |  7.3.1  |   9.0    |
+|    cuda9.0_cudnn7_avx_openblas | [fluid_inference.zip](https://paddle-wheel.bj.bcebos.com/1.8.4/win-infer/open/post97/fluid_inference_install_dir.zip) | MSVC 2015 update 3 |  CMake v3.16.0  |  7.3.1  |   9.0    |
+|    cuda10.0_cudnn7_avx_mkl | [fluid_inference.zip](https://paddle-wheel.bj.bcebos.com/1.8.4/win-infer/mkl/post107/fluid_inference_install_dir.zip) | MSVC 2015 update 3 |  CMake v3.16.0  |  7.4.1  |   10.0    |
 
 ### Hardware Environment
 

doc/fluid/api/distributed.rst

+==================
+paddle.distributed
+==================
+
+..  toctree::
+    :maxdepth: 1
+
+    distributed/get_rank.rst
+    distributed/get_world_size.rst
+    distributed/init_parallel_env.rst
+    distributed/ParallelEnv.rst
+    distributed/prepare_context.rst
+    distributed/spawn.rst

doc/fluid/api/distributed/ParallelEnv.rst

+.. _api_distributed_ParallelEnv:
+
+ParallelEnv
+-------------------------------
+:doc_source: paddle.fluid.dygraph.parallel.ParallelEnv
\ No newline at end of file

doc/fluid/api/distributed/get_rank.rst

+..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
+    !DO NOT EDIT THIS FILE MANUALLY!
+
+.. _api_distributed_get_rank:
+
+get_rank
+--------
+
+..  autofunction:: paddle.distributed.get_rank
+    :noindex:
\ No newline at end of file

doc/fluid/api/distributed/get_world_size.rst

+..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
+    !DO NOT EDIT THIS FILE MANUALLY!
+
+.. _api_distributed_get_world_size:
+
+get_world_size
+--------------
+
+..  autofunction:: paddle.distributed.get_world_size
+    :noindex:
\ No newline at end of file

doc/fluid/api/distributed/init_parallel_env.rst

+..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
+    !DO NOT EDIT THIS FILE MANUALLY!
+
+.. _api_distributed_init_parallel_env:
+
+init_parallel_env
+-----------------
+
+..  autofunction:: paddle.distributed.init_parallel_env
+    :noindex:
\ No newline at end of file

doc/fluid/api/distributed/prepare_context.rst

+.. _api_distributed_prepare_context:
+
+prepare_context
+-------------------------------
+:doc_source: paddle.fluid.dygraph.parallel.prepare_context

doc/fluid/api/distributed/spawn.rst

+..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
+    !DO NOT EDIT THIS FILE MANUALLY!
+
+.. _api_distributed_spawn:
+
+spawn
+-----
+
+..  autofunction:: paddle.distributed.spawn
+    :noindex:
\ No newline at end of file

doc/fluid/api/dygraph.rst

@@ -5,7 +5,6 @@ fluid.dygraph
 ..  toctree::
     :maxdepth: 1
 
-    dygraph/BackwardStrategy.rst
     dygraph/BatchNorm.rst
     dygraph/BilinearTensorProduct.rst
     dygraph/Conv2D.rst

doc/fluid/api/gen_doc.py

@@ -21,24 +21,38 @@ import contextlib
 import paddle.fluid as fluid
 import paddle.tensor as tensor
 import paddle.nn as nn
-import paddle.complex as complex
+import paddle.optimizer as optimizer
+
+#import paddle.complex as complex
 #import paddle.framework as framework
 
+
 def parse_arg():
     parser = argparse.ArgumentParser()
     parser.add_argument('--submodules', nargs="*")
     parser.add_argument(
-        '--module_name', type=str, help='Generate the documentation of which module')
+        '--module_name',
+        type=str,
+        help='Generate the documentation of which module')
     parser.add_argument(
         '--module_prefix', type=str, help='Generate the prefix of module')
     parser.add_argument(
-        '--output', type=str, help='Output file or output directory for output rst')
+        '--output',
+        type=str,
+        help='Output file or output directory for output rst')
     parser.add_argument(
-        '--output_name', type=str, help='Output file or output directory for output rst')
+        '--output_name',
+        type=str,
+        help='Output file or output directory for output rst')
     parser.add_argument(
-        '--output_dir', type=str, help='Output file or output directory for output rst')
+        '--output_dir',
+        type=str,
+        help='Output file or output directory for output rst')
     parser.add_argument(
-        '--to_multiple_files', type=bool, default=False, help='Whether to separate to multiple files')
+        '--to_multiple_files',
+        type=bool,
+        default=False,
+        help='Whether to separate to multiple files')
 
     return parser.parse_args()
 
@@ -53,8 +67,9 @@ def parse_arg():
         else:
             pass
 
+
 class DocGenerator(object):
-    def __init__(self, module_name=None, module_prefix=None): 
+    def __init__(self, module_name=None, module_prefix=None):
         self.module_name = module_name
         self.module_prefix = module_prefix
         self.stream = None
@@ -62,7 +77,7 @@ class DocGenerator(object):
     @contextlib.contextmanager
     def guard(self, filename):
         assert self.stream is None, "stream must be None"
-        self.stream = open(filename, 'w') 
+        self.stream = open(filename, 'w')
         yield
         self.stream.close()
         self.stream = None
@@ -70,14 +85,15 @@ class DocGenerator(object):
     def print_submodule(self, submodule_name):
         submodule = getattr(self.module, submodule_name)
         if submodule is None:
-            raise ValueError("Cannot find submodule {0}".format(submodule_name))
+            raise ValueError(
+                "Cannot find submodule {0}".format(submodule_name))
         self.print_section(submodule_name)
 
-        for item in sorted(submodule.__all__,key=str.lower):
+        for item in sorted(submodule.__all__, key=str.lower):
             self.print_item(item)
 
     def print_current_module(self):
-        for item in sorted(self.module.__all__,key=str.lower):
+        for item in sorted(self.module.__all__, key=str.lower):
             self.print_item(item)
 
     def print_section(self, name):
@@ -91,7 +107,7 @@ class DocGenerator(object):
             self.print_method(name)
         else:
             self.stream.close()
-            path = os.getcwd()+"/"+output_name+"/"+name+".rst"
+            path = os.getcwd() + "/" + output_name + "/" + name + ".rst"
             if name != "PipeReader":
                 os.remove(path)
 
@@ -149,7 +165,9 @@ class DocGenerator(object):
         self.stream.write(".. _api_{0}_{1}:\n\n".format("_".join(
             self.module_prefix.split(".")), name))
 
-def generate_doc(module_name, module_prefix, output, output_name, to_multiple_files, output_dir):
+
+def generate_doc(module_name, module_prefix, output, output_name,
+                 to_multiple_files, output_dir):
     if module_name == "":
         module_name = None
 
@@ -176,13 +194,14 @@ def generate_doc(module_name, module_prefix, output, output_name, to_multiple_fi
     else:
         gen.module_prefix = output_name + "." + module_prefix
 
-    dirname = output if to_multiple_files else os.path.dirname(output) 
+    dirname = output if to_multiple_files else os.path.dirname(output)
 
     if output_dir != None:
         dirname = output_dir + "/" + dirname
         output = output_dir + "/" + output
 
-    if len(dirname) > 0 and (not os.path.exists(dirname) or not os.path.isdir(dirname)): 
+    if len(dirname) > 0 and (not os.path.exists(dirname) or
+                             not os.path.isdir(dirname)):
         os.makedirs(dirname)
 
     if not to_multiple_files:
@@ -191,7 +210,7 @@ def generate_doc(module_name, module_prefix, output, output_name, to_multiple_fi
             prefix_len = len(gen.module_prefix)
             assert gen.module_prefix == gen.module_name[0:prefix_len],    \
                 "module_prefix must be prefix of module_name"
-            diff_name = gen.module_name[prefix_len+1:]
+            diff_name = gen.module_name[prefix_len + 1:]
             if diff_name != "":
                 header_name = diff_name
     else:
@@ -203,7 +222,7 @@ def generate_doc(module_name, module_prefix, output, output_name, to_multiple_fi
             gen._print_header_(header_name, dot='=', is_title=True)
             gen.print_current_module()
     else:
-        apis = sorted(gen.module.__all__,key=str.lower)
+        apis = sorted(gen.module.__all__, key=str.lower)
         for api in apis:
             header_name = api
             with gen.guard(os.path.join(output, api + '.rst')):
@@ -213,7 +232,8 @@ def generate_doc(module_name, module_prefix, output, output_name, to_multiple_fi
 
 def main():
     args = parse_arg()
-    generate_doc(args.module_name, args.module_prefix, args.output, args.output_name, args.to_multiple_files, args.output_dir)
+    generate_doc(args.module_name, args.module_prefix, args.output,
+                 args.output_name, args.to_multiple_files, args.output_dir)
 
 
 if __name__ == '__main__':

doc/fluid/api/index_en.rst

@@ -8,12 +8,14 @@ API Reference
     ../api_guides/index_en.rst
     dataset.rst
     declarative.rst
+    distributed.rst
     framework.rst
     imperative.rst
     io.rst
     metric.rst
     nn.rst
     optimizer.rst
+    static.rst
     tensor.rst
     fluid.rst
     backward.rst

doc/fluid/api/nn.rst

@@ -172,3 +172,4 @@ paddle.nn
     nn/AdaptiveAvgPool2d.rst
     nn/AdaptiveAvgPool3d.rst
     nn/layer/activation/Sigmoid.rst
+    nn/Bilinear.rst
\ No newline at end of file

doc/fluid/api/nn/Bilinear.rst

+.. _api_nn_Bilinear:
+
+Bilinear
+-------------------------------
+
+..  autofunction:: paddle.nn.Bilinear
+    :noindex:
+

doc/fluid/api/nn/functional.rst

@@ -5,9 +5,12 @@ functional
 ..  toctree::
     :maxdepth: 1
 
+    functional/binary_cross_entropy_with_logits.rst
     functional/l1_loss.rst
     functional/nll_loss.rst
     functional/mse_loss.rst
+    functional/one_hot.rst
     functional/ctc_loss.rst
     functional/adaptive_avg_pool2d.rst
     functional/adaptive_avg_pool3d.rst
+    functional/bilinear.rst

doc/fluid/api/nn/functional/bilinear.rst

+.. _api_nn_functional_bilinear:
+
+bilinear
+--------------------
+
+..  autofunction:: paddle.nn.functional.bilinear
+    :noindex:
+

doc/fluid/api/nn/functional/binary_cross_entropy_with_logits.rst

+.. _api_nn_functional_binary_cross_entropy_with_logits:
+
+binary_cross_entropy_with_logits
+--------------------------------
+
+..  autoclass:: paddle.nn.functional.binary_cross_entropy_with_logits
+    :members:
+    :inherited-members:
+    :noindex:
+

doc/fluid/api/nn/functional/one_hot.rst

+.. _api_nn_functional_one_hot:
+
+one_hot
+---------
+
+..  autofunction:: paddle.nn.functional.one_hot
+    :noindex:

doc/fluid/api/nn/loss.rst

@@ -6,8 +6,10 @@ loss
     :maxdepth: 1
 
     loss/BCELoss.rst
+    loss/BCEWithLogitsLoss.rst
     loss/CrossEntropyLoss.rst
     loss/L1Loss.rst
     loss/MSELoss.rst
     loss/NLLLoss.rst
+    loss/SmoothL1Loss.rst
     loss/CTCLoss.rst

doc/fluid/api/optimizer/AdamOptimizer.rst → doc/fluid/api/nn/loss/BCEWithLogitsLoss.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_fluid_optimizer_AdamOptimizer:
+.. _api_nn_loss_BCEWithLogitsLoss:
 
-AdamOptimizer
--------------
+BCEWithLogitsLoss
+-------------------------------
 
-..  autoclass:: paddle.fluid.optimizer.AdamOptimizer
+..  autoclass:: paddle.nn.loss.BCEWithLogitsLoss
     :members:
     :inherited-members:
-    :exclude-members: apply_gradients, apply_optimize, backward, load
     :noindex:
 

doc/fluid/api/optimizer/AdamaxOptimizer.rst → doc/fluid/api/nn/loss/SmoothL1Loss.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_fluid_optimizer_AdamaxOptimizer:
+.. _api_nn_loss_SmoothL1Loss:
 
-AdamaxOptimizer
----------------
+SmoothL1Loss
+-------------------------------
 
-..  autoclass:: paddle.fluid.optimizer.AdamaxOptimizer
+..  autoclass:: paddle.nn.loss.SmoothL1Loss
     :members:
     :inherited-members:
-    :exclude-members: apply_gradients, apply_optimize, backward, load
     :noindex:
 

doc/fluid/api/optimizer.rst

-===============
+======================
 paddle.optimizer
-===============
+======================
 
 ..  toctree::
     :maxdepth: 1
@@ -11,8 +11,7 @@ paddle.optimizer
     optimizer/AdagradOptimizer.rst
     optimizer/Adam.rst
     optimizer/Adamax.rst
-    optimizer/AdamaxOptimizer.rst
-    optimizer/AdamOptimizer.rst
+    optimizer/AdamW.rst
     optimizer/DecayedAdagrad.rst
     optimizer/DecayedAdagradOptimizer.rst
     optimizer/DGCMomentumOptimizer.rst
@@ -29,6 +28,19 @@ paddle.optimizer
     optimizer/Momentum.rst
     optimizer/MomentumOptimizer.rst
     optimizer/RecomputeOptimizer.rst
-    optimizer/RMSPropOptimizer.rst
+    optimizer/RMSProp.rst
     optimizer/SGD.rst
     optimizer/SGDOptimizer.rst
+    optimizer/Optimizer.rst
+    optimizer/NoamLR.rst
+    optimizer/PiecewiseLR.rst
+    optimizer/NaturalExpLR.rst
+    optimizer/InverseTimeLR.rst
+    optimizer/PolynomialLR.rst
+    optimizer/LinearLrWarmup.rst
+    optimizer/ExponentialLR.rst
+    optimizer/MultiStepLR.rst
+    optimizer/StepLR.rst
+    optimizer/LambdaLR.rst
+    optimizer/ReduceLROnPlateau.rst
+    optimizer/CosineAnnealingLR.rst

doc/fluid/api/optimizer/Adadelta.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_fluid_optimizer_Adadelta:
+.. _api_optimizer_Adadelta:
 
 Adadelta
 --------
 
-..  autoclass:: paddle.fluid.optimizer.Adadelta
+..  autoclass:: paddle.optimizer.Adadelta
     :members:
     :inherited-members:
-    :exclude-members: apply_gradients, apply_optimize, backward, load
     :noindex:
 

doc/fluid/api/optimizer/AdadeltaOptimizer.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_fluid_optimizer_AdadeltaOptimizer:
+.. _api_optimizer_AdadeltaOptimizer:
 
 AdadeltaOptimizer
 -----------------
 
-..  autoclass:: paddle.fluid.optimizer.AdadeltaOptimizer
+..  autoclass:: paddle.optimizer.AdadeltaOptimizer
     :members:
     :inherited-members:
-    :exclude-members: apply_gradients, apply_optimize, backward, load
     :noindex:
 

doc/fluid/api/optimizer/Adagrad.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_fluid_optimizer_Adagrad:
+.. _api_optimizer_Adagrad:
 
 Adagrad
 -------
 
-..  autoclass:: paddle.fluid.optimizer.Adagrad
+..  autoclass:: paddle.optimizer.Adagrad
     :members:
     :inherited-members:
-    :exclude-members: apply_gradients, apply_optimize, backward, load
     :noindex:
 

doc/fluid/api/optimizer/AdagradOptimizer.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_fluid_optimizer_AdagradOptimizer:
+.. _api_optimizer_AdagradOptimizer:
 
 AdagradOptimizer
 ----------------
 
-..  autoclass:: paddle.fluid.optimizer.AdagradOptimizer
+..  autoclass:: paddle.optimizer.AdagradOptimizer
     :members:
     :inherited-members:
-    :exclude-members: apply_gradients, apply_optimize, backward, load
     :noindex:
 

doc/fluid/api/optimizer/Adam.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_fluid_optimizer_Adam:
+.. _api_optimizer_Adam:
 
 Adam
 ----
 
-..  autoclass:: paddle.fluid.optimizer.Adam
+..  autoclass:: paddle.optimizer.Adam
     :members:
     :inherited-members:
-    :exclude-members: apply_gradients, apply_optimize, backward, load
     :noindex:
 

doc/fluid/api/dygraph/BackwardStrategy.rst → doc/fluid/api/optimizer/AdamW.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_fluid_dygraph_BackwardStrategy:
+.. _api_optimizer_AdamW:
 
-BackwardStrategy
-----------------
+AdamW
+-----
 
-..  autoclass:: paddle.fluid.dygraph.BackwardStrategy
+..  autoclass:: paddle.optimizer.AdamW
     :members:
+    :inherited-members:
     :noindex:
 

doc/fluid/api/optimizer/Adamax.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_fluid_optimizer_Adamax:
+.. _api_optimizer_Adamax:
 
 Adamax
 ------
 
-..  autoclass:: paddle.fluid.optimizer.Adamax
+..  autoclass:: paddle.optimizer.Adamax
     :members:
     :inherited-members:
-    :exclude-members: apply_gradients, apply_optimize, backward, load
     :noindex:
 

doc/fluid/api/optimizer/RMSPropOptimizer.rst → doc/fluid/api/optimizer/CosineAnnealingLR.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_fluid_optimizer_RMSPropOptimizer:
+.. _api_optimizer_CosineAnnealingLR:
 
-RMSPropOptimizer
-----------------
+CosineAnnealingLR
+-------------------
 
-..  autoclass:: paddle.fluid.optimizer.RMSPropOptimizer
+..  autoclass:: paddle.optimizer.CosineAnnealingLR
     :members:
     :inherited-members:
-    :exclude-members: apply_gradients, apply_optimize, backward, load
+    :exclude-members: set_dict, set_state_dict, state_dict
     :noindex:
 

doc/fluid/api/optimizer/DGCMomentumOptimizer.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_fluid_optimizer_DGCMomentumOptimizer:
+.. _api_optimizer_DGCMomentumOptimizer:
 
 DGCMomentumOptimizer
 --------------------
 
-..  autoclass:: paddle.fluid.optimizer.DGCMomentumOptimizer
+..  autoclass:: paddle.optimizer.DGCMomentumOptimizer
     :members:
     :inherited-members:
-    :exclude-members: apply_gradients, apply_optimize, backward, load
     :noindex:
 

doc/fluid/api/optimizer/DecayedAdagrad.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_fluid_optimizer_DecayedAdagrad:
+.. _api_optimizer_DecayedAdagrad:
 
 DecayedAdagrad
 --------------
 
-..  autoclass:: paddle.fluid.optimizer.DecayedAdagrad
+..  autoclass:: paddle.optimizer.DecayedAdagrad
     :members:
     :inherited-members:
-    :exclude-members: apply_gradients, apply_optimize, backward, load
     :noindex:
 

doc/fluid/api/optimizer/DecayedAdagradOptimizer.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_fluid_optimizer_DecayedAdagradOptimizer:
+.. _api_optimizer_DecayedAdagradOptimizer:
 
 DecayedAdagradOptimizer
 -----------------------
 
-..  autoclass:: paddle.fluid.optimizer.DecayedAdagradOptimizer
+..  autoclass:: paddle.optimizer.DecayedAdagradOptimizer
     :members:
     :inherited-members:
-    :exclude-members: apply_gradients, apply_optimize, backward, load
     :noindex:
 

doc/fluid/api/optimizer/Dpsgd.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_fluid_optimizer_Dpsgd:
+.. _api_optimizer_Dpsgd:
 
 Dpsgd
 -----
 
-..  autoclass:: paddle.fluid.optimizer.Dpsgd
+..  autoclass:: paddle.optimizer.Dpsgd
     :members:
     :inherited-members:
-    :exclude-members: apply_gradients, apply_optimize, backward, load
     :noindex:
 

doc/fluid/api/optimizer/DpsgdOptimizer.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_fluid_optimizer_DpsgdOptimizer:
+.. _api_optimizer_DpsgdOptimizer:
 
 DpsgdOptimizer
 --------------
 
-..  autoclass:: paddle.fluid.optimizer.DpsgdOptimizer
+..  autoclass:: paddle.optimizer.DpsgdOptimizer
     :members:
     :inherited-members:
-    :exclude-members: apply_gradients, apply_optimize, backward, load
     :noindex:
 

doc/fluid/api/optimizer/ExponentialLR.rst

+..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
+    !DO NOT EDIT THIS FILE MANUALLY!
+
+.. _api_optimizer_ExponentialLR:
+
+ExponentialLR
+-------------------
+
+..  autoclass:: paddle.optimizer.ExponentialLR
+    :members:
+    :inherited-members:
+    :exclude-members: set_dict, set_state_dict, state_dict
+    :noindex:
+

doc/fluid/api/optimizer/ExponentialMovingAverage.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_fluid_optimizer_ExponentialMovingAverage:
+.. _api_optimizer_ExponentialMovingAverage:
 
 ExponentialMovingAverage
 ------------------------
 
-..  autoclass:: paddle.fluid.optimizer.ExponentialMovingAverage
+..  autoclass:: paddle.optimizer.ExponentialMovingAverage
     :members:
     :inherited-members:
-    :exclude-members: apply_gradients, apply_optimize, backward, load
     :noindex:
 

doc/fluid/api/optimizer/Ftrl.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_fluid_optimizer_Ftrl:
+.. _api_optimizer_Ftrl:
 
 Ftrl
 ----
 
-..  autoclass:: paddle.fluid.optimizer.Ftrl
+..  autoclass:: paddle.optimizer.Ftrl
     :members:
     :inherited-members:
-    :exclude-members: apply_gradients, apply_optimize, backward, load
     :noindex:
 

doc/fluid/api/optimizer/FtrlOptimizer.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_fluid_optimizer_FtrlOptimizer:
+.. _api_optimizer_FtrlOptimizer:
 
 FtrlOptimizer
 -------------
 
-..  autoclass:: paddle.fluid.optimizer.FtrlOptimizer
+..  autoclass:: paddle.optimizer.FtrlOptimizer
     :members:
     :inherited-members:
-    :exclude-members: apply_gradients, apply_optimize, backward, load
     :noindex:
 

doc/fluid/api/optimizer/InverseTimeLR.rst

+..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
+    !DO NOT EDIT THIS FILE MANUALLY!
+
+.. _api_optimizer_InverseTimeLR:
+
+InverseTimeLR
+-------------------
+
+..  autoclass:: paddle.optimizer.InverseTimeLR
+    :members:
+    :inherited-members:
+    :exclude-members: set_dict, set_state_dict, state_dict
+    :noindex:
+

doc/fluid/api/optimizer/LambOptimizer.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_fluid_optimizer_LambOptimizer:
+.. _api_optimizer_LambOptimizer:
 
 LambOptimizer
 -------------
 
-..  autoclass:: paddle.fluid.optimizer.LambOptimizer
+..  autoclass:: paddle.optimizer.LambOptimizer
     :members:
     :inherited-members:
-    :exclude-members: apply_gradients, apply_optimize, backward, load
     :noindex:
 

doc/fluid/api/optimizer/LambdaLR.rst

+..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
+    !DO NOT EDIT THIS FILE MANUALLY!
+
+.. _api_optimizer_LambdaLR:
+
+LambdaLR
+-------------------
+
+..  autoclass:: paddle.optimizer.LambdaLR
+    :members:
+    :inherited-members:
+    :exclude-members: set_dict, set_state_dict, state_dict
+    :noindex:
+

doc/fluid/api/optimizer/LarsMomentum.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_fluid_optimizer_LarsMomentum:
+.. _api_optimizer_LarsMomentum:
 
 LarsMomentum
 ------------
 
-..  autoclass:: paddle.fluid.optimizer.LarsMomentum
+..  autoclass:: paddle.optimizer.LarsMomentum
     :members:
     :inherited-members:
-    :exclude-members: apply_gradients, apply_optimize, backward, load
     :noindex:
 

doc/fluid/api/optimizer/LarsMomentumOptimizer.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_fluid_optimizer_LarsMomentumOptimizer:
+.. _api_optimizer_LarsMomentumOptimizer:
 
 LarsMomentumOptimizer
 ---------------------
 
-..  autoclass:: paddle.fluid.optimizer.LarsMomentumOptimizer
+..  autoclass:: paddle.optimizer.LarsMomentumOptimizer
     :members:
     :inherited-members:
-    :exclude-members: apply_gradients, apply_optimize, backward, load
     :noindex:
 

doc/fluid/api/optimizer/LinearLrWarmup.rst

+..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
+    !DO NOT EDIT THIS FILE MANUALLY!
+
+.. _api_optimizer_LinearLrWarmup:
+
+LinearLrWarmup
+-------------------
+
+..  autoclass:: paddle.optimizer.LinearLrWarmup
+    :members:
+    :inherited-members:
+    :exclude-members: set_dict, set_state_dict, state_dict
+    :noindex:
+

doc/fluid/api/optimizer/LookaheadOptimizer.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_fluid_optimizer_LookaheadOptimizer:
+.. _api_optimizer_LookaheadOptimizer:
 
 LookaheadOptimizer
 ------------------
 
-..  autoclass:: paddle.fluid.optimizer.LookaheadOptimizer
+..  autoclass:: paddle.optimizer.LookaheadOptimizer
     :members:
     :inherited-members:
-    :exclude-members: apply_gradients, apply_optimize, backward, load
     :noindex:
 

doc/fluid/api/optimizer/ModelAverage.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_fluid_optimizer_ModelAverage:
+.. _api_optimizer_ModelAverage:
 
 ModelAverage
 ------------
 
-..  autoclass:: paddle.fluid.optimizer.ModelAverage
+..  autoclass:: paddle.optimizer.ModelAverage
     :members:
     :inherited-members:
-    :exclude-members: apply_gradients, apply_optimize, backward, load
     :noindex:
 

doc/fluid/api/optimizer/Momentum.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_fluid_optimizer_Momentum:
+.. _api_optimizer_Momentum:
 
 Momentum
 --------
 
-..  autoclass:: paddle.fluid.optimizer.Momentum
+..  autoclass:: paddle.optimizer.Momentum
     :members:
     :inherited-members:
-    :exclude-members: apply_gradients, apply_optimize, backward, load
     :noindex:
 

doc/fluid/api/optimizer/MomentumOptimizer.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_fluid_optimizer_MomentumOptimizer:
+.. _api_optimizer_MomentumOptimizer:
 
 MomentumOptimizer
 -----------------
 
-..  autoclass:: paddle.fluid.optimizer.MomentumOptimizer
+..  autoclass:: paddle.optimizer.MomentumOptimizer
     :members:
     :inherited-members:
-    :exclude-members: apply_gradients, apply_optimize, backward, load
     :noindex:
 

doc/fluid/api/optimizer/MultiStepLR.rst

+..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
+    !DO NOT EDIT THIS FILE MANUALLY!
+
+.. _api_optimizer_MultiStepLR:
+
+MultiStepLR
+-------------------
+
+..  autoclass:: paddle.optimizer.MultiStepLR
+    :members:
+    :inherited-members:
+    :exclude-members: set_dict, set_state_dict, state_dict
+    :noindex:
+

doc/fluid/api/optimizer/NaturalExpLR.rst

+..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
+    !DO NOT EDIT THIS FILE MANUALLY!
+
+.. _api_optimizer_NaturalExpLR:
+
+NaturalExpLR
+-------------------
+
+..  autoclass:: paddle.optimizer.NaturalExpLR
+    :members:
+    :inherited-members:
+    :exclude-members: set_dict, set_state_dict, state_dict
+    :noindex:
+

doc/fluid/api/optimizer/NoamLR.rst

+..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
+    !DO NOT EDIT THIS FILE MANUALLY!
+
+.. _api_optimizer_NoamLR:
+
+NoamLR
+-------------------
+
+..  autoclass:: paddle.optimizer.NoamLR
+    :members:
+    :inherited-members:
+    :exclude-members: set_dict, set_state_dict, state_dict
+    :noindex:
+

doc/fluid/api/optimizer/Optimizer.rst

+..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
+    !DO NOT EDIT THIS FILE MANUALLY!
+
+.. _api_optimizer_Optimizer:
+
+Optimizer
+---------
+
+..  autoclass:: paddle.optimizer.Optimizer
+    :members:
+    :inherited-members:
+    :noindex:
+

doc/fluid/api/optimizer/PiecewiseLR.rst

+..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
+    !DO NOT EDIT THIS FILE MANUALLY!
+
+.. _api_optimizer_PiecewiseLR:
+
+PiecewiseLR
+-------------------
+
+..  autoclass:: paddle.optimizer.PiecewiseLR
+    :members:
+    :inherited-members:
+    :exclude-members: set_dict, set_state_dict, state_dict
+    :noindex:
+

doc/fluid/api/optimizer/PolynomialLR.rst

+..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
+    !DO NOT EDIT THIS FILE MANUALLY!
+
+.. _api_optimizer_PolynomialLR:
+
+PolynomialLR
+-------------------
+
+..  autoclass:: paddle.optimizer.PolynomialLR
+    :members:
+    :inherited-members:
+    :exclude-members: set_dict, set_state_dict, state_dict
+    :noindex:
+

doc/fluid/api/optimizer/RMSProp.rst

+..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
+    !DO NOT EDIT THIS FILE MANUALLY!
+
+.. _api_optimizer_RMSProp:
+
+RMSProp
+-------
+
+..  autoclass:: paddle.optimizer.RMSProp
+    :members:
+    :inherited-members:
+    :noindex:
+

doc/fluid/api/optimizer/RecomputeOptimizer.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_fluid_optimizer_RecomputeOptimizer:
+.. _api_optimizer_RecomputeOptimizer:
 
 RecomputeOptimizer
 ------------------
 
-..  autoclass:: paddle.fluid.optimizer.RecomputeOptimizer
+..  autoclass:: paddle.optimizer.RecomputeOptimizer
     :members:
     :inherited-members:
-    :exclude-members: apply_gradients, apply_optimize, backward, load
     :noindex:
 

doc/fluid/api/optimizer/ReduceLROnPlateau.rst

+..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
+    !DO NOT EDIT THIS FILE MANUALLY!
+
+.. _api_optimizer_ReduceLROnPlateau:
+
+ReduceLROnPlateau
+-------------------
+
+..  autoclass:: paddle.optimizer.ReduceLROnPlateau
+    :members:
+    :inherited-members:
+    :exclude-members: set_dict, set_state_dict, state_dict
+    :noindex:
+

doc/fluid/api/optimizer/SGD.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_fluid_optimizer_SGD:
+.. _api_optimizer_SGD:
 
 SGD
 ---
 
-..  autoclass:: paddle.fluid.optimizer.SGD
+..  autoclass:: paddle.optimizer.SGD
     :members:
     :inherited-members:
-    :exclude-members: apply_gradients, apply_optimize, backward, load
     :noindex:
 

doc/fluid/api/optimizer/SGDOptimizer.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_fluid_optimizer_SGDOptimizer:
+.. _api_optimizer_SGDOptimizer:
 
 SGDOptimizer
 ------------
 
-..  autoclass:: paddle.fluid.optimizer.SGDOptimizer
+..  autoclass:: paddle.optimizer.SGDOptimizer
     :members:
     :inherited-members:
-    :exclude-members: apply_gradients, apply_optimize, backward, load
     :noindex:
 

doc/fluid/api/optimizer/StepLR.rst

+..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
+    !DO NOT EDIT THIS FILE MANUALLY!
+
+.. _api_optimizer_StepLR:
+
+StepLR
+-------------------
+
+..  autoclass:: paddle.optimizer.StepLR
+    :members:
+    :inherited-members:
+    :exclude-members: set_dict, set_state_dict, state_dict
+    :noindex:
+

doc/fluid/api/paddle.rst

@@ -37,6 +37,7 @@ paddle
 	paddle/CUDAPinnedPlace.rst
 	paddle/CUDAPlace.rst
 	paddle/cumsum.rst
+	paddle/DataParallel.rst
 	paddle/default_main_program.rst
 	paddle/default_startup_program.rst
 	paddle/diag.rst
@@ -97,6 +98,7 @@ paddle
 	paddle/logical_xor.rst
 	paddle/logsumexp.rst
 	paddle/manual_seed.rst
+	paddle/masked_select.rst
 	paddle/matmul.rst
 	paddle/max.rst
 	paddle/maximum.rst

doc/fluid/api/paddle/DataParallel.rst

+.. _api_paddle_DataParallel:
+
+DataParallel
+-------------------------------
+:doc_source: paddle.fluid.dygraph.parallel.DataParallel
+
+

doc/fluid/api/static.rst

+=======================
+paddle.static
+=======================
+
+
+
+
+..  toctree::
+    :maxdepth: 1
+
+    static/data.rst
+    static/InputSpec.rst
\ No newline at end of file

doc/fluid/api/static/InputSpec.rst

+.. _api_static_InputSpec:
+
+InputSpec
+------------
+
+..  autoclass:: paddle.static.InputSpec
+    :members:
+    :noindex:
\ No newline at end of file

doc/fluid/api/static/data.rst

+.. _api_static_data:
+
+data
+------------
+
+..  autofunction:: paddle.static.data
+    :noindex:
\ No newline at end of file

doc/fluid/api/tensor.rst

@@ -42,6 +42,7 @@ paddle.tensor
     tensor/flatten.rst
     tensor/floor.rst
     tensor/full.rst
+    tensor/full_like.rst
     tensor/gather.rst
     tensor/gather_nd.rst
     tensor/greater_equal.rst
@@ -66,6 +67,7 @@ paddle.tensor
     tensor/logical_or.rst
     tensor/logical_xor.rst
     tensor/math.rst
+    tensor/masked_select.rst
     tensor/max.rst
     tensor/maximum.rst
     tensor/mean.rst

doc/fluid/api/tensor/chunk.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_tensor_chunk:
+.. _api_tensor_manipulation_chunk:
 
 chunk
----
+-------
 
-..  autofunction:: paddle.tensor.chunk
+..  autofunction:: paddle.tensor.manipulation.chunk
     :noindex:
 

doc/fluid/api/tensor/concat.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_tensor_concat:
+.. _api_tensor_manipulation_concat:
 
 concat
 --------
 
-..  autofunction:: paddle.tensor.concat
+..  autofunction:: paddle.tensor.manipulation.concat
     :noindex:
 

doc/fluid/api/tensor/eye.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_tensor_eye:
+.. _api_tensor_creation_eye:
 
 eye
 --------
 
-..  autofunction:: paddle.tensor.full
+..  autofunction:: paddle.tensor.creation.eye
     :noindex:
 

doc/fluid/api/tensor/full.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_tensor_full:
+.. _api_tensor_creation_full:
 
 full
 --------
 
-..  autofunction:: paddle.tensor.full
+..  autofunction:: paddle.tensor.creation.full
     :noindex:

doc/fluid/api/tensor/full_like.rst

+..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
+    !DO NOT EDIT THIS FILE MANUALLY!
+
+.. _api_tensor_creation_full_like:
+
+full_like
+------------
+
+..  autofunction:: paddle.tensor.creation.full_like
+    :noindex:

doc/fluid/api/tensor/gather.rst

-.. _api_tensor_cn_gather:
+..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
+    !DO NOT EDIT THIS FILE MANUALLY!
+
+.. _api_tensor_manipulation_gather:
 
 gather
--------------------------------
-:doc_source: paddle.fluid.layers.gather
+--------
+
+..  autofunction:: paddle.tensor.manipulation.gather
+    :noindex:
 
 

doc/fluid/api/tensor/gather_nd.rst

-.. _api_tensor_cn_gather_nd:
+..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
+    !DO NOT EDIT THIS FILE MANUALLY!
+
+.. _api_tensor_manipulation_gather_nd:
 
 gather_nd
--------------------------------
-:doc_source: paddle.fluid.layers.gather_nd
+----------
 
+..  autofunction:: paddle.tensor.manipulation.gather_nd
+    :noindex:
 

doc/fluid/api/tensor/index_select.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_tensor_index_select:
+.. _api_tensor_search_index_select:
 
 index_select
 -------------
 
-..  autofunction:: paddle.tensor.index_select
+..  autofunction:: paddle.tensor.search.index_select
     :noindex:
 

doc/fluid/api/tensor/linspace.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_tensor_linspace:
+.. _api_tensor_creation_linspace:
 
 linspace
 --------
 
-..  autofunction:: paddle.tensor.linspace
+..  autofunction:: paddle.tensor.creation.linspace
     :noindex:
 

doc/fluid/api/tensor/masked_select.rst

+..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
+    !DO NOT EDIT THIS FILE MANUALLY!
+
+.. _api_tensor_search_masked_select:
+
+masked_select
+---------------
+
+..  autofunction:: paddle.tensor.search.masked_select
+    :noindex:
+

doc/fluid/api/tensor/math.rst

@@ -22,6 +22,7 @@ math
     math/min.rst
     math/mm.rst
     math/mul.rst
+    math/multiply.rst
     math/pow.rst
     math/prod.rst
     math/sign.rst

doc/fluid/api/tensor/math/multiply.rst

+..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
+    !DO NOT EDIT THIS FILE MANUALLY!
+.. _api_tensor_math_multiply:
+
+multiply
+--------
+
+ ..  autofunction:: paddle.tensor.math.multiply
+    :noindex:
\ No newline at end of file

doc/fluid/api/tensor/numel.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_tensor_numel:
+.. _api_tensor_stat_numel:
 
 numel
 -------
 
-..  autofunction:: paddle.tensor.numel
+..  autofunction:: paddle.tensor.stat.numel
     :noindex:
 

doc/fluid/api/tensor/ones.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_tensor_ones:
+.. _api_tensor_creation_ones:
 
 ones
 --------
 
-..  autofunction:: paddle.tensor.ones
+..  autofunction:: paddle.tensor.creation_ones
     :noindex:
 
 

doc/fluid/api/tensor/split.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_tensor_split:
+.. _api_tensor_manipulation_split:
 
 split
 --------
 
-..  autofunction:: paddle.tensor.split
+..  autofunction:: paddle.tensor.manipulation.split
     :noindex:
 

doc/fluid/api/tensor/topk.rst

@@ -2,6 +2,6 @@
 
 topk
 -------------------------------
-:doc_source: paddle.fluid.layers.topk
+:doc_source: paddle.tensor.topk
 
 

doc/fluid/api/tensor/unique.rst

-.. _api_tensor_cn_unique:
+..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
+    !DO NOT EDIT THIS FILE MANUALLY!
 
-unique
--------------------------------
-:doc_source: paddle.fluid.layers.unique
+.. _api_tensor_manipulation_unique:
 
+unique
+--------
 
+..  autofunction:: paddle.tensor.manipulation.unique
+   

doc/fluid/api/tensor/zeros.rst

 ..  THIS FILE IS GENERATED BY `gen_doc.{py|sh}`
     !DO NOT EDIT THIS FILE MANUALLY!
 
-.. _api_tensor_zeros:
+.. _api_tensor_creation_zeros:
 
 zeros
 --------
 
-..  autofunction:: paddle.tensor.zeros
+..  autofunction:: paddle.tensor.creation.zeros
     :noindex:
 
 

doc/fluid/api_cn/api_tree_cn.rst

@@ -13,7 +13,9 @@ API接口
     data/dataset_cn.rst
     data_feeder_cn.rst
     dataset_cn.rst
+    distributed_cn.rst
     dygraph_cn.rst
+    static_cn.rst
     executor_cn.rst
     initializer_cn.rst
     io_cn.rst
@@ -25,5 +27,6 @@ API接口
     regularizer_cn.rst
     transpiler_cn.rst
     unique_name_cn.rst
+    static_cn.rst
 
 

doc/fluid/api_cn/distributed_cn.rst

+=======================
+paddle.distributed
+=======================
+
+
+
+
+..  toctree::
+    :maxdepth: 1
+
+    distributed_cn/all_gather_cn.rst
+    distributed_cn/all_reduce_cn.rst
+    distributed_cn/barrier_cn.rst
+    distributed_cn/broadcast_cn.rst
+    distributed_cn/get_rank_cn.rst
+    distributed_cn/get_world_size_cn.rst
+    distributed_cn/init_parallel_env_cn.rst
+    distributed_cn/ParallelEnv_cn.rst
+    distributed_cn/prepare_context_cn.rst
+    distributed_cn/reduce_cn.rst
+    distributed_cn/scatter_cn.rst
+    distributed_cn/spawn_cn.rst

doc/fluid/api_cn/distributed_cn/ParallelEnv_cn.rst

+.. _cn_api_distributed_ParallelEnv:
+
+ParallelEnv
+-------------------------------
+:doc_source: paddle.fluid.dygraph.parallel.ParallelEnv
\ No newline at end of file

doc/fluid/api_cn/distributed_cn/all_gather_cn.rst

+.. _cn_api_distributed_all_gather:
+
+all_gather
+-------------------------------
+
+
+.. py:function:: paddle.distributed.all_gather(tensor_list, tensor, group=0)
+
+进程组内所有进程的指定tensor进行聚合操作，并返回给所有进程聚合的结果。
+
+参数
+:::::::::
+    - tensor_list (list) - 操作的输出Tensor列表。列表中的每个元素均为Tensor，每个Tensor的数据类型为：float16、float32、float64、int32、int64。
+    - tensor (Tensor) - 操作的输入Tensor。Tensor的数据类型为：float16、float32、float64、int32、int64。
+    - group (int，可选) - 工作的进程组编号，默认为0。
+
+返回
+:::::::::
+无
+
+代码示例
+:::::::::
+.. code-block:: python
+
+        import numpy as np
+        import paddle
+        from paddle.distributed import init_parallel_env
+
+        paddle.disable_static()
+        paddle.set_device('gpu:%d'%paddle.distributed.ParallelEnv().dev_id)
+        init_parallel_env()
+        tensor_list = []
+        if paddle.distributed.ParallelEnv().local_rank == 0:
+            np_data1 = np.array([[4, 5, 6], [4, 5, 6]])
+            np_data2 = np.array([[4, 5, 6], [4, 5, 6]])
+            data1 = paddle.to_tensor(np_data1)
+            data2 = paddle.to_tensor(np_data2)
+            paddle.distributed.all_gather(tensor_list, data1)
+        else:
+            np_data1 = np.array([[1, 2, 3], [1, 2, 3]])
+            np_data2 = np.array([[1, 2, 3], [1, 2, 3]])
+            data1 = paddle.to_tensor(np_data1)
+            data2 = paddle.to_tensor(np_data2)
+            paddle.distributed.all_gather(tensor_list, data2)

doc/fluid/api_cn/distributed_cn/all_reduce_cn.rst

+.. _cn_api_distributed_all_reduce:
+
+all_reduce
+-------------------------------
+
+
+.. py:function:: paddle.distributed.all_reduce(tensor, op=ReduceOp.SUM, group=0)
+
+进程组内所有进程的指定tensor进行归约操作，并返回给所有进程归约的结果。
+
+参数
+:::::::::
+    - tensor (Tensor) - 操作的输入Tensor，同时也会将归约结果返回至此Tensor中。Tensor的数据类型为：float16、float32、float64、int32、int64。
+    - op (ReduceOp.SUM|ReduceOp.MAX|ReduceOp.Min|ReduceOp.PROD，可选) - 归约的具体操作，比如求和，取最大值，取最小值和求乘积，默认为求和归约。
+    - group (int，可选) - 工作的进程组编号，默认为0。
+
+返回
+:::::::::
+无
+
+代码示例
+:::::::::
+.. code-block:: python
+
+        import numpy as np
+        import paddle
+        from paddle.distributed import ReduceOp
+        from paddle.distributed import init_parallel_env
+
+        paddle.disable_static()
+        paddle.set_device('gpu:%d'%paddle.distributed.ParallelEnv().dev_id)
+        init_parallel_env()
+        if paddle.distributed.ParallelEnv().local_rank == 0:
+            np_data = np.array([[4, 5, 6], [4, 5, 6]])
+        else:
+            np_data = np.array([[1, 2, 3], [1, 2, 3]])
+        data = paddle.to_tensor(np_data)
+        paddle.distributed.all_reduce(data)
+        out = data.numpy()
+        # [[5, 7, 9], [5, 7, 9]]

doc/fluid/api_cn/distributed_cn/barrier_cn.rst

+.. _cn_api_distributed_barrier:
+
+barrier
+-------------------------------
+
+
+.. py:function:: paddle.distributed.barrier(group=0)
+
+同步进程组内的所有进程。
+
+参数
+:::::::::
+    - group (int，可选) - 工作的进程组编号，默认为0。
+
+返回
+:::::::::
+无
+
+代码示例
+:::::::::
+.. code-block:: python
+
+        import paddle
+        from paddle.distributed import init_parallel_env
+
+        paddle.disable_static()
+        paddle.set_device('gpu:%d'%paddle.distributed.ParallelEnv().dev_id)
+        init_parallel_env()
+        paddle.distributed.barrier()

doc/fluid/api_cn/distributed_cn/broadcast_cn.rst

+.. _cn_api_distributed_broadcast:
+
+broadcast
+-------------------------------
+
+
+.. py:function:: paddle.distributed.broadcast(tensor, src, group=0)
+
+广播一个Tensor给其他所有进程
+
+参数
+:::::::::
+    - tensor (Tensor) - 如果当前进程编号是源，那么这个Tensor变量将被发送给其他进程，否则这个Tensor将接收源发送过来的数据。Tensor的数据类型为：float16、float32、float64、int32、int64。
+    - src (int) - 发送源的进程编号。
+    - group (int，可选) - 工作的进程组编号，默认为0。
+
+返回
+:::::::::
+无
+
+代码示例
+:::::::::
+.. code-block:: python
+
+        import numpy as np
+        import paddle
+        from paddle.distributed import init_parallel_env
+
+        paddle.disable_static()
+        paddle.set_device('gpu:%d'%paddle.distributed.ParallelEnv().dev_id)
+        init_parallel_env()
+        if paddle.distributed.ParallelEnv().local_rank == 0:
+            np_data = np.array([[4, 5, 6], [4, 5, 6]])
+        else:
+            np_data = np.array([[1, 2, 3], [1, 2, 3]])
+        data = paddle.to_tensor(np_data)
+        paddle.distributed.broadcast(data, 1)
+        out = data.numpy()
+        # [[1, 2, 3], [1, 2, 3]]

doc/fluid/api_cn/distributed_cn/get_rank_cn.rst

+.. _cn_api_distributed_get_rank:
+
+get_rank
+----------
+
+..  py:function:: paddle.distributed.get_rank()
+
+返回当前进程的rank。
+
+当前进程rank的值等于环境变量 ``PADDLE_TRAINER_ID`` 的值，默认值为0。
+
+返回
+:::::::::
+(int) 当前进程的rank。
+
+代码示例
+:::::::::
+.. code-block:: python
+
+    import paddle
+    import paddle.distributed as dist
+
+    # execute this command in terminal: export PADDLE_TRAINER_ID=0
+    print("The rank is %d" % dist.get_rank())
+    # The rank is 0

doc/fluid/api_cn/distributed_cn/get_world_size_cn.rst

+.. _cn_api_distributed_get_world_size:
+
+get_world_size
+----------------
+
+.. py:function:: paddle.distributed.get_world_size()
+
+返回参与当前任务的进程数。
+
+当前进程数等于环境变量 ``PADDLE_TRAINERS_NUM`` 的值，默认值为1。
+
+返回
+:::::::::
+(int) 参与任务的进程数。
+
+代码示例
+:::::::::
+.. code-block:: python
+
+    import paddle
+    import paddle.distributed as dist
+
+    # execute this command in terminal: export PADDLE_TRAINERS_NUM=4
+    print("The world_size is %d" % dist.get_world_size())
+    # The world_size is 4

doc/fluid/api_cn/distributed_cn/init_parallel_env_cn.rst

+.. _cn_api_distributed_init_parallel_env:
+
+init_parallel_env
+-----------------
+
+.. py:function:: paddle.distributed.init_parallel_env()
+
+初始化动态图模式下的并行训练环境。
+
+.. note::
+    目前仅支持初始化GPU训练环境，使用NCCL进行通信。
+
+返回
+:::::::::
+无
+
+代码示例
+:::::::::
+.. code-block:: python
+
+    import paddle
+    import paddle.nn as nn
+    import paddle.optimizer as opt
+    import paddle.distributed as dist
+
+    class LinearNet(nn.Layer):
+        def __init__(self):
+            super(LinearNet, self).__init__()
+            self._linear1 = nn.Linear(10, 10)
+            self._linear2 = nn.Linear(10, 1)
+            
+        def forward(self, x):
+            return self._linear2(self._linear1(x))
+
+    def train():
+        # 1. enable dynamic mode
+        paddle.disable_static()
+        
+        # 2. initialize parallel environment
+        dist.init_parallel_env()
+
+        # 3. create data parallel layer & optimizer
+        layer = LinearNet()
+        dp_layer = paddle.DataParallel(layer)
+
+        loss_fn = nn.MSELoss()
+        adam = opt.Adam(
+            learning_rate=0.001, parameters=dp_layer.parameters())
+
+        # 4. run layer
+        inputs = paddle.randn([10, 10], 'float32')
+        outputs = dp_layer(inputs)
+        labels = paddle.randn([10, 1], 'float32')
+        loss = loss_fn(outputs, labels)
+        
+        loss = dp_layer.scale_loss(loss)
+        loss.backward()
+        dp_layer.apply_collective_grads()
+
+        adam.step()
+        adam.clear_grad()
+
+    if __name__ == '__main__':
+        dist.spawn(train)

doc/fluid/api_cn/distributed_cn/prepare_context_cn.rst

+.. _cn_api_distributed_prepare_context:
+
+prepare_context
+-------------------------------
+:doc_source: paddle.fluid.dygraph.parallel.prepare_context