Add TranslatedLayer.program api & update example (#2541)

* add program api & update example

* replace correct jit.save/load

* update program api example

doc/paddle/api/paddle/fluid/dygraph/io/TranslatedLayer_cn.rst

@@ -15,70 +15,180 @@ TranslatedLayer
     .. code-block:: python
 
         import numpy as np
-        import paddle.fluid as fluid
-        from paddle.fluid.dygraph import Linear
-        from paddle.fluid.dygraph import declarative
-        BATCH_SIZE = 32
-        BATCH_NUM = 20
-        def random_batch_reader():
-            def _get_random_images_and_labels(image_shape, label_shape):
-                image = np.random.random(size=image_shape).astype('float32')
-                label = np.random.random(size=label_shape).astype('int64')
+        import paddle
+        import paddle.nn as nn
+        import paddle.optimizer as opt
+
+        BATCH_SIZE = 16
+        BATCH_NUM = 4
+        EPOCH_NUM = 4
+
+        IMAGE_SIZE = 784
+        CLASS_NUM = 10
+
+        # define a random dataset
+        class RandomDataset(paddle.io.Dataset):
+            def __init__(self, num_samples):
+                self.num_samples = num_samples
+
+            def __getitem__(self, idx):
+                image = np.random.random([IMAGE_SIZE]).astype('float32')
+                label = np.random.randint(0, CLASS_NUM - 1, (1, )).astype('int64')
                 return image, label
-            def __reader__():
-                for _ in range(BATCH_NUM):
-                    batch_image, batch_label = _get_random_images_and_labels(
-                        [BATCH_SIZE, 784], [BATCH_SIZE, 1])
-                    yield batch_image, batch_label
-            return __reader__
-        class LinearNet(fluid.dygraph.Layer):
-            def __init__(self, in_size, out_size):
+
+            def __len__(self):
+                return self.num_samples
+
+        class LinearNet(nn.Layer):
+            def __init__(self):
                 super(LinearNet, self).__init__()
-                self._linear = Linear(in_size, out_size)
-            @declarative
+                self._linear = nn.Linear(IMAGE_SIZE, CLASS_NUM)
+
+            @paddle.jit.to_static
             def forward(self, x):
                 return self._linear(x)
-        # 开启命令式编程模式
-        fluid.enable_dygraph() 
-        # 1. 训练存储模型.
-        # 创建网络
-        net = LinearNet(784, 1)
-        adam = fluid.optimizer.AdamOptimizer(learning_rate=0.1, parameter_list=net.parameters())
-        # 创建DataLoader
-        train_loader = fluid.io.DataLoader.from_generator(capacity=5)
-        train_loader.set_batch_generator(random_batch_reader())
-        # 训练
-        for data in train_loader():
-            img, label = data
-            label.stop_gradient = True
-            cost = net(img)
-            loss = fluid.layers.cross_entropy(cost, label)
-            avg_loss = fluid.layers.mean(loss)
-            avg_loss.backward()
-            adam.minimize(avg_loss)
-            net.clear_gradients()
+
+        def train(layer, loader, loss_fn, opt):
+            for epoch_id in range(EPOCH_NUM):
+                for batch_id, (image, label) in enumerate(loader()):
+                    out = layer(image)
+                    loss = loss_fn(out, label)
+                    loss.backward()
+                    opt.step()
+                    opt.clear_grad()
+                    print("Epoch {} batch {}: loss = {}".format(
+                        epoch_id, batch_id, np.mean(loss.numpy())))
+
+        # enable dygraph mode
+        place = paddle.CPUPlace()
+        paddle.disable_static(place) 
+
+        # 1. train & save model.
+
+        # create network
+        layer = LinearNet()
+        loss_fn = nn.CrossEntropyLoss()
+        adam = opt.Adam(learning_rate=0.001, parameters=layer.parameters())
+
+        # create data loader
+        dataset = RandomDataset(BATCH_NUM * BATCH_SIZE)
+        loader = paddle.io.DataLoader(dataset,
+            places=place,
+            batch_size=BATCH_SIZE,
+            shuffle=True,
+            drop_last=True,
+            num_workers=2)
+
+        # train
+        train(layer, loader, loss_fn, adam)
+
+        # save
         model_path = "linear.example.model"
-        fluid.dygraph.jit.save(
-            layer=net,
-            model_path=model_path,
-            input_spec=[img])
-        # 2. 载入模型构建TranslatedLayer
-        translated_layer = fluid.dygraph.jit.load(model_path)
-        # 预测
+        paddle.jit.save(layer, model_path)
+
+        # 2. load model as TranslatedLayer
+
+        # load
+        translated_layer = paddle.jit.load(model_path)
+
+        # inference
         translated_layer.eval()
-        x = fluid.dygraph.to_variable(np.random.random((1, 784)).astype('float32'))
+        x = paddle.randn([1, IMAGE_SIZE], 'float32')
         pred = translated_layer(x)
-        # fine-tune训练
+
+        # fine-tune
         translated_layer.train()
-        adam = fluid.optimizer.AdamOptimizer(learning_rate=0.1, parameter_list=translated_layer.parameters())
-        train_loader = fluid.io.DataLoader.from_generator(capacity=5)
-        train_loader.set_batch_generator(random_batch_reader())
-        for data in train_loader():
-            img, label = data
-            label.stop_gradient = True
-            cost = translated_layer(img)
-            loss = fluid.layers.cross_entropy(cost, label)
-            avg_loss = fluid.layers.mean(loss)
-            avg_loss.backward()
-            adam.minimize(avg_loss)
-            translated_layer.clear_gradients()
+        adam = opt.Adam(learning_rate=0.001, parameters=translated_layer.parameters())
+        train(translated_layer, loader, loss_fn, adam)
+
+
+.. py:method:: program(method_name='forward'):
+
+获取TranslatedLayer中指定方法对应的Program。
+
+参数：
+    - **method_name** (string) - 要获取的Porgram对应的方法名。默认值为"forward"。
+
+返回：Program
+
+返回类型：Program
+
+**示例代码：**
+    .. code-block:: python
+
+        import numpy as np
+        import paddle
+        import paddle.nn as nn
+        import paddle.optimizer as opt
+
+        BATCH_SIZE = 16
+        BATCH_NUM = 4
+        EPOCH_NUM = 4
+
+        IMAGE_SIZE = 784
+        CLASS_NUM = 10
+
+        # define a random dataset
+        class RandomDataset(paddle.io.Dataset):
+            def __init__(self, num_samples):
+                self.num_samples = num_samples
+
+            def __getitem__(self, idx):
+                image = np.random.random([IMAGE_SIZE]).astype('float32')
+                label = np.random.randint(0, CLASS_NUM - 1, (1, )).astype('int64')
+                return image, label
+
+            def __len__(self):
+                return self.num_samples
+
+        class LinearNet(nn.Layer):
+            def __init__(self):
+                super(LinearNet, self).__init__()
+                self._linear = nn.Linear(IMAGE_SIZE, CLASS_NUM)
+
+            @paddle.jit.to_static
+            def forward(self, x):
+                return self._linear(x)
+
+        def train(layer, loader, loss_fn, opt):
+            for epoch_id in range(EPOCH_NUM):
+                for batch_id, (image, label) in enumerate(loader()):
+                    out = layer(image)
+                    loss = loss_fn(out, label)
+                    loss.backward()
+                    opt.step()
+                    opt.clear_grad()
+                    print("Epoch {} batch {}: loss = {}".format(
+                        epoch_id, batch_id, np.mean(loss.numpy())))
+
+        # enable dygraph mode
+        place = paddle.CPUPlace()
+        paddle.disable_static(place) 
+
+        # create network
+        layer = LinearNet()
+        loss_fn = nn.CrossEntropyLoss()
+        adam = opt.Adam(learning_rate=0.001, parameters=layer.parameters())
+
+        # create data loader
+        dataset = RandomDataset(BATCH_NUM * BATCH_SIZE)
+        loader = paddle.io.DataLoader(dataset,
+            places=place,
+            batch_size=BATCH_SIZE,
+            shuffle=True,
+            drop_last=True,
+            num_workers=2)
+
+        # train
+        train(layer, loader, loss_fn, adam)
+
+        # save
+        model_path = "linear.example.model"
+        paddle.jit.save(layer, model_path)
+
+        # load
+        translated_layer = paddle.jit.load(model_path)
+
+        # get program
+        program = translated_layer.program()
+        print(program)

doc/paddle/api/paddle/jit/load_cn.rst

-.. _cn_api_fluid_load:
+.. _cn_api_fluid_dygraph_jit_load:
 
 load
--------------------------------
+-----------------
 
-.. py:function:: paddle.fluid.load(program, model_path, executor=None, var_list=None)
+.. py:function:: paddle.fluid.dygraph.jit.load(model_path, configs=None)
 
-:api_attr: 声明式编程模式（静态图)
+:api_attr: 命令式编程模式（动态图)
 
+将接口 :ref:`cn_api_fluid_dygraph_jit_save` 或者 :ref:`cn_api_fluid_io_save_inference_model` 存储的模型载入为 :ref:`cn_api_fluid_dygraph_TranslatedLayer` ，用于预测推理或者fine-tune训练。
 
+.. note::
+    由于一些历史原因，如果载入的模型是通过 :ref:`cn_api_fluid_io_save_inference_model` 存储的，
+    在使用它进行fine-tune训练时会存在一些局限：
+    1. 命令式编程模式不支持 ``LoDTensor`` ，所有原先输入变量或者参数依赖于LoD信息的模型暂时无法使用；
+    2. 所有存储模型的feed变量都需要被传入 ``Translatedlayer`` 的forward方法；
+    3. 原模型变量的 ``stop_gradient`` 信息已丢失且无法准确恢复；
+    4. 原模型参数的 ``trainable`` 信息已丢失且无法准确恢复。
 
-该接口从Program中过滤出参数和优化器信息，然后从文件中获取相应的值。
+参数：
+    - **model_path** (str) - 存储模型的目录。
+    - **configs** (SaveLoadConfig, 可选) - 用于指定额外配置选项的 :ref:`cn_api_fluid_dygraph_jit_SaveLoadConfig` 对象。默认为 ``None``。
 
-如果Program和加载的文件之间参数的维度或数据类型不匹配，将引发异常。
+返回：TranslatedLayer - 一个能够执行存储模型的 ``Layer`` 对象。
 
-该函数还可以加载用[save_params，save_persistables，save_vars]接口保存的模型文件。
-当[save_params，save_persistables，save_vars]保存的模型格式为单个大文件时，var_list不能为None。
+**示例代码**
 
-参数:
- - **program**  ( :ref:`cn_api_fluid_Program` ) – 要加载的Program。
- - **model_path**  (str) – 保存Program的目录名称+文件前缀。格式为 ``目录名称/文件前缀`` 。
- - **executor** (Executor, 可选) - 当startup program没有运行时，用于初始化参数的Executor。默认值：None。
- - **var_list** (list, 可选) - 指定加载的变量列表，该参数只在加载旧接口[save_params，save_persistables，save_vars]保存的模型文件时使用。当加载的是多个小文件时，变量列表可以是所有加载文件中变量的子集；当加载的单个大文件时，变量列表必须和加载文件中的变量保持一致。
+1. 载入由接口 :ref:`cn_api_fluid_dygraph_jit_save` 存储的模型进行预测推理及fine-tune训练。
 
-返回: 无
+    .. code-block:: python
 
-**代码示例**
-
-.. code-block:: python
-
-    # example1
+        import numpy as np
         import paddle.fluid as fluid
-
-    x = fluid.data( name="x", shape=[10, 10], dtype='float32')
-    y = fluid.layers.fc(x, 10)
-    z = fluid.layers.fc(y, 10)
-    place = fluid.CPUPlace()
-    exe = fluid.Executor(place)
-    exe.run(fluid.default_startup_program())
-    fluid.save(fluid.default_main_program(), "./test_path")
-    fluid.load(fluid.default_main_program(), "./test_path")
-
-    # example2
-    # 注意example1和example2应该分开执行，避免干扰。
+        from paddle.fluid.dygraph import Linear
+        from paddle.fluid.dygraph import declarative
+        BATCH_SIZE = 32
+        BATCH_NUM = 20
+        def random_batch_reader():
+            def _get_random_images_and_labels(image_shape, label_shape):
+                image = np.random.random(size=image_shape).astype('float32')
+                label = np.random.random(size=label_shape).astype('int64')
+                return image, label
+            def __reader__():
+                for _ in range(BATCH_NUM):
+                    batch_image, batch_label = _get_random_images_and_labels(
+                        [BATCH_SIZE, 784], [BATCH_SIZE, 1])
+                    yield batch_image, batch_label
+            return __reader__
+        class LinearNet(fluid.dygraph.Layer):
+            def __init__(self, in_size, out_size):
+                super(LinearNet, self).__init__()
+                self._linear = Linear(in_size, out_size)
+            @declarative
+            def forward(self, x):
+                return self._linear(x)
+        # 开启命令式编程模式
+        fluid.enable_dygraph() 
+        # 1. 训练存储模型.
+        # 创建网络
+        net = LinearNet(784, 1)
+        adam = fluid.optimizer.AdamOptimizer(learning_rate=0.1, parameter_list=net.parameters())
+        # 创建DataLoader
+        train_loader = fluid.io.DataLoader.from_generator(capacity=5)
+        train_loader.set_batch_generator(random_batch_reader())
+        # 训练
+        for data in train_loader():
+            img, label = data
+            label.stop_gradient = True
+            cost = net(img)
+            loss = fluid.layers.cross_entropy(cost, label)
+            avg_loss = fluid.layers.mean(loss)
+            avg_loss.backward()
+            adam.minimize(avg_loss)
+            net.clear_gradients()
+        model_path = "linear.example.model"
+        fluid.dygraph.jit.save(
+            layer=net,
+            model_path=model_path,
+            input_spec=[img])
+        # 2. 载入模型 & 预测
+        # 载入模型
+        infer_net = fluid.dygraph.jit.load(model_path)
+        # 预测
+        x = fluid.dygraph.to_variable(np.random.random((1, 784)).astype('float32'))
+        pred = infer_net(x)
+        # 3. 载入模型 & fine-tune训练
+        # 载入模型
+        train_net = fluid.dygraph.jit.load(model_path)
+        train_net.train()
+        adam = fluid.optimizer.AdamOptimizer(learning_rate=0.1, parameter_list=train_net.parameters())
+        # 创建DataLoader
+        train_loader = fluid.io.DataLoader.from_generator(capacity=5)
+        train_loader.set_batch_generator(random_batch_reader())
+        # fine-tune训练
+        for data in train_loader():
+            img, label = data
+            label.stop_gradient = True
+            cost = train_net(img)
+            loss = fluid.layers.cross_entropy(cost, label)
+            avg_loss = fluid.layers.mean(loss)
+            avg_loss.backward()
+            adam.minimize(avg_loss)
+            train_net.clear_gradients()
+
+
+2. 载入由接口 :ref:`cn_api_fluid_io_save_inference_model` 存储的模型进行预测推理及fine-tune训练。
+
+    .. code-block:: python
+
+        import numpy as np
         import paddle.fluid as fluid
-
-    x = fluid.data( name="x", shape=[10, 10], dtype='float32')
-    y = fluid.layers.fc(x, 10)
-    z = fluid.layers.fc(y, 10)
+        BATCH_SIZE = 32
+        BATCH_NUM = 20
+        def random_batch_reader():
+            def _get_random_images_and_labels(image_shape, label_shape):
+                image = np.random.random(size=image_shape).astype('float32')
+                label = np.random.random(size=label_shape).astype('int64')
+                return image, label
+            def __reader__():
+                for _ in range(BATCH_NUM):
+                    batch_image, batch_label = _get_random_images_and_labels(
+                        [BATCH_SIZE, 784], [BATCH_SIZE, 1])
+                    yield batch_image, batch_label
+            return __reader__
+        img = fluid.data(name='img', shape=[None, 784], dtype='float32')
+        label = fluid.data(name='label', shape=[None, 1], dtype='int64')
+        pred = fluid.layers.fc(input=img, size=10, act='softmax')
+        loss = fluid.layers.cross_entropy(input=pred, label=label)
+        avg_loss = fluid.layers.mean(loss)
+        optimizer = fluid.optimizer.SGD(learning_rate=0.001)
+        optimizer.minimize(avg_loss)
         place = fluid.CPUPlace()
         exe = fluid.Executor(place)
         exe.run(fluid.default_startup_program())
-    fluid.save(fluid.default_main_program(), "./test_path")
-    fluid.load(fluid.default_main_program(), "./test_path", exe)
-
+        loader = fluid.io.DataLoader.from_generator(
+            feed_list=[img, label], capacity=5, iterable=True)
+        loader.set_batch_generator(random_batch_reader(), places=place)
+        # 1. 训练 & 存储预测模型
+        for data in loader():
+            exe.run(
+                fluid.default_main_program(),
+                feed=data, 
+                fetch_list=[avg_loss])
+        model_path = "fc.example.model"
+        fluid.io.save_inference_model(
+            model_path, ["img"], [pred], exe)
+        # 开启命令式编程模式
+        fluid.enable_dygraph() 
+        # 2. 载入模型 & 预测
+        fc = fluid.dygraph.jit.load(model_path)
+        x = fluid.dygraph.to_variable(np.random.random((1, 784)).astype('float32'))
+        pred = fc(x)
+        # 3. 载入模型 & fine-tune训练
+        fc = fluid.dygraph.jit.load(model_path)
+        fc.train()
+        sgd = fluid.optimizer.SGD(learning_rate=0.001,
+                                    parameter_list=fc.parameters())
+        train_loader = fluid.io.DataLoader.from_generator(capacity=5)
+        train_loader.set_batch_generator(
+            random_batch_reader(), places=place)
+        for data in train_loader():
+            img, label = data
+            label.stop_gradient = True
+            cost = fc(img)
+            loss = fluid.layers.cross_entropy(cost, label)
+            avg_loss = fluid.layers.mean(loss)
+            avg_loss.backward()
+            sgd.minimize(avg_loss)

doc/paddle/api/paddle/jit/save_cn.rst

-.. _cn_api_fluid_save:
+.. _cn_api_fluid_dygraph_jit_save:
 
 save
--------------------------------
+-----------------
 
+.. py:function:: paddle.fluid.dygraph.jit.save(layer, model_path, input_spec=None, configs=None)
 
-.. py:function:: paddle.fluid.save(program, model_path)
+将输入的经过 ``@declarative`` 装饰的 :ref:`cn_api_fluid_dygraph_Layer` 存储为 :ref:`cn_api_fluid_dygraph_TranslatedLayer` 格式的模型，
+载入后可用于预测推理或者fine-tune训练。
 
-:api_attr: 声明式编程模式（静态图)
-:alias_main: paddle.save
-:alias: paddle.save,paddle.tensor.save,paddle.tensor.io.save
-:old_api: paddle.fluid.save
+该接口将会将输入 :ref:`cn_api_fluid_dygraph_Layer` 转写后的模型结构 ``Program`` 和所有必要的持久参数变量存储至输入路径 ``model_path`` 中。
 
+默认存储的 ``Program`` 文件名为 ``__model__``， 默认存储持久参数变量的文件名为 ``__variables__``，
+同时会将变量的一些描述信息存储至文件 ``__variables.info__``，这些额外的信息将在fine-tune训练中使用。
 
+存储的模型能够被以下API载入使用：
+  - :ref:`cn_api_fluid_dygraph_jit_load`
+  - :ref:`cn_api_fluid_io_load_inference_model` （需要配置参数 ``params_filename='__variables__'`` ）
+  - 其他预测库API
 
-该接口将传入的参数、优化器信息和网络描述保存到 ``model_path`` 。
+参数：
+    - **layer** (Layer) - 需要存储的 :ref:`cn_api_fluid_dygraph_Layer` 对象。输入的 ``Layer`` 需要经过 ``@declarative`` 装饰。
+    - **model_path** (str) - 存储模型的目录。
+    - **input_spec** (list[Variable], 可选) - 描述存储模型的输入。此参数是传入当前存储的 ``TranslatedLayer`` forward方法的一个示例输入。如果为 ``None`` ，所有原 ``Layer`` forward方法的输入变量将都会被配置为存储模型的输入变量。默认为 ``None``。
+    - **configs** (SaveLoadConfig, 可选) - 用于指定额外配置选项的 :ref:`cn_api_fluid_dygraph_jit_SaveLoadConfig` 对象。默认为 ``None``。
 
-参数包含所有的可训练 :ref:`cn_api_fluid_Variable` ，将保存到后缀为 ``.pdparams`` 的文件中。
+返回：无
 
-优化器信息包含优化器使用的所有变量。对于Adam优化器，包含beta1、beta2、momentum等。
-所有信息将保存到后缀为 ``.pdopt`` 的文件中。（如果优化器没有需要保存的变量（如sgd），则不会生成）。
-
-网络描述是程序的描述。它只用于部署。描述将保存到后缀为 ``.pdmodel`` 的文件中。
-
-参数:
- - **program**  ( :ref:`cn_api_fluid_Program` ) – 要保存的Program。
- - **model_path**  (str) – 保存program的文件前缀。格式为 ``目录名称/文件前缀``。如果文件前缀为空字符串，会引发异常。
-
-返回: 无
-
-**代码示例**
+**示例代码**
 
 .. code-block:: python
 
+    import numpy as np
     import paddle.fluid as fluid
-
-    x = fluid.data(name="x", shape=[10, 10], dtype='float32')
-    y = fluid.layers.fc(x, 10)
-    z = fluid.layers.fc(y, 10)
-    place = fluid.CPUPlace()
-    exe = fluid.Executor(place)
-    exe.run(fluid.default_startup_program())
-
-    fluid.save(fluid.default_main_program(), "./test_path")
-
-
-
-
-
-
-
+    from paddle.fluid.dygraph import Linear
+    from paddle.fluid.dygraph import declarative
+    BATCH_SIZE = 32
+    BATCH_NUM = 20
+    def random_batch_reader():
+        def _get_random_images_and_labels(image_shape, label_shape):
+            image = np.random.random(size=image_shape).astype('float32')
+            label = np.random.random(size=label_shape).astype('int64')
+            return image, label
+        def __reader__():
+            for _ in range(BATCH_NUM):
+                batch_image, batch_label = _get_random_images_and_labels(
+                    [BATCH_SIZE, 784], [BATCH_SIZE, 1])
+                yield batch_image, batch_label
+        return __reader__
+    class LinearNet(fluid.dygraph.Layer):
+        def __init__(self, in_size, out_size):
+            super(LinearNet, self).__init__()
+            self._linear = Linear(in_size, out_size)
+        @declarative
+        def forward(self, x):
+            return self._linear(x)
+    # 开启命令式编程模式
+    fluid.enable_dygraph() 
+    # 创建网络
+    net = LinearNet(784, 1)
+    adam = fluid.optimizer.AdamOptimizer(learning_rate=0.1, parameter_list=net.parameters())
+    # 创建DataLoader
+    train_loader = fluid.io.DataLoader.from_generator(capacity=5)
+    train_loader.set_batch_generator(random_batch_reader())
+    # 训练
+    for data in train_loader():
+        img, label = data
+        label.stop_gradient = True
+        cost = net(img)
+        loss = fluid.layers.cross_entropy(cost, label)
+        avg_loss = fluid.layers.mean(loss)
+        avg_loss.backward()
+        adam.minimize(avg_loss)
+        net.clear_gradients()
+    # 存储模型
+    model_path = "linear.example.model"
+    fluid.dygraph.jit.save(
+        layer=net,
+        model_path=model_path,
+        input_spec=[img])