【fluid clean】 move out LayerList, ParameterList, Sequential from fluid (#48197)

* fluid clean move out LayerList, ParameterList, Sequential from fluid

* copy Sequential from fluid to paddle.nn

* replace fluid LayerList,ParamterList, Sequential in unit test file

* add omit unit test file use paddle.nn.Sequential

* remove numpy and use paddle.uniform instead

python/paddle/fluid/contrib/slim/tests/imperative_test_utils.py

@@ -17,7 +17,7 @@ import logging
 import paddle
 import paddle.fluid as fluid
 from paddle.fluid import core
-from paddle.fluid.dygraph.container import Sequential
+from paddle.nn import Sequential
 from paddle.nn import ReLU, ReLU6, LeakyReLU, Sigmoid, Softmax, PReLU
 from paddle.nn import Linear, Conv2D, Softmax, BatchNorm2D, MaxPool2D
 from paddle.nn import BatchNorm1D

python/paddle/fluid/contrib/slim/tests/test_imperative_out_scale.py

@@ -27,7 +27,7 @@ from paddle.fluid import core
 from paddle.fluid.optimizer import AdamOptimizer
 from paddle.fluid.framework import IrGraph, _test_eager_guard
 from paddle.fluid.contrib.slim.quantization import ImperativeQuantAware
-from paddle.fluid.dygraph.container import Sequential
+from paddle.nn import Sequential
 from paddle.fluid.dygraph.io import INFER_MODEL_SUFFIX, INFER_PARAMS_SUFFIX
 from paddle.nn.layer import ReLU, LeakyReLU, Sigmoid, Softmax, PReLU
 from paddle.nn import Linear, Conv2D, Softmax, BatchNorm2D, MaxPool2D

python/paddle/fluid/contrib/slim/tests/test_imperative_qat.py

@@ -25,7 +25,7 @@ import paddle.fluid as fluid
 from paddle.fluid import core
 from paddle.fluid.optimizer import AdamOptimizer
 from paddle.fluid.contrib.slim.quantization import ImperativeQuantAware
-from paddle.fluid.dygraph.container import Sequential
+from paddle.nn import Sequential
 from paddle.nn import Linear, Conv2D, Softmax, Conv2DTranspose
 from paddle.fluid.log_helper import get_logger
 from paddle.fluid.dygraph.io import INFER_MODEL_SUFFIX, INFER_PARAMS_SUFFIX

python/paddle/fluid/contrib/slim/tests/test_imperative_qat_lsq.py

@@ -29,7 +29,7 @@ from paddle.fluid.optimizer import (
     MomentumOptimizer,
 )
 from paddle.fluid.contrib.slim.quantization import ImperativeQuantAware
-from paddle.fluid.dygraph.container import Sequential
+from paddle.nn import Sequential
 from paddle.nn import ReLU, ReLU6, LeakyReLU, Sigmoid, Softmax, PReLU
 from paddle.nn import Linear, Conv2D, Softmax, BatchNorm2D, MaxPool2D
 from paddle.fluid.log_helper import get_logger

python/paddle/fluid/dygraph/container.py

@@ -12,15 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from collections import OrderedDict
-from ..framework import Parameter
 from .layers import Layer
-from .base import param_guard
 
 __all__ = [
     'Sequential',
-    'ParameterList',
-    'LayerList',
 ]
 
 
@@ -97,240 +92,3 @@ class Sequential(Layer):
         for layer in self._sub_layers.values():
             input = layer(input)
         return input
-
-
-class ParameterList(Layer):
-    """ParameterList Container.
-
-    This container acts like a Python list, but parameters it contains will be properly added.
-
-    Parameters:
-        parameters (iterable, optional): Iterable Parameters to be added
-
-    Examples:
-        .. code-block:: python
-
-            import paddle
-            import numpy as np
-
-            class MyLayer(paddle.nn.Layer):
-                def __init__(self, num_stacked_param):
-                    super().__init__()
-                    # create ParameterList with iterable Parameters
-                    self.params = paddle.nn.ParameterList(
-                        [paddle.create_parameter(
-                            shape=[2, 2], dtype='float32')] * num_stacked_param)
-
-                def forward(self, x):
-                    for i, p in enumerate(self.params):
-                        tmp = self._helper.create_variable_for_type_inference('float32')
-                        self._helper.append_op(
-                            type="mul",
-                            inputs={"X": x,
-                                    "Y": p},
-                            outputs={"Out": tmp},
-                            attrs={"x_num_col_dims": 1,
-                                    "y_num_col_dims": 1})
-                        x = tmp
-                    return x
-
-            data_np = np.random.uniform(-1, 1, [5, 2]).astype('float32')
-            x = paddle.to_tensor(data_np)
-            num_stacked_param = 4
-            model = MyLayer(num_stacked_param)
-            print(len(model.params))  # 4
-            res = model(x)
-            print(res.shape)  # [5, 2]
-
-            replaced_param = paddle.create_parameter(shape=[2, 3], dtype='float32')
-            model.params[num_stacked_param - 1] = replaced_param  # replace last param
-            res = model(x)
-            print(res.shape)  # [5, 3]
-            model.params.append(paddle.create_parameter(shape=[3, 4], dtype='float32'))  # append param
-            print(len(model.params))  # 5
-            res = model(x)
-            print(res.shape)  # [5, 4]
-    """
-
-    def __init__(self, parameters=None):
-        super().__init__()
-        if parameters is not None:
-            for idx, param in enumerate(parameters):
-                assert isinstance(param, Parameter)
-                self.add_parameter(str(idx), param)
-
-    def __getitem__(self, idx):
-        with param_guard(self._parameters):
-            return self._parameters[str(idx)]
-
-    def __setitem__(self, idx, param):
-        assert isinstance(param, Parameter)
-        setattr(self, str(idx), param)
-
-    def __len__(self):
-        return len(self._parameters)
-
-    def __iter__(self):
-        with param_guard(self._parameters):
-            return iter(self._parameters.values())
-
-    def append(self, parameter):
-        """Appends a given parameter at the end of the list.
-
-        Parameters:
-            parameter (Parameter): parameter to append
-        """
-        idx = len(self._parameters)
-        self.add_parameter(str(idx), parameter)
-        return self
-
-
-class LayerList(Layer):
-    """
-    LayerList holds sublayers, and sublayers it contains are properly registered.
-    Holded sublayers can be indexed like a regular python list.
-
-    Parameters:
-        sublayers (iterable of Layer, optional): sublayers to hold
-
-    Examples:
-        .. code-block:: python
-
-            import paddle
-            import numpy as np
-
-            class MyLayer(paddle.nn.Layer):
-                def __init__(self):
-                    super().__init__()
-                    self.linears = paddle.nn.LayerList(
-                        [paddle.nn.Linear(10, 10) for i in range(10)])
-
-                def forward(self, x):
-                    # LayerList can act as an iterable, or be indexed using ints
-                    for i, l in enumerate(self.linears):
-                        x = self.linears[i // 2](x) + l(x)
-                    return x
-    """
-
-    def __init__(self, sublayers=None):
-        super().__init__()
-        if sublayers is not None:
-            for idx, layer in enumerate(sublayers):
-                self.add_sublayer(str(idx), layer)
-
-    def _get_abs_idx(self, idx):
-        if isinstance(idx, int):
-            if not (-len(self) <= idx < len(self)):
-                raise IndexError(
-                    'index {} is out of range, should be an integer in range [{}, {})'.format(
-                        idx, -len(self), len(self)
-                    )
-                )
-            if idx < 0:
-                idx += len(self)
-        return idx
-
-    def __getitem__(self, idx):
-        if isinstance(idx, slice):
-            return self.__class__(list(self._sub_layers.values())[idx])
-        else:
-            idx = self._get_abs_idx(idx)
-            return self._sub_layers[str(idx)]
-
-    def __setitem__(self, idx, sublayer):
-        idx = self._get_abs_idx(idx)
-        return setattr(self, str(idx), sublayer)
-
-    def __delitem__(self, idx):
-        if isinstance(idx, slice):
-            for k in range(len(self._sub_layers))[idx]:
-                delattr(self, str(k))
-        else:
-            idx = self._get_abs_idx(idx)
-            delattr(self, str(idx))
-        str_indices = [str(i) for i in range(len(self._sub_layers))]
-        self._sub_layers = OrderedDict(
-            list(zip(str_indices, self._sub_layers.values()))
-        )
-
-    def __len__(self):
-        return len(self._sub_layers)
-
-    def __iter__(self):
-        return iter(self._sub_layers.values())
-
-    def append(self, sublayer):
-        """
-        Appends a sublayer to the end of the list.
-
-        Parameters:
-            sublayer (Layer): sublayer to append
-
-        Examples:
-            .. code-block:: python
-
-                import paddle
-
-                linears = paddle.nn.LayerList([paddle.nn.Linear(10, 10) for i in range(10)])
-                another = paddle.nn.Linear(10, 10)
-                linears.append(another)
-                print(len(linears))  # 11
-        """
-        self.add_sublayer(str(len(self)), sublayer)
-        return self
-
-    def insert(self, index, sublayer):
-        """
-        Insert a sublayer before a given index in the list.
-
-        Parameters:
-            index (int): index to insert.
-            sublayer (Layer): sublayer to insert
-
-        Examples:
-            .. code-block:: python
-
-                import paddle
-
-                linears = paddle.nn.LayerList([paddle.nn.Linear(10, 10) for i in range(10)])
-                another = paddle.nn.Linear(10, 10)
-                linears.insert(3, another)
-                print(linears[3] is another)  # True
-                another = paddle.nn.Linear(10, 10)
-                linears.insert(-1, another)
-                print(linears[-2] is another) # True
-        """
-        assert isinstance(index, int) and -len(self._sub_layers) <= index < len(
-            self._sub_layers
-        ), "index should be an integer in range [{}, {})".format(
-            -len(self), len(self)
-        )
-
-        index = self._get_abs_idx(index)
-        for i in range(len(self._sub_layers), index, -1):
-            self._sub_layers[str(i)] = self._sub_layers[str(i - 1)]
-        self._sub_layers[str(index)] = sublayer
-
-    def extend(self, sublayers):
-        """
-        Appends sublayers to the end of the list.
-
-        Parameters:
-            sublayers (iterable of Layer): iterable of sublayers to append
-
-        Examples:
-            .. code-block:: python
-
-                import paddle
-
-                linears = paddle.nn.LayerList([paddle.nn.Linear(10, 10) for i in range(10)])
-                another_list = paddle.nn.LayerList([paddle.nn.Linear(10, 10) for i in range(5)])
-                linears.extend(another_list)
-                print(len(linears))  # 15
-                print(another_list[0] is linears[10])  # True
-        """
-        offset = len(self)
-        for i, sublayer in enumerate(sublayers):
-            idx = str(offset + i)
-            self.add_sublayer(idx, sublayer)
-        return self

python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_embedding.py

@@ -19,7 +19,7 @@ import random
 import paddle
 import paddle.distributed as dist
 import paddle.distributed.fleet as fleet
-from paddle.fluid.dygraph.container import Sequential
+from paddle.nn import Sequential
 from paddle.distributed.fleet.meta_parallel import PipelineLayer
 from paddle.fluid.dygraph.layers import Layer
 import paddle.nn as nn

python/paddle/fluid/tests/unittests/hybrid_parallel_pp_layer.py

@@ -15,7 +15,7 @@
 import unittest
 import numpy as np
 from paddle.distributed import fleet
-from paddle.fluid.dygraph.container import Sequential
+from paddle.nn import Sequential
 import paddle.nn as nn
 from paddle.fluid.dygraph.layers import Layer
 from paddle.distributed.fleet.meta_parallel import LayerDesc, PipelineLayer

python/paddle/fluid/tests/unittests/test_imperative_container_layerlist.py

@@ -31,11 +31,6 @@ class MyLayer(fluid.Layer):
 
 
 class TestImperativeContainer(unittest.TestCase):
-    def fluid_dygraph_list(self):
-        return fluid.dygraph.LayerList(
-            [fluid.dygraph.Linear(2**i, 2 ** (i + 1)) for i in range(6)]
-        )
-
     def paddle_imperative_list(self):
         return paddle.nn.LayerList(
             [fluid.dygraph.Linear(2**i, 2 ** (i + 1)) for i in range(6)]
@@ -45,11 +40,7 @@ class TestImperativeContainer(unittest.TestCase):
         data_np = np.random.uniform(-1, 1, [5, 1]).astype('float32')
         with fluid.dygraph.guard():
             x = fluid.dygraph.to_variable(data_np)
-            layerlist = (
-                self.fluid_dygraph_list()
-                if use_fluid_api
-                else self.paddle_imperative_list()
-            )
+            layerlist = self.paddle_imperative_list()
             size = len(layerlist)
 
             model = MyLayer(layerlist)

python/paddle/fluid/tests/unittests/test_imperative_container_parameterlist.py

@@ -24,17 +24,7 @@ class MyLayer(fluid.Layer):
     def __init__(self, num_stacked_param, use_fluid_api):
         super().__init__()
         # create ParameterList with iterable Parameters
-        self.params = (
-            self.fluid_dygraph_ParameterList(num_stacked_param)
-            if use_fluid_api
-            else self.paddle_imperative_ParameterList(num_stacked_param)
-        )
-
-    def fluid_dygraph_ParameterList(self, num_stacked_param):
-        return fluid.dygraph.ParameterList(
-            [fluid.layers.create_parameter(shape=[2, 2], dtype='float32')]
-            * num_stacked_param
-        )
+        self.params = self.paddle_imperative_ParameterList(num_stacked_param)
 
     def paddle_imperative_ParameterList(self, num_stacked_param):
         return paddle.nn.ParameterList(

python/paddle/fluid/tests/unittests/test_imperative_named_members.py

@@ -37,7 +37,7 @@ class TestImperativeNamedSubLayers(unittest.TestCase):
             fc1 = fluid.Linear(10, 3)
             fc2 = fluid.Linear(3, 10, bias_attr=False)
             custom = MyLayer(3, 10)
-            model = fluid.dygraph.Sequential(fc1, fc2, custom)
+            model = paddle.nn.Sequential(fc1, fc2, custom)
             named_sublayers = model.named_sublayers()
             list_named_sublayers = list(named_sublayers)
 

python/paddle/fluid/tests/unittests/test_imperative_star_gan_with_gradient_penalty.py

@@ -268,7 +268,7 @@ class Generator(fluid.dygraph.Layer):
             cur_channels *= 2
             sub_layers.append(sub_layer)
 
-        self._conv0 = fluid.dygraph.Sequential(*sub_layers)
+        self._conv0 = paddle.nn.Sequential(*sub_layers)
 
         repeat_num = cfg.g_repeat_num
         sub_layers = []
@@ -278,7 +278,7 @@ class Generator(fluid.dygraph.Layer):
             )
             sub_layers.append(res_block)
 
-        self._res_block = fluid.dygraph.Sequential(*sub_layers)
+        self._res_block = paddle.nn.Sequential(*sub_layers)
 
         cur_channels = cfg.g_base_dims * 4
         sub_layers = []
@@ -296,7 +296,7 @@ class Generator(fluid.dygraph.Layer):
             cur_channels = cfg.g_base_dims * rate
             sub_layers.append(deconv)
 
-        self._deconv = fluid.dygraph.Sequential(*sub_layers)
+        self._deconv = paddle.nn.Sequential(*sub_layers)
 
         self._conv1 = Conv2DLayer(
             num_channels=cur_channels,
@@ -353,7 +353,7 @@ class Discriminator(fluid.dygraph.Layer):
             cur_dim *= 2
             sub_layers.append(sub_layer)
 
-        self._conv0 = fluid.dygraph.Sequential(*sub_layers)
+        self._conv0 = paddle.nn.Sequential(*sub_layers)
 
         kernel_size = int(cfg.image_size / np.power(2, repeat_num))
 

python/paddle/nn/__init__.py

@@ -15,9 +15,9 @@
 # TODO: import all neural network related api under this directory,
 # including layers, linear, conv, rnn etc.
 from ..fluid.dygraph.layers import Layer  # noqa: F401
-from ..fluid.dygraph.container import LayerList  # noqa: F401
-from ..fluid.dygraph.container import ParameterList  # noqa: F401
-from ..fluid.dygraph.container import Sequential  # noqa: F401
+from .layer.container import LayerList  # noqa: F401
+from .layer.container import ParameterList  # noqa: F401
+from .layer.container import Sequential  # noqa: F401
 
 from .clip import ClipGradByGlobalNorm  # noqa: F401
 from .clip import ClipGradByNorm  # noqa: F401

python/paddle/nn/layer/container.py

@@ -15,6 +15,8 @@
 from collections import OrderedDict
 from .. import Layer
 from collections.abc import Iterable, Mapping
+from ...fluid.framework import Parameter
+from ...fluid.dygraph.base import param_guard
 
 __all__ = []
 
@@ -295,3 +297,310 @@ class LayerDict(Layer):
                         + ", which must be 2."
                     )
                 self.add_sublayer(kv[0], kv[1])
+
+
+class Sequential(Layer):
+    """Sequential container.
+    Sub layers will be added to this container in the order of argument in the constructor.
+    The argument passed to the constructor can be iterable Layers or iterable name Layer pairs.
+
+    Parameters:
+        layers(Layer|list|tuple): Layer or list/tuple of iterable name Layer pair.
+
+    Examples:
+        .. code-block:: python
+
+            import paddle
+
+            data = paddle.uniform(shape=[30, 10], dtype='float32')
+            # create Sequential with iterable Layers
+            model1 = paddle.nn.Sequential(
+                paddle.nn.Linear(10, 1), paddle.nn.Linear(1, 2)
+            )
+            model1[0]  # access the first layer
+            res1 = model1(data)  # sequential execution
+
+            # create Sequential with name Layer pairs
+            model2 = paddle.nn.Sequential(
+                ('l1', paddle.nn.Linear(10, 2)),
+                ('l2', paddle.nn.Linear(2, 3))
+            )
+            model2['l1']  # access l1 layer
+            model2.add_sublayer('l3', paddle.nn.Linear(3, 3))  # add sublayer
+            res2 = model2(data)  # sequential execution
+
+    """
+
+    def __init__(self, *layers):
+        super().__init__()
+        if len(layers) > 0 and isinstance(layers[0], (list, tuple)):
+            for name, layer in layers:
+                self.add_sublayer(name, layer)
+        else:
+            for idx, layer in enumerate(layers):
+                self.add_sublayer(str(idx), layer)
+
+    def __getitem__(self, name):
+        if isinstance(name, slice):
+            return self.__class__(*(list(self._sub_layers.values())[name]))
+        elif isinstance(name, str):
+            return self._sub_layers[name]
+        else:
+            if name >= len(self._sub_layers):
+                raise IndexError('index {} is out of range'.format(name))
+            elif name < 0 and name >= -len(self._sub_layers):
+                name += len(self._sub_layers)
+            elif name < -len(self._sub_layers):
+                raise IndexError('index {} is out of range'.format(name))
+            return list(self._sub_layers.values())[name]
+
+    def __setitem__(self, name, layer):
+        assert isinstance(layer, Layer)
+        setattr(self, str(name), layer)
+
+    def __delitem__(self, name):
+        name = str(name)
+        assert name in self._sub_layers
+        del self._sub_layers[name]
+
+    def __len__(self):
+        return len(self._sub_layers)
+
+    def forward(self, input):
+        for layer in self._sub_layers.values():
+            input = layer(input)
+        return input
+
+
+class ParameterList(Layer):
+    """ParameterList Container.
+
+    This container acts like a Python list, but parameters it contains will be properly added.
+
+    Parameters:
+        parameters (iterable, optional): Iterable Parameters to be added
+
+    Examples:
+        .. code-block:: python
+
+            import paddle
+
+            class MyLayer(paddle.nn.Layer):
+                def __init__(self, num_stacked_param):
+                    super().__init__()
+                    # create ParameterList with iterable Parameters
+                    self.params = paddle.nn.ParameterList(
+                        [paddle.create_parameter(
+                            shape=[2, 2], dtype='float32')] * num_stacked_param)
+
+                def forward(self, x):
+                    for i, p in enumerate(self.params):
+                        tmp = self._helper.create_variable_for_type_inference('float32')
+                        self._helper.append_op(
+                            type="mul",
+                            inputs={"X": x,
+                                    "Y": p},
+                            outputs={"Out": tmp},
+                            attrs={"x_num_col_dims": 1,
+                                    "y_num_col_dims": 1})
+                        x = tmp
+                    return x
+
+            x = paddle.uniform(shape=[5, 2], dtype='float32')
+            num_stacked_param = 4
+            model = MyLayer(num_stacked_param)
+            print(len(model.params))  # 4
+            res = model(x)
+            print(res.shape)  # [5, 2]
+
+            replaced_param = paddle.create_parameter(shape=[2, 3], dtype='float32')
+            model.params[num_stacked_param - 1] = replaced_param  # replace last param
+            res = model(x)
+            print(res.shape)  # [5, 3]
+            model.params.append(paddle.create_parameter(shape=[3, 4], dtype='float32'))  # append param
+            print(len(model.params))  # 5
+            res = model(x)
+            print(res.shape)  # [5, 4]
+    """
+
+    def __init__(self, parameters=None):
+        super().__init__()
+        if parameters is not None:
+            for idx, param in enumerate(parameters):
+                assert isinstance(param, Parameter)
+                self.add_parameter(str(idx), param)
+
+    def __getitem__(self, idx):
+        with param_guard(self._parameters):
+            return self._parameters[str(idx)]
+
+    def __setitem__(self, idx, param):
+        assert isinstance(param, Parameter)
+        setattr(self, str(idx), param)
+
+    def __len__(self):
+        return len(self._parameters)
+
+    def __iter__(self):
+        with param_guard(self._parameters):
+            return iter(self._parameters.values())
+
+    def append(self, parameter):
+        """Appends a given parameter at the end of the list.
+
+        Parameters:
+            parameter (Parameter): parameter to append
+        """
+        idx = len(self._parameters)
+        self.add_parameter(str(idx), parameter)
+        return self
+
+
+class LayerList(Layer):
+    """
+    LayerList holds sublayers, and sublayers it contains are properly registered.
+    Holded sublayers can be indexed like a regular python list.
+
+    Parameters:
+        sublayers (iterable of Layer, optional): sublayers to hold
+
+    Examples:
+        .. code-block:: python
+
+            import paddle
+
+            class MyLayer(paddle.nn.Layer):
+                def __init__(self):
+                    super().__init__()
+                    self.linears = paddle.nn.LayerList(
+                        [paddle.nn.Linear(10, 10) for i in range(10)])
+
+                def forward(self, x):
+                    # LayerList can act as an iterable, or be indexed using ints
+                    for i, l in enumerate(self.linears):
+                        x = self.linears[i // 2](x) + l(x)
+                    return x
+    """
+
+    def __init__(self, sublayers=None):
+        super().__init__()
+        if sublayers is not None:
+            for idx, layer in enumerate(sublayers):
+                self.add_sublayer(str(idx), layer)
+
+    def _get_abs_idx(self, idx):
+        if isinstance(idx, int):
+            if not (-len(self) <= idx < len(self)):
+                raise IndexError(
+                    'index {} is out of range, should be an integer in range [{}, {})'.format(
+                        idx, -len(self), len(self)
+                    )
+                )
+            if idx < 0:
+                idx += len(self)
+        return idx
+
+    def __getitem__(self, idx):
+        if isinstance(idx, slice):
+            return self.__class__(list(self._sub_layers.values())[idx])
+        else:
+            idx = self._get_abs_idx(idx)
+            return self._sub_layers[str(idx)]
+
+    def __setitem__(self, idx, sublayer):
+        idx = self._get_abs_idx(idx)
+        return setattr(self, str(idx), sublayer)
+
+    def __delitem__(self, idx):
+        if isinstance(idx, slice):
+            for k in range(len(self._sub_layers))[idx]:
+                delattr(self, str(k))
+        else:
+            idx = self._get_abs_idx(idx)
+            delattr(self, str(idx))
+        str_indices = [str(i) for i in range(len(self._sub_layers))]
+        self._sub_layers = OrderedDict(
+            list(zip(str_indices, self._sub_layers.values()))
+        )
+
+    def __len__(self):
+        return len(self._sub_layers)
+
+    def __iter__(self):
+        return iter(self._sub_layers.values())
+
+    def append(self, sublayer):
+        """
+        Appends a sublayer to the end of the list.
+
+        Parameters:
+            sublayer (Layer): sublayer to append
+
+        Examples:
+            .. code-block:: python
+
+                import paddle
+
+                linears = paddle.nn.LayerList([paddle.nn.Linear(10, 10) for i in range(10)])
+                another = paddle.nn.Linear(10, 10)
+                linears.append(another)
+                print(len(linears))  # 11
+        """
+        self.add_sublayer(str(len(self)), sublayer)
+        return self
+
+    def insert(self, index, sublayer):
+        """
+        Insert a sublayer before a given index in the list.
+
+        Parameters:
+            index (int): index to insert.
+            sublayer (Layer): sublayer to insert
+
+        Examples:
+            .. code-block:: python
+
+                import paddle
+
+                linears = paddle.nn.LayerList([paddle.nn.Linear(10, 10) for i in range(10)])
+                another = paddle.nn.Linear(10, 10)
+                linears.insert(3, another)
+                print(linears[3] is another)  # True
+                another = paddle.nn.Linear(10, 10)
+                linears.insert(-1, another)
+                print(linears[-2] is another) # True
+        """
+        assert isinstance(index, int) and -len(self._sub_layers) <= index < len(
+            self._sub_layers
+        ), "index should be an integer in range [{}, {})".format(
+            -len(self), len(self)
+        )
+
+        index = self._get_abs_idx(index)
+        for i in range(len(self._sub_layers), index, -1):
+            self._sub_layers[str(i)] = self._sub_layers[str(i - 1)]
+        self._sub_layers[str(index)] = sublayer
+
+    def extend(self, sublayers):
+        """
+        Appends sublayers to the end of the list.
+
+        Parameters:
+            sublayers (iterable of Layer): iterable of sublayers to append
+
+        Examples:
+            .. code-block:: python
+
+                import paddle
+
+                linears = paddle.nn.LayerList([paddle.nn.Linear(10, 10) for i in range(10)])
+                another_list = paddle.nn.LayerList([paddle.nn.Linear(10, 10) for i in range(5)])
+                linears.extend(another_list)
+                print(len(linears))  # 15
+                print(another_list[0] is linears[10])  # True
+        """
+        offset = len(self)
+        for i, sublayer in enumerate(sublayers):
+            idx = str(offset + i)
+            self.add_sublayer(idx, sublayer)
+        return self

python/paddle/nn/layer/rnn.py

@@ -20,7 +20,8 @@ import paddle
 from paddle import framework
 from paddle.nn import functional as F
 from paddle.nn import initializer as I
-from paddle.nn import Layer, LayerList
+from paddle.nn import Layer
+from .container import LayerList
 from paddle.fluid.layers import utils
 from paddle.fluid.layers.utils import flatten, map_structure
 from paddle import _C_ops, _legacy_C_ops

python/paddle/nn/layer/transformer.py

@@ -24,7 +24,8 @@ from .norm import LayerNorm
 from .. import functional as F
 from ... import tensor
 from ...fluid import layers
-from .. import Layer, LayerList
+from .. import Layer
+from .container import LayerList
 from ...framework import ParamAttr
 from paddle.fluid.data_feeder import convert_dtype