From a914d68eb70c2fbb0cf5e044c8d110dd78a43e49 Mon Sep 17 00:00:00 2001 From: GGBond8488 <33050871+GGBond8488@users.noreply.github.com> Date: Wed, 23 Nov 2022 15:40:30 +0800 Subject: [PATCH] =?UTF-8?q?=E3=80=90fluid=20clean=E3=80=91=20move=20out=20?= =?UTF-8?q?LayerList,=20ParameterList,=20Sequential=20from=20fluid=20(#481?= =?UTF-8?q?97)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fluid clean move out LayerList, ParameterList, Sequential from fluid * copy Sequential from fluid to paddle.nn * replace fluid LayerList,ParamterList, Sequential in unit test file * add omit unit test file use paddle.nn.Sequential * remove numpy and use paddle.uniform instead --- .../slim/tests/imperative_test_utils.py | 2 +- .../slim/tests/test_imperative_out_scale.py | 2 +- .../contrib/slim/tests/test_imperative_qat.py | 2 +- .../slim/tests/test_imperative_qat_lsq.py | 2 +- python/paddle/fluid/dygraph/container.py | 242 -------------- .../fleet/hybrid_parallel_pp_embedding.py | 2 +- .../unittests/hybrid_parallel_pp_layer.py | 2 +- .../test_imperative_container_layerlist.py | 11 +- ...test_imperative_container_parameterlist.py | 12 +- .../test_imperative_named_members.py | 2 +- ...perative_star_gan_with_gradient_penalty.py | 8 +- python/paddle/nn/__init__.py | 6 +- python/paddle/nn/layer/container.py | 309 ++++++++++++++++++ python/paddle/nn/layer/rnn.py | 3 +- python/paddle/nn/layer/transformer.py | 3 +- 15 files changed, 329 insertions(+), 279 deletions(-) diff --git a/python/paddle/fluid/contrib/slim/tests/imperative_test_utils.py b/python/paddle/fluid/contrib/slim/tests/imperative_test_utils.py index 91c2dacfdd9..1a5f52b0406 100644 --- a/python/paddle/fluid/contrib/slim/tests/imperative_test_utils.py +++ b/python/paddle/fluid/contrib/slim/tests/imperative_test_utils.py @@ -17,7 +17,7 @@ import logging import paddle import paddle.fluid as fluid from paddle.fluid import core -from paddle.fluid.dygraph.container import Sequential +from paddle.nn import Sequential from paddle.nn import ReLU, ReLU6, LeakyReLU, Sigmoid, Softmax, PReLU from paddle.nn import Linear, Conv2D, Softmax, BatchNorm2D, MaxPool2D from paddle.nn import BatchNorm1D diff --git a/python/paddle/fluid/contrib/slim/tests/test_imperative_out_scale.py b/python/paddle/fluid/contrib/slim/tests/test_imperative_out_scale.py index 3dd871a64e2..02b19947ec9 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_imperative_out_scale.py +++ b/python/paddle/fluid/contrib/slim/tests/test_imperative_out_scale.py @@ -27,7 +27,7 @@ from paddle.fluid import core from paddle.fluid.optimizer import AdamOptimizer from paddle.fluid.framework import IrGraph, _test_eager_guard from paddle.fluid.contrib.slim.quantization import ImperativeQuantAware -from paddle.fluid.dygraph.container import Sequential +from paddle.nn import Sequential from paddle.fluid.dygraph.io import INFER_MODEL_SUFFIX, INFER_PARAMS_SUFFIX from paddle.nn.layer import ReLU, LeakyReLU, Sigmoid, Softmax, PReLU from paddle.nn import Linear, Conv2D, Softmax, BatchNorm2D, MaxPool2D diff --git a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat.py b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat.py index e037cea8f55..aff07fb397c 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat.py +++ b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat.py @@ -25,7 +25,7 @@ import paddle.fluid as fluid from paddle.fluid import core from paddle.fluid.optimizer import AdamOptimizer from paddle.fluid.contrib.slim.quantization import ImperativeQuantAware -from paddle.fluid.dygraph.container import Sequential +from paddle.nn import Sequential from paddle.nn import Linear, Conv2D, Softmax, Conv2DTranspose from paddle.fluid.log_helper import get_logger from paddle.fluid.dygraph.io import INFER_MODEL_SUFFIX, INFER_PARAMS_SUFFIX diff --git a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_lsq.py b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_lsq.py index bee201ee790..fbf1f4e7f51 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_lsq.py +++ b/python/paddle/fluid/contrib/slim/tests/test_imperative_qat_lsq.py @@ -29,7 +29,7 @@ from paddle.fluid.optimizer import ( MomentumOptimizer, ) from paddle.fluid.contrib.slim.quantization import ImperativeQuantAware -from paddle.fluid.dygraph.container import Sequential +from paddle.nn import Sequential from paddle.nn import ReLU, ReLU6, LeakyReLU, Sigmoid, Softmax, PReLU from paddle.nn import Linear, Conv2D, Softmax, BatchNorm2D, MaxPool2D from paddle.fluid.log_helper import get_logger diff --git a/python/paddle/fluid/dygraph/container.py b/python/paddle/fluid/dygraph/container.py index 254e13555b2..af0f043495c 100644 --- a/python/paddle/fluid/dygraph/container.py +++ b/python/paddle/fluid/dygraph/container.py @@ -12,15 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from collections import OrderedDict -from ..framework import Parameter from .layers import Layer -from .base import param_guard __all__ = [ 'Sequential', - 'ParameterList', - 'LayerList', ] @@ -97,240 +92,3 @@ class Sequential(Layer): for layer in self._sub_layers.values(): input = layer(input) return input - - -class ParameterList(Layer): - """ParameterList Container. - - This container acts like a Python list, but parameters it contains will be properly added. - - Parameters: - parameters (iterable, optional): Iterable Parameters to be added - - Examples: - .. code-block:: python - - import paddle - import numpy as np - - class MyLayer(paddle.nn.Layer): - def __init__(self, num_stacked_param): - super().__init__() - # create ParameterList with iterable Parameters - self.params = paddle.nn.ParameterList( - [paddle.create_parameter( - shape=[2, 2], dtype='float32')] * num_stacked_param) - - def forward(self, x): - for i, p in enumerate(self.params): - tmp = self._helper.create_variable_for_type_inference('float32') - self._helper.append_op( - type="mul", - inputs={"X": x, - "Y": p}, - outputs={"Out": tmp}, - attrs={"x_num_col_dims": 1, - "y_num_col_dims": 1}) - x = tmp - return x - - data_np = np.random.uniform(-1, 1, [5, 2]).astype('float32') - x = paddle.to_tensor(data_np) - num_stacked_param = 4 - model = MyLayer(num_stacked_param) - print(len(model.params)) # 4 - res = model(x) - print(res.shape) # [5, 2] - - replaced_param = paddle.create_parameter(shape=[2, 3], dtype='float32') - model.params[num_stacked_param - 1] = replaced_param # replace last param - res = model(x) - print(res.shape) # [5, 3] - model.params.append(paddle.create_parameter(shape=[3, 4], dtype='float32')) # append param - print(len(model.params)) # 5 - res = model(x) - print(res.shape) # [5, 4] - """ - - def __init__(self, parameters=None): - super().__init__() - if parameters is not None: - for idx, param in enumerate(parameters): - assert isinstance(param, Parameter) - self.add_parameter(str(idx), param) - - def __getitem__(self, idx): - with param_guard(self._parameters): - return self._parameters[str(idx)] - - def __setitem__(self, idx, param): - assert isinstance(param, Parameter) - setattr(self, str(idx), param) - - def __len__(self): - return len(self._parameters) - - def __iter__(self): - with param_guard(self._parameters): - return iter(self._parameters.values()) - - def append(self, parameter): - """Appends a given parameter at the end of the list. - - Parameters: - parameter (Parameter): parameter to append - """ - idx = len(self._parameters) - self.add_parameter(str(idx), parameter) - return self - - -class LayerList(Layer): - """ - LayerList holds sublayers, and sublayers it contains are properly registered. - Holded sublayers can be indexed like a regular python list. - - Parameters: - sublayers (iterable of Layer, optional): sublayers to hold - - Examples: - .. code-block:: python - - import paddle - import numpy as np - - class MyLayer(paddle.nn.Layer): - def __init__(self): - super().__init__() - self.linears = paddle.nn.LayerList( - [paddle.nn.Linear(10, 10) for i in range(10)]) - - def forward(self, x): - # LayerList can act as an iterable, or be indexed using ints - for i, l in enumerate(self.linears): - x = self.linears[i // 2](x) + l(x) - return x - """ - - def __init__(self, sublayers=None): - super().__init__() - if sublayers is not None: - for idx, layer in enumerate(sublayers): - self.add_sublayer(str(idx), layer) - - def _get_abs_idx(self, idx): - if isinstance(idx, int): - if not (-len(self) <= idx < len(self)): - raise IndexError( - 'index {} is out of range, should be an integer in range [{}, {})'.format( - idx, -len(self), len(self) - ) - ) - if idx < 0: - idx += len(self) - return idx - - def __getitem__(self, idx): - if isinstance(idx, slice): - return self.__class__(list(self._sub_layers.values())[idx]) - else: - idx = self._get_abs_idx(idx) - return self._sub_layers[str(idx)] - - def __setitem__(self, idx, sublayer): - idx = self._get_abs_idx(idx) - return setattr(self, str(idx), sublayer) - - def __delitem__(self, idx): - if isinstance(idx, slice): - for k in range(len(self._sub_layers))[idx]: - delattr(self, str(k)) - else: - idx = self._get_abs_idx(idx) - delattr(self, str(idx)) - str_indices = [str(i) for i in range(len(self._sub_layers))] - self._sub_layers = OrderedDict( - list(zip(str_indices, self._sub_layers.values())) - ) - - def __len__(self): - return len(self._sub_layers) - - def __iter__(self): - return iter(self._sub_layers.values()) - - def append(self, sublayer): - """ - Appends a sublayer to the end of the list. - - Parameters: - sublayer (Layer): sublayer to append - - Examples: - .. code-block:: python - - import paddle - - linears = paddle.nn.LayerList([paddle.nn.Linear(10, 10) for i in range(10)]) - another = paddle.nn.Linear(10, 10) - linears.append(another) - print(len(linears)) # 11 - """ - self.add_sublayer(str(len(self)), sublayer) - return self - - def insert(self, index, sublayer): - """ - Insert a sublayer before a given index in the list. - - Parameters: - index (int): index to insert. - sublayer (Layer): sublayer to insert - - Examples: - .. code-block:: python - - import paddle - - linears = paddle.nn.LayerList([paddle.nn.Linear(10, 10) for i in range(10)]) - another = paddle.nn.Linear(10, 10) - linears.insert(3, another) - print(linears[3] is another) # True - another = paddle.nn.Linear(10, 10) - linears.insert(-1, another) - print(linears[-2] is another) # True - """ - assert isinstance(index, int) and -len(self._sub_layers) <= index < len( - self._sub_layers - ), "index should be an integer in range [{}, {})".format( - -len(self), len(self) - ) - - index = self._get_abs_idx(index) - for i in range(len(self._sub_layers), index, -1): - self._sub_layers[str(i)] = self._sub_layers[str(i - 1)] - self._sub_layers[str(index)] = sublayer - - def extend(self, sublayers): - """ - Appends sublayers to the end of the list. - - Parameters: - sublayers (iterable of Layer): iterable of sublayers to append - - Examples: - .. code-block:: python - - import paddle - - linears = paddle.nn.LayerList([paddle.nn.Linear(10, 10) for i in range(10)]) - another_list = paddle.nn.LayerList([paddle.nn.Linear(10, 10) for i in range(5)]) - linears.extend(another_list) - print(len(linears)) # 15 - print(another_list[0] is linears[10]) # True - """ - offset = len(self) - for i, sublayer in enumerate(sublayers): - idx = str(offset + i) - self.add_sublayer(idx, sublayer) - return self diff --git a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_embedding.py b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_embedding.py index 33833454c01..99bb63cd969 100644 --- a/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_embedding.py +++ b/python/paddle/fluid/tests/unittests/collective/fleet/hybrid_parallel_pp_embedding.py @@ -19,7 +19,7 @@ import random import paddle import paddle.distributed as dist import paddle.distributed.fleet as fleet -from paddle.fluid.dygraph.container import Sequential +from paddle.nn import Sequential from paddle.distributed.fleet.meta_parallel import PipelineLayer from paddle.fluid.dygraph.layers import Layer import paddle.nn as nn diff --git a/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_layer.py b/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_layer.py index d073d7267a2..b9d8db5aa87 100644 --- a/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_layer.py +++ b/python/paddle/fluid/tests/unittests/hybrid_parallel_pp_layer.py @@ -15,7 +15,7 @@ import unittest import numpy as np from paddle.distributed import fleet -from paddle.fluid.dygraph.container import Sequential +from paddle.nn import Sequential import paddle.nn as nn from paddle.fluid.dygraph.layers import Layer from paddle.distributed.fleet.meta_parallel import LayerDesc, PipelineLayer diff --git a/python/paddle/fluid/tests/unittests/test_imperative_container_layerlist.py b/python/paddle/fluid/tests/unittests/test_imperative_container_layerlist.py index f0ac7ee14c9..db86295a5aa 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_container_layerlist.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_container_layerlist.py @@ -31,11 +31,6 @@ class MyLayer(fluid.Layer): class TestImperativeContainer(unittest.TestCase): - def fluid_dygraph_list(self): - return fluid.dygraph.LayerList( - [fluid.dygraph.Linear(2**i, 2 ** (i + 1)) for i in range(6)] - ) - def paddle_imperative_list(self): return paddle.nn.LayerList( [fluid.dygraph.Linear(2**i, 2 ** (i + 1)) for i in range(6)] @@ -45,11 +40,7 @@ class TestImperativeContainer(unittest.TestCase): data_np = np.random.uniform(-1, 1, [5, 1]).astype('float32') with fluid.dygraph.guard(): x = fluid.dygraph.to_variable(data_np) - layerlist = ( - self.fluid_dygraph_list() - if use_fluid_api - else self.paddle_imperative_list() - ) + layerlist = self.paddle_imperative_list() size = len(layerlist) model = MyLayer(layerlist) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_container_parameterlist.py b/python/paddle/fluid/tests/unittests/test_imperative_container_parameterlist.py index f1cd45904de..a8739274122 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_container_parameterlist.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_container_parameterlist.py @@ -24,17 +24,7 @@ class MyLayer(fluid.Layer): def __init__(self, num_stacked_param, use_fluid_api): super().__init__() # create ParameterList with iterable Parameters - self.params = ( - self.fluid_dygraph_ParameterList(num_stacked_param) - if use_fluid_api - else self.paddle_imperative_ParameterList(num_stacked_param) - ) - - def fluid_dygraph_ParameterList(self, num_stacked_param): - return fluid.dygraph.ParameterList( - [fluid.layers.create_parameter(shape=[2, 2], dtype='float32')] - * num_stacked_param - ) + self.params = self.paddle_imperative_ParameterList(num_stacked_param) def paddle_imperative_ParameterList(self, num_stacked_param): return paddle.nn.ParameterList( diff --git a/python/paddle/fluid/tests/unittests/test_imperative_named_members.py b/python/paddle/fluid/tests/unittests/test_imperative_named_members.py index c5e39d330d1..044661940cb 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_named_members.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_named_members.py @@ -37,7 +37,7 @@ class TestImperativeNamedSubLayers(unittest.TestCase): fc1 = fluid.Linear(10, 3) fc2 = fluid.Linear(3, 10, bias_attr=False) custom = MyLayer(3, 10) - model = fluid.dygraph.Sequential(fc1, fc2, custom) + model = paddle.nn.Sequential(fc1, fc2, custom) named_sublayers = model.named_sublayers() list_named_sublayers = list(named_sublayers) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_star_gan_with_gradient_penalty.py b/python/paddle/fluid/tests/unittests/test_imperative_star_gan_with_gradient_penalty.py index 4a99e0fb63b..a75bc4b8a8e 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_star_gan_with_gradient_penalty.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_star_gan_with_gradient_penalty.py @@ -268,7 +268,7 @@ class Generator(fluid.dygraph.Layer): cur_channels *= 2 sub_layers.append(sub_layer) - self._conv0 = fluid.dygraph.Sequential(*sub_layers) + self._conv0 = paddle.nn.Sequential(*sub_layers) repeat_num = cfg.g_repeat_num sub_layers = [] @@ -278,7 +278,7 @@ class Generator(fluid.dygraph.Layer): ) sub_layers.append(res_block) - self._res_block = fluid.dygraph.Sequential(*sub_layers) + self._res_block = paddle.nn.Sequential(*sub_layers) cur_channels = cfg.g_base_dims * 4 sub_layers = [] @@ -296,7 +296,7 @@ class Generator(fluid.dygraph.Layer): cur_channels = cfg.g_base_dims * rate sub_layers.append(deconv) - self._deconv = fluid.dygraph.Sequential(*sub_layers) + self._deconv = paddle.nn.Sequential(*sub_layers) self._conv1 = Conv2DLayer( num_channels=cur_channels, @@ -353,7 +353,7 @@ class Discriminator(fluid.dygraph.Layer): cur_dim *= 2 sub_layers.append(sub_layer) - self._conv0 = fluid.dygraph.Sequential(*sub_layers) + self._conv0 = paddle.nn.Sequential(*sub_layers) kernel_size = int(cfg.image_size / np.power(2, repeat_num)) diff --git a/python/paddle/nn/__init__.py b/python/paddle/nn/__init__.py index 8e02cdffd5e..a4aaa18ea55 100644 --- a/python/paddle/nn/__init__.py +++ b/python/paddle/nn/__init__.py @@ -15,9 +15,9 @@ # TODO: import all neural network related api under this directory, # including layers, linear, conv, rnn etc. from ..fluid.dygraph.layers import Layer # noqa: F401 -from ..fluid.dygraph.container import LayerList # noqa: F401 -from ..fluid.dygraph.container import ParameterList # noqa: F401 -from ..fluid.dygraph.container import Sequential # noqa: F401 +from .layer.container import LayerList # noqa: F401 +from .layer.container import ParameterList # noqa: F401 +from .layer.container import Sequential # noqa: F401 from .clip import ClipGradByGlobalNorm # noqa: F401 from .clip import ClipGradByNorm # noqa: F401 diff --git a/python/paddle/nn/layer/container.py b/python/paddle/nn/layer/container.py index d70d8fec2ca..0a8e5ec009d 100644 --- a/python/paddle/nn/layer/container.py +++ b/python/paddle/nn/layer/container.py @@ -15,6 +15,8 @@ from collections import OrderedDict from .. import Layer from collections.abc import Iterable, Mapping +from ...fluid.framework import Parameter +from ...fluid.dygraph.base import param_guard __all__ = [] @@ -295,3 +297,310 @@ class LayerDict(Layer): + ", which must be 2." ) self.add_sublayer(kv[0], kv[1]) + + +class Sequential(Layer): + """Sequential container. + Sub layers will be added to this container in the order of argument in the constructor. + The argument passed to the constructor can be iterable Layers or iterable name Layer pairs. + + Parameters: + layers(Layer|list|tuple): Layer or list/tuple of iterable name Layer pair. + + Examples: + .. code-block:: python + + import paddle + + data = paddle.uniform(shape=[30, 10], dtype='float32') + # create Sequential with iterable Layers + model1 = paddle.nn.Sequential( + paddle.nn.Linear(10, 1), paddle.nn.Linear(1, 2) + ) + model1[0] # access the first layer + res1 = model1(data) # sequential execution + + # create Sequential with name Layer pairs + model2 = paddle.nn.Sequential( + ('l1', paddle.nn.Linear(10, 2)), + ('l2', paddle.nn.Linear(2, 3)) + ) + model2['l1'] # access l1 layer + model2.add_sublayer('l3', paddle.nn.Linear(3, 3)) # add sublayer + res2 = model2(data) # sequential execution + + """ + + def __init__(self, *layers): + super().__init__() + if len(layers) > 0 and isinstance(layers[0], (list, tuple)): + for name, layer in layers: + self.add_sublayer(name, layer) + else: + for idx, layer in enumerate(layers): + self.add_sublayer(str(idx), layer) + + def __getitem__(self, name): + if isinstance(name, slice): + return self.__class__(*(list(self._sub_layers.values())[name])) + elif isinstance(name, str): + return self._sub_layers[name] + else: + if name >= len(self._sub_layers): + raise IndexError('index {} is out of range'.format(name)) + elif name < 0 and name >= -len(self._sub_layers): + name += len(self._sub_layers) + elif name < -len(self._sub_layers): + raise IndexError('index {} is out of range'.format(name)) + return list(self._sub_layers.values())[name] + + def __setitem__(self, name, layer): + assert isinstance(layer, Layer) + setattr(self, str(name), layer) + + def __delitem__(self, name): + name = str(name) + assert name in self._sub_layers + del self._sub_layers[name] + + def __len__(self): + return len(self._sub_layers) + + def forward(self, input): + for layer in self._sub_layers.values(): + input = layer(input) + return input + + +class ParameterList(Layer): + """ParameterList Container. + + This container acts like a Python list, but parameters it contains will be properly added. + + Parameters: + parameters (iterable, optional): Iterable Parameters to be added + + Examples: + .. code-block:: python + + import paddle + + class MyLayer(paddle.nn.Layer): + def __init__(self, num_stacked_param): + super().__init__() + # create ParameterList with iterable Parameters + self.params = paddle.nn.ParameterList( + [paddle.create_parameter( + shape=[2, 2], dtype='float32')] * num_stacked_param) + + def forward(self, x): + for i, p in enumerate(self.params): + tmp = self._helper.create_variable_for_type_inference('float32') + self._helper.append_op( + type="mul", + inputs={"X": x, + "Y": p}, + outputs={"Out": tmp}, + attrs={"x_num_col_dims": 1, + "y_num_col_dims": 1}) + x = tmp + return x + + x = paddle.uniform(shape=[5, 2], dtype='float32') + num_stacked_param = 4 + model = MyLayer(num_stacked_param) + print(len(model.params)) # 4 + res = model(x) + print(res.shape) # [5, 2] + + replaced_param = paddle.create_parameter(shape=[2, 3], dtype='float32') + model.params[num_stacked_param - 1] = replaced_param # replace last param + res = model(x) + print(res.shape) # [5, 3] + model.params.append(paddle.create_parameter(shape=[3, 4], dtype='float32')) # append param + print(len(model.params)) # 5 + res = model(x) + print(res.shape) # [5, 4] + """ + + def __init__(self, parameters=None): + super().__init__() + if parameters is not None: + for idx, param in enumerate(parameters): + assert isinstance(param, Parameter) + self.add_parameter(str(idx), param) + + def __getitem__(self, idx): + with param_guard(self._parameters): + return self._parameters[str(idx)] + + def __setitem__(self, idx, param): + assert isinstance(param, Parameter) + setattr(self, str(idx), param) + + def __len__(self): + return len(self._parameters) + + def __iter__(self): + with param_guard(self._parameters): + return iter(self._parameters.values()) + + def append(self, parameter): + """Appends a given parameter at the end of the list. + + Parameters: + parameter (Parameter): parameter to append + """ + idx = len(self._parameters) + self.add_parameter(str(idx), parameter) + return self + + +class LayerList(Layer): + """ + LayerList holds sublayers, and sublayers it contains are properly registered. + Holded sublayers can be indexed like a regular python list. + + Parameters: + sublayers (iterable of Layer, optional): sublayers to hold + + Examples: + .. code-block:: python + + import paddle + + class MyLayer(paddle.nn.Layer): + def __init__(self): + super().__init__() + self.linears = paddle.nn.LayerList( + [paddle.nn.Linear(10, 10) for i in range(10)]) + + def forward(self, x): + # LayerList can act as an iterable, or be indexed using ints + for i, l in enumerate(self.linears): + x = self.linears[i // 2](x) + l(x) + return x + """ + + def __init__(self, sublayers=None): + super().__init__() + if sublayers is not None: + for idx, layer in enumerate(sublayers): + self.add_sublayer(str(idx), layer) + + def _get_abs_idx(self, idx): + if isinstance(idx, int): + if not (-len(self) <= idx < len(self)): + raise IndexError( + 'index {} is out of range, should be an integer in range [{}, {})'.format( + idx, -len(self), len(self) + ) + ) + if idx < 0: + idx += len(self) + return idx + + def __getitem__(self, idx): + if isinstance(idx, slice): + return self.__class__(list(self._sub_layers.values())[idx]) + else: + idx = self._get_abs_idx(idx) + return self._sub_layers[str(idx)] + + def __setitem__(self, idx, sublayer): + idx = self._get_abs_idx(idx) + return setattr(self, str(idx), sublayer) + + def __delitem__(self, idx): + if isinstance(idx, slice): + for k in range(len(self._sub_layers))[idx]: + delattr(self, str(k)) + else: + idx = self._get_abs_idx(idx) + delattr(self, str(idx)) + str_indices = [str(i) for i in range(len(self._sub_layers))] + self._sub_layers = OrderedDict( + list(zip(str_indices, self._sub_layers.values())) + ) + + def __len__(self): + return len(self._sub_layers) + + def __iter__(self): + return iter(self._sub_layers.values()) + + def append(self, sublayer): + """ + Appends a sublayer to the end of the list. + + Parameters: + sublayer (Layer): sublayer to append + + Examples: + .. code-block:: python + + import paddle + + linears = paddle.nn.LayerList([paddle.nn.Linear(10, 10) for i in range(10)]) + another = paddle.nn.Linear(10, 10) + linears.append(another) + print(len(linears)) # 11 + """ + self.add_sublayer(str(len(self)), sublayer) + return self + + def insert(self, index, sublayer): + """ + Insert a sublayer before a given index in the list. + + Parameters: + index (int): index to insert. + sublayer (Layer): sublayer to insert + + Examples: + .. code-block:: python + + import paddle + + linears = paddle.nn.LayerList([paddle.nn.Linear(10, 10) for i in range(10)]) + another = paddle.nn.Linear(10, 10) + linears.insert(3, another) + print(linears[3] is another) # True + another = paddle.nn.Linear(10, 10) + linears.insert(-1, another) + print(linears[-2] is another) # True + """ + assert isinstance(index, int) and -len(self._sub_layers) <= index < len( + self._sub_layers + ), "index should be an integer in range [{}, {})".format( + -len(self), len(self) + ) + + index = self._get_abs_idx(index) + for i in range(len(self._sub_layers), index, -1): + self._sub_layers[str(i)] = self._sub_layers[str(i - 1)] + self._sub_layers[str(index)] = sublayer + + def extend(self, sublayers): + """ + Appends sublayers to the end of the list. + + Parameters: + sublayers (iterable of Layer): iterable of sublayers to append + + Examples: + .. code-block:: python + + import paddle + + linears = paddle.nn.LayerList([paddle.nn.Linear(10, 10) for i in range(10)]) + another_list = paddle.nn.LayerList([paddle.nn.Linear(10, 10) for i in range(5)]) + linears.extend(another_list) + print(len(linears)) # 15 + print(another_list[0] is linears[10]) # True + """ + offset = len(self) + for i, sublayer in enumerate(sublayers): + idx = str(offset + i) + self.add_sublayer(idx, sublayer) + return self diff --git a/python/paddle/nn/layer/rnn.py b/python/paddle/nn/layer/rnn.py index aeac50d0680..c93dcae4ac0 100644 --- a/python/paddle/nn/layer/rnn.py +++ b/python/paddle/nn/layer/rnn.py @@ -20,7 +20,8 @@ import paddle from paddle import framework from paddle.nn import functional as F from paddle.nn import initializer as I -from paddle.nn import Layer, LayerList +from paddle.nn import Layer +from .container import LayerList from paddle.fluid.layers import utils from paddle.fluid.layers.utils import flatten, map_structure from paddle import _C_ops, _legacy_C_ops diff --git a/python/paddle/nn/layer/transformer.py b/python/paddle/nn/layer/transformer.py index af7fc19794a..52cffc4998a 100644 --- a/python/paddle/nn/layer/transformer.py +++ b/python/paddle/nn/layer/transformer.py @@ -24,7 +24,8 @@ from .norm import LayerNorm from .. import functional as F from ... import tensor from ...fluid import layers -from .. import Layer, LayerList +from .. import Layer +from .container import LayerList from ...framework import ParamAttr from paddle.fluid.data_feeder import convert_dtype -- GitLab