diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/simnet_dygraph_model_v2.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/simnet_dygraph_model_v2.py index 6612450b7cff8782e4bba08897bd21bf975c29ca..66efb1cdf467bd0027dba0bb899424f0a54d6ad3 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/simnet_dygraph_model_v2.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/simnet_dygraph_model_v2.py @@ -41,7 +41,8 @@ class EmbeddingLayer(object): is_sparse=True, padding_idx=self.padding_idx, param_attr=paddle.ParamAttr( - name=self.name, initializer=paddle.nn.initializer.Xavier())) + name=self.name, + initializer=paddle.nn.initializer.XavierUniform())) return emb diff --git a/python/paddle/fluid/tests/unittests/test_initializer_nn.py b/python/paddle/fluid/tests/unittests/test_initializer_nn.py index edf52c6ff902fd4786e010c90aa3c21b79ff7267..ce72b5effbc514345020e77a1b870224e2d3ac00 100644 --- a/python/paddle/fluid/tests/unittests/test_initializer_nn.py +++ b/python/paddle/fluid/tests/unittests/test_initializer_nn.py @@ -27,6 +27,12 @@ from paddle.fluid.core import VarDesc DELTA = 0.00001 +def get_uniform_min_and_max(weight): + min_value = np.min(weight) + max_value = np.max(weight) + return min_value, max_value + + def check_cast_op(op): return op.type == 'cast' and \ op.attr('in_dtype') == VarDesc.VarType.FP32 and \ @@ -196,5 +202,485 @@ class TestKaimingInitializer(unittest.TestCase): is_conv=True) +class TestUniform(unittest.TestCase): + def test_uniform_common(self, dtype="float32", seed=0): + """Test the uniform initializer with default value + """ + paddle.enable_static() + + program = framework.Program() + program.random_seed = seed + block = program.global_block() + for _ in range(2): + block.create_parameter( + dtype=dtype, + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.Uniform()) + num_ops = 2 if dtype == "float16" else 1 + self.assertEqual(len(block.ops), num_ops) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'uniform_random') + self.assertAlmostEqual(init_op.attr('min'), -1.0, delta=DELTA) + self.assertAlmostEqual(init_op.attr('max'), 1.0, delta=DELTA) + self.assertEqual(init_op.attr('seed'), seed) + + paddle.disable_static() + + return block + + def test_uniform_initializer_default_value(self, + dtype="float32", + seed=0, + min_value=-1.0, + max_vlaue=1.0): + """Test the uniform initializer with default value + """ + paddle.enable_static() + + program = framework.Program() + program.random_seed = seed + block = program.global_block() + for _ in range(2): + block.create_parameter( + dtype=dtype, + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.Uniform()) + num_ops = 2 if dtype == "float16" else 1 + self.assertEqual(len(block.ops), num_ops) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'uniform_random') + self.assertAlmostEqual(init_op.attr('min'), min_value, delta=DELTA) + self.assertAlmostEqual(init_op.attr('max'), max_vlaue, delta=DELTA) + self.assertEqual(init_op.attr('seed'), seed) + + paddle.disable_static() + + return block + + def test_uniform_initializer(self, + dtype="float32", + seed=0, + min_value=-4.2, + max_vlaue=3.1): + """Test uniform initializer with supplied attributes + """ + paddle.enable_static() + + program = framework.Program() + program.random_seed = seed + block = program.global_block() + for _ in range(2): + block.create_parameter( + dtype=dtype, + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.Uniform(min_value, max_vlaue)) + num_ops = 2 if dtype == "float16" else 1 + self.assertEqual(len(block.ops), num_ops) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'uniform_random') + self.assertAlmostEqual(init_op.attr('min'), min_value, delta=DELTA) + self.assertAlmostEqual(init_op.attr('max'), max_vlaue, delta=DELTA) + + paddle.disable_static() + + return block + + def test_uniform_initializer_two_op(self, + dtype="float32", + seed=123, + min_value=-4.2, + max_vlaue=0.0): + """Test uniform initializer with supplied attributes + """ + paddle.enable_static() + + program = framework.Program() + program.random_seed = seed + block = program.global_block() + for i in range(2): + block.create_parameter( + dtype=dtype, + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.Uniform(min_value, float(i))) + num_ops = 2 if dtype == "float16" else 1 + self.assertEqual(len(block.ops), num_ops) + init_op0 = block.ops[0] + self.assertEqual(init_op0.type, 'uniform_random') + self.assertAlmostEqual(init_op0.attr('min'), min_value, delta=DELTA) + self.assertAlmostEqual(init_op0.attr('max'), 0.0, delta=DELTA) + self.assertEqual(init_op0.attr("seed"), seed) + + paddle.disable_static() + + return block + + def test_uniform_initializer_fp16(self): + """Test uniform initializer with float16 + """ + block = self.test_uniform_initializer_default_value("float16") + self.assertTrue(check_cast_op(block.ops[1])) + block = self.test_uniform_initializer(dtype="float16") + self.assertTrue(check_cast_op(block.ops[1])) + block = self.test_uniform_initializer_two_op("float16") + self.assertTrue(check_cast_op(block.ops[1])) + + def test_uniform_initializer_dygraph(self): + """Test uniform initializer in dygraph model. + """ + paddle.disable_static() + + weight_attr = paddle.framework.ParamAttr( + name="linear_weight", + initializer=paddle.nn.initializer.Uniform( + low=-0.5, high=0.5)) + linear = paddle.nn.Linear(2, 2, weight_attr=weight_attr) + + min_value, max_value = get_uniform_min_and_max(linear.weight.numpy()) + self.assertTrue(min_value >= -0.5, + 'min value {} should >= -0.5'.format(min_value)) + self.assertTrue(max_value <= 0.5, + 'max value {} should <= 0.5'.format(max_value)) + + +class TestNormal(unittest.TestCase): + def test_normal_initializer_default_value(self): + """Test the normal initializer with default value + """ + paddle.enable_static() + + program = framework.Program() + block = program.global_block() + for _ in range(2): + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.Normal()) + self.assertEqual(len(block.ops), 1) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'gaussian_random') + self.assertAlmostEqual(init_op.attr('mean'), 0.0, delta=DELTA) + self.assertAlmostEqual(init_op.attr('std'), 1.0, delta=DELTA) + self.assertEqual(init_op.attr('seed'), 0) + + paddle.disable_static() + + def test_normal_initializer(self, dtype="float32"): + """Test normal initializer with supplied attributes + """ + paddle.enable_static() + + program = framework.Program() + block = program.global_block() + for _ in range(2): + block.create_parameter( + dtype=dtype, + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.Normal(2.3, 1.9)) + num_ops = 2 if dtype == "float16" else 1 + self.assertEqual(len(block.ops), num_ops) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'gaussian_random') + self.assertAlmostEqual(init_op.attr('mean'), 2.3, delta=DELTA) + self.assertAlmostEqual(init_op.attr('std'), 1.9, delta=DELTA) + + paddle.disable_static() + + return block + + def test_normal_initializer_fp16(self): + """Test normal initializer with float16 + """ + block = self.test_normal_initializer("float16") + self.assertTrue(check_cast_op(block.ops[1])) + + def test_normal_initializer_dygraph(self): + """Test normal initializer in dygraph model. + """ + paddle.disable_static() + + weight_attr = paddle.framework.ParamAttr( + name="linear_weight", + initializer=paddle.nn.initializer.Normal( + mean=0.0, std=2.0)) + linear = paddle.nn.Linear(2, 2, weight_attr=weight_attr) + + +class TestTruncatedNormal(unittest.TestCase): + def test_truncated_normal_initializer_default_value(self): + """Test the truncated normal initializer with default value + """ + paddle.enable_static() + + program = framework.Program() + block = program.global_block() + for _ in range(2): + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.TruncatedNormal()) + self.assertEqual(len(block.ops), 1) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'truncated_gaussian_random') + self.assertAlmostEqual(init_op.attr('mean'), 0.0, delta=DELTA) + self.assertAlmostEqual(init_op.attr('std'), 1.0, delta=DELTA) + self.assertEqual(init_op.attr('seed'), 0) + + paddle.disable_static() + + def test_truncated_normal_initializer(self, dtype="float32"): + """Test truncated normal initializer with supplied attributes + """ + paddle.enable_static() + + program = framework.Program() + block = program.global_block() + for _ in range(2): + block.create_parameter( + dtype=dtype, + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.TruncatedNormal(2.3, 1.9)) + num_ops = 2 if dtype == "float16" else 1 + self.assertEqual(len(block.ops), num_ops) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'truncated_gaussian_random') + self.assertAlmostEqual(init_op.attr('mean'), 2.3, delta=DELTA) + self.assertAlmostEqual(init_op.attr('std'), 1.9, delta=DELTA) + + paddle.disable_static() + + return block + + def test_truncated_normal_initializer_fp16(self): + """Test truncated normal initializer with float16 + """ + paddle.enable_static() + + block = self.test_truncated_normal_initializer("float16") + self.assertTrue(check_cast_op(block.ops[1])) + + def test_truncated_normal_initializer_dygraph(self): + """Test truncated normal initializer in dygraph model. + """ + paddle.disable_static() + + weight_attr = paddle.framework.ParamAttr( + name="linear_weight", + initializer=paddle.nn.initializer.TruncatedNormal( + mean=0.0, std=2.0)) + linear = paddle.nn.Linear(2, 2, weight_attr=weight_attr) + + +class TestXavierUniform(unittest.TestCase): + def test_xavier_uniform_initializer(self): + """Test Xavier initializer with uniform distribution on + for matrix multiply. + """ + paddle.enable_static() + + program = framework.Program() + block = program.global_block() + for _ in range(2): + param = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.XavierUniform()) + self.assertEqual(len(block.ops), 1) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'uniform_random') + limit = np.sqrt(6.0 / (param.shape[0] + param.shape[1])) + self.assertAlmostEqual(init_op.attr('min'), -limit, delta=DELTA) + self.assertAlmostEqual(init_op.attr('max'), limit, delta=DELTA) + self.assertEqual(init_op.attr('seed'), 0) + + paddle.disable_static() + + def test_xavier_uniform_initializer_conv(self): + """Test Xavier initializer with uniform distribution on + for convolutions. + """ + paddle.enable_static() + + program = framework.Program() + block = program.global_block() + for _ in range(2): + param = block.create_parameter( + dtype="float32", + shape=[5, 10, 15, 20], + lod_level=0, + name="param", + initializer=initializer.XavierUniform()) + self.assertEqual(len(block.ops), 1) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'uniform_random') + receptive_field_size = float(15 * 20) + limit = np.sqrt(6.0 / ( + (param.shape[0] + param.shape[1]) * receptive_field_size)) + self.assertAlmostEqual(init_op.attr('min'), -limit, delta=DELTA) + self.assertAlmostEqual(init_op.attr('max'), limit, delta=DELTA) + self.assertEqual(init_op.attr('seed'), 0) + + def test_xavier_uniform_initializer_dygraph(self): + """Test xavier uniform initializer in dygraph model. + """ + paddle.disable_static() + + weight_attr = paddle.framework.ParamAttr( + name="linear_weight", + initializer=paddle.nn.initializer.XavierUniform()) + linear = paddle.nn.Linear(2, 2, weight_attr=weight_attr) + + +class TestXavierNormal(unittest.TestCase): + def test_xavier_normal_initializer(self): + """Test Xavier initializer with normal distribution on + for matrix multiply. + """ + paddle.enable_static() + + program = framework.Program() + block = program.global_block() + for _ in range(2): + param = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.XavierNormal()) + self.assertEqual(len(block.ops), 1) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'gaussian_random') + std = np.sqrt(2.0 / (param.shape[0] + param.shape[1])) + self.assertAlmostEqual(init_op.attr('mean'), 0.0, delta=DELTA) + self.assertAlmostEqual(init_op.attr('std'), std, delta=DELTA) + self.assertEqual(init_op.attr('seed'), 0) + + paddle.disable_static() + + def test_xavier_normal_initializer_conv(self): + """Test Xavier initializer with normal distribution on + for convolutions. + """ + paddle.enable_static() + + program = framework.Program() + block = program.global_block() + for _ in range(2): + param = block.create_parameter( + dtype="float32", + shape=[5, 10, 15, 20], + lod_level=0, + name="param", + initializer=initializer.XavierNormal()) + self.assertEqual(len(block.ops), 1) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'gaussian_random') + receptive_field_size = float(15 * 20) + std = np.sqrt(2.0 / ( + (param.shape[0] + param.shape[1]) * receptive_field_size)) + self.assertAlmostEqual(init_op.attr('mean'), 0.0, delta=DELTA) + self.assertAlmostEqual(init_op.attr('std'), std, delta=DELTA) + self.assertEqual(init_op.attr('seed'), 0) + + paddle.disable_static() + + def test_xavier_normal_initializer_dygraph(self): + """Test xavier normal initializer in dygraph model. + """ + paddle.disable_static() + + weight_attr = paddle.framework.ParamAttr( + name="linear_weight", + initializer=paddle.nn.initializer.XavierNormal()) + linear = paddle.nn.Linear(2, 2, weight_attr=weight_attr) + + +class TestAssign(unittest.TestCase): + def test_assign_initializer(self, dtype="float32"): + """Test the numpy array initializer with supplied arguments + """ + paddle.enable_static() + + import numpy + program = framework.Program() + block = program.global_block() + np_array = numpy.random.random((10000)).astype(dtype) + for _ in range(2): + block.create_parameter( + dtype=np_array.dtype, + shape=np_array.shape, + lod_level=0, + name="param", + initializer=initializer.Assign(np_array)) + num_ops = 2 if dtype == "float16" else 1 + self.assertEqual(len(block.ops), num_ops) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'assign_value') + assert (init_op.attr('fp32_values') == np_array).all() + + paddle.disable_static() + + return block + + def test_assign_initializer_fp16(self): + """Test the numpy array initializer with float16 + """ + block = self.test_assign_initializer("float16") + self.assertTrue(block.ops[1]) + + def test_assign_initializer_dygraph_1(self): + """Test assign initializer in dygraph model. + """ + paddle.disable_static() + + weight_attr_1 = paddle.framework.ParamAttr( + name="linear_weight_1", + initializer=paddle.nn.initializer.Assign(np.array([2, 2]))) + linear_1 = paddle.nn.Linear(2, 2, weight_attr=weight_attr_1) + + self.assertTrue((linear_1.weight.numpy() == [2.0, 2.0]).all(), '') + + def test_assign_initializer_dygraph_2(self): + """Test assign initializer in dygraph model. + """ + paddle.disable_static() + + weight_attr_2 = paddle.framework.ParamAttr( + name="linear_weight_2", + initializer=paddle.nn.initializer.Assign([2, 2])) + linear_2 = paddle.nn.Linear(2, 2, weight_attr=weight_attr_2) + + self.assertTrue((linear_2.weight.numpy() == [2.0, 2.0]).all(), '') + + def test_assign_initializer_dygraph_3(self): + """Test assign initializer in dygraph model. + """ + paddle.disable_static() + + weight_attr_3 = paddle.framework.ParamAttr( + name="linear_weight_3", + initializer=paddle.nn.initializer.Assign(paddle.full([2], 2))) + linear_3 = paddle.nn.Linear(2, 2, weight_attr=weight_attr_3) + + self.assertTrue((linear_3.weight.numpy() == [2.0, 2.0]).all(), '') + + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/nn/initializer/__init__.py b/python/paddle/nn/initializer/__init__.py index bd072164e1b45595548e02bce2d83c4e10bfe7ef..5d80386838435f84f87fb540bd8c71ae501413a2 100644 --- a/python/paddle/nn/initializer/__init__.py +++ b/python/paddle/nn/initializer/__init__.py @@ -14,10 +14,6 @@ # TODO: define the initializers to create a Parameter in neural network from ...fluid.initializer import Bilinear #DEFINE_ALIAS -from ...fluid.initializer import Normal #DEFINE_ALIAS -from ...fluid.initializer import TruncatedNormal #DEFINE_ALIAS -from ...fluid.initializer import Uniform #DEFINE_ALIAS -from ...fluid.initializer import Xavier #DEFINE_ALIAS from . import constant from .constant import Constant #DEFINE_ALIAS @@ -26,13 +22,26 @@ from . import kaiming from .kaiming import KaimingNormal #DEFINE_ALIAS from .kaiming import KaimingUniform #DEFINE_ALIAS -__all__ = [ - 'Bilinear', - 'Normal', - 'TruncatedNormal', - 'Uniform', - 'Xavier', -] +__all__ = ['Bilinear', ] __all__ += constant.__all__ __all__ += kaiming.__all__ + +from . import xavier +from .xavier import XavierNormal #DEFINE_ALIAS +from .xavier import XavierUniform #DEFINE_ALIAS + +from . import assign +from .assign import Assign #DEFINE_ALIAS + +from . import normal +from .normal import Normal #DEFINE_ALIAS +from .normal import TruncatedNormal #DEFINE_ALIAS + +from . import uniform +from .uniform import Uniform #DEFINE_ALIAS + +__all__ += xavier.__all__ +__all__ += assign.__all__ +__all__ += normal.__all__ +__all__ += uniform.__all__ diff --git a/python/paddle/nn/initializer/assign.py b/python/paddle/nn/initializer/assign.py new file mode 100644 index 0000000000000000000000000000000000000000..ac6dc3f0f31c7a682806d1cddf41646f43d1b612 --- /dev/null +++ b/python/paddle/nn/initializer/assign.py @@ -0,0 +1,100 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ...fluid import framework +from ...fluid import core +from ...fluid import unique_name +from ...fluid.core import VarDesc +from ...fluid.data_feeder import check_type +from ...fluid.initializer import NumpyArrayInitializer + +__all__ = ['Assign'] + + +class Assign(NumpyArrayInitializer): + """Init an parameter with a numpy array, list, or tensor. + + Args: + value (Tensor|numpy.ndarray|list): numpy array, list, or tensor to initialize the parameter. + name(str, optional): The default value is None. Normally there is no need for user to set this + property. For more information, please refer to :ref:`api_guide_Name`. + + Returns: + A parameter initialized by the input numpy array, list, or tensor. + + Examples: + .. code-block:: python + + import paddle + import numpy as np + + # numpy array + data_1 = paddle.ones(shape=[1, 2], dtype='float32') + weight_attr_1 = paddle.framework.ParamAttr( + name="linear_weight_1", + initializer=paddle.nn.initializer.Assign(np.array([2, 2]))) + bias_attr_1 = paddle.framework.ParamAttr( + name="linear_bias_1", + initializer=paddle.nn.initializer.Assign(np.array([2]))) + linear_1 = paddle.nn.Linear(2, 2, weight_attr=weight_attr_1, bias_attr=bias_attr_1) + # linear_1.weight: [2. 2.] + # linear_1.bias: [2.] + + res_1 = linear(data_1) + # res_1: [6.] + + # python list + data_2 = paddle.ones(shape=[1, 2], dtype='float32') + weight_attr_2 = paddle.framework.ParamAttr( + name="linear_weight_2", + initializer=paddle.nn.initializer.Assign([2, 2])) + bias_attr_2 = paddle.framework.ParamAttr( + name="linear_bias_2", + initializer=paddle.nn.initializer.Assign([2])) + linear_2 = paddle.nn.Linear(2, 2, weight_attr=weight_attr_2, bias_attr=bias_attr_2) + # linear_2.weight: [2. 2.] + # linear_2.bias: [2.] + + res_2 = linear(data_2) + # res_2: [6.] + + # tensor + data_3 = paddle.ones(shape=[1, 2], dtype='float32') + weight_attr_3 = paddle.framework.ParamAttr( + name="linear_weight_3", + initializer=paddle.nn.initializer.Assign(paddle.full([2], 2))) + bias_attr_3 = paddle.framework.ParamAttr( + name="linear_bias_3", + initializer=paddle.nn.initializer.Assign(paddle.full([1], 2))) + linear_3 = paddle.nn.Linear(2, 2, weight_attr=weight_attr_3, bias_attr=bias_attr_3) + # linear_3.weight: [2. 2.] + # linear_3.bias: [2.] + + res_3 = linear(data_3) + # res_3: [6.] + """ + + def __init__(self, value, name=None): + import numpy + check_type(value, 'value', (numpy.ndarray, list, framework.Variable), + 'Assign') + + if (isinstance(value, list)): + value = numpy.array(value) + + # TODO: value is already is a tensor, accounting efficiency maybe it does not need to convert tensor to numpy data and then initialized. + if (isinstance(value, framework.Variable)): + value = value.numpy() + + super(Assign, self).__init__(value) diff --git a/python/paddle/nn/initializer/normal.py b/python/paddle/nn/initializer/normal.py new file mode 100644 index 0000000000000000000000000000000000000000..a572d0e2c9216040e3ffa7e1c02841ebc1fc33ae --- /dev/null +++ b/python/paddle/nn/initializer/normal.py @@ -0,0 +1,100 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ...fluid.initializer import NormalInitializer +from ...fluid.initializer import TruncatedNormalInitializer + +__all__ = ['Normal', 'TruncatedNormal'] + + +class Normal(NormalInitializer): + """The Random Normal (Gaussian) distribution initializer. + + Args: + mean (float, optional): mean of the normal distribution. The default value is 0.0. + std (float, optional): standard deviation of the normal distribution. The default value is 1.0. + name(str, optional): The default value is None. Normally there is no need for user to set this + property. For more information, please refer to :ref:`api_guide_Name`. + + Returns: + A parameter initialized by Random Normal (Gaussian) distribution. + + Examples: + .. code-block:: python + + import paddle + + data = paddle.ones(shape=[3, 1, 2], dtype='float32') + weight_attr = paddle.framework.ParamAttr( + name="linear_weight", + initializer=paddle.nn.initializer.Normal(mean=0.0, std=2.0)) + bias_attr = paddle.framework.ParamAttr( + name="linear_bias", + initializer=paddle.nn.initializer.Normal(mean=0.0, std=2.0)) + linear = paddle.nn.Linear(2, 2, weight_attr=weight_attr, bias_attr=bias_attr) + # linear.weight: [[ 2.1973135 -2.2697184] + # [-1.9104223 -1.0541488]] + # linear.bias: [ 0.7885926 -0.74719954] + + res = linear(data) + # res: [[[ 1.0754838 -4.071067 ]] + # [[ 1.0754838 -4.071067 ]] + # [[ 1.0754838 -4.071067 ]]] + """ + + def __init__(self, mean=0.0, std=1.0, name=None): + assert mean is not None, 'mean should not be None' + assert std is not None, 'std should not be None' + super(Normal, self).__init__(loc=mean, scale=std, seed=0) + + +class TruncatedNormal(TruncatedNormalInitializer): + """The Random TruncatedNormal (Gaussian) distribution initializer. + + Args: + mean (float, optional): mean of the normal distribution. The default value is 0.0. + std (float, optional): standard deviation of the normal distribution. The default value is 1.0. + name(str, optional): The default value is None. Normally there is no need for user to set this + property. For more information, please refer to :ref:`api_guide_Name`. + + Returns: + A parameter initialized by Random TruncatedNormal (Gaussian) distribution. + + Examples: + .. code-block:: python + + import paddle + + data = paddle.ones(shape=[3, 1, 2], dtype='float32') + weight_attr = paddle.framework.ParamAttr( + name="linear_weight", + initializer=paddle.nn.initializer.TruncatedNormal(mean=0.0, std=2.0)) + bias_attr = paddle.framework.ParamAttr( + name="linear_bias", + initializer=paddle.nn.initializer.TruncatedNormal(mean=0.0, std=2.0)) + linear = paddle.nn.Linear(2, 2, weight_attr=weight_attr, bias_attr=bias_attr) + # linear.weight: [[-1.0981836 1.4140984] + # [ 3.1390522 -2.8266568]] + # linear.bias: [-2.1546738 -1.6570673] + + res = linear(data) + # res: [[[-0.11380529 -3.0696259 ]] + # [[-0.11380529 -3.0696259 ]] + # [[-0.11380529 -3.0696259 ]] + """ + + def __init__(self, mean=0.0, std=1.0, name=None): + assert mean is not None, 'mean should not be None' + assert std is not None, 'std should not be None' + super(TruncatedNormal, self).__init__(loc=mean, scale=std, seed=0) diff --git a/python/paddle/nn/initializer/uniform.py b/python/paddle/nn/initializer/uniform.py new file mode 100644 index 0000000000000000000000000000000000000000..a5d7d34efcf664a5bd46d7d3f06e2c542a8b4ef9 --- /dev/null +++ b/python/paddle/nn/initializer/uniform.py @@ -0,0 +1,60 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ...fluid.initializer import UniformInitializer + +__all__ = ['Uniform'] + + +class Uniform(UniformInitializer): + """The random uniform distribution initializer. + + Args: + low (float, optional): lower boundary of the uniform distribution. The default value is -1.0. + high (float, optional): upper boundary of the uniform distribution. The default value is 1.0. + name(str, optional): The default value is None. Normally there is no need for user to set this + property. For more information, please refer to :ref:`api_guide_Name`. + + Returns: + A parameter initialized by random uniform distribution. + + Examples: + .. code-block:: python + + import paddle + + data = paddle.ones(shape=[3, 1, 2], dtype='float32') + weight_attr = paddle.framework.ParamAttr( + name="linear_weight", + initializer=paddle.nn.initializer.Uniform(low=-0.5, high=0.5)) + bias_attr = paddle.framework.ParamAttr( + name="linear_bias", + initializer=paddle.nn.initializer.Uniform(low=-0.5, high=0.5)) + linear = paddle.nn.Linear(2, 2, weight_attr=weight_attr, bias_attr=bias_attr) + # linear.weight: [[-0.46245047 0.05260676] + # [ 0.38054508 0.29169726]] + # linear.bias: [-0.2734719 0.23939109] + + res = linear(data) + # res: [[[-0.3553773 0.5836951]] + # [[-0.3553773 0.5836951]] + # [[-0.3553773 0.5836951]]] + """ + + def __init__(self, low=-1.0, high=1.0, name=None): + assert low is not None, 'low should not be None' + assert high is not None, 'high should not be None' + assert high >= low, 'high should greater or equal than low' + super(Uniform, self).__init__( + low=low, high=high, seed=0, diag_num=0, diag_step=0, diag_val=1.0) diff --git a/python/paddle/nn/initializer/xavier.py b/python/paddle/nn/initializer/xavier.py new file mode 100644 index 0000000000000000000000000000000000000000..5a4e7fec057e71b2ac9285f4c388d6b08c176444 --- /dev/null +++ b/python/paddle/nn/initializer/xavier.py @@ -0,0 +1,124 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ...fluid.initializer import XavierInitializer + +__all__ = ['XavierNormal', 'XavierUniform'] + + +class XavierNormal(XavierInitializer): + """ + This class implements the Xavier weight initializer from the paper + `Understanding the difficulty of training deep feedforward neural + networks `_ + by Xavier Glorot and Yoshua Bengio, using a normal distribution. + + The mean is 0 and the standard deviation is + + .. math:: + + \sqrt{\\frac{2.0}{fan\_in + fan\_out}} + + + Args: + fan_in (float, optional): fan_in for Xavier initialization, It is + inferred from the tensor. The default value is None. + fan_out (float, optional): fan_out for Xavier initialization, it is + inferred from the tensor. The default value is None. + name(str, optional): The default value is None. Normally there is no need for user to set this + property. For more information, please refer to :ref:`api_guide_Name`. + + Returns: + A parameter initialized by Xavier weight, using a normal distribution. + + Examples: + .. code-block:: python + + import paddle + + data = paddle.ones(shape=[3, 1, 2], dtype='float32') + weight_attr = paddle.framework.ParamAttr( + name="linear_weight", + initializer=paddle.nn.initializer.XavierNormal()) + bias_attr = paddle.framework.ParamAttr( + name="linear_bias", + initializer=paddle.nn.initializer.XavierNormal()) + linear = paddle.nn.Linear(2, 2, weight_attr=weight_attr, bias_attr=bias_attr) + # inear.weight: [[ 0.06910077 -0.18103665] + # [-0.02546741 -1.0402188 ]] + # linear.bias: [-0.5012929 0.12418364] + + res = linear(data) + # res: [[[-0.4576595 -1.0970719]] + # [[-0.4576595 -1.0970719]] + # [[-0.4576595 -1.0970719]]] + """ + + def __init__(self, fan_in=None, fan_out=None, name=None): + super(XavierNormal, self).__init__( + uniform=False, fan_in=fan_in, fan_out=fan_out, seed=0) + + +class XavierUniform(XavierInitializer): + """ + This class implements the Xavier weight initializer from the paper + `Understanding the difficulty of training deep feedforward neural + networks `_ + by Xavier Glorot and Yoshua Bengio. + + This initializer is designed to keep the scale of the gradients + approximately same in all the layers. In case of Uniform distribution, + the range is [-x, x], where + + .. math:: + + x = \sqrt{\\frac{6.0}{fan\_in + fan\_out}} + + Args: + fan_in (float, optional): fan_in for Xavier initialization, it is + inferred from the tensor. The default value is None. + fan_out (float, optional): fan_out for Xavier initialization, it is + inferred from the tensor. The default value is None. + name(str, optional): The default value is None. Normally there is no need for user to set this + property. For more information, please refer to :ref:`api_guide_Name`. + + Returns: + A parameter initialized by Xavier weight, using a uniform distribution. + + Examples: + .. code-block:: python + + import paddle + + data = paddle.ones(shape=[3, 1, 2], dtype='float32') + weight_attr = paddle.framework.ParamAttr( + name="linear_weight", + initializer=paddle.nn.initializer.XavierUniform()) + bias_attr = paddle.framework.ParamAttr( + name="linear_bias", + initializer=paddle.nn.initializer.XavierUniform()) + linear = paddle.nn.Linear(2, 2, weight_attr=weight_attr, bias_attr=bias_attr) + # linear.weight: [[-0.04229349 -1.1248565 ] + # [-0.10789523 -0.5938053 ]] + # linear.bias: [ 1.1983747 -0.40201235] + + res = linear(data) + # res: [[[ 1.0481861 -2.1206741]] + # [[ 1.0481861 -2.1206741]] + # [[ 1.0481861 -2.1206741]]] + """ + + def __init__(self, fan_in=None, fan_out=None, name=None): + super(XavierUniform, self).__init__( + uniform=True, fan_in=fan_in, fan_out=fan_out, seed=0)