diff --git a/python/paddle/fluid/__init__.py b/python/paddle/fluid/__init__.py index e136804f8a12b200e861abb3658a8f5672793c67..776a52b300fe0c7c582b59947e13e5ca98daf4e4 100644 --- a/python/paddle/fluid/__init__.py +++ b/python/paddle/fluid/__init__.py @@ -51,6 +51,7 @@ from . import trainer_desc from . import io from . import evaluator from . import initializer +from .initializer import set_global_initializer from . import layers from . import dygraph from . import contrib diff --git a/python/paddle/fluid/initializer.py b/python/paddle/fluid/initializer.py index 6c394f4c6955fdccae79fa7bca791f00bba41d38..ce975ea8423263bd099fda5b2f12add9aab088d4 100644 --- a/python/paddle/fluid/initializer.py +++ b/python/paddle/fluid/initializer.py @@ -26,9 +26,12 @@ __all__ = [ 'Constant', 'Uniform', 'Normal', 'TruncatedNormal', 'Xavier', 'Bilinear', 'MSRA', 'ConstantInitializer', 'UniformInitializer', 'NormalInitializer', 'TruncatedNormalInitializer', 'XavierInitializer', 'BilinearInitializer', - 'MSRAInitializer', 'NumpyArrayInitializer' + 'MSRAInitializer', 'NumpyArrayInitializer', 'set_global_initializer' ] +_global_weight_initializer_ = None +_global_bias_initializer_ = None + class Initializer(object): """Base class for variable initializers @@ -917,6 +920,78 @@ class NumpyArrayInitializer(Initializer): return op +def set_global_initializer(weight_init, bias_init=None): + """ + This API is used to set up global model parameter initializer in framework. + + After this API is invoked, the global initializer will takes effect in subsequent code. + + The model parameters include ``weight`` and ``bias`` . In the framework, they correspond + to ``fluid.Parameter`` , which is inherited from ``fluid.Variable`` , and is a persistable Variable. + This API only takes effect for model parameters, not for variables created through apis such as + :ref:`api_fluid_layers_create_global_var` , :ref:`api_fluid_layers_create_tensor`. + + If the initializer is also set up by ``param_attr`` or ``bias_attr`` when creating a network layer, + the global initializer setting here will not take effect because it has a lower priority. + + If you want to cancel the global initializer in framework, please set global initializer to ``None`` . + + Args: + weight_init (Initializer): set the global initializer for ``weight`` of model parameters. + bias_init (Initializer, optional): set the global initializer for ``bias`` of model parameters. + Default: None. + + Returns: + None + + Examples: + .. code-block:: python + import paddle.fluid as fluid + + fluid.set_global_initializer(fluid.initializer.Uniform(), fluid.initializer.Constant()) + x = fluid.data(name="x", shape=[1, 3, 32, 32]) + + # The weight of conv1 is initialized by Uniform + # The bias of conv1 is initialized by Constant + conv1 = fluid.layers.conv2d(x, 5, 3) + + # If set param_attr/bias_attr too, global initializer will not take effect + # The weight of conv2 is initialized by Xavier + # The bias of conv2 is initialized by Normal + conv2 = fluid.layers.conv2d(conv1, 5, 3, + param_attr=fluid.initializer.Xavier(), + bias_attr=fluid.initializer.Normal()) + + # Cancel the global initializer in framework, it will takes effect in subsequent code + fluid.set_global_initializer(None) + + + """ + check_type(weight_init, 'weight_init', (Initializer, type(None)), + 'set_global_initializer') + global _global_weight_initializer_ + _global_weight_initializer_ = weight_init + + check_type(bias_init, 'bias_init', (Initializer, type(None)), + 'set_global_initializer') + global _global_bias_initializer_ + _global_bias_initializer_ = bias_init + + +def _global_weight_initializer(): + """ + Return the global weight initializer, The user doesn't need to use it. + """ + return _global_weight_initializer_ + + +def _global_bias_initializer(): + """ + Return the global weight initializer, The user doesn't need to use it. + """ + return _global_bias_initializer_ + + # We short the class name, since users will use the initializer with the package # name. The sample code: # diff --git a/python/paddle/fluid/layer_helper_base.py b/python/paddle/fluid/layer_helper_base.py index dd83f2f04a2f82b55a845e5413e1a3d3e4a8627b..0b57b3fefd414c483c537957ed6ca3cfdd58fa65 100644 --- a/python/paddle/fluid/layer_helper_base.py +++ b/python/paddle/fluid/layer_helper_base.py @@ -21,6 +21,7 @@ from .framework import Variable, default_main_program, default_startup_program, from . import unique_name from .param_attr import ParamAttr, WeightNormParamAttr from . import core +from .initializer import _global_weight_initializer, _global_bias_initializer class LayerHelperBase(object): @@ -298,7 +299,15 @@ class LayerHelperBase(object): if not attr: return None assert isinstance(attr, ParamAttr) - suffix = 'b' if is_bias else 'w' + if is_bias: + suffix = 'b' + default_initializer = _global_bias_initializer( + ) if _global_bias_initializer() is not None else default_initializer + else: + suffix = 'w' + default_initializer = _global_weight_initializer( + ) if _global_weight_initializer( + ) is not None else default_initializer if attr.name is None: attr.name = unique_name.generate(".".join([self.name, suffix])) diff --git a/python/paddle/fluid/tests/unittests/test_initializer.py b/python/paddle/fluid/tests/unittests/test_initializer.py index 1b880569dfee0b7a20e783fae419e32db712121e..4c76af616f4a2b7a8b78967f7271270aa3875cad 100644 --- a/python/paddle/fluid/tests/unittests/test_initializer.py +++ b/python/paddle/fluid/tests/unittests/test_initializer.py @@ -17,6 +17,7 @@ from __future__ import print_function import numpy as np import unittest +import paddle.fluid as fluid import paddle.fluid.framework as framework import paddle.fluid.initializer as initializer from paddle.fluid.core import VarDesc @@ -522,5 +523,65 @@ class TestNumpyArrayInitializer(unittest.TestCase): self.assertTrue(block.ops[1]) +class TestSetGlobalInitializer(unittest.TestCase): + def test_set_global_weight_initilizer(self): + """Test Set Global Param initilizer with UniformInitializer + """ + main_prog = framework.Program() + startup_prog = framework.Program() + fluid.set_global_initializer(initializer.Uniform(low=-0.5, high=0.5)) + with fluid.program_guard(main_prog, startup_prog): + x = fluid.data(name="x", shape=[1, 3, 32, 32]) + # default initilizer of param in layers.conv2d is NormalInitializer + conv = fluid.layers.conv2d(x, 5, 3) + + block = startup_prog.global_block() + self.assertEqual(len(block.ops), 2) + + # init bias is the first op, and weight is the second + bias_init_op = block.ops[0] + self.assertEqual(bias_init_op.type, 'fill_constant') + self.assertAlmostEqual(bias_init_op.attr('value'), 0.0, delta=DELTA) + + param_init_op = block.ops[1] + self.assertEqual(param_init_op.type, 'uniform_random') + self.assertAlmostEqual(param_init_op.attr('min'), -0.5, delta=DELTA) + self.assertAlmostEqual(param_init_op.attr('max'), 0.5, delta=DELTA) + self.assertEqual(param_init_op.attr('seed'), 0) + fluid.set_global_initializer(None) + + def test_set_global_bias_initilizer(self): + """Test Set Global Bias initilizer with NormalInitializer + """ + main_prog = framework.Program() + startup_prog = framework.Program() + fluid.set_global_initializer( + initializer.Uniform( + low=-0.5, high=0.5), + bias_init=initializer.Normal( + loc=0.0, scale=2.0)) + with fluid.program_guard(main_prog, startup_prog): + x = fluid.data(name="x", shape=[1, 3, 32, 32]) + # default initilizer of bias in layers.conv2d is ConstantInitializer + conv = fluid.layers.conv2d(x, 5, 3) + + block = startup_prog.global_block() + self.assertEqual(len(block.ops), 2) + + # init bias is the first op, and weight is the second + bias_init_op = block.ops[0] + self.assertEqual(bias_init_op.type, 'gaussian_random') + self.assertAlmostEqual(bias_init_op.attr('mean'), 0.0, delta=DELTA) + self.assertAlmostEqual(bias_init_op.attr('std'), 2.0, delta=DELTA) + self.assertEqual(bias_init_op.attr('seed'), 0) + + param_init_op = block.ops[1] + self.assertEqual(param_init_op.type, 'uniform_random') + self.assertAlmostEqual(param_init_op.attr('min'), -0.5, delta=DELTA) + self.assertAlmostEqual(param_init_op.attr('max'), 0.5, delta=DELTA) + self.assertEqual(param_init_op.attr('seed'), 0) + fluid.set_global_initializer(None) + + if __name__ == '__main__': unittest.main()