add new API: set_global_initializer (#24378)

* add new api (set_global_initializer/reset_global_initializer),test=develop

* add new api (set_global_initializer/reset_global_initializer),test=develop

* fix doc and example code of set_global_initializer,test=develop

python/paddle/fluid/__init__.py

@@ -51,6 +51,7 @@ from . import trainer_desc
 from . import io
 from . import evaluator
 from . import initializer
+from .initializer import set_global_initializer
 from . import layers
 from . import dygraph
 from . import contrib

python/paddle/fluid/initializer.py

@@ -26,9 +26,12 @@ __all__ = [
     'Constant', 'Uniform', 'Normal', 'TruncatedNormal', 'Xavier', 'Bilinear',
     'MSRA', 'ConstantInitializer', 'UniformInitializer', 'NormalInitializer',
     'TruncatedNormalInitializer', 'XavierInitializer', 'BilinearInitializer',
-    'MSRAInitializer', 'NumpyArrayInitializer'
+    'MSRAInitializer', 'NumpyArrayInitializer', 'set_global_initializer'
 ]
 
+_global_weight_initializer_ = None
+_global_bias_initializer_ = None
+
 
 class Initializer(object):
     """Base class for variable initializers
@@ -917,6 +920,78 @@ class NumpyArrayInitializer(Initializer):
         return op
 
 
+def set_global_initializer(weight_init, bias_init=None):
+    """
+    This API is used to set up global model parameter initializer in framework.
+
+    After this API is invoked, the global initializer will takes effect in subsequent code.
+
+    The model parameters include ``weight`` and ``bias`` . In the framework, they correspond 
+    to ``fluid.Parameter`` , which is inherited from ``fluid.Variable`` , and is a persistable Variable.
+    This API only takes effect for model parameters, not for variables created through apis such as 
+    :ref:`api_fluid_layers_create_global_var` , :ref:`api_fluid_layers_create_tensor`.
+    
+    If the initializer is also set up by ``param_attr`` or ``bias_attr`` when creating a network layer,
+    the global initializer setting here will not take effect because it has a lower priority.
+
+    If you want to cancel the global initializer in framework, please set global initializer to ``None`` .
+
+    Args:
+        weight_init (Initializer): set the global initializer for ``weight`` of model parameters.
+        bias_init (Initializer, optional): set the global initializer for ``bias`` of model parameters. 
+            Default: None.
+
+    Returns:
+        None
+
+    Examples:
+        .. code-block:: python
+            import paddle.fluid as fluid
+
+            fluid.set_global_initializer(fluid.initializer.Uniform(), fluid.initializer.Constant())
+            x = fluid.data(name="x", shape=[1, 3, 32, 32])
+
+            # The weight of conv1 is initialized by Uniform
+            # The bias of conv1 is initialized by Constant
+            conv1 = fluid.layers.conv2d(x, 5, 3)
+
+            # If set param_attr/bias_attr too, global initializer will not take effect
+            # The weight of conv2 is initialized by Xavier
+            # The bias of conv2 is initialized by Normal
+            conv2 = fluid.layers.conv2d(conv1, 5, 3, 
+                param_attr=fluid.initializer.Xavier(), 
+                bias_attr=fluid.initializer.Normal())
+
+            # Cancel the global initializer in framework, it will takes effect in subsequent code
+            fluid.set_global_initializer(None)
+
+
+    """
+    check_type(weight_init, 'weight_init', (Initializer, type(None)),
+               'set_global_initializer')
+    global _global_weight_initializer_
+    _global_weight_initializer_ = weight_init
+
+    check_type(bias_init, 'bias_init', (Initializer, type(None)),
+               'set_global_initializer')
+    global _global_bias_initializer_
+    _global_bias_initializer_ = bias_init
+
+
+def _global_weight_initializer():
+    """
+    Return the global weight initializer, The user doesn't need to use it.
+    """
+    return _global_weight_initializer_
+
+
+def _global_bias_initializer():
+    """
+    Return the global weight initializer, The user doesn't need to use it.
+    """
+    return _global_bias_initializer_
+
+
 # We short the class name, since users will use the initializer with the package
 # name. The sample code:
 #

python/paddle/fluid/layer_helper_base.py

@@ -21,6 +21,7 @@ from .framework import Variable, default_main_program, default_startup_program,
 from . import unique_name
 from .param_attr import ParamAttr, WeightNormParamAttr
 from . import core
+from .initializer import _global_weight_initializer, _global_bias_initializer
 
 
 class LayerHelperBase(object):
@@ -298,7 +299,15 @@ class LayerHelperBase(object):
         if not attr:
             return None
         assert isinstance(attr, ParamAttr)
-        suffix = 'b' if is_bias else 'w'
+        if is_bias:
+            suffix = 'b'
+            default_initializer = _global_bias_initializer(
+            ) if _global_bias_initializer() is not None else default_initializer
+        else:
+            suffix = 'w'
+            default_initializer = _global_weight_initializer(
+            ) if _global_weight_initializer(
+            ) is not None else default_initializer
         if attr.name is None:
             attr.name = unique_name.generate(".".join([self.name, suffix]))
 

python/paddle/fluid/tests/unittests/test_initializer.py

@@ -17,6 +17,7 @@ from __future__ import print_function
 import numpy as np
 import unittest
 
+import paddle.fluid as fluid
 import paddle.fluid.framework as framework
 import paddle.fluid.initializer as initializer
 from paddle.fluid.core import VarDesc
@@ -522,5 +523,65 @@ class TestNumpyArrayInitializer(unittest.TestCase):
         self.assertTrue(block.ops[1])
 
 
+class TestSetGlobalInitializer(unittest.TestCase):
+    def test_set_global_weight_initilizer(self):
+        """Test Set Global Param initilizer with UniformInitializer
+        """
+        main_prog = framework.Program()
+        startup_prog = framework.Program()
+        fluid.set_global_initializer(initializer.Uniform(low=-0.5, high=0.5))
+        with fluid.program_guard(main_prog, startup_prog):
+            x = fluid.data(name="x", shape=[1, 3, 32, 32])
+            # default initilizer of param in layers.conv2d is NormalInitializer
+            conv = fluid.layers.conv2d(x, 5, 3)
+
+        block = startup_prog.global_block()
+        self.assertEqual(len(block.ops), 2)
+
+        # init bias is the first op, and weight is the second
+        bias_init_op = block.ops[0]
+        self.assertEqual(bias_init_op.type, 'fill_constant')
+        self.assertAlmostEqual(bias_init_op.attr('value'), 0.0, delta=DELTA)
+
+        param_init_op = block.ops[1]
+        self.assertEqual(param_init_op.type, 'uniform_random')
+        self.assertAlmostEqual(param_init_op.attr('min'), -0.5, delta=DELTA)
+        self.assertAlmostEqual(param_init_op.attr('max'), 0.5, delta=DELTA)
+        self.assertEqual(param_init_op.attr('seed'), 0)
+        fluid.set_global_initializer(None)
+
+    def test_set_global_bias_initilizer(self):
+        """Test Set Global Bias initilizer with NormalInitializer
+        """
+        main_prog = framework.Program()
+        startup_prog = framework.Program()
+        fluid.set_global_initializer(
+            initializer.Uniform(
+                low=-0.5, high=0.5),
+            bias_init=initializer.Normal(
+                loc=0.0, scale=2.0))
+        with fluid.program_guard(main_prog, startup_prog):
+            x = fluid.data(name="x", shape=[1, 3, 32, 32])
+            # default initilizer of bias in layers.conv2d is ConstantInitializer
+            conv = fluid.layers.conv2d(x, 5, 3)
+
+        block = startup_prog.global_block()
+        self.assertEqual(len(block.ops), 2)
+
+        # init bias is the first op, and weight is the second
+        bias_init_op = block.ops[0]
+        self.assertEqual(bias_init_op.type, 'gaussian_random')
+        self.assertAlmostEqual(bias_init_op.attr('mean'), 0.0, delta=DELTA)
+        self.assertAlmostEqual(bias_init_op.attr('std'), 2.0, delta=DELTA)
+        self.assertEqual(bias_init_op.attr('seed'), 0)
+
+        param_init_op = block.ops[1]
+        self.assertEqual(param_init_op.type, 'uniform_random')
+        self.assertAlmostEqual(param_init_op.attr('min'), -0.5, delta=DELTA)
+        self.assertAlmostEqual(param_init_op.attr('max'), 0.5, delta=DELTA)
+        self.assertEqual(param_init_op.attr('seed'), 0)
+        fluid.set_global_initializer(None)
+
+
 if __name__ == '__main__':
     unittest.main()