add NumpyArrayInitializer and use it to refactor nce op

python/paddle/fluid/initializer.py

@@ -24,7 +24,8 @@ __all__ = [
     'Constant', 'Uniform', 'Normal', 'TruncatedNormal', 'Xavier', 'Bilinear',
     'MSRA', 'force_init_on_cpu', 'init_on_cpu', 'ConstantInitializer',
     'UniformInitializer', 'NormalInitializer', 'TruncatedNormalInitializer',
-    'XavierInitializer', 'BilinearInitializer', 'MSRAInitializer'
+    'XavierInitializer', 'BilinearInitializer', 'MSRAInitializer',
+    'NumpyArrayInitializer'
 ]
 
 _force_init_on_cpu_ = False
@@ -683,6 +684,64 @@ class BilinearInitializer(Initializer):
         return op
 
 
+class NumpyArrayInitializer(Initializer):
+    """Init an parameter with an numpy array
+
+    Args:
+        value (numpy): numpy array to initialize the variable
+
+    Examples:
+        .. code-block:: python
+
+            fc = fluid.layers.fc(input=x, size=10,
+                param_attr=fluid.initializer.NumpyArrayInitializer(numpy.array([1,2])))
+    """
+
+    def __init__(self, value):
+        import numpy
+        assert isinstance(value, numpy.ndarray)
+        super(NumpyArrayInitializer, self).__init__()
+        self._value = value
+
+    def __call__(self, var, block):
+        """Add constant initialization ops for a variable
+
+        Args:
+            var: Variable that needs to be initialized
+            block: The block in which initialization ops
+                   should be added
+
+        Returns:
+            the initialization op
+        """
+        assert isinstance(var, framework.Variable)
+        assert isinstance(block, framework.Block)
+        # Initialization Ops should be prepended and not appended
+        dtype = framework.convert_np_dtype_to_dtype_(self._value.dtype)
+        if dtype == VarDesc.VarType.FP32:
+            value_name = "fp32_values"
+            values = [float(v) for v in self._value.flat]
+        elif dtype == VarDesc.VarType.INT32:
+            value_name = "int32_values"
+            values = [int(v) for v in self._value.flat]
+        else:
+            raise ValueError("Unsupported dtype %s", self._value.dtype)
+        if self._value.size > 1024 * 1024 * 5:
+            raise ValueError("The size of input is too big. Please consider "
+                             "saving it to file and 'load_op' to load it")
+        op = block._prepend_op(
+            type='assign_value',
+            outputs={'Out': var},
+            attrs={
+                'dtype': dtype,
+                'shape': list(input.shape),
+                value_name: values
+            },
+            stop_gradient=True)
+        var.op = op
+        return op
+
+
 # We short the class name, since users will use the initializer with the package
 # name. The sample code:
 #

python/paddle/fluid/layers/nn.py

@@ -22,7 +22,7 @@ import six
 import os
 import inspect
 from ..layer_helper import LayerHelper
-from ..initializer import Normal, Constant
+from ..initializer import Normal, Constant, NumpyArrayInitializer
 from ..framework import Variable, OpProtoHolder
 from ..param_attr import ParamAttr
 from .layer_function_generator import autodoc, templatedoc, _generate_doc_string_
@@ -5181,16 +5181,21 @@ def nce(input,
             alias_probs_[little[0]] = 1.0
             alias_[little[0]] = -1
 
-        probs = assign(
-            input=np.array(custom_dist).astype('float32'), init_once=True)
-        custom_alias = assign(
-            input=np.array(alias_).astype('int32'), init_once=True)
-        custom_alias_probs = assign(
-            input=np.array(alias_probs_).astype('float32'), init_once=True)
-
-        inputs['CustomDistProbs'] = probs
-        inputs['CustomDistAlias'] = custom_alias
-        inputs['CustomDistAliasProbs'] = custom_alias_probs
+        def _init_by_numpy_array(numpy_array):
+            ret = helper.create_parameter(
+                attr=ParamAttr(),
+                shape=numpy_array.shape,
+                dtype=numpy_array.dtype,
+                default_initializer=NumpyArrayInitializer(numpy_array))
+            ret.stop_gradient = True
+            return ret
+
+        inputs['CustomDistProbs'] = _init_by_numpy_array(
+            np.array(custom_dist).astype('float32'))
+        inputs['CustomDistAlias'] = _init_by_numpy_array(
+            np.array(alias_).astype('int32'))
+        inputs['CustomDistAliasProbs'] = _init_by_numpy_array(
+            np.array(alias_probs_).astype('float32'))
         sampler = 2
     else:
         raise Exception("Unsupported sampler type.")

python/paddle/fluid/layers/tensor.py

@@ -291,7 +291,7 @@ def sums(input, out=None):
     return out
 
 
-def assign(input, output=None, init_once=False):
+def assign(input, output=None):
     """
     **Assign**
 
@@ -300,7 +300,6 @@ def assign(input, output=None, init_once=False):
     Args:
         input(Variable|numpy.ndarray): The source variable
         output(Variable|None): The destination variable
-        init_once(bool|false): assign value into global var only in startup program.
 
     Returns:
         Variable: The destination variable that was supplied as the *output*.
@@ -314,22 +313,10 @@ def assign(input, output=None, init_once=False):
     """
     helper = LayerHelper('assign', **locals())
     if output is None:
-        if init_once:
-            output = helper.create_parameter(
-                attr=ParamAttr(),
-                shape=input.shape,
-                dtype=input.dtype,
-                default_initializer=Constant(0.0))
-            output.stop_gradient = True
-        else:
-            output = helper.create_variable_for_type_inference(
-                dtype=input.dtype)
+        output = helper.create_variable_for_type_inference(dtype=input.dtype)
     if isinstance(input, Variable):
-        if init_once:
-            raise ValueError("init once only support numpy assign!")
         helper.append_op(
             type='assign', inputs={'X': [input]}, outputs={'Out': [output]})
-
     elif isinstance(input, numpy.ndarray):
         dtype = convert_np_dtype_to_dtype_(input.dtype)
         if dtype == VarDesc.VarType.FP32:
@@ -340,28 +327,18 @@ def assign(input, output=None, init_once=False):
             values = [int(v) for v in input.flat]
         else:
             raise ValueError("Unsupported dtype %s", input.dtype)
-        if input.size > 1024 * 1024 * 5:
+        if input.size > 1024 * 1024:
             raise ValueError("The size of input is too big. Please consider "
                              "saving it to file and 'load_op' to load it")
 
-        if init_once:
-            helper.startup_program.global_block().append_op(
-                type='assign_value',
-                outputs={'Out': [output]},
-                attrs={
-                    'dtype': dtype,
-                    'shape': list(input.shape),
-                    value_name: values
-                })
-        else:
-            helper.append_op(
-                type='assign_value',
-                outputs={'Out': [output]},
-                attrs={
-                    'dtype': dtype,
-                    'shape': list(input.shape),
-                    value_name: values
-                })
+        helper.append_op(
+            type='assign_value',
+            outputs={'Out': [output]},
+            attrs={
+                'dtype': dtype,
+                'shape': list(input.shape),
+                value_name: values
+            })
     else:
         raise ValueError("Wrong type for assign input: %s" % type(input))
 

python/paddle/fluid/tests/unittests/test_layers.py

@@ -1023,18 +1023,6 @@ class TestBook(unittest.TestCase):
 
         print(str(program))
 
-    def test_assign(self):
-        import numpy as np
-        startup = Program()
-        main = Program()
-        with program_guard(main, startup):
-            probs = layers.assign(
-                input=np.random.random([1, 2]).astype('float32'),
-                init_once=True)
-
-        print(str(main))
-        print(str(startup))
-
 
 if __name__ == '__main__':
     unittest.main()