From db3b9438b7d273198dda76f6b30ab5bb678d2778 Mon Sep 17 00:00:00 2001 From: Abhinav Arora Date: Tue, 31 Oct 2017 13:28:48 -0700 Subject: [PATCH] Adding Normal distribution initializer and unit tests for python initializers (#5256) --- paddle/operators/gaussian_random_op.cc | 12 +- python/paddle/v2/framework/initializer.py | 51 +++++++- .../tests/test_gaussian_random_op.py | 2 +- .../v2/framework/tests/test_initializer.py | 120 ++++++++++++++++++ 4 files changed, 177 insertions(+), 8 deletions(-) create mode 100644 python/paddle/v2/framework/tests/test_initializer.py diff --git a/paddle/operators/gaussian_random_op.cc b/paddle/operators/gaussian_random_op.cc index 04dfdf7c48..be7f542a7a 100644 --- a/paddle/operators/gaussian_random_op.cc +++ b/paddle/operators/gaussian_random_op.cc @@ -45,14 +45,14 @@ class GaussianRandomOp : public framework::OperatorWithKernel { void InferShape(framework::InferShapeContext* ctx) const override { PADDLE_ENFORCE(ctx->HasOutput("Out"), "Output(Out) of GaussianRandomOp should not be null."); - auto dims = ctx->Attrs().Get>("dims"); + auto shape = ctx->Attrs().Get>("shape"); std::vector temp; - temp.reserve(dims.size()); - for (auto dim : dims) { + temp.reserve(shape.size()); + for (auto dim : shape) { temp.push_back(static_cast(dim)); } - PADDLE_ENFORCE(dims.size() > 0UL, - "dims can be one int or array. dims must be set."); + PADDLE_ENFORCE(shape.size() > 0UL, + "shape can be one int or array. shape must be set."); ctx->SetOutputDim("Out", framework::make_ddim(temp)); } @@ -74,7 +74,7 @@ GaussianRandom operator. Use to initialize tensor with gaussian random generator. )DOC"); - AddAttr>("dims", "The dimension of random tensor."); + AddAttr>("shape", "The dimension of random tensor."); AddAttr("mean", "mean of random tensor.").SetDefault(.0f); AddAttr("std", "std of random tensor.").SetDefault(1.0f); AddAttr("seed", diff --git a/python/paddle/v2/framework/initializer.py b/python/paddle/v2/framework/initializer.py index 377d332713..507fd16062 100644 --- a/python/paddle/v2/framework/initializer.py +++ b/python/paddle/v2/framework/initializer.py @@ -62,7 +62,7 @@ class ConstantInitializer(Initializer): class UniformInitializer(Initializer): - """Implements for random uniform distribution initializer + """Implements the random uniform distribution initializer """ def __init__(self, low=-1.0, high=1.0, seed=0): @@ -75,6 +75,7 @@ class UniformInitializer(Initializer): """ assert low is not None assert high is not None + assert high >= low assert seed is not None super(UniformInitializer, self).__init__() self._low = low @@ -107,3 +108,51 @@ class UniformInitializer(Initializer): }) var.op = op return op + + +class NormalInitializer(Initializer): + """Implements the random Normal(Gaussian) distribution initializer + """ + + def __init__(self, loc=0.0, scale=1.0, seed=0): + """Constructor for NormalInitializer + + Args: + loc: mean of the normal distribution + scale: standard deviation of the normal distribution + seed: random seed + """ + assert loc is not None + assert scale is not None + assert seed is not None + super(NormalInitializer, self).__init__() + self._mean = loc + self._std_dev = scale + self._seed = seed + + def __call__(self, var, block): + """Add normal distribution initialization ops for a variable + + Args: + var: Variable that needs to be initialized + block: The block in which initialization ops + should be added + + Returns: + the initialization op + """ + assert isinstance(var, framework.Variable) + assert isinstance(block, framework.Block) + # Initialization Ops should be prepended and not appended + op = block.prepend_op( + type="gaussian_random", + outputs={"Out": var}, + attrs={ + "shape": var.shape, + "data_type": int(var.data_type), + "mean": self._mean, + "std": self._std_dev, + "seed": self._seed + }) + var.op = op + return op diff --git a/python/paddle/v2/framework/tests/test_gaussian_random_op.py b/python/paddle/v2/framework/tests/test_gaussian_random_op.py index 8b7779667d..0dc7e091a5 100644 --- a/python/paddle/v2/framework/tests/test_gaussian_random_op.py +++ b/python/paddle/v2/framework/tests/test_gaussian_random_op.py @@ -19,7 +19,7 @@ class TestGaussianRandomOp(unittest.TestCase): op = Operator( "gaussian_random", Out='Out', - dims=[1000, 784], + shape=[1000, 784], mean=.0, std=1., seed=10) diff --git a/python/paddle/v2/framework/tests/test_initializer.py b/python/paddle/v2/framework/tests/test_initializer.py new file mode 100644 index 0000000000..f28fc8a86c --- /dev/null +++ b/python/paddle/v2/framework/tests/test_initializer.py @@ -0,0 +1,120 @@ +import unittest + +import paddle.v2.framework.framework as framework +import paddle.v2.framework.initializer as initializer + +DELTA = 0.00001 + + +class TestConstantInitializer(unittest.TestCase): + def test_constant_initializer_default_value(self): + """Test the constant initializer with default value + """ + program = framework.Program() + block = program.global_block() + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.ConstantInitializer()) + self.assertEqual(len(block.ops), 1) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'fill_constant') + self.assertAlmostEqual(init_op.attr('value'), 0.0, delta=DELTA) + + def test_constant_initializer(self): + """Test constant initializer with supplied value + """ + program = framework.Program() + block = program.global_block() + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.ConstantInitializer(2.3)) + self.assertEqual(len(block.ops), 1) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'fill_constant') + self.assertAlmostEqual(init_op.attr('value'), 2.3, delta=DELTA) + + +class TestUniformInitializer(unittest.TestCase): + def test_uniform_initializer_default_value(self): + """Test the uniform initializer with default value + """ + program = framework.Program() + block = program.global_block() + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.UniformInitializer()) + self.assertEqual(len(block.ops), 1) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'uniform_random') + self.assertAlmostEqual(init_op.attr('min'), -1.0, delta=DELTA) + self.assertAlmostEqual(init_op.attr('max'), 1.0, delta=DELTA) + self.assertEqual(init_op.attr('seed'), 0) + + def test_uniform_initializer(self): + """Test uniform initializer with supplied attributes + """ + program = framework.Program() + block = program.global_block() + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.UniformInitializer(-4.2, 3.1, 123)) + self.assertEqual(len(block.ops), 1) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'uniform_random') + self.assertAlmostEqual(init_op.attr('min'), -4.2, delta=DELTA) + self.assertAlmostEqual(init_op.attr('max'), 3.1, delta=DELTA) + self.assertEqual(init_op.attr('seed'), 123) + + +class TestNormalInitializer(unittest.TestCase): + def test_normal_initializer_default_value(self): + """Test the normal initializer with default value + """ + program = framework.Program() + block = program.global_block() + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.NormalInitializer()) + self.assertEqual(len(block.ops), 1) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'gaussian_random') + self.assertAlmostEqual(init_op.attr('mean'), 0.0, delta=DELTA) + self.assertAlmostEqual(init_op.attr('std'), 1.0, delta=DELTA) + self.assertEqual(init_op.attr('seed'), 0) + + def test_normal_initializer(self): + """Test normal initializer with supplied attributes + """ + program = framework.Program() + block = program.global_block() + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.NormalInitializer(2.3, 1.9, 123)) + self.assertEqual(len(block.ops), 1) + init_op = block.ops[0] + self.assertEqual(init_op.type, 'gaussian_random') + self.assertAlmostEqual(init_op.attr('mean'), 2.3, delta=DELTA) + self.assertAlmostEqual(init_op.attr('std'), 1.9, delta=DELTA) + self.assertEqual(init_op.attr('seed'), 123) + + +if __name__ == '__main__': + unittest.main() -- GitLab