From e9b8ebf42cfce4d86556998ca23f5961c831fbbf Mon Sep 17 00:00:00 2001 From: xuwei06 Date: Thu, 22 Feb 2018 17:18:11 -0800 Subject: [PATCH] Correctly handling variable with batch dimension for math ops. When the second argument contains batch dimension, the axis should be 0. Also makes elementwise ops more tolerant at handling tensors with trailing singular dimensions. --- paddle/fluid/operators/elementwise_op.h | 8 +- .../fluid/operators/elementwise_op_function.h | 79 +++++-------------- python/paddle/fluid/executor.py | 4 +- python/paddle/fluid/layers/math_op_patch.py | 33 ++++++-- .../unittests/test_elementwise_add_op.py | 24 ++++++ .../tests/unittests/test_math_op_patch.py | 20 +++-- 6 files changed, 94 insertions(+), 74 deletions(-) diff --git a/paddle/fluid/operators/elementwise_op.h b/paddle/fluid/operators/elementwise_op.h index 06bcd0be646..fe31bbaed44 100644 --- a/paddle/fluid/operators/elementwise_op.h +++ b/paddle/fluid/operators/elementwise_op.h @@ -65,12 +65,17 @@ smaller than or equal to the dimensions of $X$. There are two cases for this operator: 1. The shape of $Y$ is same with $X$; -2. The shape of $Y$ is a subset of $X$. +2. The shape of $Y$ is a congiguous subsequencet of $X$. The trailing dimensions + of size 1 for $Y$ will be ignored for the consideration of subsequence. + For case 2: + $Y$ will be broadcasted to match the shape of $X$ and axis should be set to index of the start dimension to broadcast $Y$ onto $X$. +If axis is -1, it is treated as axis=rank(X)-rank(Y). + For example .. code-block:: python @@ -79,6 +84,7 @@ For example shape(X) = (2, 3, 4, 5), shape(Y) = (4, 5) shape(X) = (2, 3, 4, 5), shape(Y) = (3, 4), with axis=1 shape(X) = (2, 3, 4, 5), shape(Y) = (2), with axis=0 + shape(X) = (2, 3, 4, 5), shape(Y) = (2, 1), with axis=0 Either of the inputs $X$ and $Y$ or none can carry the LoD (Level of Details) information. However, the output only shares the LoD information with input $X$. diff --git a/paddle/fluid/operators/elementwise_op_function.h b/paddle/fluid/operators/elementwise_op_function.h index 5c783035309..a14e24d14da 100644 --- a/paddle/fluid/operators/elementwise_op_function.h +++ b/paddle/fluid/operators/elementwise_op_function.h @@ -61,6 +61,19 @@ inline void get_mid_dims(const framework::DDim& x_dims, } } +inline void trim_trailing_singular_dims(framework::DDim& dims) { + // Remove trailing dimensions of size 1 for y + auto actual_dims_size = dims.size(); + for (; actual_dims_size != 0; --actual_dims_size) { + if (dims[actual_dims_size - 1] != 1) break; + } + if (actual_dims_size != dims.size()) { + auto actual_dims = framework::vectorize(dims); + actual_dims.resize(actual_dims_size); + dims = framework::make_ddim(actual_dims); + } +} + template class RowwiseTransformIterator; template @@ -263,44 +276,6 @@ class TransformFunctor { } \ } -template -void ElementwiseCompute(const framework::ExecutionContext& ctx) { - using Tensor = framework::Tensor; - - auto* x = ctx.Input("X"); - auto* y = ctx.Input("Y"); - auto* z = ctx.Output("Out"); - z->mutable_data(ctx.GetPlace()); - - auto x_dims = x->dims(); - auto y_dims = y->dims(); - PADDLE_ENFORCE_GE(x_dims.size(), y_dims.size(), - "Rank of first input must >= rank of second input."); - - if (x_dims == y_dims) { - functor f; - f.template Run(x, y, z, ctx); - return; - } - - int axis = ctx.Attr("axis"); - axis = (axis == -1 ? x_dims.size() - y_dims.size() : axis); - PADDLE_ENFORCE(axis >= 0 && axis < x_dims.size(), - "Axis should be in range [0, x_dims)"); - - int pre, n, post; - get_mid_dims(x_dims, y_dims, axis, pre, n, post); - if (post == 1) { - functor f; - f.template RunBroadCast(x, y, z, ctx, pre, n); - return; - } else { - functor f; - f.template RunBroadCast2(x, y, z, ctx, pre, n, post); - return; - } -} - #define EIGEN_ADD(x, y) ((x) + (y)) EIGEN_FUNCTOR(Add, EIGEN_ADD); @@ -516,14 +491,10 @@ void ElemwiseGradCompute(const framework::ExecutionContext& ctx, auto x_dim = x.dims(); auto y_dim = y.dims(); - if (y_dim.size() == 1 && y_dim[0] == 1) { - // y is a scalar - auto extended_dims = framework::vectorize(x_dim); - extended_dims.push_back(1); - x_dim = framework::make_ddim(extended_dims); - } - axis = (axis == -1 ? x_dim.size() - y_dim.size() : axis); + trim_trailing_singular_dims(y_dim); + axis = (y_dim.size() == 0) ? x_dim.size() : axis; + int pre, n, post; get_mid_dims(x_dim, y_dim, axis, pre, n, post); if (post == 1) { @@ -591,14 +562,9 @@ void ElementwiseGradCompute(const framework::ExecutionContext& ctx, return; } - if (y_dims.size() == 1 && y_dims[0] == 1) { - // y is a scalar - auto extended_dims = framework::vectorize(x_dims); - extended_dims.push_back(1); - x_dims = framework::make_ddim(extended_dims); - } - axis = (axis == -1 ? x_dims.size() - y_dims.size() : axis); + trim_trailing_singular_dims(y_dims); + axis = (y_dims.size() == 0) ? x_dims.size() : axis; int pre, n, post; get_mid_dims(x_dims, y_dims, axis, pre, n, post); @@ -633,16 +599,11 @@ void ElementwiseComputeEx(const framework::ExecutionContext& ctx, return; } - if (y_dims.size() == 1 && y_dims[0] == 1) { - // y is a scalar - auto extended_dims = framework::vectorize(x_dims); - extended_dims.push_back(1); - x_dims = framework::make_ddim(extended_dims); - } - axis = (axis == -1 ? x_dims.size() - y_dims.size() : axis); PADDLE_ENFORCE(axis >= 0 && axis < x_dims.size(), "Axis should be in range [0, x_dims)"); + trim_trailing_singular_dims(y_dims); + axis = (y_dims.size() == 0) ? x_dims.size() : axis; int pre, n, post; get_mid_dims(x_dims, y_dims, axis, pre, n, post); diff --git a/python/paddle/fluid/executor.py b/python/paddle/fluid/executor.py index e2749593057..e5fb0e5d628 100644 --- a/python/paddle/fluid/executor.py +++ b/python/paddle/fluid/executor.py @@ -14,7 +14,7 @@ import numpy as np import contextlib -from framework import Program, default_main_program +from framework import Program, default_main_program, Variable from . import core __all__ = [ @@ -281,6 +281,8 @@ class Executor(object): if not has_fetch_operators(global_block, fetch_list, fetch_var_name): for i, var in enumerate(fetch_list): + assert isinstance(var, Variable) or isinstance(var, str), ( + "Wrong type for fetch_list[%s]: %s" % (i, type(var))) global_block.append_op( type='fetch', inputs={'X': [var]}, diff --git a/python/paddle/fluid/layers/math_op_patch.py b/python/paddle/fluid/layers/math_op_patch.py index faccc3ddf82..08a0184c2c2 100644 --- a/python/paddle/fluid/layers/math_op_patch.py +++ b/python/paddle/fluid/layers/math_op_patch.py @@ -1,11 +1,11 @@ # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -53,12 +53,22 @@ def monkey_patch_variable(): value = float(value) tmp_name = unique_tmp_name() var = ref_var.block.create_var(name=tmp_name, dtype=dtype) + batch_dim = -1 + for i, d in enumerate(ref_var.shape): + if d < 0: + batch_dim = i + break + assert batch_dim != -1 ref_var.block.append_op( type='fill_constant_batch_size_like', outputs={'Out': [var]}, inputs={'Input': [ref_var]}, - attrs={'shape': ref_var.shape, - 'value': value}) + attrs={ + 'shape': ref_var.shape, + 'value': value, + 'input_dim_idx': batch_dim, + 'output_dim_idx': batch_dim + }) return var def astype(self, dtype): @@ -118,11 +128,20 @@ def monkey_patch_variable(): tmp_name = unique_tmp_name() out = self.block.create_var(name=tmp_name, dtype=lhs_dtype) + axis = -1 + if other_var.shape[0] == -1: + axis = 0 + assert len(self.shape) >= len(other_var.shape), ( + "The rank of the first argument of an binary operator cannot " + "be smaller than the rank of its second argument: %s vs %s" % + (len(self.shape), len(other_var.shape))) + self.block.append_op( type=op_type, inputs={'X': [self], 'Y': [other_var]}, - outputs={'Out': out}) + outputs={'Out': out}, + attrs={'axis': axis}) return out comment = OpProtoHolder.instance().get_op_proto(op_type).comment @@ -131,7 +150,7 @@ def monkey_patch_variable(): {0} Args: self(Variable): left hand variable - other_var(Variable|float|int): right hand variable + other_var(Variable|float|int): right hand variable Returns: Variable diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_add_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_add_op.py index c8e930dad76..5b2384e94d7 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_add_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_add_op.py @@ -50,6 +50,16 @@ class TestElementwiseAddOp_scalar(TestElementwiseOp): self.outputs = {'Out': self.inputs['X'] + self.inputs['Y']} +class TestElementwiseAddOp_scalar2(TestElementwiseOp): + def setUp(self): + self.op_type = "elementwise_add" + self.inputs = { + 'X': np.random.rand(2, 3, 4).astype(np.float32), + 'Y': np.random.rand(1, 1).astype(np.float32) + } + self.outputs = {'Out': self.inputs['X'] + self.inputs['Y']} + + class TestElementwiseAddOp_Vector(TestElementwiseOp): def setUp(self): self.op_type = "elementwise_add" @@ -115,6 +125,20 @@ class TestElementwiseAddOp_broadcast_3(TestElementwiseOp): } +class TestElementwiseAddOp_broadcast_4(TestElementwiseOp): + def setUp(self): + self.op_type = "elementwise_add" + self.inputs = { + 'X': np.random.rand(2, 3, 4, 5).astype(np.float32), + 'Y': np.random.rand(2, 1).astype(np.float32) + } + + self.attrs = {'axis': 0} + self.outputs = { + 'Out': self.inputs['X'] + self.inputs['Y'].reshape(2, 1, 1, 1) + } + + class TestElementwiseAddOp_rowwise_add_0(TestElementwiseOp): def setUp(self): self.op_type = "elementwise_add" diff --git a/python/paddle/fluid/tests/unittests/test_math_op_patch.py b/python/paddle/fluid/tests/unittests/test_math_op_patch.py index 6864d271e79..852a80261e0 100644 --- a/python/paddle/fluid/tests/unittests/test_math_op_patch.py +++ b/python/paddle/fluid/tests/unittests/test_math_op_patch.py @@ -1,11 +1,11 @@ # Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,13 +23,21 @@ class TestMathOpPatches(unittest.TestCase): def test_add_scalar(self): a = fluid.layers.data(name="a", shape=[1]) b = a + 10 + ab = fluid.layers.concat(input=[a, b], axis=1) + c = ab + 10 + d = ab + a + # e = a + ab place = fluid.CPUPlace() exe = fluid.Executor(place) a_np = numpy.random.random(size=[10, 1]).astype('float32') - b_np = exe.run(fluid.default_main_program(), - feed={"a": a_np}, - fetch_list=[b]) + b_np, c_np, d_np = exe.run(fluid.default_main_program(), + feed={"a": a_np}, + fetch_list=[b, c, d]) self.assertTrue(numpy.allclose(a_np + 10, b_np)) + ab_np = numpy.concatenate([a_np, b_np], axis=1) + self.assertTrue(numpy.allclose(ab_np + 10, c_np)) + d_expected = ab_np + numpy.concatenate([a_np, a_np], axis=1) + self.assertTrue(numpy.allclose(d_expected, d_np)) @decorators.prog_scope() def test_radd_scalar(self): -- GitLab