math_op_patch.py 14.2 KB
Newer Older
1
#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
2
#
Y
Yang Yu 已提交
3 4 5
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
6
#
Y
Yang Yu 已提交
7
#     http://www.apache.org/licenses/LICENSE-2.0
8
#
Y
Yang Yu 已提交
9 10 11 12 13 14
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15 16
from __future__ import print_function

17
import warnings
18 19
import inspect

20
from .. import core
21
from ..framework import Variable, unique_name
22
from .layer_function_generator import OpProtoHolder
Y
Yang Yu 已提交
23

24
_supported_int_dtype_ = [
25
    core.VarDesc.VarType.BOOL,
26 27 28 29 30 31 32
    core.VarDesc.VarType.UINT8,
    core.VarDesc.VarType.INT8,
    core.VarDesc.VarType.INT16,
    core.VarDesc.VarType.INT32,
    core.VarDesc.VarType.INT64,
]

33 34
compare_ops = ['__eq__', '__ne__', '__lt__', '__le__', '__gt__', '__ge__']

35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55
EXPRESSION_MAP = {
    "__add__": "A + B",
    "__radd__": "A += B",
    "__sub__": "A - B",
    "__rsub__": "A -= B",
    "__mul__": "A * B",
    "__rmul__": "A *= B",
    "__truediv__": "A / B",
    "__rtruediv__": "A /= B",
    "__pow__": "A ** B",
    "__rpow__": "A **= B",
    "__floordiv__": "A //B",
    "__mod__": "A % B",
    "__eq__": "A == B",
    "__ne__": "A != B",
    "__lt__": "A < B",
    "__le__": "A <= B",
    "__gt__": "A > B",
    "__ge__": "A >= B"
}

56 57
_already_patch_variable = False

Y
Yang Yu 已提交
58 59

def monkey_patch_variable():
Y
Yang Yu 已提交
60
    def unique_tmp_name():
Y
Yu Yang 已提交
61
        return unique_name.generate("tmp")
Y
Yang Yu 已提交
62 63 64 65 66 67 68 69

    def safe_get_dtype(var):
        try:
            dtype = var.dtype
        except:
            raise ValueError("Cannot get data type from %s", var.name)
        return dtype

70
    def current_block(var):
71
        return var.block.program.current_block()
72 73 74 75 76

    def create_new_tmp_var(block, dtype):
        tmp_name = unique_tmp_name()
        return block.create_var(name=tmp_name, dtype=dtype)

Y
Yang Yu 已提交
77 78
    def create_tensor(block, value, dtype, shape):
        value = float(value)
79
        var = create_new_tmp_var(block, dtype)
Y
Yang Yu 已提交
80 81 82
        block.append_op(
            type="fill_constant",
            outputs={'Out': [var]},
83 84 85 86
            attrs={
                'dtype': var.dtype,
                'shape': shape,
                'value': value,
87
                'force_cpu': False
H
Hongyu Liu 已提交
88 89 90
            },
            stop_gradient=True)
        var.stop_gradient = True
Y
Yang Yu 已提交
91 92
        return var

Y
Yang Yu 已提交
93 94 95
    def create_scalar(block, value, dtype):
        return create_tensor(block, value, dtype, shape=[1])

Y
Yang Yu 已提交
96 97 98
    def create_tensor_with_batchsize(ref_var, value, dtype):
        assert isinstance(ref_var, Variable)
        value = float(value)
99 100
        block = current_block(ref_var)
        var = create_new_tmp_var(block, dtype)
101
        batch_dim = -1
102
        out_shape = []
103 104
        for i, d in enumerate(ref_var.shape):
            if d < 0:
105 106 107 108 109 110 111
                if batch_dim < 0:
                    batch_dim = i
                    out_shape.append(d)
                else:
                    out_shape.append(1)
            else:
                out_shape.append(d)
112
        assert batch_dim != -1
113
        block.append_op(
Y
Yang Yu 已提交
114 115 116
            type='fill_constant_batch_size_like',
            outputs={'Out': [var]},
            inputs={'Input': [ref_var]},
117
            attrs={
118
                'shape': out_shape,
119 120 121
                'value': value,
                'input_dim_idx': batch_dim,
                'output_dim_idx': batch_dim
H
Hongyu Liu 已提交
122 123 124 125
            },
            stop_gradient=True)

        var.stop_gradient = True
Y
Yang Yu 已提交
126 127 128 129
        return var

    def astype(self, dtype):
        """
J
Jiabin Yang 已提交
130 131 132
        **Notes**:
            **The variable must be a** :ref:`api_fluid_Tensor`

Y
Yang Yu 已提交
133
        Cast a variable to a specified data type.
J
Jiabin Yang 已提交
134

Y
Yang Yu 已提交
135
        Args:
J
Jiabin Yang 已提交
136

Y
Yang Yu 已提交
137
            self(Variable): The source variable
J
Jiabin Yang 已提交
138 139

            dtype: The target data type
Y
Yang Yu 已提交
140 141

        Returns:
J
Jiabin Yang 已提交
142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171
            Variable: Variable with new dtype

        Examples:
            In Static Graph Mode:

            .. code-block:: python

                import paddle.fluid as fluid

                startup_prog = fluid.Program()
                main_prog = fluid.Program()
                with fluid.program_guard(startup_prog, main_prog):
                    original_variable = fluid.data(name = "new_variable", shape=[2,2], dtype='float32')
                    new_variable = original_variable.astype('int64')
                    print("new var's dtype is: {}".format(new_variable.dtype))

            In Dygraph Mode:

            .. code-block:: python

                import paddle.fluid as fluid
                import numpy as np

                x = np.ones([2, 2], np.float32)
                with fluid.dygraph.guard():
                    original_variable = fluid.dygraph.to_variable(x)
                    print("original var's dtype is: {}, numpy dtype is {}".format(original_variable.dtype, original_variable.numpy().dtype))
                    new_variable = original_variable.astype('int64')
                    print("new var's dtype is: {}, numpy dtype is {}".format(new_variable.dtype, new_variable.numpy().dtype))

Y
Yang Yu 已提交
172
        """
173 174 175
        block = current_block(self)
        out = create_new_tmp_var(block, dtype)
        block.append_op(
Y
Yang Yu 已提交
176 177 178 179 180
            type="cast",
            inputs={"X": [self]},
            outputs={"Out": [out]},
            attrs={"in_dtype": self.dtype,
                   "out_dtype": out.dtype})
181
        out.stop_gradient = self.stop_gradient
Y
Yang Yu 已提交
182 183
        return out

184
    def _scalar_op_(var, scale, bias):
185 186 187 188 189 190 191 192 193 194
        block = current_block(var)
        out = create_new_tmp_var(block, var.dtype)
        block.append_op(
            type="scale",
            inputs={"X": [var]},
            outputs={"Out": [out]},
            attrs={"scale": scale,
                   "bias": bias})
        return out

195
    def _neg_(var):
196
        return _scalar_op_(var, -1.0, 0.0)
197

198 199
    def _scalar_add_(var, value):
        return _scalar_op_(var, 1.0, value)
200

201 202
    def _scalar_sub_(var, value):
        return _scalar_op_(var, 1.0, -value)
203

204 205
    def _scalar_rsub_(var, value):
        return _scalar_op_(var, -1.0, value)
206

207 208
    def _scalar_mul_(var, value):
        return _scalar_op_(var, value, 0.0)
209

210 211 212 213
    def _binary_creator_(method_name,
                         op_type,
                         reverse=False,
                         scalar_method=None):
Y
Yang Yu 已提交
214
        def __impl__(self, other_var):
215 216 217 218 219 220 221 222 223
            # 1. scalar exists cases
            # we need combine the tensor.dtype and scalar.dtype, cast correct object
            if isinstance(other_var, float):
                # in all cases(+, -, *, /, **, //, %), we need cast tensor.dtype to float
                if self.dtype in _supported_int_dtype_:
                    self = astype(self, 'float32')
                # here use `scale` replace `elementwise` to get better performance
                # but only +, -, *, / can use this method
                if scalar_method is not None:
224
                    return scalar_method(self, other_var)
225 226 227 228 229 230 231 232 233 234 235 236 237 238
            elif isinstance(other_var, int):
                # in all cases(+, -, *, /, **, //, %), we can cast it to float
                # because the output tensor.dtype depend on the type of input tensor
                other_var = float(other_var)
                # division is a special case
                # NOTE(chenweihang): because we cast tensor to float32 instead float64,
                # the division result can only guarantee the numerical accuracy of 6 digits 
                # after the decimal point. The result of numpy calculation is of float64 type, 
                # so the calculation result here and the calculation result of numpy are 
                # different after 6 decimal point. If necessary, we can also use float64 here.
                # torch's behavior here is consistent with ours
                if op_type == 'elementwise_div' and self.dtype in _supported_int_dtype_:
                    self = astype(self, 'float32')
                # here use `scale` replace `elementwise` to get better performance
239 240 241 242
                # but only +, -, * can use this method
                # NOTE(chentianyu03): / can not use `scale` method,because the result of
                # `scale` method (self*(1/other_var)) do not exactly equal with the result 
                # of `elementwise_div` method.
243 244 245 246 247
                if scalar_method is not None:
                    return scalar_method(self, other_var)
            else:
                # do nothing
                pass
248

249
            # 2. create variable for scalar
Y
Yang Yu 已提交
250 251 252 253 254 255 256 257 258 259
            lhs_dtype = safe_get_dtype(self)
            if not isinstance(other_var, Variable):
                if reverse:
                    has_batch_size = False
                    for elem in self.shape:
                        if elem < 0:
                            has_batch_size = True
                            break
                    if not has_batch_size:
                        other_var = create_tensor(
260
                            current_block(self),
Y
Yang Yu 已提交
261 262 263 264 265 266 267
                            other_var,
                            dtype=lhs_dtype,
                            shape=self.shape)
                    else:
                        other_var = create_tensor_with_batchsize(
                            self, other_var, lhs_dtype)
                else:
268
                    # add fill_op to current_block
Y
Yang Yu 已提交
269
                    other_var = create_scalar(
270
                        current_block(self), value=other_var, dtype=lhs_dtype)
Y
Yang Yu 已提交
271

272
            # 3. unify right var type to left var
Y
Yang Yu 已提交
273 274 275 276 277 278 279 280
            rhs_dtype = safe_get_dtype(other_var)
            if lhs_dtype != rhs_dtype:
                other_var = astype(other_var, lhs_dtype)
            if reverse:
                tmp = self
                self = other_var
                other_var = tmp

281 282 283 284 285 286
            # NOTE(zhiqiu): the output of compare operator should be bool.
            if method_name in compare_ops:
                out = create_new_tmp_var(current_block(self), dtype="bool")
            else:
                out = create_new_tmp_var(current_block(self), dtype=lhs_dtype)

287 288
            axis = -1
            if other_var.shape[0] == -1:
289 290 291
                stack = inspect.stack()[1]
                file_name = stack[1]
                line_num = stack[2]
292
                warnings.warn(
293 294 295 296 297
                    "%s:%s\nThe behavior of expression %s has been unified with %s(X, Y, axis=-1) from Paddle 2.0. "
                    "If your code works well in the older versions but crashes in this version, try to use "
                    "%s(X, Y, axis=0) instead of %s. This transitional warning will be dropped in the future."
                    % (file_name, line_num, EXPRESSION_MAP[method_name],
                       op_type, op_type, EXPRESSION_MAP[method_name]))
298
            current_block(self).append_op(
Y
Yang Yu 已提交
299 300 301
                type=op_type,
                inputs={'X': [self],
                        'Y': [other_var]},
302
                outputs={'Out': out},
303
                attrs={'axis': axis})
Y
Yang Yu 已提交
304 305 306 307 308 309 310 311
            return out

        comment = OpProtoHolder.instance().get_op_proto(op_type).comment

        __impl__.__doc__ = """
        {0}
        Args:
            self(Variable): left hand variable
312
            other_var(Variable|float|int): right hand variable
Y
Yang Yu 已提交
313 314 315 316 317 318 319

        Returns:
            Variable
        """.format(comment)
        __impl__.__name__ = method_name
        return __impl__

320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337
    variable_methods = [
        #   b=-a
        ('__neg__', _neg_),
        ('astype', astype),
        ('__add__', _binary_creator_('__add__', 'elementwise_add', False,
                                     _scalar_add_)),
        #  a+b == b+a. Do not need to reverse explicitly
        ('__radd__',
         _binary_creator_('__radd__', 'elementwise_add', False, _scalar_add_)),
        ('__sub__', _binary_creator_('__sub__', 'elementwise_sub', False,
                                     _scalar_sub_)),
        ('__rsub__', _binary_creator_('__rsub__', 'elementwise_sub', True,
                                      _scalar_rsub_)),
        ('__mul__', _binary_creator_('__mul__', 'elementwise_mul', False,
                                     _scalar_mul_)),
        #  a*b == b*a. Do not need to reverse explicitly
        ('__rmul__',
         _binary_creator_('__rmul__', 'elementwise_mul', False, _scalar_mul_)),
S
ShenLiang 已提交
338
        ('__truediv__', _binary_creator_('__truediv__', 'elementwise_div',
339
                                         False, None)),
S
ShenLiang 已提交
340 341
        ('__rtruediv__', _binary_creator_('__rtruediv__', 'elementwise_div',
                                          True, None)),
342 343 344 345
        ('__pow__', _binary_creator_('__pow__', 'elementwise_pow', False,
                                     None)),
        ('__rpow__', _binary_creator_('__rpow__', 'elementwise_pow', True,
                                      None)),
S
ShenLiang 已提交
346 347 348 349
        ('__floordiv__', _binary_creator_('__floordiv__',
                                          'elementwise_floordiv', False, None)),
        ('__mod__', _binary_creator_('__mod__', 'elementwise_mod', False,
                                     None)),
350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366
        #  for logical compare
        ('__eq__', _binary_creator_('__eq__', 'equal', False, None)),
        ('__ne__', _binary_creator_('__ne__', 'not_equal', False, None)),
        ('__lt__', _binary_creator_('__lt__', 'less_than', False, None)),
        ('__le__', _binary_creator_('__le__', 'less_equal', False, None)),
        ('__gt__', _binary_creator_('__gt__', 'greater_than', False, None)),
        ('__ge__', _binary_creator_('__ge__', 'greater_equal', False, None))
    ]

    global _already_patch_variable
    if not _already_patch_variable:
        for method in variable_methods:
            method_name = method[0]
            method_impl = method[1]
            setattr(Variable, method_name, method_impl)
    else:
        import paddle.tensor
367
        for method_name in paddle.tensor.tensor_method_func:
368 369 370 371
            if hasattr(Variable, method_name): continue
            method_impl = getattr(paddle.tensor, method_name, None)
            if method_impl: setattr(Variable, method_name, method_impl)

372 373 374 375
        for magic_method, origin_method in paddle.tensor.magic_method_func:
            impl = getattr(paddle.tensor, origin_method, None)
            if impl: setattr(Variable, magic_method, impl)

376
    _already_patch_variable = True