op_test.py 64.3 KB
Newer Older
1
#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
D
dzhwinter 已提交
2
#
D
dzhwinter 已提交
3 4 5
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
D
dzhwinter 已提交
6
#
D
dzhwinter 已提交
7
#     http://www.apache.org/licenses/LICENSE-2.0
D
dzhwinter 已提交
8
#
D
dzhwinter 已提交
9 10 11 12 13 14
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15 16
from __future__ import print_function

B
baojun 已提交
17
import os
18
import unittest
19
import warnings
20
import numpy as np
21
import random
M
minqiyang 已提交
22
import six
23
import time
24
import itertools
Y
Yu Yang 已提交
25
import collections
M
minqiyang 已提交
26
from collections import defaultdict
27 28 29

import paddle.fluid as fluid
import paddle.fluid.core as core
30 31 32
from paddle.fluid.backward import append_backward
from paddle.fluid.op import Operator
from paddle.fluid.executor import Executor
33
from paddle.fluid.framework import Program, OpProtoHolder, Variable
34
from testsuite import create_op, set_input, append_input_output, append_loss_ops
35
from paddle.fluid import unique_name
36
from white_list import op_accuracy_white_list, check_shape_white_list, compile_vs_runtime_white_list, no_check_set_white_list
37
from white_list import op_threshold_white_list, no_grad_set_white_list
38 39


40 41 42 43 44 45 46 47
def _set_use_system_allocator(value=None):
    USE_SYSTEM_ALLOCATOR_FLAG = "FLAGS_use_system_allocator"
    old_value = core.globals()[USE_SYSTEM_ALLOCATOR_FLAG]
    value = old_value if value is None else value
    core.globals()[USE_SYSTEM_ALLOCATOR_FLAG] = value
    return old_value


48 49 50 51
def randomize_probability(batch_size, class_num, dtype='float32'):
    prob = np.random.uniform(
        0.1, 1.0, size=(batch_size, class_num)).astype(dtype)
    prob_sum = prob.sum(axis=1)
M
minqiyang 已提交
52
    for i in six.moves.xrange(len(prob)):
53 54 55 56
        prob[i] /= prob_sum[i]
    return prob


57 58
def get_numeric_gradient(place,
                         scope,
59 60 61
                         op,
                         inputs,
                         input_to_check,
Y
Yancey 已提交
62
                         output_names,
63
                         delta=0.005,
C
chengduo 已提交
64
                         in_place=False):
Y
Yu Yang 已提交
65
    # FIXME: change this method by compile time concepts
66
    set_input(scope, op, inputs, place)
67 68

    def product(dim):
M
minqiyang 已提交
69
        return six.moves.reduce(lambda a, b: a * b, dim, 1)
70 71

    tensor_to_check = scope.find_var(input_to_check).get_tensor()
Y
yuyang18 已提交
72 73
    tensor_size = product(tensor_to_check.shape())
    tensor_to_check_dtype = tensor_to_check._dtype()
74
    if tensor_to_check_dtype == core.VarDesc.VarType.FP32:
75
        tensor_to_check_dtype = np.float32
76
    elif tensor_to_check_dtype == core.VarDesc.VarType.FP64:
77
        tensor_to_check_dtype = np.float64
D
dzhwinter 已提交
78 79 80 81
    elif tensor_to_check_dtype == core.VarDesc.VarType.FP16:
        tensor_to_check_dtype = np.float16
        # set delta as np.float16, will automatic convert to float32, float64
        delta = np.array(delta).astype(np.float16)
82 83 84 85
    else:
        raise ValueError("Not supported data type " + str(
            tensor_to_check_dtype))

C
chengduo 已提交
86 87 88 89 90 91 92 93 94
    def get_output():
        sum = []
        op.run(scope, place)
        for output_name in output_names:
            sum.append(
                np.array(scope.find_var(output_name).get_tensor()).astype(
                    tensor_to_check_dtype).mean())
        return tensor_to_check_dtype(np.array(sum).sum() / len(output_names))

95 96 97
    gradient_flat = np.zeros(shape=(tensor_size, ), dtype=tensor_to_check_dtype)

    def __get_elem__(tensor, i):
D
dzhwinter 已提交
98 99 100 101 102
        if tensor_to_check_dtype == np.float16:
            numpy_tensor = np.array(tensor).astype(np.float16)
            numpy_tensor = numpy_tensor.flatten()
            return numpy_tensor[i]
        elif tensor_to_check_dtype == np.float32:
Y
yuyang18 已提交
103
            return tensor._get_float_element(i)
104
        else:
Y
yuyang18 已提交
105
            return tensor._get_double_element(i)
106 107

    def __set_elem__(tensor, i, e):
D
dzhwinter 已提交
108 109 110 111 112
        if tensor_to_check_dtype == np.float16:
            numpy_tensor = np.array(tensor).astype(np.float16)
            shape = numpy_tensor.shape
            numpy_tensor = numpy_tensor.flatten()
            numpy_tensor[i] = e
113
            numpy_tensor = numpy_tensor.reshape(shape)
D
dzhwinter 已提交
114 115
            tensor.set(numpy_tensor, place)
        elif tensor_to_check_dtype == np.float32:
Y
yuyang18 已提交
116
            tensor._set_float_element(i, e)
117
        else:
Y
yuyang18 已提交
118
            tensor._set_double_element(i, e)
119

120 121
    # we only compute gradient of one element each time.
    # we use a for loop to compute the gradient of every element.
M
minqiyang 已提交
122
    for i in six.moves.xrange(tensor_size):
123
        if in_place:
124
            set_input(scope, op, inputs, place)
125 126

        # get one input element throw it's index i.
127
        origin = __get_elem__(tensor_to_check, i)
128 129
        # add delta to it, run op and then get the sum of the result tensor.
        x_pos = origin + delta
130
        __set_elem__(tensor_to_check, i, x_pos)
131 132 133
        y_pos = get_output()

        if in_place:
134
            set_input(scope, op, inputs, place)
135 136

        x_neg = origin - delta
137
        __set_elem__(tensor_to_check, i, x_neg)
138 139
        y_neg = get_output()

140
        __set_elem__(tensor_to_check, i, origin)
141 142
        gradient_flat[i] = (y_pos - y_neg) / delta / 2

Y
yuyang18 已提交
143
    return gradient_flat.reshape(tensor_to_check.shape())
144 145


146 147
def skip_check_grad_ci(reason=None):
    """Decorator to skip check_grad CI.
C
cc 已提交
148

149
       Check_grad is required for Op test cases. However, there are some special
C
cc 已提交
150
       cases that do not need to do check_grad. This decorator is used to skip the
151
       check_grad of the above cases.
C
cc 已提交
152 153

       Note: the execution of unit test will not be skipped. It just avoids check_grad
154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169
       checking in tearDownClass method by setting a `no_need_check_grad` flag.

       Example:
           @skip_check_grad_ci(reason="For inference, check_grad is not required.")
           class TestInference(OpTest):
    """
    if not isinstance(reason, str):
        raise AssertionError("The reason for skipping check_grad is required.")

    def wrapper(cls):
        cls.no_need_check_grad = True
        return cls

    return wrapper


170
class OpTest(unittest.TestCase):
171 172 173 174 175
    @classmethod
    def setUpClass(cls):
        '''Fix random seeds to remove randomness from tests'''
        cls._np_rand_state = np.random.get_state()
        cls._py_rand_state = random.getstate()
176
        cls.call_once = False
177
        cls.dtype = None
178
        cls.outputs = {}
179
        cls.input_shape_is_large = True
180 181 182 183

        np.random.seed(123)
        random.seed(124)

184 185
        cls._use_system_allocator = _set_use_system_allocator(True)

186 187
    @classmethod
    def tearDownClass(cls):
Y
yuyang18 已提交
188
        """Restore random seeds"""
189 190 191
        np.random.set_state(cls._np_rand_state)
        random.setstate(cls._py_rand_state)

192 193
        _set_use_system_allocator(cls._use_system_allocator)

194 195 196 197
        def is_empty_grad_op(op_type):
            all_op_kernels = core._get_all_register_op_kernels()
            grad_op = op_type + '_grad'
            if grad_op in all_op_kernels.keys():
J
juncaipeng 已提交
198
                if is_mkldnn_op_test():
199 200 201 202 203 204 205 206
                    grad_op_kernels = all_op_kernels[grad_op]
                    for grad_op_kernel in grad_op_kernels:
                        if 'MKLDNN' in grad_op_kernel:
                            return False
                else:
                    return False
            return True

J
juncaipeng 已提交
207
        def is_mkldnn_op_test():
208
            return hasattr(cls, "use_mkldnn") and cls.use_mkldnn == True
J
juncaipeng 已提交
209

210 211
        if not hasattr(cls, "op_type"):
            raise AssertionError(
212 213
                "This test do not have op_type in class attrs, "
                "please set self.__class__.op_type=the_real_op_type manually.")
214

J
juncaipeng 已提交
215 216
        # case in NO_FP64_CHECK_GRAD_CASES and op in NO_FP64_CHECK_GRAD_OP_LIST should be fixed
        if not hasattr(cls, "no_need_check_grad") \
217
            and not is_empty_grad_op(cls.op_type):
J
juncaipeng 已提交
218
            if cls.dtype is None or \
219 220
                (cls.dtype == np.float16 \
                    and cls.op_type not in op_accuracy_white_list.NO_FP16_CHECK_GRAD_OP_LIST \
J
juncaipeng 已提交
221 222 223 224
                    and not hasattr(cls, "exist_check_grad")):
                raise AssertionError("This test of %s op needs check_grad." %
                                     cls.op_type)

225
            # check for op test with fp64 precision, but not check mkldnn op test for now
J
juncaipeng 已提交
226 227
            if cls.dtype in [np.float32, np.float64] \
                and cls.op_type not in op_accuracy_white_list.NO_FP64_CHECK_GRAD_OP_LIST \
228
                and not hasattr(cls, 'exist_fp64_check_grad') \
J
juncaipeng 已提交
229
                and not is_mkldnn_op_test():
J
juncaipeng 已提交
230 231 232 233
                raise AssertionError(
                    "This test of %s op needs check_grad with fp64 precision." %
                    cls.op_type)

234
            if not cls.input_shape_is_large \
235 236 237 238
                and cls.op_type not in check_shape_white_list.NEED_TO_FIX_OP_LIST:
                raise AssertionError(
                    "Input's shape should be large than or equal to 100 for " +
                    cls.op_type + " Op.")
239

240 241 242 243 244 245
    def try_call_once(self, data_type):
        if not self.call_once:
            self.call_once = True
            self.dtype = data_type

    def infer_dtype_from_inputs_outputs(self, inputs, outputs):
J
juncaipeng 已提交
246 247 248 249
        def is_np_data(input):
            return isinstance(input, (np.ndarray, np.generic))

        def infer_dtype(numpy_dict, dtype_set):
250 251 252
            assert isinstance(
                numpy_dict,
                dict), "self.inputs, self.outputs must be numpy_dict"
J
juncaipeng 已提交
253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279
            # the inputs are as follows:
            # case 1: inputs = {'X': x}
            # case 2: inputs = {'X': (x, x_lod)}
            # case 3: inputs = {"X": [("x0", x0), ("x1", x1), ("x2", x2)]}
            # case 4: inputs = {'X': [("x1", (x1, [x1_lod1])), ("x2", (x2, [x2_.lod2]))]}
            # TODO(juncaipeng) infer dtype from inputs maybe obtain wrong type.
            for _, var_value in six.iteritems(numpy_dict):
                if is_np_data(var_value):  # case 1
                    dtype_set.add(var_value.dtype)
                elif isinstance(var_value, (list, tuple)):  # case 2, 3, 4
                    for sub_val_value in var_value:
                        if is_np_data(sub_val_value):  # case 2
                            dtype_set.add(sub_val_value.dtype)
                        elif len(sub_val_value) > 1 and is_np_data(
                                sub_val_value[1]):  # case 3
                            dtype_set.add(sub_val_value[1].dtype)
                        elif len(sub_val_value) > 1 and isinstance(sub_val_value[1], (list, tuple)) \
                            and is_np_data(sub_val_value[1][0]): # case 4
                            dtype_set.add(sub_val_value[1][0].dtype)

        # infer dtype from inputs, and dtype means the precision of the test
        # collect dtype of all inputs
        dtype_set = set()
        infer_dtype(inputs, dtype_set)
        dtype_list = [
            np.dtype(np.float64), np.dtype(np.float32), np.dtype(np.float16),
            np.dtype(np.int64), np.dtype(np.int32), np.dtype(np.int16),
280
            np.dtype(np.int8), np.dtype(np.uint8), np.dtype(np.bool)
J
juncaipeng 已提交
281 282 283 284 285 286
        ]
        # check the dtype in dtype_list in order, select the first dtype that in dtype_set
        for dtype in dtype_list:
            if dtype in dtype_set:
                self.dtype = dtype
                break
287 288
        # save dtype in class attr
        self.__class__.dtype = self.dtype
289

Y
Yang Yang(Tony) 已提交
290 291 292 293 294 295
    def feed_var(self, input_vars, place):
        feed_map = {}
        for var_name in input_vars:
            if isinstance(input_vars[var_name], list):
                for name, np_value in self.inputs[var_name]:
                    tensor = core.LoDTensor()
296
                    if isinstance(np_value, tuple):
297
                        tensor.set(np_value[0], place)
298
                        tensor.set_recursive_sequence_lengths(np_value[1])
299
                    else:
300
                        tensor.set(np_value, place)
Y
Yang Yang(Tony) 已提交
301 302 303 304
                    feed_map[name] = tensor
            else:
                tensor = core.LoDTensor()
                if isinstance(self.inputs[var_name], tuple):
305
                    tensor.set(self.inputs[var_name][0], place)
306 307
                    tensor.set_recursive_sequence_lengths(self.inputs[var_name][
                        1])
Y
Yang Yang(Tony) 已提交
308
                else:
309
                    tensor.set(self.inputs[var_name], place)
Y
Yang Yang(Tony) 已提交
310 311 312 313
                feed_map[var_name] = tensor

        return feed_map

314
    def _append_ops(self, block):
J
juncaipeng 已提交
315
        self.__class__.op_type = self.op_type  # for ci check, please not delete it for now
316 317 318 319
        if (hasattr(self, "use_mkldnn") and self.use_mkldnn == True) or \
            (hasattr(self, "attrs") and "use_mkldnn" in self.attrs and \
                    self.attrs["use_mkldnn"] == True):
            self.__class__.use_mkldnn = True
C
cc 已提交
320

Y
Yang Yang(Tony) 已提交
321
        op_proto = OpProtoHolder.instance().get_op_proto(self.op_type)
322 323 324 325 326 327
        "infer datatype from inputs and outputs for this test case"
        self.infer_dtype_from_inputs_outputs(self.inputs, self.outputs)
        inputs = append_input_output(block, op_proto, self.inputs, True,
                                     self.dtype)
        outputs = append_input_output(block, op_proto, self.outputs, False,
                                      self.dtype)
P
phlrain 已提交
328 329 330 331 332 333 334 335 336

        if hasattr(self, "cache_name_list"):
            for name in self.cache_name_list:
                inputs[name] = block.create_var(
                    name=name,
                    persistable=True,
                    type=core.VarDesc.VarType.RAW,
                    stop_gradient=True)

Y
Yang Yang(Tony) 已提交
337 338 339 340 341
        op = block.append_op(
            type=self.op_type,
            inputs=inputs,
            outputs=outputs,
            attrs=self.attrs if hasattr(self, "attrs") else dict())
C
cc 已提交
342
        # infer variable type and infer shape in compile-time
Q
QI JUN 已提交
343 344
        op.desc.infer_var_type(block.desc)
        op.desc.infer_shape(block.desc)
Y
Yang Yang(Tony) 已提交
345

346 347
        return op

348 349
    def _get_io_vars(self, block, numpy_inputs):
        inputs = {}
M
minqiyang 已提交
350
        for name, value in six.iteritems(numpy_inputs):
351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369
            if isinstance(value, list):
                var_list = [
                    block.var(sub_name) for sub_name, sub_value in value
                ]
                inputs[name] = var_list
            else:
                inputs[name] = block.var(name)
        return inputs

    def _get_inputs(self, block):
        return self._get_io_vars(block, self.inputs)

    def _get_outputs(self, block):
        return self._get_io_vars(block, self.outputs)

    def calc_output(self, place):
        outs, _ = self._calc_output(place)
        return outs

M
minqiyang 已提交
370 371 372 373
    def _create_var_from_numpy(self, value):
        if isinstance(value, tuple):
            data = value[0]
            lod = value[1]
L
lujun 已提交
374
            v = fluid.dygraph.base.to_variable(value=data)
375
            v.value().get_tensor().set_recursive_sequence_lengths(lod)
M
minqiyang 已提交
376 377
            return v
        else:
L
lujun 已提交
378
            return fluid.dygraph.base.to_variable(value)
M
minqiyang 已提交
379

380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397
    def get_sequence_batch_size_1_input(self, lod=None, shape=None):
        """Get LoD input data whose batch size is 1.
        All sequence related OP unittests should call this function to contain the case of batch size = 1.
        Args:
            lod (list[list of int], optional): Length-based LoD, length of lod[0] should be 1. Default: [[13]].
            shape (list, optional): Shape of input, shape[0] should be equals to lod[0][0]. Default: [13, 23].
        Returns:
            tuple (ndarray, lod) : LoD input data whose batch size is 1.
        """
        if lod is None:
            lod = [[13]]
        if shape is None:
            shape = [13, 23]
        assert len(lod[0]) == 1
        assert lod[0][0] == shape[0]
        x = np.random.uniform(0.1, 1, shape).astype('float32')
        return (x, lod)

398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433
    def lod_has_single_zero(self, lod):
        for i in range(len(lod) - 2):
            if lod[i] != 0 and lod[i + 1] == 0 and lod[i + 2] != 0:
                return True
        return False

    def lod_has_continuous_zero(self, lod):
        for i in range(len(lod) - 3):
            if lod[i] != 0 and lod[i + 1] == 0 and lod[i + 2] == 0 and lod[
                    i + 3] != 0:
                return True
        return False

    def get_sequence_instance_size_0_input(self, lod=None, shape=None):
        """Get LoD input data whose instance size is 0.
        All sequence related OP unittests should call this function to contain the case of instance size is 0.
        Args:
            lod (list[list of int], optional): Length-based LoD, lod[0]'s size must at least eight, lod[0] must at least two zeros at the beginning and at least two zeros at the end, the middle position of lod[0] contains a single zero and multiple zero. Default: [[0, 0, 4, 0, 3, 0, 0, 5, 0, 0]].
            shape (list, optional): Shape of input, shape[0] should be equals to lod[0][0]. Default: [13, 23].
        Returns:
            tuple (ndarray, lod): LoD input data whose instance size is 0.
        """
        if lod is None:
            lod = [[0, 0, 4, 0, 3, 0, 0, 5, 0, 0]]
        if shape is None:
            shape = [12, 10]
        assert len(lod[0]) >= 8
        assert lod[0][0] == 0 and lod[0][1] == 0 and lod[0][-1] == 0 and lod[0][
            -2] == 0
        assert self.lod_has_single_zero(lod[0]) is True
        assert self.lod_has_continuous_zero(lod[0]) is True
        assert sum(lod[0]) == shape[0]

        x = np.random.uniform(0.1, 1, shape).astype('float32')
        return (x, lod)

434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449
    def append_input_output_for_dygraph(self, op_proto, np_list, is_input,
                                        if_return_inputs_grad_dict, block):
        def create_var(np_value, name, is_input, if_return_inputs_grad_dict):
            np_value_temp = np_value
            has_lod = False
            lod_temp = None
            if isinstance(np_value, tuple):
                np_value_temp = np_value[0]
                has_lod = True
                lod_temp = np_value[1]

            if is_input:
                v = self._create_var_from_numpy(np_value_temp)
                if if_return_inputs_grad_dict:
                    v.stop_gradient = False
                if has_lod:
450
                    v.value().get_tensor().set_recursive_sequence_lengths(
451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511
                        lod_temp)
            else:
                v = block.create_var(
                    name=name,
                    dtype=np_value_temp.dtype,
                    type=core.VarDesc.VarType.LOD_TENSOR,
                    persistable=False,
                    stop_gradient=False)

            return v

        # prepare variable for input or output
        var_dict = defaultdict(list)
        if if_return_inputs_grad_dict:
            inputs_grad_dict = defaultdict()
        proto_list = op_proto.inputs if is_input else op_proto.outputs
        for var_proto in proto_list:
            name = var_proto.name
            if (name not in np_list) and var_proto.dispensable:
                continue
            if name not in np_list:
                assert var_proto.intermediate, "{} not found".format(name)
                v = block.create_var(
                    dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR)
                var_dict[name].append(v)
                if if_return_inputs_grad_dict:
                    inputs_grad_dict[name] = v
                continue
            if var_proto.duplicable:
                assert isinstance(
                    np_list[name],
                    list), "Duplicable {} should be set as list".format(name)
                var_list = []
                slot_name = name
                for (name, np_value) in np_list[name]:
                    v = create_var(np_value, name, is_input,
                                   if_return_inputs_grad_dict)
                    var_list.append(v)
                    if if_return_inputs_grad_dict:
                        inputs_grad_dict[name] = v
                var_dict[slot_name] = var_list
            else:
                nplist_value_temp = None
                name_temp = None
                if isinstance(np_list[name], list):
                    nplist_value_temp = np_list[name][0]
                    name_temp = name
                else:
                    nplist_value_temp = np_list[name]
                    name_temp = unique_name.generate("%s_out" % (name))
                v = create_var(nplist_value_temp, name_temp, is_input,
                               if_return_inputs_grad_dict)
                var_dict[name].append(v)
                if if_return_inputs_grad_dict:
                    inputs_grad_dict[name] = v

        if if_return_inputs_grad_dict:
            return var_dict, inputs_grad_dict
        else:
            return var_dict

L
lujun 已提交
512
    def _calc_dygraph_output(self, place, parallel=False, no_check_set=None):
J
juncaipeng 已提交
513
        self.__class__.op_type = self.op_type  # for ci check, please not delete it for now
L
lujun 已提交
514
        with fluid.dygraph.base.guard(place=place):
M
minqiyang 已提交
515 516
            block = fluid.default_main_program().global_block()

517
            op_proto = OpProtoHolder.instance().get_op_proto(self.op_type)
M
minqiyang 已提交
518

519 520 521
            # prepare input variable
            inputs = self.append_input_output_for_dygraph(op_proto, self.inputs,
                                                          True, False, block)
M
minqiyang 已提交
522 523

            # prepare output variable
524 525 526 527 528 529 530 531 532
            outputs = self.append_input_output_for_dygraph(
                op_proto, self.outputs, False, False, block)

            # prepare attrbutes
            attrs_outputs = {}
            if hasattr(self, "attrs"):
                for attrs_name in self.attrs:
                    if self.attrs[attrs_name] is not None:
                        attrs_outputs[attrs_name] = self.attrs[attrs_name]
M
minqiyang 已提交
533 534 535 536
            block.append_op(
                type=self.op_type,
                inputs=inputs,
                outputs=outputs,
537
                attrs=attrs_outputs if hasattr(self, "attrs") else None)
M
minqiyang 已提交
538
            return outputs
539

540 541 542 543 544 545
    def _calc_output(self,
                     place,
                     parallel=False,
                     no_check_set=None,
                     loss=None,
                     enable_inplace=None,
546
                     for_inplace_test=None):
547 548
        program = Program()
        block = program.global_block()
549
        op = self._append_ops(block)
550 551 552 553 554

        inputs = self._get_inputs(block)
        outputs = self._get_outputs(block)
        feed_map = self.feed_var(inputs, place)

555
        if for_inplace_test:
C
cc 已提交
556 557
            # Some variables' tensors hold no buffer (tensor's _holder is NULL), like XShape in reshape2 op,
            # and the shapes of those variables contain 0 (eg. Xshape.shape = [0, 2, 5]).
558 559
            # Set persistable for those variables in order to get them from global_scope for inplace grad test directly other than feed them,
            # since feed op calls check_memory_size() which fails when tensor's holder_ is NULL.
560 561
            for out_name in op.output_arg_names:
                var = block.var(out_name)
562 563
                if 0 in var.shape:
                    var.persistable = True
564
        original_program = program
565 566
        if parallel:
            use_cuda = False
567
            if isinstance(place, fluid.CUDAPlace):
568
                use_cuda = True
569 570 571
            compiled_prog = fluid.CompiledProgram(program).with_data_parallel(
                loss_name=loss.name if loss else None, places=place)
            program = compiled_prog
572 573 574 575
        fetch_list = getattr(self, "fetch_list", [])
        # if the fetch_list is customized by user, we use it directly.
        # if not, fill the fetch_list by the user configured outputs in test.
        if len(fetch_list) == 0:
M
minqiyang 已提交
576
            for var_name, var in six.iteritems(outputs):
577 578
                if no_check_set is not None and var_name in no_check_set:
                    continue
Y
Yang Yang(Tony) 已提交
579 580
                if isinstance(var, list):
                    for v in var:
581
                        fetch_list.append(v.name)
Y
Yang Yang(Tony) 已提交
582
                else:
583
                    fetch_list.append(var.name)
584 585 586 587
        # if the fetch_list still empty, fill the fetch_list by the operator output.
        if len(fetch_list) == 0:
            for out_name, out_dup in Operator.get_op_outputs(self.op_type):
                fetch_list.append(str(out_name))
588 589 590 591 592 593 594 595 596

        if enable_inplace is not None:
            build_strategy = fluid.BuildStrategy()
            build_strategy.enable_inplace = enable_inplace

            compiled_prog = fluid.CompiledProgram(program).with_data_parallel(
                build_strategy=build_strategy, places=place)
            program = compiled_prog

597
        executor = Executor(place)
598 599 600 601
        outs = executor.run(program,
                            feed=feed_map,
                            fetch_list=fetch_list,
                            return_numpy=False)
602 603
        self.op = op
        self.program = original_program
604 605 606 607
        if for_inplace_test:
            return outs, fetch_list, feed_map, original_program, op.desc
        else:
            return outs, fetch_list
Y
Yang Yang(Tony) 已提交
608

609 610 611 612 613 614 615 616 617
    def _compare_expect_and_actual_outputs(self,
                                           place,
                                           fetch_list,
                                           expect_outs,
                                           actual_outs,
                                           inplace_atol=None):
        """Compare expect outs and actual outs of an tested op.

        Args:
C
cc 已提交
618
            place (CPUPlace | CUDAPlace): The place where the op runs.
619 620 621 622 623 624 625 626 627 628
            fetch_list (list): The outputs of tested op.
            expect_outs (list): The expect outs of tested op.
            actual_outs (list): The actual outs of tested op.
            inplace_atol (float): The tolerable error, only set when tested op doesn't ensure computational consistency, like group_norm op.

        Returns:
            None.
        """
        # compare expect_outs and actual_outs
        for i, name in enumerate(fetch_list):
C
cc 已提交
629
            # Note(zhiqiu): inplace_atol should be only set when op doesn't ensure
L
Leo Chen 已提交
630 631 632
            # computational consistency.
            # When inplace_atol is not None, the inplace check uses numpy.allclose
            # to check inplace result instead of numpy.array_equal.
633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658
            if inplace_atol is not None:
                self.assertTrue(
                    np.allclose(
                        np.array(expect_outs[i]),
                        np.array(actual_outs[i]),
                        atol=inplace_atol),
                    "Output (" + name + ") has diff at " + str(place) +
                    " when using and not using inplace" + "\nExpect " +
                    str(expect_outs[i]) + "\n" + "But Got" + str(actual_outs[i])
                    + " in class " + self.__class__.__name__)
            else:
                self.assertTrue(
                    np.array_equal(
                        np.array(expect_outs[i]), np.array(actual_outs[i])),
                    "Output (" + name + ") has diff at " + str(place) +
                    " when using and not using inplace" + "\nExpect " +
                    str(expect_outs[i]) + "\n" + "But Got" + str(actual_outs[i])
                    + " in class " + self.__class__.__name__ + '\n')

    def _construct_grad_program_from_forward(self, fwd_program, grad_op_desc,
                                             op_grad_to_var):
        """Generate grad_program which contains the grad_op.

        Args:
            fwd_program (tuple): The program that contains grad_op_desc's corresponding forward op.
            grad_op_desc (OpDesc): The OpDesc of grad op.
C
cc 已提交
659
            op_grad_to_var (dict): The relation of variables in grad op and its forward op.
660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685

        Returns:
            grad_program (program): The program which contains the grad_op.
        """
        grad_program = Program()
        grad_block = grad_program.global_block()
        new_op_desc = grad_block.desc.append_op()
        new_op_desc.copy_from(grad_op_desc)
        grad_program._sync_with_cpp()

        # Create grad vars based on fwd vars (shape and dtype)
        for arg in grad_op_desc.input_arg_names(
        ) + grad_op_desc.output_arg_names():
            fwd_var_name = op_grad_to_var.get(arg, None)
            if fwd_var_name is None:
                fwd_var_name = arg
            fwd_var = fwd_program.global_block().vars.get(fwd_var_name)
            assert fwd_var is not None, "{} cannot be found".format(
                fwd_var_name)
            grad_var = grad_block.create_var(
                name=arg,
                dtype=fwd_var.dtype,
                shape=fwd_var.shape,
                type=fwd_var.type,
                persistable=False)

C
cc 已提交
686 687
            # Some variables' tensors hold no buffer (tensor's _holder is NULL), like XShape in reshape2 op,
            # and the shapes of those variables contain 0 (eg. Xshape.shape = [0, 2, 5]).
688 689 690 691 692 693 694 695 696 697 698 699 700 701 702
            # Set persistable for those variables in order to get them from global_scope for inplace grad test directly other than feed them,
            # since feed op calls check_memory_size() which fails when tensor's holder_ is NULL.
            if 0 in grad_var.shape:
                grad_var.persistable = True
        grad_program._sync_with_cpp()
        return grad_program

    def _construct_grad_feed_map_from_forward(self, place, fwd_res,
                                              grad_op_desc, op_grad_to_var):
        """Generate grad_feed_map for grad_program.

        since we don`t really check gradient accuracy, but check the consistency when using and not using inplace,
        we use fwd outs (also inputs sometimes) to construct grad inputs.

        Args:
C
cc 已提交
703
            place (CPUPlace | CUDAPlace): The place where the op runs.
704 705 706
            fwd_res (tuple): The outputs of its forward op, in the same form as returns of _calc_outputs() when for_inplace_test is True.
                i.e., tuple(fwd_outs, fwd_fetch_list, fwd_feed_map, fwd_program, fwd_op_desc)
            grad_op_desc (OpDesc): The OpDesc of grad op.
C
cc 已提交
707
            op_grad_to_var (dict): The relation of variables in grad op and its fwd_op.
708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738

        Returns:
            grad_feed_map (dict): The feed_map of grad_op.
        """
        fwd_outs, fwd_fetch_list, fwd_feed_map, fwd_program, fwd_op_desc = fwd_res
        p = core.Place()
        p.set_place(place)
        grad_feed_map = {}
        for arg in grad_op_desc.input_arg_names():
            if arg in fwd_feed_map.keys():
                grad_feed_map[arg] = fwd_feed_map[arg]._copy(p)
            else:
                fwd_var_name = op_grad_to_var.get(arg, None)
                if fwd_var_name is None:
                    fwd_var_name = arg

                for i, out_name in enumerate(fwd_fetch_list):
                    if out_name == fwd_var_name:
                        # don't feed variables whose tensors hold no buffer (shape contains 0 like shape = [0,2,5] and holder_ is NULL), like XShape in reshape2 op.
                        # get them from global_scope directly since we have set them persistable in fwd execution
                        if 0 in fwd_program.global_block().var(out_name).shape:
                            continue
                        else:
                            grad_feed_map[arg] = fwd_outs[i]._copy(p)
        return grad_feed_map

    def _get_need_run_ops(self, op_desc, fwd_op_desc=None):
        """Postorder traversal of the 'grad' tree to get all ops that need to run during inplace test.
        An op needs to run druing inplace check if,
        (1) it has infer_inplace,
        (2) it has infer_inplace in its grad descendants. (since we need its outputs as to construct its grad's inputs)
C
cc 已提交
739

740
        Args:
C
cc 已提交
741 742
            op_desc (OpDesc): The op_desc of current op.
            fwd_op_desc (OpDesc): The op_desc of current op's forward op, None if current op has no forward op.
743
                Eg. relu's fwd_op is None, relu_grad's fwd_op is relu, relu_grad_grad's fwd_op is relu_grad, etc.
C
cc 已提交
744

745 746 747 748 749 750 751 752 753 754 755 756 757 758
        Returns:
            need_run_ops (list[(op_desc, fwd_op_desc)]): The ops that need to run during inplace test.
        """
        need_run_ops = []
        visited_ops = []

        def _dfs_grad_op(op_desc, fwd_op_desc=None):
            visited_ops.append(op_desc.type())
            has_infer_inplace = fluid.core.has_infer_inplace(op_desc.type())
            has_grad_op_maker = fluid.core.has_grad_op_maker(op_desc.type())
            has_infer_inplace_in_grad_descendants = False
            if not has_grad_op_maker:
                has_infer_inplace_in_descendants = False
            else:
C
cc 已提交
759
                # get grad_op_desc
760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784
                grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc(
                    op_desc, set(), [])
                if not grad_op_desc_list:
                    has_infer_inplace_in_grad_descendants = False
                else:
                    for i, grad_op_desc in enumerate(grad_op_desc_list):
                        if grad_op_desc.type(
                        ) not in visited_ops and _dfs_grad_op(
                                grad_op_desc, fwd_op_desc=op_desc):
                            has_infer_inplace_in_grad_descendants = True
            if has_infer_inplace or has_infer_inplace_in_grad_descendants:
                need_run_ops.append((op_desc, fwd_op_desc))
                return True
            else:
                return False

        _dfs_grad_op(op_desc, fwd_op_desc=fwd_op_desc)
        return need_run_ops

    def _check_forward_inplace(self,
                               place,
                               no_check_set=None,
                               inplace_atol=None):
        """Chech the inplace correctness of given op (self.op_type).
        Run the op twice with same inputs, one enable inplace and another disable, compare their outputs.
C
cc 已提交
785

786
        Args:
C
cc 已提交
787
            place (CPUPlace | CUDAPlace): The place where the op runs.
788 789 790 791
            no_check_set (list): The names of outputs that needn't check, like XShape of reshape op.
            inplace_atol (float): The tolerable error, only set when op doesn't ensure computational consistency, like group_norm op.

        Returns:
C
cc 已提交
792 793
            expect_res (tuple(outs, fetch_list, feed_map, program, op_desc)): The results of given op.
                We return this to construct grad_program and grad_feed_map for grad inplace check.
794 795
        """
        # _calc_output() returns in the form tuple(outs, fetch_list, feed_map, program, op_desc) when for_inplace_test=True.
796 797 798 799 800 801 802 803 804 805
        expect_res = self._calc_output(
            place,
            no_check_set=no_check_set,
            enable_inplace=False,
            for_inplace_test=True)
        actual_res = self._calc_output(
            place,
            no_check_set=no_check_set,
            enable_inplace=True,
            for_inplace_test=True)
806
        # compare expect_outs and actual_outs
807 808 809 810 811 812
        self._compare_expect_and_actual_outputs(
            place,
            expect_res[1],
            expect_res[0],
            actual_res[0],
            inplace_atol=inplace_atol)
813 814 815 816 817 818 819 820 821 822 823 824 825
        return expect_res

    def _calc_grad_output(self,
                          place,
                          fwd_res,
                          grad_op_desc,
                          enable_inplace=None):
        """Calculate grad_output for given grad_op_desc.

        since we don`t really check gradient accuracy, but check the consistency when using and not using inplace,
        we use fwd outs (also inputs sometimes) to construct grad inputs.

        Args:
C
cc 已提交
826
            place (CPUPlace | CUDAPlace): The place where the op runs.
827 828 829 830 831 832 833 834 835
            fwd_res (tuple): The outputs of its forward op, in the same form as returns of _calc_outputs() when for_inplace_test is True.
                i.e., tuple(fwd_outs, fwd_fetch_list, fwd_feed_map, fwd_program, fwd_op_desc).
            grad_op_desc (OpDesc): The OpDesc of grad op.
            enable_inplace (bool): Enable inplace or not.

        Returns:
            res (tuple(outs, fetch_list, feed_map, program, op_desc)): The results of given grad_op_desc.
        """
        fwd_outs, fwd_fetch_list, fwd_feed_map, fwd_program, fwd_op_desc = fwd_res
836
        grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc(fwd_op_desc,
837
                                                                  set(), [])
838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869
        grad_program = self._construct_grad_program_from_forward(
            fwd_program, grad_op_desc, op_grad_to_var)
        grad_feed_map = self._construct_grad_feed_map_from_forward(
            place, fwd_res, grad_op_desc, op_grad_to_var)
        grad_fetch_list = grad_op_desc.output_arg_names()
        exe = Executor(place)
        program = grad_program
        if enable_inplace is not None:
            build_strategy = fluid.BuildStrategy()
            build_strategy.enable_inplace = enable_inplace
            compiled_program = fluid.CompiledProgram(
                grad_program).with_data_parallel(
                    loss_name="", build_strategy=build_strategy, places=place)
            program = compiled_program
        outs = exe.run(program,
                       feed=grad_feed_map,
                       fetch_list=grad_fetch_list,
                       return_numpy=False)
        return outs, grad_fetch_list, grad_feed_map, grad_program, grad_op_desc

    def _check_grad_inplace(self,
                            place,
                            fwd_res,
                            grad_op_desc,
                            inplace_atol=None):
        """Chech the inplace correctness of given grad_op_desc.

        Run the grad op twice with same inputs, one enable inplace and another disable, compare their outputs.
        It works like _check_forward_inplace, but the way to construct program and feed_map differs.
        So we define a new function for grad, grad_grad, etc.

        Args:
C
cc 已提交
870
            place (CPUPlace | CUDAPlace): The place where the op runs.
871 872 873 874 875 876
            fwd_res (tuple): The outputs of its forward op, in the same form as returns of _calc_outputs() when for_inplace_test is True.
                i.e., tuple(fwd_outs, fwd_fetch_list, fwd_feed_map, fwd_program, fwd_op_desc).
            grad_op_desc (OpDesc): The OpDesc of grad op.
            inplace_atol (float): The tolerable error, only set when op doesn't ensure computational consistency, like group_norm op.

        Returns:
C
cc 已提交
877 878
            expect_res (tuple(outs, fetch_list, feed_map, program, op_desc)): The results of given op.
                We return this to construct grad_program and grad_feed_map for grad inplace check.
879 880 881 882 883 884 885 886 887 888 889 890
        """
        expect_res = self._calc_grad_output(
            place, fwd_res, grad_op_desc, enable_inplace=False)
        actual_res = self._calc_grad_output(
            place, fwd_res, grad_op_desc, enable_inplace=True)
        self._compare_expect_and_actual_outputs(
            place,
            expect_res[1],
            expect_res[0],
            actual_res[0],
            inplace_atol=inplace_atol)
        return expect_res
891

892 893 894 895 896 897 898 899 900 901
    def check_inplace_output_with_place(self,
                                        place,
                                        no_check_set=None,
                                        inplace_atol=None):
        """Chech the inplace correctness of given op, its grad op, its grad_grad op, etc.

        (1) Get all ops need to run. (see conditions in _get_need_run_ops())
        (2) Run op in need_run_ops, and do inplace check if it has infer_inplace.

        Args:
C
cc 已提交
902
            place (CPUPlace | CUDAPlace): The place where the op runs.
903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930
            no_check_set (list): The names of outputs that needn't check, like XShape of reshape op.
            inplace_atol (float): The tolerable error, only set when op doesn't ensure computational consistency, like group_norm op.

        Returns:
            None
        """
        has_infer_inplace = fluid.core.has_infer_inplace(self.op_type)
        has_grad_op_maker = fluid.core.has_grad_op_maker(self.op_type)

        fwd_res = self._calc_output(
            place, no_check_set=no_check_set, for_inplace_test=True)
        op_desc = fwd_res[4]
        need_run_ops = self._get_need_run_ops(op_desc)

        res = {}
        for op_desc, father_op_desc in reversed(need_run_ops):
            # The first one is the forward op
            has_infer_inplace = fluid.core.has_infer_inplace(op_desc.type())
            if op_desc.type() == self.op_type:
                if has_infer_inplace:
                    res[op_desc] = self._check_forward_inplace(
                        place,
                        no_check_set=no_check_set,
                        inplace_atol=inplace_atol)
                else:
                    res[op_desc] = self._calc_output(
                        place, no_check_set=no_check_set, for_inplace_test=True)
            else:
931 932
                # TODO(zhiqiu): enhance inplace_grad test for ops (sum and activation) using mkldnn
                # skip op that use_mkldnn currently
933
                flags_use_mkldnn = fluid.core.globals()["FLAGS_use_mkldnn"]
934 935 936 937 938 939 940 941 942 943 944 945
                attrs_use_mkldnn = hasattr(
                    self,
                    'attrs') and bool(self.attrs.get('use_mkldnn', False))
                if flags_use_mkldnn or attrs_use_mkldnn:
                    warnings.warn(
                        "check inplace_grad for ops using mkldnn is not supported"
                    )
                    continue
                if has_infer_inplace:
                    fwd_res = res[father_op_desc]
                    res[op_desc] = self._check_grad_inplace(
                        place, fwd_res, op_desc, inplace_atol=inplace_atol)
946
                else:
947 948
                    res[op_desc] = self._calc_grad_output(place, fwd_res,
                                                          op_desc)
949

950 951
    def check_output_with_place(self,
                                place,
952
                                atol=0,
953
                                no_check_set=None,
M
minqiyang 已提交
954
                                equal_nan=False,
955
                                check_dygraph=True,
956
                                inplace_atol=None):
957 958 959 960 961
        self.infer_dtype_from_inputs_outputs(self.inputs, self.outputs)
        if self.dtype == np.float64 and \
            self.op_type not in op_threshold_white_list.NEED_FIX_FP64_CHECK_OUTPUT_THRESHOLD_OP_LIST:
            atol = 0

962 963 964 965
        if no_check_set is not None:
            if self.op_type not in no_check_set_white_list.no_check_set_white_list:
                raise AssertionError(
                    "no_check_set of op %s must be set to None." % self.op_type)
966

L
lujun 已提交
967 968
        if check_dygraph:
            dygraph_outs = self._calc_dygraph_output(
M
minqiyang 已提交
969
                place, no_check_set=no_check_set)
970
        outs, fetch_list = self._calc_output(place, no_check_set=no_check_set)
Y
Yang Yang(Tony) 已提交
971
        for out_name, out_dup in Operator.get_op_outputs(self.op_type):
972 973
            if out_name not in self.outputs:
                continue
974 975
            if no_check_set is not None and out_name in no_check_set:
                continue
976

977 978 979 980 981 982 983 984 985 986 987 988
            def find_imperative_actual(target_name, dygraph_outs, place):
                with fluid.dygraph.base.guard(place=place):
                    for name in dygraph_outs:
                        if name == target_name:
                            return dygraph_outs[name][0]
                        var_list = dygraph_outs[name]
                        for i, var in enumerate(var_list):
                            if var.name == target_name:
                                return dygraph_outs[name][i]
                    self.assertTrue(False, "Found failed {} {}".format(
                        dygraph_outs.keys(), target_name))

Y
Yang Yang(Tony) 已提交
989 990
            def find_actual(target_name, fetch_list):
                found = [
991 992
                    i for i, var_name in enumerate(fetch_list)
                    if var_name == target_name
Y
Yang Yang(Tony) 已提交
993 994 995 996 997 998
                ]
                self.assertTrue(
                    len(found) == 1, "Found {} {}".format(
                        len(found), target_name))
                return found[0]

999 1000
            if out_dup:
                sub_out = self.outputs[out_name]
Y
Yancey 已提交
1001 1002 1003
                if not isinstance(sub_out, list):
                    raise AssertionError("sub_out type %s is not list",
                                         type(sub_out))
1004 1005
                for item in sub_out:
                    sub_out_name, expect = item[0], item[1]
L
lujun 已提交
1006
                    if check_dygraph:
1007 1008
                        imperative_actual = find_imperative_actual(
                            sub_out_name, dygraph_outs, place)
1009 1010
                        imperative_actual_t = np.array(imperative_actual.value()
                                                       .get_tensor())
Y
Yang Yang(Tony) 已提交
1011
                    idx = find_actual(sub_out_name, fetch_list)
Q
QI JUN 已提交
1012 1013
                    actual = outs[idx]
                    actual_t = np.array(actual)
1014 1015
                    expect_t = expect[0] \
                        if isinstance(expect, tuple) else expect
1016 1017
                    self.assertTrue(
                        np.allclose(
1018
                            actual_t, expect_t, atol=atol, equal_nan=equal_nan),
Y
Yang Yang(Tony) 已提交
1019 1020
                        "Output (" + sub_out_name + ") has diff at " +
                        str(place))
L
lujun 已提交
1021
                    if check_dygraph:
M
minqiyang 已提交
1022 1023 1024 1025 1026 1027 1028
                        self.assertTrue(
                            np.allclose(
                                imperative_actual_t,
                                expect_t,
                                atol=atol,
                                equal_nan=equal_nan),
                            "Output (" + sub_out_name + ") has diff at " +
L
lujun 已提交
1029
                            str(place) + " in dygraph mode")
1030 1031
                    if isinstance(expect, tuple):
                        self.assertListEqual(
1032 1033
                            actual.recursive_sequence_lengths(), expect[1],
                            "Output (" + sub_out_name +
Q
QI JUN 已提交
1034
                            ") has different lod at " + str(place))
1035 1036
                        if check_dygraph:
                            self.assertListEqual(
1037
                                imperative_actual.value().get_tensor()
1038 1039 1040 1041
                                .recursive_sequence_lengths(), expect[1],
                                "Output (" + out_name +
                                ") has different lod at " + str(place) +
                                " in dygraph mode")
1042
            else:
L
lujun 已提交
1043
                if check_dygraph:
1044 1045
                    imperative_actual = find_imperative_actual(
                        out_name, dygraph_outs, place)
1046 1047
                    imperative_actual_t = np.array(imperative_actual.value()
                                                   .get_tensor())
Y
Yang Yang(Tony) 已提交
1048
                idx = find_actual(out_name, fetch_list)
Q
QI JUN 已提交
1049 1050
                actual = outs[idx]
                actual_t = np.array(actual)
1051
                expect = self.outputs[out_name]
1052
                expect_t = expect[0] if isinstance(expect, tuple) else expect
1053 1054
                self.assertTrue(
                    np.allclose(
1055
                        actual_t, expect_t, atol=atol, equal_nan=equal_nan),
E
emailweixu 已提交
1056
                    "Output (" + out_name + ") has diff at " + str(place) +
D
dzhwinter 已提交
1057
                    "\nExpect " + str(expect_t) + "\n" + "But Got" +
1058
                    str(actual_t) + " in class " + self.__class__.__name__)
L
lujun 已提交
1059
                if check_dygraph:
1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075
                    if six.moves.reduce(
                            lambda x, y: x * y, imperative_actual_t.shape,
                            1) == 0 and six.moves.reduce(
                                lambda x, y: x * y, expect_t.shape, 1) == 0:
                        pass
                    else:
                        self.assertTrue(
                            np.allclose(
                                imperative_actual_t,
                                expect_t,
                                atol=atol,
                                equal_nan=equal_nan),
                            "Output (" + out_name + ") has diff at " +
                            str(place) + "\nExpect " + str(expect_t) + "\n" +
                            "But Got" + str(imperative_actual_t) + " in class "
                            + self.__class__.__name__)
1076
                if isinstance(expect, tuple):
1077 1078
                    self.assertListEqual(actual.recursive_sequence_lengths(),
                                         expect[1], "Output (" + out_name +
1079
                                         ") has different lod at " + str(place))
L
lujun 已提交
1080
                    if check_dygraph:
M
minqiyang 已提交
1081
                        self.assertListEqual(
1082
                            imperative_actual.value().get_tensor()
M
minqiyang 已提交
1083 1084
                            .recursive_sequence_lengths(), expect[1],
                            "Output (" + out_name + ") has different lod at " +
L
lujun 已提交
1085
                            str(place) + " in dygraph mode")
1086

C
cc 已提交
1087
        # Note(zhiqiu): inplace_atol should be only set when op doesn't ensure
L
Leo Chen 已提交
1088 1089
        # computational consistency.
        # For example, group_norm uses AtomicAdd on CUDAPlace, which do not ensure
C
cc 已提交
1090
        # computation order when multiple threads write the same address. So the
L
Leo Chen 已提交
1091 1092 1093
        # result of group_norm is non-deterministic when datatype is float.
        # When inplace_atol is not None, the inplace check uses numpy.allclose
        # to check inplace result instead of numpy.array_equal.
1094 1095
        if inplace_atol is not None:
            warnings.warn(
L
Leo Chen 已提交
1096 1097
                "inplace_atol should only be set when op doesn't ensure computational consistency, please check it!"
            )
1098
        # Check inplace for given op, its grad op, its grad_grad op, etc.
C
cc 已提交
1099
        # No effect on original OpTest
1100 1101 1102
        self.check_inplace_output_with_place(
            place, no_check_set=no_check_set, inplace_atol=inplace_atol)

1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150
        if check_dygraph:
            return outs, dygraph_outs, fetch_list
        else:
            return outs, fetch_list

    def check_compile_vs_runtime(self, fetch_list, fetch_outs):
        def find_fetch_index(target_name, fetch_list):
            found = [
                i for i, var_name in enumerate(fetch_list)
                if var_name == target_name
            ]
            if len(found) == 0:
                return -1
            else:
                self.assertTrue(
                    len(found) == 1,
                    "Found {} {}".format(len(found), target_name))
                return found[0]

        for name in self.op.desc.output_names():
            var_names = self.op.desc.output(name)
            for var_name in var_names:
                i = find_fetch_index(var_name, fetch_list)
                if i == -1:
                    # The output is dispensiable or intermediate.
                    break
                out = fetch_outs[i]
                if isinstance(out, core.LoDTensor):
                    lod_level_runtime = len(out.lod())
                else:
                    if isinstance(out, core.LoDTensorArray):
                        warnings.warn(
                            "The check of LoDTensorArray's lod_level is not implemented now!"
                        )
                    lod_level_runtime = 0

                var = self.program.global_block().var(var_name)
                if var.type == core.VarDesc.VarType.LOD_TENSOR:
                    lod_level_compile = var.lod_level
                else:
                    lod_level_compile = 0
                self.assertEqual(
                    lod_level_compile, lod_level_runtime,
                    "The lod_level of Output (" + name +
                    ") is different between compile-time and runtime (" +
                    str(lod_level_compile) + " vs " + str(lod_level_runtime) +
                    ")")

1151
    def _get_places(self):
D
dzhwinter 已提交
1152 1153 1154 1155 1156 1157
        if self.dtype == np.float16:
            if core.is_compiled_with_cuda() and core.op_support_gpu(
                    self.op_type):
                place = core.CUDAPlace(0)
                if core.is_float16_supported(place):
                    return [place]
W
Wu Yi 已提交
1158 1159
                else:
                    return []
D
dzhwinter 已提交
1160 1161
            else:
                return []
1162
        places = [fluid.CPUPlace()]
1163 1164 1165
        cpu_only = self._cpu_only if hasattr(self, '_cpu_only') else False
        if core.is_compiled_with_cuda() and core.op_support_gpu(self.op_type)\
           and not cpu_only:
D
dzhwinter 已提交
1166
            places.append(core.CUDAPlace(0))
1167 1168
        return places

M
minqiyang 已提交
1169 1170 1171 1172
    def check_output(self,
                     atol=1e-5,
                     no_check_set=None,
                     equal_nan=False,
1173
                     check_dygraph=True,
1174
                     inplace_atol=None):
1175
        self.__class__.op_type = self.op_type
1176 1177 1178 1179
        if (hasattr(self, "use_mkldnn") and self.use_mkldnn == True) or \
            (hasattr(self, "attrs") and "use_mkldnn" in self.attrs and \
                    self.attrs["use_mkldnn"] == True):
            self.__class__.use_mkldnn = True
C
cc 已提交
1180

1181
        places = self._get_places()
Q
qijun 已提交
1182
        for place in places:
1183 1184 1185 1186 1187 1188
            res = self.check_output_with_place(place, atol, no_check_set,
                                               equal_nan, check_dygraph)
            if check_dygraph:
                outs, dygraph_outs, fetch_list = res
            else:
                outs, fetch_list = res
1189
            if self.op_type not in compile_vs_runtime_white_list.COMPILE_RUN_OP_WHITE_LIST:
1190
                self.check_compile_vs_runtime(fetch_list, outs)
Q
qijun 已提交
1191

1192
    def check_output_customized(self, checker):
1193
        places = self._get_places()
1194 1195 1196
        for place in places:
            outs = self.calc_output(place)
            outs = [np.array(out) for out in outs]
1197
            outs.sort(key=len)
1198 1199
            checker(outs)

D
Dun 已提交
1200 1201
    def _assert_is_close(self, numeric_grads, analytic_grads, names,
                         max_relative_error, msg_prefix):
1202

M
minqiyang 已提交
1203
        for a, b, name in six.moves.zip(numeric_grads, analytic_grads, names):
1204 1205 1206 1207 1208 1209
            # It asserts np.abs(a - b) / np.abs(a) < max_relative_error, in which
            # max_relative_error is 1e-7. According to the value of np.abs(a), we
            # change np.abs(a) to achieve dynamic threshold. For example, if
            # the value of np.abs(a) is between 1e-10 and 1e-8, we set np.abs(a)*=1e4.
            # Therefore, it asserts np.abs(a - b) / (np.abs(a)*1e4) < max_relative_error,
            # which is the same as np.abs(a - b) / np.abs(a) < max_relative_error*1e4.
1210
            abs_a = np.abs(a)
1211 1212 1213 1214 1215 1216 1217
            if self.dtype == np.float64 and \
                self.op_type not in op_threshold_white_list.NEED_FIX_FP64_CHECK_GRAD_THRESHOLD_OP_LIST:
                abs_a[abs_a < 1e-10] = 1e-3
                abs_a[np.logical_and(abs_a > 1e-10, abs_a <= 1e-8)] *= 1e4
                abs_a[np.logical_and(abs_a > 1e-8, abs_a <= 1e-6)] *= 1e2
            else:
                abs_a[abs_a < 1e-3] = 1
1218 1219 1220 1221 1222 1223

            diff_mat = np.abs(a - b) / abs_a
            max_diff = np.max(diff_mat)

            def err_msg():
                offset = np.argmax(diff_mat > max_relative_error)
1224 1225 1226
                return ("Operator %s error, %s variable %s (shape: %s, dtype: %s) max gradient diff %e over limit %e, "
                    "the first error element is %d, expected %e, but got %e.") \
                    % (self.op_type, msg_prefix, name, str(a.shape), self.dtype, max_diff, max_relative_error,
1227
                    offset, a.flatten()[offset], b.flatten()[offset])
1228 1229 1230

            self.assertLessEqual(max_diff, max_relative_error, err_msg())

1231 1232 1233 1234 1235 1236 1237
    def _check_grad_helper(self):
        self.infer_dtype_from_inputs_outputs(self.inputs, self.outputs)
        self.__class__.op_type = self.op_type
        self.__class__.exist_check_grad = True
        if self.dtype == np.float64:
            self.__class__.exist_fp64_check_grad = True

1238 1239
    def check_grad(self,
                   inputs_to_check,
Y
Yancey 已提交
1240
                   output_names,
1241
                   no_grad_set=None,
1242
                   numeric_grad_delta=0.005,
1243
                   in_place=False,
Q
Qiao Longfei 已提交
1244
                   max_relative_error=0.005,
1245 1246
                   user_defined_grads=None,
                   check_dygraph=True):
1247
        self._check_grad_helper()
1248
        places = self._get_places()
1249 1250 1251 1252
        for place in places:
            self.check_grad_with_place(place, inputs_to_check, output_names,
                                       no_grad_set, numeric_grad_delta,
                                       in_place, max_relative_error,
1253
                                       user_defined_grads, check_dygraph)
1254 1255 1256 1257 1258 1259 1260 1261 1262

    def check_grad_with_place(self,
                              place,
                              inputs_to_check,
                              output_names,
                              no_grad_set=None,
                              numeric_grad_delta=0.005,
                              in_place=False,
                              max_relative_error=0.005,
1263 1264
                              user_defined_grads=None,
                              check_dygraph=True):
1265
        self.scope = core.Scope()
Q
qijun 已提交
1266
        op_inputs = self.inputs if hasattr(self, "inputs") else dict()
1267
        op_outputs = self.outputs if hasattr(self, "outputs") else dict()
Q
qijun 已提交
1268
        op_attrs = self.attrs if hasattr(self, "attrs") else dict()
P
phlrain 已提交
1269

1270
        self._check_grad_helper()
1271 1272 1273 1274
        if self.dtype == np.float64 and \
            self.op_type not in op_threshold_white_list.NEED_FIX_FP64_CHECK_GRAD_THRESHOLD_OP_LIST:
            numeric_grad_delta = 1e-5
            max_relative_error = 1e-7
1275

P
phlrain 已提交
1276 1277 1278 1279 1280 1281 1282 1283 1284 1285
        cache_list = None
        if hasattr(self, "cache_name_list"):
            cache_list = self.cache_name_list
        self.op = create_op(
            self.scope,
            self.op_type,
            op_inputs,
            op_outputs,
            op_attrs,
            cache_list=cache_list)
Y
Yu Yang 已提交
1286

1287 1288
        if no_grad_set is None:
            no_grad_set = set()
1289 1290 1291 1292 1293 1294
        else:
            if (self.op_type not in no_grad_set_white_list.NEED_TO_FIX_OP_LIST
                ) and (self.op_type not in
                       no_grad_set_white_list.NOT_CHECK_OP_LIST):
                raise AssertionError("no_grad_set must be None, op_type is " +
                                     self.op_type + " Op.")
1295

1296 1297 1298 1299 1300 1301 1302 1303
        for input_to_check in inputs_to_check:
            set_input(self.scope, self.op, self.inputs, place)
            tensor_to_check = self.scope.find_var(input_to_check).get_tensor()
            tensor_size = six.moves.reduce(lambda a, b: a * b,
                                           tensor_to_check.shape(), 1)
            if tensor_size < 100:
                self.__class__.input_shape_is_large = False

Y
Yancey 已提交
1304 1305 1306
        if not type(output_names) is list:
            output_names = [output_names]

Q
Qiao Longfei 已提交
1307
        numeric_grads = user_defined_grads or [
1308
            get_numeric_gradient(
1309
                place,
1310 1311 1312 1313
                self.scope,
                self.op,
                self.inputs,
                input_to_check,
Y
Yancey 已提交
1314
                output_names,
1315
                delta=numeric_grad_delta,
C
chengduo 已提交
1316
                in_place=in_place) for input_to_check in inputs_to_check
1317
        ]
1318 1319
        analytic_grads = self._get_gradient(inputs_to_check, place,
                                            output_names, no_grad_set)
D
Dun 已提交
1320 1321 1322
        self._assert_is_close(numeric_grads, analytic_grads, inputs_to_check,
                              max_relative_error,
                              "Gradient Check On %s" % str(place))
Q
qijun 已提交
1323

1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431
        if check_dygraph:
            dygraph_grad = self._get_dygraph_grad(inputs_to_check, place,
                                                  output_names, no_grad_set)
            self._assert_is_close(numeric_grads, dygraph_grad, inputs_to_check,
                                  max_relative_error,
                                  "Gradient Check On %s" % str(place))

    def _find_var_in_dygraph(self, output_vars, name):
        if name in output_vars:
            return output_vars[name]
        else:
            for output_vars_index in output_vars:
                for output_vars_selected in output_vars[output_vars_index]:
                    if output_vars_selected.name == name:
                        return output_vars_selected

    def _get_dygraph_grad(self,
                          inputs_to_check,
                          place,
                          output_names,
                          no_grad_set=None):
        with fluid.dygraph.base.guard(place=place):
            block = fluid.default_main_program().global_block()

            op_proto = OpProtoHolder.instance().get_op_proto(self.op_type)

            # prepare input variable
            inputs, inputs_grad_dict = self.append_input_output_for_dygraph(
                op_proto, self.inputs, True, True, block)

            # prepare output variable
            outputs = self.append_input_output_for_dygraph(
                op_proto, self.outputs, False, False, block)

            # prepare attrbutes
            attrs_outputs = {}
            if hasattr(self, "attrs"):
                for attrs_name in self.attrs:
                    if self.attrs[attrs_name] is not None:
                        attrs_outputs[attrs_name] = self.attrs[attrs_name]
            block.append_op(
                type=self.op_type,
                inputs=inputs,
                outputs=outputs,
                attrs=attrs_outputs if hasattr(self, "attrs") else None)

            outputs_valid = {}
            for output_name in output_names:
                outputs_valid[output_name] = self._find_var_in_dygraph(
                    outputs, output_name)

            if len(outputs_valid) == 1:
                loss = block.create_var(
                    dtype=self.dtype,
                    type=core.VarDesc.VarType.LOD_TENSOR,
                    persistable=False,
                    stop_gradient=False,
                    shape=[1])
                for outputs_valid_key in outputs_valid:
                    block.append_op(
                        type="mean",
                        inputs={"X": outputs_valid[outputs_valid_key]},
                        outputs={"Out": [loss]},
                        attrs=None)
            else:
                avg_sum = []
                for cur_loss in outputs_valid:
                    cur_avg_loss = block.create_var(
                        dtype=self.dtype,
                        type=core.VarDesc.VarType.LOD_TENSOR,
                        persistable=False,
                        stop_gradient=False)
                    block.append_op(
                        type="mean",
                        inputs={"X": outputs_valid[cur_loss]},
                        outputs={"Out": [cur_avg_loss]},
                        attrs=None)
                    avg_sum.append(cur_avg_loss)
                loss_sum = block.create_var(
                    dtype=self.dtype,
                    type=core.VarDesc.VarType.LOD_TENSOR,
                    persistable=False,
                    stop_gradient=False,
                    shape=[1])
                block.append_op(
                    type='sum',
                    inputs={"X": avg_sum},
                    outputs={"Out": loss_sum},
                    attrs=None)
                loss = block.create_var(
                    dtype=self.dtype,
                    type=core.VarDesc.VarType.LOD_TENSOR,
                    persistable=False,
                    stop_gradient=False,
                    shape=[1])
                block.append_op(
                    type='scale',
                    inputs={"X": loss_sum},
                    outputs={"Out": loss},
                    attrs={'scale': 1.0 / float(len(avg_sum))})
            loss.backward()

            fetch_list_grad = []
            for inputs_to_check_name in inputs_to_check:
                a = inputs_grad_dict[inputs_to_check_name].gradient()
                fetch_list_grad.append(a)
            return fetch_list_grad

Y
Yu Yang 已提交
1432 1433 1434 1435 1436
    @staticmethod
    def _numpy_to_lod_tensor(np_value, lod, place):
        tensor = core.LoDTensor()
        tensor.set(np_value, place)
        if lod is not None:
1437
            tensor.set_recursive_sequence_lengths(lod)
Y
Yu Yang 已提交
1438 1439
        return tensor

K
Kexin Zhao 已提交
1440
    @staticmethod
K
Kexin Zhao 已提交
1441 1442
    def np_dtype_to_fluid_dtype(input):
        return input
K
Kexin Zhao 已提交
1443

D
dzhwinter 已提交
1444 1445 1446 1447 1448 1449 1450 1451
    @staticmethod
    def fluid_dtype_to_np_dtype(self, dtype):
        return dtype

    @staticmethod
    def np_value_to_fluid_value(input):
        return input

1452 1453 1454 1455 1456 1457
    def _get_gradient(self,
                      input_to_check,
                      place,
                      output_names,
                      no_grad_set,
                      parallel=False):
Y
Yu Yang 已提交
1458 1459
        prog = Program()
        block = prog.global_block()
1460 1461
        self._append_ops(block)
        loss = append_loss_ops(block, output_names)
F
fengjiayi 已提交
1462
        param_grad_list = append_backward(
Y
Yu Yang 已提交
1463 1464
            loss=loss, parameter_list=input_to_check, no_grad_set=no_grad_set)

1465 1466
        inputs = self._get_inputs(block)
        feed_dict = self.feed_var(inputs, place)
Y
Yu Yang 已提交
1467 1468

        fetch_list = [g for p, g in param_grad_list]
1469 1470
        if parallel:
            use_cuda = False
1471
            if isinstance(place, fluid.CUDAPlace):
1472
                use_cuda = True
1473 1474 1475 1476
            compiled_prog = fluid.CompiledProgram(prog).with_data_parallel(
                loss_name=loss.name, places=place)
            prog = compiled_prog
        executor = fluid.Executor(place)
1477 1478 1479
        return list(
            map(np.array,
                executor.run(prog, feed_dict, fetch_list, return_numpy=False)))