op_test.py 71.2 KB
Newer Older
1
#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
D
dzhwinter 已提交
2
#
D
dzhwinter 已提交
3 4 5
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
D
dzhwinter 已提交
6
#
D
dzhwinter 已提交
7
#     http://www.apache.org/licenses/LICENSE-2.0
D
dzhwinter 已提交
8
#
D
dzhwinter 已提交
9 10 11 12 13 14
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15 16
from __future__ import print_function

B
baojun 已提交
17
import os
18
import unittest
19
import warnings
20
import numpy as np
21
import random
M
minqiyang 已提交
22
import six
23
import struct
24
import time
25
import itertools
Y
Yu Yang 已提交
26
import collections
M
minqiyang 已提交
27
from collections import defaultdict
28

29
import paddle
30 31
import paddle.fluid as fluid
import paddle.fluid.core as core
32 33 34
from paddle.fluid.backward import append_backward
from paddle.fluid.op import Operator
from paddle.fluid.executor import Executor
35
from paddle.fluid.framework import Program, OpProtoHolder, Variable
36
from testsuite import create_op, set_input, append_input_output, append_loss_ops
37
from paddle.fluid import unique_name
38
from white_list import op_accuracy_white_list, check_shape_white_list, compile_vs_runtime_white_list, no_check_set_white_list
39
from white_list import op_threshold_white_list, no_grad_set_white_list
40 41


42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83
def check_out_dtype(api_fn, in_specs, expect_dtypes, target_index=0, **configs):
    """
    Determines whether dtype of output tensor is as expected.

    Args:
        api_fn(callable):  paddle api function
        in_specs(list[tuple]): list of shape and dtype information for constructing input tensor of api_fn, such as [(shape, dtype), (shape, dtype)].
        expected_dtype(list[str]): expected dtype of output tensor.
        target_index(int): indicate which one from in_specs to infer the dtype of output.
        config(dict): other arguments of paddle api function

    Example:
        check_out_dtype(fluid.layers.pad_constant_like, [([2,3,2,3], 'float64'), ([1, 3, 1,3], )], ['float32', 'float64', 'int64'], target_index=1, pad_value=0.)

    """
    paddle.enable_static()
    for i, expect_dtype in enumerate(expect_dtypes):
        with paddle.static.program_guard(paddle.static.Program()):
            input_t = []
            for index, spec in enumerate(in_specs):
                if len(spec) == 1:
                    shape = spec[0]
                    dtype = expect_dtype if target_index == index else 'float32'
                elif len(spec) == 2:
                    shape, dtype = spec
                else:
                    raise ValueError(
                        "Value of in_specs[{}] should contains two elements: [shape, dtype]".
                        format(index))
                input_t.append(
                    paddle.static.data(
                        name='data_%s' % index, shape=shape, dtype=dtype))

            out = api_fn(*input_t, **configs)
            out_dtype = fluid.data_feeder.convert_dtype(out.dtype)

            if out_dtype != expect_dtype:
                raise ValueError(
                    "Expected out.dtype is {}, but got {} from {}.".format(
                        expect_dtype, out_dtype, api_fn.__name__))


84 85 86 87 88 89 90 91
def _set_use_system_allocator(value=None):
    USE_SYSTEM_ALLOCATOR_FLAG = "FLAGS_use_system_allocator"
    old_value = core.globals()[USE_SYSTEM_ALLOCATOR_FLAG]
    value = old_value if value is None else value
    core.globals()[USE_SYSTEM_ALLOCATOR_FLAG] = value
    return old_value


92 93 94 95
def randomize_probability(batch_size, class_num, dtype='float32'):
    prob = np.random.uniform(
        0.1, 1.0, size=(batch_size, class_num)).astype(dtype)
    prob_sum = prob.sum(axis=1)
M
minqiyang 已提交
96
    for i in six.moves.xrange(len(prob)):
97 98 99 100
        prob[i] /= prob_sum[i]
    return prob


101 102
def get_numeric_gradient(place,
                         scope,
103 104 105
                         op,
                         inputs,
                         input_to_check,
Y
Yancey 已提交
106
                         output_names,
107
                         delta=0.005,
C
chengduo 已提交
108
                         in_place=False):
Y
Yu Yang 已提交
109
    # FIXME: change this method by compile time concepts
110
    set_input(scope, op, inputs, place)
111 112

    def product(dim):
M
minqiyang 已提交
113
        return six.moves.reduce(lambda a, b: a * b, dim, 1)
114 115

    tensor_to_check = scope.find_var(input_to_check).get_tensor()
Y
yuyang18 已提交
116 117
    tensor_size = product(tensor_to_check.shape())
    tensor_to_check_dtype = tensor_to_check._dtype()
118
    if tensor_to_check_dtype == core.VarDesc.VarType.FP32:
119
        tensor_to_check_dtype = np.float32
120
    elif tensor_to_check_dtype == core.VarDesc.VarType.FP64:
121
        tensor_to_check_dtype = np.float64
D
dzhwinter 已提交
122 123 124 125
    elif tensor_to_check_dtype == core.VarDesc.VarType.FP16:
        tensor_to_check_dtype = np.float16
        # set delta as np.float16, will automatic convert to float32, float64
        delta = np.array(delta).astype(np.float16)
126 127 128 129
    else:
        raise ValueError("Not supported data type " + str(
            tensor_to_check_dtype))

C
chengduo 已提交
130 131 132 133 134 135 136 137 138
    def get_output():
        sum = []
        op.run(scope, place)
        for output_name in output_names:
            sum.append(
                np.array(scope.find_var(output_name).get_tensor()).astype(
                    tensor_to_check_dtype).mean())
        return tensor_to_check_dtype(np.array(sum).sum() / len(output_names))

139 140 141
    gradient_flat = np.zeros(shape=(tensor_size, ), dtype=tensor_to_check_dtype)

    def __get_elem__(tensor, i):
D
dzhwinter 已提交
142 143 144 145 146
        if tensor_to_check_dtype == np.float16:
            numpy_tensor = np.array(tensor).astype(np.float16)
            numpy_tensor = numpy_tensor.flatten()
            return numpy_tensor[i]
        elif tensor_to_check_dtype == np.float32:
Y
yuyang18 已提交
147
            return tensor._get_float_element(i)
148
        elif tensor_to_check_dtype == np.float64:
Y
yuyang18 已提交
149
            return tensor._get_double_element(i)
150 151 152
        else:
            raise TypeError("Unsupported test data type %s." %
                            tensor_to_check_dtype)
153 154

    def __set_elem__(tensor, i, e):
D
dzhwinter 已提交
155 156 157 158 159
        if tensor_to_check_dtype == np.float16:
            numpy_tensor = np.array(tensor).astype(np.float16)
            shape = numpy_tensor.shape
            numpy_tensor = numpy_tensor.flatten()
            numpy_tensor[i] = e
160
            numpy_tensor = numpy_tensor.reshape(shape)
D
dzhwinter 已提交
161 162
            tensor.set(numpy_tensor, place)
        elif tensor_to_check_dtype == np.float32:
Y
yuyang18 已提交
163
            tensor._set_float_element(i, e)
164
        elif tensor_to_check_dtype == np.float64:
Y
yuyang18 已提交
165
            tensor._set_double_element(i, e)
166 167 168
        else:
            raise TypeError("Unsupported test data type %s." %
                            tensor_to_check_dtype)
169

170 171
    # we only compute gradient of one element each time.
    # we use a for loop to compute the gradient of every element.
M
minqiyang 已提交
172
    for i in six.moves.xrange(tensor_size):
173
        if in_place:
174
            set_input(scope, op, inputs, place)
175 176

        # get one input element throw it's index i.
177
        origin = __get_elem__(tensor_to_check, i)
178 179
        # add delta to it, run op and then get the sum of the result tensor.
        x_pos = origin + delta
180
        __set_elem__(tensor_to_check, i, x_pos)
181 182 183
        y_pos = get_output()

        if in_place:
184
            set_input(scope, op, inputs, place)
185 186

        x_neg = origin - delta
187
        __set_elem__(tensor_to_check, i, x_neg)
188 189
        y_neg = get_output()

190
        __set_elem__(tensor_to_check, i, origin)
191 192
        gradient_flat[i] = (y_pos - y_neg) / delta / 2

Y
yuyang18 已提交
193
    return gradient_flat.reshape(tensor_to_check.shape())
194 195


196 197
def skip_check_grad_ci(reason=None):
    """Decorator to skip check_grad CI.
C
cc 已提交
198

199
       Check_grad is required for Op test cases. However, there are some special
C
cc 已提交
200
       cases that do not need to do check_grad. This decorator is used to skip the
201
       check_grad of the above cases.
C
cc 已提交
202 203

       Note: the execution of unit test will not be skipped. It just avoids check_grad
204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219
       checking in tearDownClass method by setting a `no_need_check_grad` flag.

       Example:
           @skip_check_grad_ci(reason="For inference, check_grad is not required.")
           class TestInference(OpTest):
    """
    if not isinstance(reason, str):
        raise AssertionError("The reason for skipping check_grad is required.")

    def wrapper(cls):
        cls.no_need_check_grad = True
        return cls

    return wrapper


220 221 222 223 224 225 226 227 228 229 230 231
def copy_bits_from_float_to_uint16(f):
    return struct.unpack('<I', struct.pack('<f', f))[0] >> 16


def convert_float_to_uint16(float_list):
    new_output = []
    for x in np.nditer(float_list):
        new_output.append(np.uint16(copy_bits_from_float_to_uint16(x)))

    return np.reshape(new_output, float_list.shape).view(np.uint16)


232
class OpTest(unittest.TestCase):
233 234 235 236 237
    @classmethod
    def setUpClass(cls):
        '''Fix random seeds to remove randomness from tests'''
        cls._np_rand_state = np.random.get_state()
        cls._py_rand_state = random.getstate()
238
        cls.call_once = False
239
        cls.dtype = None
240
        cls.outputs = {}
241
        cls.input_shape_is_large = True
242 243 244 245

        np.random.seed(123)
        random.seed(124)

246 247
        cls._use_system_allocator = _set_use_system_allocator(True)

248 249
    @classmethod
    def tearDownClass(cls):
Y
yuyang18 已提交
250
        """Restore random seeds"""
251 252 253
        np.random.set_state(cls._np_rand_state)
        random.setstate(cls._py_rand_state)

254 255
        _set_use_system_allocator(cls._use_system_allocator)

256 257 258 259
        def is_empty_grad_op(op_type):
            all_op_kernels = core._get_all_register_op_kernels()
            grad_op = op_type + '_grad'
            if grad_op in all_op_kernels.keys():
J
juncaipeng 已提交
260
                if is_mkldnn_op_test():
261 262 263 264 265 266 267 268
                    grad_op_kernels = all_op_kernels[grad_op]
                    for grad_op_kernel in grad_op_kernels:
                        if 'MKLDNN' in grad_op_kernel:
                            return False
                else:
                    return False
            return True

269 270 271
        def is_xpu_op_test():
            return hasattr(cls, "use_xpu") and cls.use_xpu == True

J
juncaipeng 已提交
272
        def is_mkldnn_op_test():
273
            return hasattr(cls, "use_mkldnn") and cls.use_mkldnn == True
J
juncaipeng 已提交
274

275 276
        if not hasattr(cls, "op_type"):
            raise AssertionError(
277 278
                "This test do not have op_type in class attrs, "
                "please set self.__class__.op_type=the_real_op_type manually.")
279

J
juncaipeng 已提交
280 281
        # case in NO_FP64_CHECK_GRAD_CASES and op in NO_FP64_CHECK_GRAD_OP_LIST should be fixed
        if not hasattr(cls, "no_need_check_grad") \
282
            and not is_empty_grad_op(cls.op_type):
J
juncaipeng 已提交
283
            if cls.dtype is None or \
284 285
                (cls.dtype == np.float16 \
                    and cls.op_type not in op_accuracy_white_list.NO_FP16_CHECK_GRAD_OP_LIST \
J
juncaipeng 已提交
286 287 288 289
                    and not hasattr(cls, "exist_check_grad")):
                raise AssertionError("This test of %s op needs check_grad." %
                                     cls.op_type)

290
            # check for op test with fp64 precision, but not check mkldnn op test for now
J
juncaipeng 已提交
291 292
            if cls.dtype in [np.float32, np.float64] \
                and cls.op_type not in op_accuracy_white_list.NO_FP64_CHECK_GRAD_OP_LIST \
293
                and not hasattr(cls, 'exist_fp64_check_grad') \
294
                and not is_xpu_op_test() \
J
juncaipeng 已提交
295
                and not is_mkldnn_op_test():
J
juncaipeng 已提交
296 297 298 299
                raise AssertionError(
                    "This test of %s op needs check_grad with fp64 precision." %
                    cls.op_type)

300
            if not cls.input_shape_is_large \
301 302 303 304
                and cls.op_type not in check_shape_white_list.NEED_TO_FIX_OP_LIST:
                raise AssertionError(
                    "Input's shape should be large than or equal to 100 for " +
                    cls.op_type + " Op.")
305

306 307 308 309 310
    def try_call_once(self, data_type):
        if not self.call_once:
            self.call_once = True
            self.dtype = data_type

311 312 313 314 315
    def is_bfloat16_op(self):
        return self.dtype == np.uint16 or (
            hasattr(self, 'mkldnn_data_type') and
            getattr(self, 'mkldnn_data_type') is "bfloat16")

316
    def infer_dtype_from_inputs_outputs(self, inputs, outputs):
J
juncaipeng 已提交
317 318 319 320
        def is_np_data(input):
            return isinstance(input, (np.ndarray, np.generic))

        def infer_dtype(numpy_dict, dtype_set):
321 322 323
            assert isinstance(
                numpy_dict,
                dict), "self.inputs, self.outputs must be numpy_dict"
J
juncaipeng 已提交
324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349
            # the inputs are as follows:
            # case 1: inputs = {'X': x}
            # case 2: inputs = {'X': (x, x_lod)}
            # case 3: inputs = {"X": [("x0", x0), ("x1", x1), ("x2", x2)]}
            # case 4: inputs = {'X': [("x1", (x1, [x1_lod1])), ("x2", (x2, [x2_.lod2]))]}
            # TODO(juncaipeng) infer dtype from inputs maybe obtain wrong type.
            for _, var_value in six.iteritems(numpy_dict):
                if is_np_data(var_value):  # case 1
                    dtype_set.add(var_value.dtype)
                elif isinstance(var_value, (list, tuple)):  # case 2, 3, 4
                    for sub_val_value in var_value:
                        if is_np_data(sub_val_value):  # case 2
                            dtype_set.add(sub_val_value.dtype)
                        elif len(sub_val_value) > 1 and is_np_data(
                                sub_val_value[1]):  # case 3
                            dtype_set.add(sub_val_value[1].dtype)
                        elif len(sub_val_value) > 1 and isinstance(sub_val_value[1], (list, tuple)) \
                            and is_np_data(sub_val_value[1][0]): # case 4
                            dtype_set.add(sub_val_value[1][0].dtype)

        # infer dtype from inputs, and dtype means the precision of the test
        # collect dtype of all inputs
        dtype_set = set()
        infer_dtype(inputs, dtype_set)
        dtype_list = [
            np.dtype(np.float64), np.dtype(np.float32), np.dtype(np.float16),
350 351 352
            np.dtype(np.int64), np.dtype(np.int32), np.dtype(np.uint16),
            np.dtype(np.int16), np.dtype(np.int8), np.dtype(np.uint8),
            np.dtype(np.bool)
J
juncaipeng 已提交
353 354 355 356 357 358
        ]
        # check the dtype in dtype_list in order, select the first dtype that in dtype_set
        for dtype in dtype_list:
            if dtype in dtype_set:
                self.dtype = dtype
                break
359 360
        # save dtype in class attr
        self.__class__.dtype = self.dtype
361

Y
Yang Yang(Tony) 已提交
362 363 364 365 366 367
    def feed_var(self, input_vars, place):
        feed_map = {}
        for var_name in input_vars:
            if isinstance(input_vars[var_name], list):
                for name, np_value in self.inputs[var_name]:
                    tensor = core.LoDTensor()
368
                    if isinstance(np_value, tuple):
369
                        tensor.set(np_value[0], place)
370
                        tensor.set_recursive_sequence_lengths(np_value[1])
371
                    else:
372
                        tensor.set(np_value, place)
Y
Yang Yang(Tony) 已提交
373 374 375 376
                    feed_map[name] = tensor
            else:
                tensor = core.LoDTensor()
                if isinstance(self.inputs[var_name], tuple):
377
                    tensor.set(self.inputs[var_name][0], place)
378 379
                    tensor.set_recursive_sequence_lengths(self.inputs[var_name][
                        1])
Y
Yang Yang(Tony) 已提交
380
                else:
381
                    tensor.set(self.inputs[var_name], place)
Y
Yang Yang(Tony) 已提交
382 383 384
                feed_map[var_name] = tensor
        return feed_map

385
    def _append_ops(self, block):
J
juncaipeng 已提交
386
        self.__class__.op_type = self.op_type  # for ci check, please not delete it for now
387 388 389 390
        if (hasattr(self, "use_mkldnn") and self.use_mkldnn == True) or \
            (hasattr(self, "attrs") and "use_mkldnn" in self.attrs and \
                    self.attrs["use_mkldnn"] == True):
            self.__class__.use_mkldnn = True
C
cc 已提交
391

392 393 394 395 396
        if (hasattr(self, "use_xpu") and self.use_xpu == True) or \
            (hasattr(self, "attrs") and "use_xpu" in self.attrs and \
                    self.attrs["use_xpu"] == True):
            self.__class__.use_xpu = True

Y
Yang Yang(Tony) 已提交
397
        op_proto = OpProtoHolder.instance().get_op_proto(self.op_type)
398 399 400 401 402 403
        "infer datatype from inputs and outputs for this test case"
        self.infer_dtype_from_inputs_outputs(self.inputs, self.outputs)
        inputs = append_input_output(block, op_proto, self.inputs, True,
                                     self.dtype)
        outputs = append_input_output(block, op_proto, self.outputs, False,
                                      self.dtype)
P
phlrain 已提交
404 405 406 407 408 409 410 411 412

        if hasattr(self, "cache_name_list"):
            for name in self.cache_name_list:
                inputs[name] = block.create_var(
                    name=name,
                    persistable=True,
                    type=core.VarDesc.VarType.RAW,
                    stop_gradient=True)

Y
Yang Yang(Tony) 已提交
413 414 415 416 417
        op = block.append_op(
            type=self.op_type,
            inputs=inputs,
            outputs=outputs,
            attrs=self.attrs if hasattr(self, "attrs") else dict())
C
cc 已提交
418
        # infer variable type and infer shape in compile-time
Q
QI JUN 已提交
419 420
        op.desc.infer_var_type(block.desc)
        op.desc.infer_shape(block.desc)
Y
Yang Yang(Tony) 已提交
421

422 423
        return op

424 425
    def _get_io_vars(self, block, numpy_inputs):
        inputs = {}
M
minqiyang 已提交
426
        for name, value in six.iteritems(numpy_inputs):
427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445
            if isinstance(value, list):
                var_list = [
                    block.var(sub_name) for sub_name, sub_value in value
                ]
                inputs[name] = var_list
            else:
                inputs[name] = block.var(name)
        return inputs

    def _get_inputs(self, block):
        return self._get_io_vars(block, self.inputs)

    def _get_outputs(self, block):
        return self._get_io_vars(block, self.outputs)

    def calc_output(self, place):
        outs, _ = self._calc_output(place)
        return outs

M
minqiyang 已提交
446 447 448 449
    def _create_var_from_numpy(self, value):
        if isinstance(value, tuple):
            data = value[0]
            lod = value[1]
L
lujun 已提交
450
            v = fluid.dygraph.base.to_variable(value=data)
451
            v.value().get_tensor().set_recursive_sequence_lengths(lod)
M
minqiyang 已提交
452 453
            return v
        else:
L
lujun 已提交
454
            return fluid.dygraph.base.to_variable(value)
M
minqiyang 已提交
455

456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473
    def get_sequence_batch_size_1_input(self, lod=None, shape=None):
        """Get LoD input data whose batch size is 1.
        All sequence related OP unittests should call this function to contain the case of batch size = 1.
        Args:
            lod (list[list of int], optional): Length-based LoD, length of lod[0] should be 1. Default: [[13]].
            shape (list, optional): Shape of input, shape[0] should be equals to lod[0][0]. Default: [13, 23].
        Returns:
            tuple (ndarray, lod) : LoD input data whose batch size is 1.
        """
        if lod is None:
            lod = [[13]]
        if shape is None:
            shape = [13, 23]
        assert len(lod[0]) == 1
        assert lod[0][0] == shape[0]
        x = np.random.uniform(0.1, 1, shape).astype('float32')
        return (x, lod)

474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509
    def lod_has_single_zero(self, lod):
        for i in range(len(lod) - 2):
            if lod[i] != 0 and lod[i + 1] == 0 and lod[i + 2] != 0:
                return True
        return False

    def lod_has_continuous_zero(self, lod):
        for i in range(len(lod) - 3):
            if lod[i] != 0 and lod[i + 1] == 0 and lod[i + 2] == 0 and lod[
                    i + 3] != 0:
                return True
        return False

    def get_sequence_instance_size_0_input(self, lod=None, shape=None):
        """Get LoD input data whose instance size is 0.
        All sequence related OP unittests should call this function to contain the case of instance size is 0.
        Args:
            lod (list[list of int], optional): Length-based LoD, lod[0]'s size must at least eight, lod[0] must at least two zeros at the beginning and at least two zeros at the end, the middle position of lod[0] contains a single zero and multiple zero. Default: [[0, 0, 4, 0, 3, 0, 0, 5, 0, 0]].
            shape (list, optional): Shape of input, shape[0] should be equals to lod[0][0]. Default: [13, 23].
        Returns:
            tuple (ndarray, lod): LoD input data whose instance size is 0.
        """
        if lod is None:
            lod = [[0, 0, 4, 0, 3, 0, 0, 5, 0, 0]]
        if shape is None:
            shape = [12, 10]
        assert len(lod[0]) >= 8
        assert lod[0][0] == 0 and lod[0][1] == 0 and lod[0][-1] == 0 and lod[0][
            -2] == 0
        assert self.lod_has_single_zero(lod[0]) is True
        assert self.lod_has_continuous_zero(lod[0]) is True
        assert sum(lod[0]) == shape[0]

        x = np.random.uniform(0.1, 1, shape).astype('float32')
        return (x, lod)

510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525
    def append_input_output_for_dygraph(self, op_proto, np_list, is_input,
                                        if_return_inputs_grad_dict, block):
        def create_var(np_value, name, is_input, if_return_inputs_grad_dict):
            np_value_temp = np_value
            has_lod = False
            lod_temp = None
            if isinstance(np_value, tuple):
                np_value_temp = np_value[0]
                has_lod = True
                lod_temp = np_value[1]

            if is_input:
                v = self._create_var_from_numpy(np_value_temp)
                if if_return_inputs_grad_dict:
                    v.stop_gradient = False
                if has_lod:
526
                    v.value().get_tensor().set_recursive_sequence_lengths(
527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587
                        lod_temp)
            else:
                v = block.create_var(
                    name=name,
                    dtype=np_value_temp.dtype,
                    type=core.VarDesc.VarType.LOD_TENSOR,
                    persistable=False,
                    stop_gradient=False)

            return v

        # prepare variable for input or output
        var_dict = defaultdict(list)
        if if_return_inputs_grad_dict:
            inputs_grad_dict = defaultdict()
        proto_list = op_proto.inputs if is_input else op_proto.outputs
        for var_proto in proto_list:
            name = var_proto.name
            if (name not in np_list) and var_proto.dispensable:
                continue
            if name not in np_list:
                assert var_proto.intermediate, "{} not found".format(name)
                v = block.create_var(
                    dtype='float32', type=core.VarDesc.VarType.LOD_TENSOR)
                var_dict[name].append(v)
                if if_return_inputs_grad_dict:
                    inputs_grad_dict[name] = v
                continue
            if var_proto.duplicable:
                assert isinstance(
                    np_list[name],
                    list), "Duplicable {} should be set as list".format(name)
                var_list = []
                slot_name = name
                for (name, np_value) in np_list[name]:
                    v = create_var(np_value, name, is_input,
                                   if_return_inputs_grad_dict)
                    var_list.append(v)
                    if if_return_inputs_grad_dict:
                        inputs_grad_dict[name] = v
                var_dict[slot_name] = var_list
            else:
                nplist_value_temp = None
                name_temp = None
                if isinstance(np_list[name], list):
                    nplist_value_temp = np_list[name][0]
                    name_temp = name
                else:
                    nplist_value_temp = np_list[name]
                    name_temp = unique_name.generate("%s_out" % (name))
                v = create_var(nplist_value_temp, name_temp, is_input,
                               if_return_inputs_grad_dict)
                var_dict[name].append(v)
                if if_return_inputs_grad_dict:
                    inputs_grad_dict[name] = v

        if if_return_inputs_grad_dict:
            return var_dict, inputs_grad_dict
        else:
            return var_dict

L
lujun 已提交
588
    def _calc_dygraph_output(self, place, parallel=False, no_check_set=None):
J
juncaipeng 已提交
589
        self.__class__.op_type = self.op_type  # for ci check, please not delete it for now
L
lujun 已提交
590
        with fluid.dygraph.base.guard(place=place):
M
minqiyang 已提交
591 592
            block = fluid.default_main_program().global_block()

593
            op_proto = OpProtoHolder.instance().get_op_proto(self.op_type)
M
minqiyang 已提交
594

595 596 597
            # prepare input variable
            inputs = self.append_input_output_for_dygraph(op_proto, self.inputs,
                                                          True, False, block)
M
minqiyang 已提交
598 599

            # prepare output variable
600 601 602 603 604 605 606 607 608
            outputs = self.append_input_output_for_dygraph(
                op_proto, self.outputs, False, False, block)

            # prepare attrbutes
            attrs_outputs = {}
            if hasattr(self, "attrs"):
                for attrs_name in self.attrs:
                    if self.attrs[attrs_name] is not None:
                        attrs_outputs[attrs_name] = self.attrs[attrs_name]
M
minqiyang 已提交
609 610 611 612
            block.append_op(
                type=self.op_type,
                inputs=inputs,
                outputs=outputs,
613
                attrs=attrs_outputs if hasattr(self, "attrs") else None)
M
minqiyang 已提交
614
            return outputs
615

616 617 618 619 620 621
    def _calc_output(self,
                     place,
                     parallel=False,
                     no_check_set=None,
                     loss=None,
                     enable_inplace=None,
622
                     for_inplace_test=None):
623 624
        program = Program()
        block = program.global_block()
625
        op = self._append_ops(block)
626 627 628 629 630

        inputs = self._get_inputs(block)
        outputs = self._get_outputs(block)
        feed_map = self.feed_var(inputs, place)

631
        if for_inplace_test:
C
cc 已提交
632 633
            # Some variables' tensors hold no buffer (tensor's _holder is NULL), like XShape in reshape2 op,
            # and the shapes of those variables contain 0 (eg. Xshape.shape = [0, 2, 5]).
634 635
            # Set persistable for those variables in order to get them from global_scope for inplace grad test directly other than feed them,
            # since feed op calls check_memory_size() which fails when tensor's holder_ is NULL.
636 637
            for out_name in op.output_arg_names:
                var = block.var(out_name)
638 639
                if 0 in var.shape:
                    var.persistable = True
640
        original_program = program
641 642
        if parallel:
            use_cuda = False
643
            if isinstance(place, fluid.CUDAPlace):
644
                use_cuda = True
645 646 647
            compiled_prog = fluid.CompiledProgram(program).with_data_parallel(
                loss_name=loss.name if loss else None, places=place)
            program = compiled_prog
648 649 650 651
        fetch_list = getattr(self, "fetch_list", [])
        # if the fetch_list is customized by user, we use it directly.
        # if not, fill the fetch_list by the user configured outputs in test.
        if len(fetch_list) == 0:
M
minqiyang 已提交
652
            for var_name, var in six.iteritems(outputs):
653 654
                if no_check_set is not None and var_name in no_check_set:
                    continue
Y
Yang Yang(Tony) 已提交
655 656
                if isinstance(var, list):
                    for v in var:
657
                        fetch_list.append(v.name)
Y
Yang Yang(Tony) 已提交
658
                else:
659
                    fetch_list.append(var.name)
660 661 662 663
        # if the fetch_list still empty, fill the fetch_list by the operator output.
        if len(fetch_list) == 0:
            for out_name, out_dup in Operator.get_op_outputs(self.op_type):
                fetch_list.append(str(out_name))
664 665 666 667 668 669 670 671 672

        if enable_inplace is not None:
            build_strategy = fluid.BuildStrategy()
            build_strategy.enable_inplace = enable_inplace

            compiled_prog = fluid.CompiledProgram(program).with_data_parallel(
                build_strategy=build_strategy, places=place)
            program = compiled_prog

673
        executor = Executor(place)
674 675 676 677
        outs = executor.run(program,
                            feed=feed_map,
                            fetch_list=fetch_list,
                            return_numpy=False)
678 679
        self.op = op
        self.program = original_program
680 681 682 683
        if for_inplace_test:
            return outs, fetch_list, feed_map, original_program, op.desc
        else:
            return outs, fetch_list
Y
Yang Yang(Tony) 已提交
684

685 686 687 688 689 690 691 692 693
    def _compare_expect_and_actual_outputs(self,
                                           place,
                                           fetch_list,
                                           expect_outs,
                                           actual_outs,
                                           inplace_atol=None):
        """Compare expect outs and actual outs of an tested op.

        Args:
C
cc 已提交
694
            place (CPUPlace | CUDAPlace): The place where the op runs.
695 696 697 698 699 700 701 702 703 704
            fetch_list (list): The outputs of tested op.
            expect_outs (list): The expect outs of tested op.
            actual_outs (list): The actual outs of tested op.
            inplace_atol (float): The tolerable error, only set when tested op doesn't ensure computational consistency, like group_norm op.

        Returns:
            None.
        """
        # compare expect_outs and actual_outs
        for i, name in enumerate(fetch_list):
C
cc 已提交
705
            # Note(zhiqiu): inplace_atol should be only set when op doesn't ensure
L
Leo Chen 已提交
706 707 708
            # computational consistency.
            # When inplace_atol is not None, the inplace check uses numpy.allclose
            # to check inplace result instead of numpy.array_equal.
709 710
            expect_out = np.array(expect_outs[i])
            actual_out = np.array(actual_outs[i])
711 712 713
            if inplace_atol is not None:
                self.assertTrue(
                    np.allclose(
714
                        expect_out, actual_out, atol=inplace_atol),
715 716
                    "Output (" + name + ") has diff at " + str(place) +
                    " when using and not using inplace" + "\nExpect " +
717 718
                    str(expect_out) + "\n" + "But Got" + str(actual_out) +
                    " in class " + self.__class__.__name__)
719 720
            else:
                self.assertTrue(
721
                    np.array_equal(expect_out, actual_out),
722 723
                    "Output (" + name + ") has diff at " + str(place) +
                    " when using and not using inplace" + "\nExpect " +
724 725
                    str(expect_out) + "\n" + "But Got" + str(actual_out) +
                    " in class " + self.__class__.__name__ + '\n')
726 727 728 729 730 731 732 733

    def _construct_grad_program_from_forward(self, fwd_program, grad_op_desc,
                                             op_grad_to_var):
        """Generate grad_program which contains the grad_op.

        Args:
            fwd_program (tuple): The program that contains grad_op_desc's corresponding forward op.
            grad_op_desc (OpDesc): The OpDesc of grad op.
C
cc 已提交
734
            op_grad_to_var (dict): The relation of variables in grad op and its forward op.
735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760

        Returns:
            grad_program (program): The program which contains the grad_op.
        """
        grad_program = Program()
        grad_block = grad_program.global_block()
        new_op_desc = grad_block.desc.append_op()
        new_op_desc.copy_from(grad_op_desc)
        grad_program._sync_with_cpp()

        # Create grad vars based on fwd vars (shape and dtype)
        for arg in grad_op_desc.input_arg_names(
        ) + grad_op_desc.output_arg_names():
            fwd_var_name = op_grad_to_var.get(arg, None)
            if fwd_var_name is None:
                fwd_var_name = arg
            fwd_var = fwd_program.global_block().vars.get(fwd_var_name)
            assert fwd_var is not None, "{} cannot be found".format(
                fwd_var_name)
            grad_var = grad_block.create_var(
                name=arg,
                dtype=fwd_var.dtype,
                shape=fwd_var.shape,
                type=fwd_var.type,
                persistable=False)

C
cc 已提交
761 762
            # Some variables' tensors hold no buffer (tensor's _holder is NULL), like XShape in reshape2 op,
            # and the shapes of those variables contain 0 (eg. Xshape.shape = [0, 2, 5]).
763 764 765 766 767 768 769 770 771 772 773 774 775 776 777
            # Set persistable for those variables in order to get them from global_scope for inplace grad test directly other than feed them,
            # since feed op calls check_memory_size() which fails when tensor's holder_ is NULL.
            if 0 in grad_var.shape:
                grad_var.persistable = True
        grad_program._sync_with_cpp()
        return grad_program

    def _construct_grad_feed_map_from_forward(self, place, fwd_res,
                                              grad_op_desc, op_grad_to_var):
        """Generate grad_feed_map for grad_program.

        since we don`t really check gradient accuracy, but check the consistency when using and not using inplace,
        we use fwd outs (also inputs sometimes) to construct grad inputs.

        Args:
C
cc 已提交
778
            place (CPUPlace | CUDAPlace): The place where the op runs.
779 780 781
            fwd_res (tuple): The outputs of its forward op, in the same form as returns of _calc_outputs() when for_inplace_test is True.
                i.e., tuple(fwd_outs, fwd_fetch_list, fwd_feed_map, fwd_program, fwd_op_desc)
            grad_op_desc (OpDesc): The OpDesc of grad op.
C
cc 已提交
782
            op_grad_to_var (dict): The relation of variables in grad op and its fwd_op.
783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813

        Returns:
            grad_feed_map (dict): The feed_map of grad_op.
        """
        fwd_outs, fwd_fetch_list, fwd_feed_map, fwd_program, fwd_op_desc = fwd_res
        p = core.Place()
        p.set_place(place)
        grad_feed_map = {}
        for arg in grad_op_desc.input_arg_names():
            if arg in fwd_feed_map.keys():
                grad_feed_map[arg] = fwd_feed_map[arg]._copy(p)
            else:
                fwd_var_name = op_grad_to_var.get(arg, None)
                if fwd_var_name is None:
                    fwd_var_name = arg

                for i, out_name in enumerate(fwd_fetch_list):
                    if out_name == fwd_var_name:
                        # don't feed variables whose tensors hold no buffer (shape contains 0 like shape = [0,2,5] and holder_ is NULL), like XShape in reshape2 op.
                        # get them from global_scope directly since we have set them persistable in fwd execution
                        if 0 in fwd_program.global_block().var(out_name).shape:
                            continue
                        else:
                            grad_feed_map[arg] = fwd_outs[i]._copy(p)
        return grad_feed_map

    def _get_need_run_ops(self, op_desc, fwd_op_desc=None):
        """Postorder traversal of the 'grad' tree to get all ops that need to run during inplace test.
        An op needs to run druing inplace check if,
        (1) it has infer_inplace,
        (2) it has infer_inplace in its grad descendants. (since we need its outputs as to construct its grad's inputs)
C
cc 已提交
814

815
        Args:
C
cc 已提交
816 817
            op_desc (OpDesc): The op_desc of current op.
            fwd_op_desc (OpDesc): The op_desc of current op's forward op, None if current op has no forward op.
818
                Eg. relu's fwd_op is None, relu_grad's fwd_op is relu, relu_grad_grad's fwd_op is relu_grad, etc.
C
cc 已提交
819

820 821 822 823 824 825 826 827 828 829 830 831 832 833
        Returns:
            need_run_ops (list[(op_desc, fwd_op_desc)]): The ops that need to run during inplace test.
        """
        need_run_ops = []
        visited_ops = []

        def _dfs_grad_op(op_desc, fwd_op_desc=None):
            visited_ops.append(op_desc.type())
            has_infer_inplace = fluid.core.has_infer_inplace(op_desc.type())
            has_grad_op_maker = fluid.core.has_grad_op_maker(op_desc.type())
            has_infer_inplace_in_grad_descendants = False
            if not has_grad_op_maker:
                has_infer_inplace_in_descendants = False
            else:
C
cc 已提交
834
                # get grad_op_desc
835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859
                grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc(
                    op_desc, set(), [])
                if not grad_op_desc_list:
                    has_infer_inplace_in_grad_descendants = False
                else:
                    for i, grad_op_desc in enumerate(grad_op_desc_list):
                        if grad_op_desc.type(
                        ) not in visited_ops and _dfs_grad_op(
                                grad_op_desc, fwd_op_desc=op_desc):
                            has_infer_inplace_in_grad_descendants = True
            if has_infer_inplace or has_infer_inplace_in_grad_descendants:
                need_run_ops.append((op_desc, fwd_op_desc))
                return True
            else:
                return False

        _dfs_grad_op(op_desc, fwd_op_desc=fwd_op_desc)
        return need_run_ops

    def _check_forward_inplace(self,
                               place,
                               no_check_set=None,
                               inplace_atol=None):
        """Chech the inplace correctness of given op (self.op_type).
        Run the op twice with same inputs, one enable inplace and another disable, compare their outputs.
C
cc 已提交
860

861
        Args:
C
cc 已提交
862
            place (CPUPlace | CUDAPlace): The place where the op runs.
863 864 865 866
            no_check_set (list): The names of outputs that needn't check, like XShape of reshape op.
            inplace_atol (float): The tolerable error, only set when op doesn't ensure computational consistency, like group_norm op.

        Returns:
C
cc 已提交
867 868
            expect_res (tuple(outs, fetch_list, feed_map, program, op_desc)): The results of given op.
                We return this to construct grad_program and grad_feed_map for grad inplace check.
869 870
        """
        # _calc_output() returns in the form tuple(outs, fetch_list, feed_map, program, op_desc) when for_inplace_test=True.
871 872 873 874 875 876 877 878 879 880
        expect_res = self._calc_output(
            place,
            no_check_set=no_check_set,
            enable_inplace=False,
            for_inplace_test=True)
        actual_res = self._calc_output(
            place,
            no_check_set=no_check_set,
            enable_inplace=True,
            for_inplace_test=True)
881
        # compare expect_outs and actual_outs
882 883 884 885 886 887
        self._compare_expect_and_actual_outputs(
            place,
            expect_res[1],
            expect_res[0],
            actual_res[0],
            inplace_atol=inplace_atol)
888 889 890 891 892 893 894 895 896 897 898 899 900
        return expect_res

    def _calc_grad_output(self,
                          place,
                          fwd_res,
                          grad_op_desc,
                          enable_inplace=None):
        """Calculate grad_output for given grad_op_desc.

        since we don`t really check gradient accuracy, but check the consistency when using and not using inplace,
        we use fwd outs (also inputs sometimes) to construct grad inputs.

        Args:
C
cc 已提交
901
            place (CPUPlace | CUDAPlace): The place where the op runs.
902 903 904 905 906 907 908 909 910
            fwd_res (tuple): The outputs of its forward op, in the same form as returns of _calc_outputs() when for_inplace_test is True.
                i.e., tuple(fwd_outs, fwd_fetch_list, fwd_feed_map, fwd_program, fwd_op_desc).
            grad_op_desc (OpDesc): The OpDesc of grad op.
            enable_inplace (bool): Enable inplace or not.

        Returns:
            res (tuple(outs, fetch_list, feed_map, program, op_desc)): The results of given grad_op_desc.
        """
        fwd_outs, fwd_fetch_list, fwd_feed_map, fwd_program, fwd_op_desc = fwd_res
911
        grad_op_desc_list, op_grad_to_var = core.get_grad_op_desc(fwd_op_desc,
912
                                                                  set(), [])
913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944
        grad_program = self._construct_grad_program_from_forward(
            fwd_program, grad_op_desc, op_grad_to_var)
        grad_feed_map = self._construct_grad_feed_map_from_forward(
            place, fwd_res, grad_op_desc, op_grad_to_var)
        grad_fetch_list = grad_op_desc.output_arg_names()
        exe = Executor(place)
        program = grad_program
        if enable_inplace is not None:
            build_strategy = fluid.BuildStrategy()
            build_strategy.enable_inplace = enable_inplace
            compiled_program = fluid.CompiledProgram(
                grad_program).with_data_parallel(
                    loss_name="", build_strategy=build_strategy, places=place)
            program = compiled_program
        outs = exe.run(program,
                       feed=grad_feed_map,
                       fetch_list=grad_fetch_list,
                       return_numpy=False)
        return outs, grad_fetch_list, grad_feed_map, grad_program, grad_op_desc

    def _check_grad_inplace(self,
                            place,
                            fwd_res,
                            grad_op_desc,
                            inplace_atol=None):
        """Chech the inplace correctness of given grad_op_desc.

        Run the grad op twice with same inputs, one enable inplace and another disable, compare their outputs.
        It works like _check_forward_inplace, but the way to construct program and feed_map differs.
        So we define a new function for grad, grad_grad, etc.

        Args:
C
cc 已提交
945
            place (CPUPlace | CUDAPlace): The place where the op runs.
946 947 948 949 950 951
            fwd_res (tuple): The outputs of its forward op, in the same form as returns of _calc_outputs() when for_inplace_test is True.
                i.e., tuple(fwd_outs, fwd_fetch_list, fwd_feed_map, fwd_program, fwd_op_desc).
            grad_op_desc (OpDesc): The OpDesc of grad op.
            inplace_atol (float): The tolerable error, only set when op doesn't ensure computational consistency, like group_norm op.

        Returns:
C
cc 已提交
952 953
            expect_res (tuple(outs, fetch_list, feed_map, program, op_desc)): The results of given op.
                We return this to construct grad_program and grad_feed_map for grad inplace check.
954 955 956 957 958 959 960 961 962 963 964 965
        """
        expect_res = self._calc_grad_output(
            place, fwd_res, grad_op_desc, enable_inplace=False)
        actual_res = self._calc_grad_output(
            place, fwd_res, grad_op_desc, enable_inplace=True)
        self._compare_expect_and_actual_outputs(
            place,
            expect_res[1],
            expect_res[0],
            actual_res[0],
            inplace_atol=inplace_atol)
        return expect_res
966

967 968 969 970 971 972 973 974 975 976
    def check_inplace_output_with_place(self,
                                        place,
                                        no_check_set=None,
                                        inplace_atol=None):
        """Chech the inplace correctness of given op, its grad op, its grad_grad op, etc.

        (1) Get all ops need to run. (see conditions in _get_need_run_ops())
        (2) Run op in need_run_ops, and do inplace check if it has infer_inplace.

        Args:
C
cc 已提交
977
            place (CPUPlace | CUDAPlace): The place where the op runs.
978 979 980 981 982 983 984 985 986 987 988 989 990 991 992
            no_check_set (list): The names of outputs that needn't check, like XShape of reshape op.
            inplace_atol (float): The tolerable error, only set when op doesn't ensure computational consistency, like group_norm op.

        Returns:
            None
        """
        has_infer_inplace = fluid.core.has_infer_inplace(self.op_type)
        has_grad_op_maker = fluid.core.has_grad_op_maker(self.op_type)

        fwd_res = self._calc_output(
            place, no_check_set=no_check_set, for_inplace_test=True)
        op_desc = fwd_res[4]
        need_run_ops = self._get_need_run_ops(op_desc)

        res = {}
993 994
        if hasattr(self, 'attrs') and bool(self.attrs.get('use_xpu', False)):
            return
995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007
        for op_desc, father_op_desc in reversed(need_run_ops):
            # The first one is the forward op
            has_infer_inplace = fluid.core.has_infer_inplace(op_desc.type())
            if op_desc.type() == self.op_type:
                if has_infer_inplace:
                    res[op_desc] = self._check_forward_inplace(
                        place,
                        no_check_set=no_check_set,
                        inplace_atol=inplace_atol)
                else:
                    res[op_desc] = self._calc_output(
                        place, no_check_set=no_check_set, for_inplace_test=True)
            else:
1008 1009
                # TODO(zhiqiu): enhance inplace_grad test for ops (sum and activation) using mkldnn
                # skip op that use_mkldnn currently
1010
                flags_use_mkldnn = fluid.core.globals()["FLAGS_use_mkldnn"]
1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022
                attrs_use_mkldnn = hasattr(
                    self,
                    'attrs') and bool(self.attrs.get('use_mkldnn', False))
                if flags_use_mkldnn or attrs_use_mkldnn:
                    warnings.warn(
                        "check inplace_grad for ops using mkldnn is not supported"
                    )
                    continue
                if has_infer_inplace:
                    fwd_res = res[father_op_desc]
                    res[op_desc] = self._check_grad_inplace(
                        place, fwd_res, op_desc, inplace_atol=inplace_atol)
1023
                else:
1024 1025
                    res[op_desc] = self._calc_grad_output(place, fwd_res,
                                                          op_desc)
1026

1027 1028
    def check_output_with_place(self,
                                place,
1029
                                atol=0,
1030
                                no_check_set=None,
M
minqiyang 已提交
1031
                                equal_nan=False,
1032
                                check_dygraph=True,
1033
                                inplace_atol=None):
1034 1035 1036 1037 1038
        self.infer_dtype_from_inputs_outputs(self.inputs, self.outputs)
        if self.dtype == np.float64 and \
            self.op_type not in op_threshold_white_list.NEED_FIX_FP64_CHECK_OUTPUT_THRESHOLD_OP_LIST:
            atol = 0

1039 1040 1041 1042 1043 1044 1045 1046
        if self.is_bfloat16_op():
            check_dygraph = False
            if hasattr(self, 'force_fp32_output') and getattr(
                    self, 'force_fp32_output'):
                atol = 1e-2
            else:
                atol = 2

1047 1048 1049 1050
        if no_check_set is not None:
            if self.op_type not in no_check_set_white_list.no_check_set_white_list:
                raise AssertionError(
                    "no_check_set of op %s must be set to None." % self.op_type)
1051

L
lujun 已提交
1052 1053
        if check_dygraph:
            dygraph_outs = self._calc_dygraph_output(
M
minqiyang 已提交
1054
                place, no_check_set=no_check_set)
1055
        outs, fetch_list = self._calc_output(place, no_check_set=no_check_set)
Y
Yang Yang(Tony) 已提交
1056
        for out_name, out_dup in Operator.get_op_outputs(self.op_type):
1057 1058
            if out_name not in self.outputs:
                continue
1059 1060
            if no_check_set is not None and out_name in no_check_set:
                continue
1061

1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073
            def find_imperative_actual(target_name, dygraph_outs, place):
                with fluid.dygraph.base.guard(place=place):
                    for name in dygraph_outs:
                        if name == target_name:
                            return dygraph_outs[name][0]
                        var_list = dygraph_outs[name]
                        for i, var in enumerate(var_list):
                            if var.name == target_name:
                                return dygraph_outs[name][i]
                    self.assertTrue(False, "Found failed {} {}".format(
                        dygraph_outs.keys(), target_name))

Y
Yang Yang(Tony) 已提交
1074 1075
            def find_actual(target_name, fetch_list):
                found = [
1076 1077
                    i for i, var_name in enumerate(fetch_list)
                    if var_name == target_name
Y
Yang Yang(Tony) 已提交
1078 1079 1080 1081 1082 1083
                ]
                self.assertTrue(
                    len(found) == 1, "Found {} {}".format(
                        len(found), target_name))
                return found[0]

1084 1085
            if out_dup:
                sub_out = self.outputs[out_name]
Y
Yancey 已提交
1086 1087 1088
                if not isinstance(sub_out, list):
                    raise AssertionError("sub_out type %s is not list",
                                         type(sub_out))
1089 1090
                for item in sub_out:
                    sub_out_name, expect = item[0], item[1]
L
lujun 已提交
1091
                    if check_dygraph:
1092 1093
                        imperative_actual = find_imperative_actual(
                            sub_out_name, dygraph_outs, place)
1094 1095
                        imperative_actual_t = np.array(imperative_actual.value()
                                                       .get_tensor())
Y
Yang Yang(Tony) 已提交
1096
                    idx = find_actual(sub_out_name, fetch_list)
Q
QI JUN 已提交
1097 1098
                    actual = outs[idx]
                    actual_t = np.array(actual)
1099 1100
                    expect_t = expect[0] \
                        if isinstance(expect, tuple) else expect
1101 1102
                    self.assertTrue(
                        np.allclose(
1103
                            actual_t, expect_t, atol=atol, equal_nan=equal_nan),
Y
Yang Yang(Tony) 已提交
1104 1105
                        "Output (" + sub_out_name + ") has diff at " +
                        str(place))
L
lujun 已提交
1106
                    if check_dygraph:
M
minqiyang 已提交
1107 1108 1109 1110 1111 1112 1113
                        self.assertTrue(
                            np.allclose(
                                imperative_actual_t,
                                expect_t,
                                atol=atol,
                                equal_nan=equal_nan),
                            "Output (" + sub_out_name + ") has diff at " +
L
lujun 已提交
1114
                            str(place) + " in dygraph mode")
1115 1116
                    if isinstance(expect, tuple):
                        self.assertListEqual(
1117 1118
                            actual.recursive_sequence_lengths(), expect[1],
                            "Output (" + sub_out_name +
Q
QI JUN 已提交
1119
                            ") has different lod at " + str(place))
1120 1121
                        if check_dygraph:
                            self.assertListEqual(
1122
                                imperative_actual.value().get_tensor()
1123 1124 1125 1126
                                .recursive_sequence_lengths(), expect[1],
                                "Output (" + out_name +
                                ") has different lod at " + str(place) +
                                " in dygraph mode")
1127
            else:
L
lujun 已提交
1128
                if check_dygraph:
1129 1130
                    imperative_actual = find_imperative_actual(
                        out_name, dygraph_outs, place)
1131 1132
                    imperative_actual_t = np.array(imperative_actual.value()
                                                   .get_tensor())
Y
Yang Yang(Tony) 已提交
1133
                idx = find_actual(out_name, fetch_list)
Q
QI JUN 已提交
1134 1135
                actual = outs[idx]
                actual_t = np.array(actual)
1136
                expect = self.outputs[out_name]
1137
                expect_t = expect[0] if isinstance(expect, tuple) else expect
1138 1139
                self.assertTrue(
                    np.allclose(
1140
                        actual_t, expect_t, atol=atol, equal_nan=equal_nan),
E
emailweixu 已提交
1141
                    "Output (" + out_name + ") has diff at " + str(place) +
D
dzhwinter 已提交
1142
                    "\nExpect " + str(expect_t) + "\n" + "But Got" +
1143
                    str(actual_t) + " in class " + self.__class__.__name__)
L
lujun 已提交
1144
                if check_dygraph:
1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160
                    if six.moves.reduce(
                            lambda x, y: x * y, imperative_actual_t.shape,
                            1) == 0 and six.moves.reduce(
                                lambda x, y: x * y, expect_t.shape, 1) == 0:
                        pass
                    else:
                        self.assertTrue(
                            np.allclose(
                                imperative_actual_t,
                                expect_t,
                                atol=atol,
                                equal_nan=equal_nan),
                            "Output (" + out_name + ") has diff at " +
                            str(place) + "\nExpect " + str(expect_t) + "\n" +
                            "But Got" + str(imperative_actual_t) + " in class "
                            + self.__class__.__name__)
1161
                if isinstance(expect, tuple):
1162 1163
                    self.assertListEqual(actual.recursive_sequence_lengths(),
                                         expect[1], "Output (" + out_name +
1164
                                         ") has different lod at " + str(place))
L
lujun 已提交
1165
                    if check_dygraph:
M
minqiyang 已提交
1166
                        self.assertListEqual(
1167
                            imperative_actual.value().get_tensor()
M
minqiyang 已提交
1168 1169
                            .recursive_sequence_lengths(), expect[1],
                            "Output (" + out_name + ") has different lod at " +
L
lujun 已提交
1170
                            str(place) + " in dygraph mode")
1171

C
cc 已提交
1172
        # Note(zhiqiu): inplace_atol should be only set when op doesn't ensure
L
Leo Chen 已提交
1173 1174
        # computational consistency.
        # For example, group_norm uses AtomicAdd on CUDAPlace, which do not ensure
C
cc 已提交
1175
        # computation order when multiple threads write the same address. So the
L
Leo Chen 已提交
1176 1177 1178
        # result of group_norm is non-deterministic when datatype is float.
        # When inplace_atol is not None, the inplace check uses numpy.allclose
        # to check inplace result instead of numpy.array_equal.
1179 1180
        if inplace_atol is not None:
            warnings.warn(
L
Leo Chen 已提交
1181 1182
                "inplace_atol should only be set when op doesn't ensure computational consistency, please check it!"
            )
1183
        # Check inplace for given op, its grad op, its grad_grad op, etc.
C
cc 已提交
1184
        # No effect on original OpTest
1185 1186 1187 1188
        # Currently not support ParallelExecutor on XPUPlace.
        if not paddle.is_compiled_with_xpu():
            self.check_inplace_output_with_place(
                place, no_check_set=no_check_set, inplace_atol=inplace_atol)
1189

1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237
        if check_dygraph:
            return outs, dygraph_outs, fetch_list
        else:
            return outs, fetch_list

    def check_compile_vs_runtime(self, fetch_list, fetch_outs):
        def find_fetch_index(target_name, fetch_list):
            found = [
                i for i, var_name in enumerate(fetch_list)
                if var_name == target_name
            ]
            if len(found) == 0:
                return -1
            else:
                self.assertTrue(
                    len(found) == 1,
                    "Found {} {}".format(len(found), target_name))
                return found[0]

        for name in self.op.desc.output_names():
            var_names = self.op.desc.output(name)
            for var_name in var_names:
                i = find_fetch_index(var_name, fetch_list)
                if i == -1:
                    # The output is dispensiable or intermediate.
                    break
                out = fetch_outs[i]
                if isinstance(out, core.LoDTensor):
                    lod_level_runtime = len(out.lod())
                else:
                    if isinstance(out, core.LoDTensorArray):
                        warnings.warn(
                            "The check of LoDTensorArray's lod_level is not implemented now!"
                        )
                    lod_level_runtime = 0

                var = self.program.global_block().var(var_name)
                if var.type == core.VarDesc.VarType.LOD_TENSOR:
                    lod_level_compile = var.lod_level
                else:
                    lod_level_compile = 0
                self.assertEqual(
                    lod_level_compile, lod_level_runtime,
                    "The lod_level of Output (" + name +
                    ") is different between compile-time and runtime (" +
                    str(lod_level_compile) + " vs " + str(lod_level_runtime) +
                    ")")

1238
    def _get_places(self):
D
dzhwinter 已提交
1239 1240 1241 1242 1243 1244
        if self.dtype == np.float16:
            if core.is_compiled_with_cuda() and core.op_support_gpu(
                    self.op_type):
                place = core.CUDAPlace(0)
                if core.is_float16_supported(place):
                    return [place]
W
Wu Yi 已提交
1245 1246
                else:
                    return []
D
dzhwinter 已提交
1247 1248
            else:
                return []
1249
        places = [fluid.CPUPlace()]
1250 1251 1252
        cpu_only = self._cpu_only if hasattr(self, '_cpu_only') else False
        if core.is_compiled_with_cuda() and core.op_support_gpu(self.op_type)\
           and not cpu_only:
D
dzhwinter 已提交
1253
            places.append(core.CUDAPlace(0))
1254 1255
        return places

M
minqiyang 已提交
1256 1257 1258 1259
    def check_output(self,
                     atol=1e-5,
                     no_check_set=None,
                     equal_nan=False,
1260
                     check_dygraph=True,
1261
                     inplace_atol=None):
1262
        self.__class__.op_type = self.op_type
1263 1264 1265 1266
        if (hasattr(self, "use_mkldnn") and self.use_mkldnn == True) or \
            (hasattr(self, "attrs") and "use_mkldnn" in self.attrs and \
                    self.attrs["use_mkldnn"] == True):
            self.__class__.use_mkldnn = True
C
cc 已提交
1267

1268 1269 1270 1271 1272
        if (hasattr(self, "use_xpu") and self.use_xpu == True) or \
            (hasattr(self, "attrs") and "use_xpu" in self.attrs and \
                    self.attrs["use_xpu"] == True):
            self.__class__.use_xpu = True

1273
        places = self._get_places()
Q
qijun 已提交
1274
        for place in places:
1275 1276 1277 1278 1279 1280
            res = self.check_output_with_place(place, atol, no_check_set,
                                               equal_nan, check_dygraph)
            if check_dygraph:
                outs, dygraph_outs, fetch_list = res
            else:
                outs, fetch_list = res
1281
            if self.op_type not in compile_vs_runtime_white_list.COMPILE_RUN_OP_WHITE_LIST:
1282
                self.check_compile_vs_runtime(fetch_list, outs)
Q
qijun 已提交
1283

1284
    def check_output_customized(self, checker):
1285
        places = self._get_places()
1286 1287 1288
        for place in places:
            outs = self.calc_output(place)
            outs = [np.array(out) for out in outs]
1289
            outs.sort(key=len)
1290 1291
            checker(outs)

D
Dun 已提交
1292 1293
    def _assert_is_close(self, numeric_grads, analytic_grads, names,
                         max_relative_error, msg_prefix):
1294

M
minqiyang 已提交
1295
        for a, b, name in six.moves.zip(numeric_grads, analytic_grads, names):
1296 1297 1298 1299 1300 1301
            # It asserts np.abs(a - b) / np.abs(a) < max_relative_error, in which
            # max_relative_error is 1e-7. According to the value of np.abs(a), we
            # change np.abs(a) to achieve dynamic threshold. For example, if
            # the value of np.abs(a) is between 1e-10 and 1e-8, we set np.abs(a)*=1e4.
            # Therefore, it asserts np.abs(a - b) / (np.abs(a)*1e4) < max_relative_error,
            # which is the same as np.abs(a - b) / np.abs(a) < max_relative_error*1e4.
1302
            abs_a = np.abs(a)
1303 1304 1305 1306 1307 1308 1309
            if self.dtype == np.float64 and \
                self.op_type not in op_threshold_white_list.NEED_FIX_FP64_CHECK_GRAD_THRESHOLD_OP_LIST:
                abs_a[abs_a < 1e-10] = 1e-3
                abs_a[np.logical_and(abs_a > 1e-10, abs_a <= 1e-8)] *= 1e4
                abs_a[np.logical_and(abs_a > 1e-8, abs_a <= 1e-6)] *= 1e2
            else:
                abs_a[abs_a < 1e-3] = 1
1310 1311 1312 1313 1314 1315

            diff_mat = np.abs(a - b) / abs_a
            max_diff = np.max(diff_mat)

            def err_msg():
                offset = np.argmax(diff_mat > max_relative_error)
1316 1317 1318
                return ("Operator %s error, %s variable %s (shape: %s, dtype: %s) max gradient diff %e over limit %e, "
                    "the first error element is %d, expected %e, but got %e.") \
                    % (self.op_type, msg_prefix, name, str(a.shape), self.dtype, max_diff, max_relative_error,
1319
                    offset, a.flatten()[offset], b.flatten()[offset])
1320 1321 1322

            self.assertLessEqual(max_diff, max_relative_error, err_msg())

1323 1324 1325 1326 1327 1328 1329
    def _check_grad_helper(self):
        self.infer_dtype_from_inputs_outputs(self.inputs, self.outputs)
        self.__class__.op_type = self.op_type
        self.__class__.exist_check_grad = True
        if self.dtype == np.float64:
            self.__class__.exist_fp64_check_grad = True

1330 1331
    def check_grad(self,
                   inputs_to_check,
Y
Yancey 已提交
1332
                   output_names,
1333
                   no_grad_set=None,
1334
                   numeric_grad_delta=0.005,
1335
                   in_place=False,
Q
Qiao Longfei 已提交
1336
                   max_relative_error=0.005,
1337
                   user_defined_grads=None,
1338
                   user_defined_grad_outputs=None,
1339
                   check_dygraph=True):
1340
        self._check_grad_helper()
1341
        places = self._get_places()
1342
        for place in places:
1343 1344 1345 1346
            self.check_grad_with_place(
                place, inputs_to_check, output_names, no_grad_set,
                numeric_grad_delta, in_place, max_relative_error,
                user_defined_grads, user_defined_grad_outputs, check_dygraph)
1347 1348 1349 1350 1351 1352 1353 1354 1355

    def check_grad_with_place(self,
                              place,
                              inputs_to_check,
                              output_names,
                              no_grad_set=None,
                              numeric_grad_delta=0.005,
                              in_place=False,
                              max_relative_error=0.005,
1356
                              user_defined_grads=None,
1357
                              user_defined_grad_outputs=None,
1358
                              check_dygraph=True):
1359
        self.scope = core.Scope()
Q
qijun 已提交
1360
        op_inputs = self.inputs if hasattr(self, "inputs") else dict()
1361
        op_outputs = self.outputs if hasattr(self, "outputs") else dict()
Q
qijun 已提交
1362
        op_attrs = self.attrs if hasattr(self, "attrs") else dict()
P
phlrain 已提交
1363

1364
        self._check_grad_helper()
1365 1366 1367 1368
        if self.dtype == np.float64 and \
            self.op_type not in op_threshold_white_list.NEED_FIX_FP64_CHECK_GRAD_THRESHOLD_OP_LIST:
            numeric_grad_delta = 1e-5
            max_relative_error = 1e-7
1369

P
phlrain 已提交
1370 1371 1372
        cache_list = None
        if hasattr(self, "cache_name_list"):
            cache_list = self.cache_name_list
1373 1374 1375 1376 1377 1378 1379

        # oneDNN numeric gradient should use CPU kernel
        use_onednn = False
        if "use_mkldnn" in op_attrs and op_attrs["use_mkldnn"] == True:
            op_attrs["use_mkldnn"] = False
            use_onednn = True

P
phlrain 已提交
1380 1381 1382 1383 1384 1385 1386
        self.op = create_op(
            self.scope,
            self.op_type,
            op_inputs,
            op_outputs,
            op_attrs,
            cache_list=cache_list)
Y
Yu Yang 已提交
1387

1388 1389 1390
        if use_onednn:
            op_attrs["use_mkldnn"] = True

1391 1392
        if no_grad_set is None:
            no_grad_set = set()
1393 1394
        else:
            if (self.op_type not in no_grad_set_white_list.NEED_TO_FIX_OP_LIST
1395 1396 1397
                ) and (
                    self.op_type not in no_grad_set_white_list.NOT_CHECK_OP_LIST
                ) and (not self.is_bfloat16_op()):
1398 1399
                raise AssertionError("no_grad_set must be None, op_type is " +
                                     self.op_type + " Op.")
1400

1401 1402 1403 1404 1405 1406 1407 1408
        for input_to_check in inputs_to_check:
            set_input(self.scope, self.op, self.inputs, place)
            tensor_to_check = self.scope.find_var(input_to_check).get_tensor()
            tensor_size = six.moves.reduce(lambda a, b: a * b,
                                           tensor_to_check.shape(), 1)
            if tensor_size < 100:
                self.__class__.input_shape_is_large = False

Y
Yancey 已提交
1409 1410 1411
        if not type(output_names) is list:
            output_names = [output_names]

Q
Qiao Longfei 已提交
1412
        numeric_grads = user_defined_grads or [
1413
            get_numeric_gradient(
1414
                place,
1415 1416 1417 1418
                self.scope,
                self.op,
                self.inputs,
                input_to_check,
Y
Yancey 已提交
1419
                output_names,
1420
                delta=numeric_grad_delta,
C
chengduo 已提交
1421
                in_place=in_place) for input_to_check in inputs_to_check
1422
        ]
1423

1424
        analytic_grads = self._get_gradient(inputs_to_check, place,
1425 1426
                                            output_names, no_grad_set,
                                            user_defined_grad_outputs)
D
Dun 已提交
1427 1428 1429
        self._assert_is_close(numeric_grads, analytic_grads, inputs_to_check,
                              max_relative_error,
                              "Gradient Check On %s" % str(place))
Q
qijun 已提交
1430

1431
        if check_dygraph:
1432 1433 1434
            dygraph_grad = self._get_dygraph_grad(
                inputs_to_check, place, output_names, user_defined_grad_outputs,
                no_grad_set)
1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451
            self._assert_is_close(numeric_grads, dygraph_grad, inputs_to_check,
                                  max_relative_error,
                                  "Gradient Check On %s" % str(place))

    def _find_var_in_dygraph(self, output_vars, name):
        if name in output_vars:
            return output_vars[name]
        else:
            for output_vars_index in output_vars:
                for output_vars_selected in output_vars[output_vars_index]:
                    if output_vars_selected.name == name:
                        return output_vars_selected

    def _get_dygraph_grad(self,
                          inputs_to_check,
                          place,
                          output_names,
1452
                          user_defined_grad_outputs=None,
1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483
                          no_grad_set=None):
        with fluid.dygraph.base.guard(place=place):
            block = fluid.default_main_program().global_block()

            op_proto = OpProtoHolder.instance().get_op_proto(self.op_type)

            # prepare input variable
            inputs, inputs_grad_dict = self.append_input_output_for_dygraph(
                op_proto, self.inputs, True, True, block)

            # prepare output variable
            outputs = self.append_input_output_for_dygraph(
                op_proto, self.outputs, False, False, block)

            # prepare attrbutes
            attrs_outputs = {}
            if hasattr(self, "attrs"):
                for attrs_name in self.attrs:
                    if self.attrs[attrs_name] is not None:
                        attrs_outputs[attrs_name] = self.attrs[attrs_name]
            block.append_op(
                type=self.op_type,
                inputs=inputs,
                outputs=outputs,
                attrs=attrs_outputs if hasattr(self, "attrs") else None)

            outputs_valid = {}
            for output_name in output_names:
                outputs_valid[output_name] = self._find_var_in_dygraph(
                    outputs, output_name)

1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517
            if user_defined_grad_outputs is None:
                if len(outputs_valid) == 1:
                    loss = block.create_var(
                        dtype=self.dtype,
                        type=core.VarDesc.VarType.LOD_TENSOR,
                        persistable=False,
                        stop_gradient=False,
                        shape=[1])
                    for outputs_valid_key in outputs_valid:
                        block.append_op(
                            type="mean",
                            inputs={"X": outputs_valid[outputs_valid_key]},
                            outputs={"Out": [loss]},
                            attrs=None)
                else:
                    avg_sum = []
                    for cur_loss in outputs_valid:
                        cur_avg_loss = block.create_var(
                            dtype=self.dtype,
                            type=core.VarDesc.VarType.LOD_TENSOR,
                            persistable=False,
                            stop_gradient=False)
                        block.append_op(
                            type="mean",
                            inputs={"X": outputs_valid[cur_loss]},
                            outputs={"Out": [cur_avg_loss]},
                            attrs=None)
                        avg_sum.append(cur_avg_loss)
                    loss_sum = block.create_var(
                        dtype=self.dtype,
                        type=core.VarDesc.VarType.LOD_TENSOR,
                        persistable=False,
                        stop_gradient=False,
                        shape=[1])
1518
                    block.append_op(
1519 1520 1521
                        type='sum',
                        inputs={"X": avg_sum},
                        outputs={"Out": loss_sum},
1522
                        attrs=None)
1523
                    loss = block.create_var(
1524 1525 1526
                        dtype=self.dtype,
                        type=core.VarDesc.VarType.LOD_TENSOR,
                        persistable=False,
1527 1528
                        stop_gradient=False,
                        shape=[1])
1529
                    block.append_op(
1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546
                        type='scale',
                        inputs={"X": loss_sum},
                        outputs={"Out": loss},
                        attrs={'scale': 1.0 / float(len(avg_sum))})
                loss.backward()
                fetch_list_grad = []
                for inputs_to_check_name in inputs_to_check:
                    a = inputs_grad_dict[inputs_to_check_name].gradient()
                    fetch_list_grad.append(a)
                return fetch_list_grad
            else:
                # user_defined_grad_outputs here are numpy arrays
                if not isinstance(user_defined_grad_outputs, list):
                    user_defined_grad_outputs = [user_defined_grad_outputs]
                grad_outputs = []
                for grad_out_value in user_defined_grad_outputs:
                    grad_outputs.append(paddle.to_tensor(grad_out_value))
C
chentianyu03 已提交
1547 1548 1549 1550
                # delete the inputs which no need to calculate grad
                for no_grad_val in no_grad_set:
                    del (inputs[no_grad_val])

1551 1552 1553 1554 1555
                grad_inputs = paddle.grad(
                    outputs=fluid.layers.utils.flatten(outputs),
                    inputs=fluid.layers.utils.flatten(inputs),
                    grad_outputs=grad_outputs)
                return [grad.numpy() for grad in grad_inputs]
1556

Y
Yu Yang 已提交
1557 1558 1559 1560 1561
    @staticmethod
    def _numpy_to_lod_tensor(np_value, lod, place):
        tensor = core.LoDTensor()
        tensor.set(np_value, place)
        if lod is not None:
1562
            tensor.set_recursive_sequence_lengths(lod)
Y
Yu Yang 已提交
1563 1564
        return tensor

K
Kexin Zhao 已提交
1565
    @staticmethod
K
Kexin Zhao 已提交
1566 1567
    def np_dtype_to_fluid_dtype(input):
        return input
K
Kexin Zhao 已提交
1568

D
dzhwinter 已提交
1569 1570 1571 1572 1573 1574 1575 1576
    @staticmethod
    def fluid_dtype_to_np_dtype(self, dtype):
        return dtype

    @staticmethod
    def np_value_to_fluid_value(input):
        return input

1577 1578 1579 1580 1581
    def _get_gradient(self,
                      input_to_check,
                      place,
                      output_names,
                      no_grad_set,
1582
                      user_defined_grad_outputs=None,
1583
                      parallel=False):
Y
Yu Yang 已提交
1584
        prog = Program()
1585
        scope = core.Scope()
Y
Yu Yang 已提交
1586
        block = prog.global_block()
1587
        self._append_ops(block)
Y
Yu Yang 已提交
1588

1589
        inputs = self._get_inputs(block)
1590
        outputs = self._get_outputs(block)
1591
        feed_dict = self.feed_var(inputs, place)
Y
Yu Yang 已提交
1592

1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623
        if user_defined_grad_outputs is None:
            loss = append_loss_ops(block, output_names)
            param_grad_list = append_backward(
                loss=loss,
                parameter_list=input_to_check,
                no_grad_set=no_grad_set)
            fetch_list = [g for p, g in param_grad_list]
        else:
            assert parallel is False, "unsupported parallel mode when giving custom grad outputs."
            # user_defined_grad_outputs here are numpy arrays
            if not isinstance(user_defined_grad_outputs, list):
                user_defined_grad_outputs = [user_defined_grad_outputs]
            grad_outputs = []
            for grad_out_value in user_defined_grad_outputs:
                # `presistable` is used to avoid executor create new var in local scope
                var = block.create_var(
                    shape=grad_out_value.shape,
                    dtype=grad_out_value.dtype,
                    persistable=True)
                true_var = scope.var(var.name)
                tensor = true_var.get_tensor()
                tensor.set(grad_out_value, place)
                grad_outputs.append(var)
            targets = [
                outputs[name] for name in outputs if name in output_names
            ]
            inputs = [inputs[name] for name in inputs if name in input_to_check]
            grad_inputs = paddle.static.gradients(targets, inputs, grad_outputs,
                                                  no_grad_set)
            fetch_list = grad_inputs

1624 1625
        if parallel:
            use_cuda = False
1626
            if isinstance(place, fluid.CUDAPlace):
1627
                use_cuda = True
1628 1629 1630 1631
            compiled_prog = fluid.CompiledProgram(prog).with_data_parallel(
                loss_name=loss.name, places=place)
            prog = compiled_prog
        executor = fluid.Executor(place)
1632 1633
        return list(
            map(np.array,
1634 1635 1636 1637 1638
                executor.run(prog,
                             feed_dict,
                             fetch_list,
                             scope=scope,
                             return_numpy=False)))