op_test.py 25.5 KB
Newer Older
1
#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
D
dzhwinter 已提交
2
#
D
dzhwinter 已提交
3 4 5
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
D
dzhwinter 已提交
6
#
D
dzhwinter 已提交
7
#     http://www.apache.org/licenses/LICENSE-2.0
D
dzhwinter 已提交
8
#
D
dzhwinter 已提交
9 10 11 12 13 14
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15 16
from __future__ import print_function

B
baojun 已提交
17
import os
18 19
import unittest
import numpy as np
20
import random
M
minqiyang 已提交
21
import six
22
import time
23
import itertools
Y
Yu Yang 已提交
24
import collections
M
minqiyang 已提交
25
from collections import defaultdict
26 27 28

import paddle.fluid as fluid
import paddle.fluid.core as core
29 30 31
from paddle.fluid.backward import append_backward
from paddle.fluid.op import Operator
from paddle.fluid.executor import Executor
32
from paddle.fluid.framework import Program, OpProtoHolder, Variable
33
from testsuite import create_op, set_input, append_input_output, append_loss_ops
34 35


36 37 38 39
def randomize_probability(batch_size, class_num, dtype='float32'):
    prob = np.random.uniform(
        0.1, 1.0, size=(batch_size, class_num)).astype(dtype)
    prob_sum = prob.sum(axis=1)
M
minqiyang 已提交
40
    for i in six.moves.xrange(len(prob)):
41 42 43 44
        prob[i] /= prob_sum[i]
    return prob


45 46
def get_numeric_gradient(place,
                         scope,
47 48 49
                         op,
                         inputs,
                         input_to_check,
Y
Yancey 已提交
50
                         output_names,
51
                         delta=0.005,
C
chengduo 已提交
52
                         in_place=False):
Y
Yu Yang 已提交
53
    # FIXME: change this method by compile time concepts
54
    set_input(scope, op, inputs, place)
55 56

    def product(dim):
M
minqiyang 已提交
57
        return six.moves.reduce(lambda a, b: a * b, dim, 1)
58 59

    tensor_to_check = scope.find_var(input_to_check).get_tensor()
Y
yuyang18 已提交
60 61
    tensor_size = product(tensor_to_check.shape())
    tensor_to_check_dtype = tensor_to_check._dtype()
62
    if tensor_to_check_dtype == core.VarDesc.VarType.FP32:
63
        tensor_to_check_dtype = np.float32
64
    elif tensor_to_check_dtype == core.VarDesc.VarType.FP64:
65
        tensor_to_check_dtype = np.float64
D
dzhwinter 已提交
66 67 68 69
    elif tensor_to_check_dtype == core.VarDesc.VarType.FP16:
        tensor_to_check_dtype = np.float16
        # set delta as np.float16, will automatic convert to float32, float64
        delta = np.array(delta).astype(np.float16)
70 71 72 73
    else:
        raise ValueError("Not supported data type " + str(
            tensor_to_check_dtype))

C
chengduo 已提交
74 75 76 77 78 79 80 81 82
    def get_output():
        sum = []
        op.run(scope, place)
        for output_name in output_names:
            sum.append(
                np.array(scope.find_var(output_name).get_tensor()).astype(
                    tensor_to_check_dtype).mean())
        return tensor_to_check_dtype(np.array(sum).sum() / len(output_names))

83 84 85
    gradient_flat = np.zeros(shape=(tensor_size, ), dtype=tensor_to_check_dtype)

    def __get_elem__(tensor, i):
D
dzhwinter 已提交
86 87 88 89 90
        if tensor_to_check_dtype == np.float16:
            numpy_tensor = np.array(tensor).astype(np.float16)
            numpy_tensor = numpy_tensor.flatten()
            return numpy_tensor[i]
        elif tensor_to_check_dtype == np.float32:
Y
yuyang18 已提交
91
            return tensor._get_float_element(i)
92
        else:
Y
yuyang18 已提交
93
            return tensor._get_double_element(i)
94 95

    def __set_elem__(tensor, i, e):
D
dzhwinter 已提交
96 97 98 99 100 101 102 103
        if tensor_to_check_dtype == np.float16:
            numpy_tensor = np.array(tensor).astype(np.float16)
            shape = numpy_tensor.shape
            numpy_tensor = numpy_tensor.flatten()
            numpy_tensor[i] = e
            numpy_tensor = numpy_tensor.reshape(shape).view(np.uint16)
            tensor.set(numpy_tensor, place)
        elif tensor_to_check_dtype == np.float32:
Y
yuyang18 已提交
104
            tensor._set_float_element(i, e)
105
        else:
Y
yuyang18 已提交
106
            tensor._set_double_element(i, e)
107

108 109
    # we only compute gradient of one element each time.
    # we use a for loop to compute the gradient of every element.
M
minqiyang 已提交
110
    for i in six.moves.xrange(tensor_size):
111
        if in_place:
112
            set_input(scope, op, inputs, place)
113 114

        # get one input element throw it's index i.
115
        origin = __get_elem__(tensor_to_check, i)
116 117
        # add delta to it, run op and then get the sum of the result tensor.
        x_pos = origin + delta
118
        __set_elem__(tensor_to_check, i, x_pos)
119 120 121
        y_pos = get_output()

        if in_place:
122
            set_input(scope, op, inputs, place)
123 124

        x_neg = origin - delta
125
        __set_elem__(tensor_to_check, i, x_neg)
126 127
        y_neg = get_output()

128
        __set_elem__(tensor_to_check, i, origin)
129 130
        gradient_flat[i] = (y_pos - y_neg) / delta / 2

Y
yuyang18 已提交
131
    return gradient_flat.reshape(tensor_to_check.shape())
132 133 134


class OpTest(unittest.TestCase):
135 136 137 138 139
    @classmethod
    def setUpClass(cls):
        '''Fix random seeds to remove randomness from tests'''
        cls._np_rand_state = np.random.get_state()
        cls._py_rand_state = random.getstate()
140 141 142
        cls.call_once = False
        cls.dtype = "float32"
        cls.outputs = {}
143 144 145 146 147 148

        np.random.seed(123)
        random.seed(124)

    @classmethod
    def tearDownClass(cls):
Y
yuyang18 已提交
149
        """Restore random seeds"""
150 151 152
        np.random.set_state(cls._np_rand_state)
        random.setstate(cls._py_rand_state)

153 154 155 156
    def try_call_once(self, data_type):
        if not self.call_once:
            self.call_once = True
            self.dtype = data_type
D
dzhwinter 已提交
157 158 159 160 161
            # See the comment of np_dtype_to_fluid_dtype
            # If the input type is uint16, we assume use float16
            # for lodtensor dtype.
            if self.dtype == np.uint16:
                self.dtype == np.float16
162 163 164 165 166 167

    def infer_dtype_from_inputs_outputs(self, inputs, outputs):
        def infer_dtype(numpy_dict):
            assert isinstance(
                numpy_dict,
                dict), "self.inputs, self.outputs must be numpy_dict"
M
minqiyang 已提交
168
            for var_name, var_value in six.iteritems(numpy_dict):
169 170 171 172 173 174 175 176 177 178 179 180 181 182
                if isinstance(var_value, (np.ndarray, np.generic)):
                    self.try_call_once(var_value.dtype)
                elif isinstance(var_value, (list, tuple)):
                    # the case of self.inputs = {"X": [("x0", x0), ("x1", x1), ("x2", x2)]}
                    if len(var_value) > 1 and isinstance(var_value[1], (
                            np.ndarray, np.generic)):
                        instance = var_value[1]
                        self.try_call_once(instance[1].dtype)
                else:
                    self.try_call_once("float32")

        infer_dtype(inputs)
        infer_dtype(outputs)

Y
Yang Yang(Tony) 已提交
183 184 185 186 187 188
    def feed_var(self, input_vars, place):
        feed_map = {}
        for var_name in input_vars:
            if isinstance(input_vars[var_name], list):
                for name, np_value in self.inputs[var_name]:
                    tensor = core.LoDTensor()
189
                    if isinstance(np_value, tuple):
D
dzhwinter 已提交
190 191
                        tensor.set(
                            OpTest.np_value_to_fluid_value(np_value[0]), place)
192
                        tensor.set_recursive_sequence_lengths(np_value[1])
193
                    else:
D
dzhwinter 已提交
194 195
                        tensor.set(
                            OpTest.np_value_to_fluid_value(np_value), place)
Y
Yang Yang(Tony) 已提交
196 197 198 199
                    feed_map[name] = tensor
            else:
                tensor = core.LoDTensor()
                if isinstance(self.inputs[var_name], tuple):
D
dzhwinter 已提交
200 201 202
                    tensor.set(
                        OpTest.np_value_to_fluid_value(self.inputs[var_name][
                            0]), place)
203 204
                    tensor.set_recursive_sequence_lengths(self.inputs[var_name][
                        1])
Y
Yang Yang(Tony) 已提交
205
                else:
D
dzhwinter 已提交
206 207 208
                    tensor.set(
                        OpTest.np_value_to_fluid_value(self.inputs[var_name]),
                        place)
Y
Yang Yang(Tony) 已提交
209 210 211 212
                feed_map[var_name] = tensor

        return feed_map

213
    def _append_ops(self, block):
Y
Yang Yang(Tony) 已提交
214
        op_proto = OpProtoHolder.instance().get_op_proto(self.op_type)
215 216 217 218 219 220
        "infer datatype from inputs and outputs for this test case"
        self.infer_dtype_from_inputs_outputs(self.inputs, self.outputs)
        inputs = append_input_output(block, op_proto, self.inputs, True,
                                     self.dtype)
        outputs = append_input_output(block, op_proto, self.outputs, False,
                                      self.dtype)
P
phlrain 已提交
221 222 223 224 225 226 227 228 229

        if hasattr(self, "cache_name_list"):
            for name in self.cache_name_list:
                inputs[name] = block.create_var(
                    name=name,
                    persistable=True,
                    type=core.VarDesc.VarType.RAW,
                    stop_gradient=True)

Y
Yang Yang(Tony) 已提交
230 231 232 233 234
        op = block.append_op(
            type=self.op_type,
            inputs=inputs,
            outputs=outputs,
            attrs=self.attrs if hasattr(self, "attrs") else dict())
Q
QI JUN 已提交
235 236 237
        # infer variable type and infer shape in compile-time
        op.desc.infer_var_type(block.desc)
        op.desc.infer_shape(block.desc)
Y
Yang Yang(Tony) 已提交
238

239 240
    def _get_io_vars(self, block, numpy_inputs):
        inputs = {}
M
minqiyang 已提交
241
        for name, value in six.iteritems(numpy_inputs):
242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260
            if isinstance(value, list):
                var_list = [
                    block.var(sub_name) for sub_name, sub_value in value
                ]
                inputs[name] = var_list
            else:
                inputs[name] = block.var(name)
        return inputs

    def _get_inputs(self, block):
        return self._get_io_vars(block, self.inputs)

    def _get_outputs(self, block):
        return self._get_io_vars(block, self.outputs)

    def calc_output(self, place):
        outs, _ = self._calc_output(place)
        return outs

M
minqiyang 已提交
261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317
    def _create_var_from_numpy(self, value):
        if isinstance(value, tuple):
            data = value[0]
            lod = value[1]
            v = fluid.imperative.base.to_variable(value=data)
            v._ivar.value().get_tensor().set_recursive_sequence_lengths(lod)
            return v
        else:
            return fluid.imperative.base.to_variable(value)

    def _calc_imperative_output(self, place, parallel=False, no_check_set=None):
        with fluid.imperative.base.guard(place=place):
            block = fluid.default_main_program().global_block()

            # prepare input variable
            inputs = defaultdict(list)
            for name, np_value in six.iteritems(self.inputs):
                if not isinstance(np_value, list):
                    np_value = [np_value]

                for i in range(len(np_value)):
                    inputs[name].append(
                        self._create_var_from_numpy(np_value[i]))

            # prepare output variable
            outputs = defaultdict(list)
            for name, np_value in six.iteritems(self.outputs):
                if not isinstance(np_value, list):
                    np_value = [np_value]

                for i in range(len(np_value)):
                    value = np_value[i]
                    if isinstance(value, tuple):
                        v = block.create_var(
                            name="%s_out%d" % (name, i),
                            dtype=value[0].dtype,
                            type=core.VarDesc.VarType.LOD_TENSOR,
                            persistable=False,
                            stop_gradient=False)
                        v._ivar.value().get_tensor(
                        ).set_recursive_sequence_lengths(value[1])
                    else:
                        v = block.create_var(
                            name="%s_out%d" % (name, i),
                            dtype=value.dtype,
                            type=core.VarDesc.VarType.LOD_TENSOR,
                            persistable=False,
                            stop_gradient=False)
                    outputs[name].append(v)

            block.append_op(
                type=self.op_type,
                inputs=inputs,
                outputs=outputs,
                attrs=self.attrs)

            return outputs
318

M
minqiyang 已提交
319
    def _calc_output(self, place, parallel=False, no_check_set=None):
320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340
        program = Program()
        block = program.global_block()
        self._append_ops(block)

        inputs = self._get_inputs(block)
        outputs = self._get_outputs(block)
        feed_map = self.feed_var(inputs, place)

        if parallel:
            use_cuda = False
            if isinstance(place, fluid.CUDAPlace(0)):
                use_cuda = True
            executor = fluid.ParallelExecutor(
                use_cuda=use_cuda, loss_name=loss.name, main_program=program)
        else:
            executor = Executor(place)

        fetch_list = getattr(self, "fetch_list", [])
        # if the fetch_list is customized by user, we use it directly.
        # if not, fill the fetch_list by the user configured outputs in test.
        if len(fetch_list) == 0:
M
minqiyang 已提交
341
            for var_name, var in six.iteritems(outputs):
342 343
                if no_check_set is not None and var_name in no_check_set:
                    continue
Y
Yang Yang(Tony) 已提交
344 345 346 347 348
                if isinstance(var, list):
                    for v in var:
                        fetch_list.append(v)
                else:
                    fetch_list.append(var)
349 350 351 352 353
        # if the fetch_list still empty, fill the fetch_list by the operator output.
        if len(fetch_list) == 0:
            for out_name, out_dup in Operator.get_op_outputs(self.op_type):
                fetch_list.append(str(out_name))
        # fetch_list = map(block.var, fetch_list)
W
Wu Yi 已提交
354
        if not isinstance(fetch_list[0], fluid.framework.Variable):
355
            fetch_list = list(map(block.var, fetch_list))
356 357 358 359
        outs = executor.run(program,
                            feed=feed_map,
                            fetch_list=fetch_list,
                            return_numpy=False)
360
        return outs, fetch_list
Y
Yang Yang(Tony) 已提交
361

362 363 364 365
    def check_output_with_place(self,
                                place,
                                atol,
                                no_check_set=None,
M
minqiyang 已提交
366 367 368 369 370
                                equal_nan=False,
                                check_imperative=False):
        if check_imperative:
            imperative_outs = self._calc_imperative_output(
                place, no_check_set=no_check_set)
371
        outs, fetch_list = self._calc_output(place, no_check_set=no_check_set)
M
minqiyang 已提交
372

Y
Yang Yang(Tony) 已提交
373
        for out_name, out_dup in Operator.get_op_outputs(self.op_type):
374 375
            if out_name not in self.outputs:
                continue
376 377
            if no_check_set is not None and out_name in no_check_set:
                continue
378

Y
Yang Yang(Tony) 已提交
379 380 381 382 383 384 385 386 387 388
            def find_actual(target_name, fetch_list):
                found = [
                    i for i, var in enumerate(fetch_list)
                    if var.name == target_name
                ]
                self.assertTrue(
                    len(found) == 1, "Found {} {}".format(
                        len(found), target_name))
                return found[0]

389 390
            if out_dup:
                sub_out = self.outputs[out_name]
Y
Yancey 已提交
391 392 393
                if not isinstance(sub_out, list):
                    raise AssertionError("sub_out type %s is not list",
                                         type(sub_out))
394 395
                for item in sub_out:
                    sub_out_name, expect = item[0], item[1]
M
minqiyang 已提交
396 397 398 399
                    if check_imperative:
                        imperative_actual = imperative_outs[sub_out_name][0]
                        imperative_actual_t = np.array(
                            imperative_actual._ivar.value().get_tensor())
Y
Yang Yang(Tony) 已提交
400
                    idx = find_actual(sub_out_name, fetch_list)
Q
QI JUN 已提交
401 402
                    actual = outs[idx]
                    actual_t = np.array(actual)
403 404
                    expect_t = expect[0] \
                        if isinstance(expect, tuple) else expect
405 406
                    self.assertTrue(
                        np.allclose(
407
                            actual_t, expect_t, atol=atol, equal_nan=equal_nan),
Y
Yang Yang(Tony) 已提交
408 409
                        "Output (" + sub_out_name + ") has diff at " +
                        str(place))
M
minqiyang 已提交
410 411 412 413 414 415 416 417
                    self.assertTrue(
                        np.allclose(
                            imperative_actual_t,
                            expect_t,
                            atol=atol,
                            equal_nan=equal_nan),
                        "Output (" + sub_out_name + ") has diff at " +
                        str(place) + " in imperative mode")
418 419
                    if isinstance(expect, tuple):
                        self.assertListEqual(
420 421
                            actual.recursive_sequence_lengths(), expect[1],
                            "Output (" + sub_out_name +
Q
QI JUN 已提交
422
                            ") has different lod at " + str(place))
423
            else:
M
minqiyang 已提交
424 425 426 427
                if check_imperative:
                    imperative_actual = imperative_outs[out_name][0]
                    imperative_actual_t = np.array(
                        imperative_actual._ivar.value().get_tensor())
Y
Yang Yang(Tony) 已提交
428
                idx = find_actual(out_name, fetch_list)
Q
QI JUN 已提交
429 430
                actual = outs[idx]
                actual_t = np.array(actual)
431
                expect = self.outputs[out_name]
432
                expect_t = expect[0] if isinstance(expect, tuple) else expect
433 434
                self.assertTrue(
                    np.allclose(
435
                        actual_t, expect_t, atol=atol, equal_nan=equal_nan),
E
emailweixu 已提交
436
                    "Output (" + out_name + ") has diff at " + str(place) +
D
dzhwinter 已提交
437
                    "\nExpect " + str(expect_t) + "\n" + "But Got" +
438
                    str(actual_t) + " in class " + self.__class__.__name__)
M
minqiyang 已提交
439 440 441 442 443 444 445 446 447 448
                self.assertTrue(
                    np.allclose(
                        imperative_actual_t,
                        expect_t,
                        atol=atol,
                        equal_nan=equal_nan),
                    "Output (" + out_name + ") has diff at " + str(place) +
                    "\nExpect " + str(expect_t) + "\n" + "But Got" +
                    str(imperative_actual_t) + " in class " +
                    self.__class__.__name__)
449
                if isinstance(expect, tuple):
450 451
                    self.assertListEqual(actual.recursive_sequence_lengths(),
                                         expect[1], "Output (" + out_name +
452
                                         ") has different lod at " + str(place))
M
minqiyang 已提交
453 454 455 456 457
                    if check_imperative:
                        self.assertListEqual(
                            imperative_actual._ivar.value().get_tensor()
                            .recursive_sequence_lengths(), expect[1], "Output ("
                            + out_name + ") has different lod at " + str(place))
458

459
    def _get_places(self):
D
dzhwinter 已提交
460 461 462 463 464 465
        if self.dtype == np.float16:
            if core.is_compiled_with_cuda() and core.op_support_gpu(
                    self.op_type):
                place = core.CUDAPlace(0)
                if core.is_float16_supported(place):
                    return [place]
W
Wu Yi 已提交
466 467
                else:
                    return []
D
dzhwinter 已提交
468 469
            else:
                return []
470
        places = [fluid.CPUPlace()]
471
        cpu_only = self._cpu_only if hasattr(self, '_cpu_only') else False
B
baojun 已提交
472 473 474
        use_ngraph = bool(os.getenv("FLAGS_use_ngraph", False))
        if use_ngraph:
            cpu_only = True
475 476
        if core.is_compiled_with_cuda() and core.op_support_gpu(self.op_type)\
           and not cpu_only:
D
dzhwinter 已提交
477
            places.append(core.CUDAPlace(0))
478 479
        return places

M
minqiyang 已提交
480 481 482 483 484
    def check_output(self,
                     atol=1e-5,
                     no_check_set=None,
                     equal_nan=False,
                     check_imperative=False):
485
        places = self._get_places()
Q
qijun 已提交
486
        for place in places:
M
minqiyang 已提交
487 488
            self.check_output_with_place(place, atol, no_check_set, equal_nan,
                                         check_imperative)
Q
qijun 已提交
489

490
    def check_output_customized(self, checker):
491
        places = self._get_places()
492 493 494
        for place in places:
            outs = self.calc_output(place)
            outs = [np.array(out) for out in outs]
495
            outs.sort(key=len)
496 497
            checker(outs)

D
Dun 已提交
498 499
    def _assert_is_close(self, numeric_grads, analytic_grads, names,
                         max_relative_error, msg_prefix):
500

M
minqiyang 已提交
501
        for a, b, name in six.moves.zip(numeric_grads, analytic_grads, names):
502 503 504 505 506 507 508 509
            abs_a = np.abs(a)
            abs_a[abs_a < 1e-3] = 1

            diff_mat = np.abs(a - b) / abs_a
            max_diff = np.max(diff_mat)

            def err_msg():
                offset = np.argmax(diff_mat > max_relative_error)
510
                return ("%s Variable %s max gradient diff %f over limit %f, "
D
dzhwinter 已提交
511 512 513
                        "the first error element is %d, expected %f, but got %f"
                        ) % (msg_prefix, name, max_diff, max_relative_error,
                             offset, a.flatten()[offset], b.flatten()[offset])
514 515 516 517 518

            self.assertLessEqual(max_diff, max_relative_error, err_msg())

    def check_grad(self,
                   inputs_to_check,
Y
Yancey 已提交
519
                   output_names,
520
                   no_grad_set=None,
521
                   numeric_grad_delta=0.005,
522
                   in_place=False,
Q
Qiao Longfei 已提交
523
                   max_relative_error=0.005,
C
chengduo 已提交
524
                   user_defined_grads=None):
525
        places = self._get_places()
526 527 528 529
        for place in places:
            self.check_grad_with_place(place, inputs_to_check, output_names,
                                       no_grad_set, numeric_grad_delta,
                                       in_place, max_relative_error,
C
chengduo 已提交
530
                                       user_defined_grads)
531 532 533 534 535 536 537 538 539

    def check_grad_with_place(self,
                              place,
                              inputs_to_check,
                              output_names,
                              no_grad_set=None,
                              numeric_grad_delta=0.005,
                              in_place=False,
                              max_relative_error=0.005,
C
chengduo 已提交
540
                              user_defined_grads=None):
541
        self.scope = core.Scope()
Q
qijun 已提交
542
        op_inputs = self.inputs if hasattr(self, "inputs") else dict()
543
        op_outputs = self.outputs if hasattr(self, "outputs") else dict()
Q
qijun 已提交
544
        op_attrs = self.attrs if hasattr(self, "attrs") else dict()
P
phlrain 已提交
545 546 547 548 549 550 551 552 553 554 555

        cache_list = None
        if hasattr(self, "cache_name_list"):
            cache_list = self.cache_name_list
        self.op = create_op(
            self.scope,
            self.op_type,
            op_inputs,
            op_outputs,
            op_attrs,
            cache_list=cache_list)
Y
Yu Yang 已提交
556

557 558 559
        if no_grad_set is None:
            no_grad_set = set()

Y
Yancey 已提交
560 561 562
        if not type(output_names) is list:
            output_names = [output_names]

Q
Qiao Longfei 已提交
563
        numeric_grads = user_defined_grads or [
564
            get_numeric_gradient(
565
                place,
566 567 568 569
                self.scope,
                self.op,
                self.inputs,
                input_to_check,
Y
Yancey 已提交
570
                output_names,
571
                delta=numeric_grad_delta,
C
chengduo 已提交
572
                in_place=in_place) for input_to_check in inputs_to_check
573
        ]
574 575 576
        analytic_grads = self._get_gradient(inputs_to_check, place,
                                            output_names, no_grad_set)

D
Dun 已提交
577 578 579
        self._assert_is_close(numeric_grads, analytic_grads, inputs_to_check,
                              max_relative_error,
                              "Gradient Check On %s" % str(place))
Q
qijun 已提交
580

Y
Yu Yang 已提交
581 582 583 584 585
    @staticmethod
    def _numpy_to_lod_tensor(np_value, lod, place):
        tensor = core.LoDTensor()
        tensor.set(np_value, place)
        if lod is not None:
586
            tensor.set_recursive_sequence_lengths(lod)
Y
Yu Yang 已提交
587 588
        return tensor

K
Kexin Zhao 已提交
589
    @staticmethod
K
Kexin Zhao 已提交
590 591
    def np_dtype_to_fluid_dtype(input):
        """Change the dtype of float16 numpy array
K
Kexin Zhao 已提交
592

593
        numpy float16 is binded to paddle::platform::float16
K
Kexin Zhao 已提交
594
        in tensor_py.h via the help of uint16 data type since
595
        the internal memory representation of float16 is
K
Kexin Zhao 已提交
596 597
        uint16_t in paddle and np.uint16 in numpy, which are
        themselves binded together by pybind.
K
Kexin Zhao 已提交
598 599 600 601 602

        Args:
            input: input numpy array

        Returns:
603
            input: The dtype of input will be changed to np.uint16 if
K
Kexin Zhao 已提交
604
                it is originally np.float16, such that the internal memory
605
                of input will be reinterpreted as of dtype np.uint16.
K
Kexin Zhao 已提交
606 607
        """
        if input.dtype == np.float16:
K
Kexin Zhao 已提交
608 609
            input.dtype = np.uint16
        return input
K
Kexin Zhao 已提交
610

D
dzhwinter 已提交
611 612 613 614 615 616 617 618 619 620 621 622 623 624 625
    @staticmethod
    def fluid_dtype_to_np_dtype(self, dtype):
        """
        See above, convert the dtype to normal type.
        """
        if dtype == np.uint16:
            dtype = np.float16
        return dtype

    @staticmethod
    def np_value_to_fluid_value(input):
        if input.dtype == np.float16:
            input = input.view(np.uint16)
        return input

626 627 628 629 630 631
    def _get_gradient(self,
                      input_to_check,
                      place,
                      output_names,
                      no_grad_set,
                      parallel=False):
Y
Yu Yang 已提交
632 633
        prog = Program()
        block = prog.global_block()
634 635
        self._append_ops(block)
        loss = append_loss_ops(block, output_names)
F
fengjiayi 已提交
636
        param_grad_list = append_backward(
Y
Yu Yang 已提交
637 638
            loss=loss, parameter_list=input_to_check, no_grad_set=no_grad_set)

639 640
        inputs = self._get_inputs(block)
        feed_dict = self.feed_var(inputs, place)
Y
Yu Yang 已提交
641 642

        fetch_list = [g for p, g in param_grad_list]
643 644 645 646 647
        if parallel:
            use_cuda = False
            if isinstance(place, fluid.CUDAPlace(0)):
                use_cuda = True
            executor = fluid.ParallelExecutor(
D
dzhwinter 已提交
648
                use_cuda=use_cuda, loss_name=loss.name, main_program=prog)
649 650
        else:
            executor = Executor(place)
651 652 653
        return list(
            map(np.array,
                executor.run(prog, feed_dict, fetch_list, return_numpy=False)))