op_test.py 20.6 KB
Newer Older
1
#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
D
dzhwinter 已提交
2
#
D
dzhwinter 已提交
3 4 5
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
D
dzhwinter 已提交
6
#
D
dzhwinter 已提交
7
#     http://www.apache.org/licenses/LICENSE-2.0
D
dzhwinter 已提交
8
#
D
dzhwinter 已提交
9 10 11 12 13 14
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15 16
import unittest
import numpy as np
17
import random
18
import itertools
Q
Qiao Longfei 已提交
19
import paddle.v2.fluid.core as core
Y
Yu Yang 已提交
20
import collections
F
fengjiayi 已提交
21
from paddle.v2.fluid.backward import append_backward
Q
Qiao Longfei 已提交
22 23 24
from paddle.v2.fluid.op import Operator
from paddle.v2.fluid.executor import Executor
from paddle.v2.fluid.framework import Program, OpProtoHolder
25 26


27 28 29 30 31 32 33 34 35
def randomize_probability(batch_size, class_num, dtype='float32'):
    prob = np.random.uniform(
        0.1, 1.0, size=(batch_size, class_num)).astype(dtype)
    prob_sum = prob.sum(axis=1)
    for i in xrange(len(prob)):
        prob[i] /= prob_sum[i]
    return prob


Q
qijun 已提交
36
def create_op(scope, op_type, inputs, outputs, attrs):
37 38
    kwargs = dict()

Y
Yu Yang 已提交
39
    def __create_var__(name, var_name):
Q
QI JUN 已提交
40
        scope.var(var_name).get_tensor()
Y
Yu Yang 已提交
41 42
        kwargs[name].append(var_name)

Q
qijun 已提交
43
    for in_name, in_dup in Operator.get_op_inputs(op_type):
44 45 46 47
        if in_name in inputs:
            kwargs[in_name] = []
            if in_dup:
                sub_in = inputs[in_name]
48 49
                for item in sub_in:
                    sub_in_name, _ = item[0], item[1]
Y
Yu Yang 已提交
50
                    __create_var__(in_name, sub_in_name)
51
            else:
Y
Yu Yang 已提交
52
                __create_var__(in_name, in_name)
53

Q
qijun 已提交
54
    for out_name, out_dup in Operator.get_op_outputs(op_type):
55 56 57
        if out_name in outputs:
            kwargs[out_name] = []
            if out_dup:
58
                sub_out = outputs[out_name]
59 60
                for item in sub_out:
                    sub_out_name, _ = item[0], item[1]
Y
Yu Yang 已提交
61
                    __create_var__(out_name, sub_out_name)
62
            else:
Y
Yu Yang 已提交
63
                __create_var__(out_name, out_name)
64

Q
qijun 已提交
65
    for attr_name in Operator.get_op_attr_names(op_type):
Q
qijun 已提交
66 67
        if attr_name in attrs:
            kwargs[attr_name] = attrs[attr_name]
68

69 70 71 72
    return Operator(op_type, **kwargs)


def set_input(scope, op, inputs, place):
Y
Yu Yang 已提交
73
    def __set_input__(var_name, var):
74 75 76 77 78 79 80 81 82 83 84
        if isinstance(var, tuple) or isinstance(var, np.ndarray):
            tensor = scope.find_var(var_name).get_tensor()
            if isinstance(var, tuple):
                tensor.set_lod(var[1])
                var = var[0]
            tensor.set_dims(var.shape)
            tensor.set(var, place)
        elif isinstance(var, float):
            scope.find_var(var_name).set_float(var)
        elif isinstance(var, int):
            scope.find_var(var_name).set_int(var)
Y
Yu Yang 已提交
85

Q
qijun 已提交
86
    for in_name, in_dup in Operator.get_op_inputs(op.type()):
87 88 89
        if in_name in inputs:
            if in_dup:
                sub_in = inputs[in_name]
90 91
                for item in sub_in:
                    sub_in_name, sub_in_val = item[0], item[1]
Y
Yu Yang 已提交
92
                    __set_input__(sub_in_name, sub_in_val)
93
            else:
Y
Yu Yang 已提交
94
                __set_input__(in_name, inputs[in_name])
95 96


97 98
def get_numeric_gradient(place,
                         scope,
99 100 101
                         op,
                         inputs,
                         input_to_check,
Y
Yancey 已提交
102
                         output_names,
103 104
                         delta=0.005,
                         in_place=False):
Y
Yu Yang 已提交
105
    # FIXME: change this method by compile time concepts
106
    set_input(scope, op, inputs, place)
107 108 109 110 111

    def product(dim):
        return reduce(lambda a, b: a * b, dim, 1)

    def get_output():
Y
Yu Yang 已提交
112
        sum = []
Y
Yancey 已提交
113
        for output_name in output_names:
114
            op.run(scope, place)
Y
Yu Yang 已提交
115 116 117
            sum.append(
                np.array(scope.find_var(output_name).get_tensor()).mean())
        return np.array(sum).mean()
118 119 120

    tensor_to_check = scope.find_var(input_to_check).get_tensor()
    tensor_size = product(tensor_to_check.get_dims())
121
    tensor_to_check_dtype = tensor_to_check.dtype()
122
    if tensor_to_check_dtype == core.VarDesc.VarType.FP32:
123
        tensor_to_check_dtype = np.float32
124
    elif tensor_to_check_dtype == core.VarDesc.VarType.FP64:
125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143
        tensor_to_check_dtype = np.float64
    else:
        raise ValueError("Not supported data type " + str(
            tensor_to_check_dtype))

    gradient_flat = np.zeros(shape=(tensor_size, ), dtype=tensor_to_check_dtype)

    def __get_elem__(tensor, i):
        if tensor_to_check_dtype == np.float32:
            return tensor.get_float_element(i)
        else:
            return tensor.get_double_element(i)

    def __set_elem__(tensor, i, e):
        if tensor_to_check_dtype == np.float32:
            tensor.set_float_element(i, e)
        else:
            tensor.set_double_element(i, e)

144 145 146 147
    # we only compute gradient of one element each time.
    # we use a for loop to compute the gradient of every element.
    for i in xrange(tensor_size):
        if in_place:
148
            set_input(scope, op, inputs, place)
149 150

        # get one input element throw it's index i.
151
        origin = __get_elem__(tensor_to_check, i)
152 153
        # add delta to it, run op and then get the sum of the result tensor.
        x_pos = origin + delta
154
        __set_elem__(tensor_to_check, i, x_pos)
155 156 157
        y_pos = get_output()

        if in_place:
158
            set_input(scope, op, inputs, place)
159 160

        x_neg = origin - delta
161
        __set_elem__(tensor_to_check, i, x_neg)
162 163
        y_neg = get_output()

164
        __set_elem__(tensor_to_check, i, origin)
165 166 167 168 169
        gradient_flat[i] = (y_pos - y_neg) / delta / 2

    return gradient_flat.reshape(tensor_to_check.get_dims())


Y
Yang Yang(Tony) 已提交
170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196
def append_input_output(block, op_proto, np_list, is_input):
    '''Insert VarDesc and generate Python variable instance'''
    proto_list = op_proto.inputs if is_input else op_proto.outputs

    def create_var(block, name, np_list, var_proto):
        if name not in np_list:
            assert var_proto.intermediate, "{} not found".format(name)
            shape = None
            lod_level = None
        else:
            np_value = np_list[name]
            if isinstance(np_value, tuple):
                shape = list(np_value[0].shape)
                lod_level = len(np_value[1])
            else:
                shape = list(np_value.shape)
                lod_level = 0
        return block.create_var(
            dtype="float32", shape=shape, lod_level=lod_level, name=name)

    var_dict = {}
    for var_proto in proto_list:
        var_name = str(var_proto.name)
        if is_input:
            if (var_name not in np_list) and var_proto.dispensable:
                continue
            assert (var_name in np_list) or (var_proto.dispensable), \
197
                "Missing {} as input".format(var_name)
Y
Yang Yang(Tony) 已提交
198 199 200 201 202 203 204 205 206 207 208 209 210 211
        if var_proto.duplicable:
            assert isinstance(np_list[var_name], list), \
                "Duplicable {} should be set as list".format(var_name)
            var_list = []
            for (name, np_value) in np_list[var_name]:
                var_list.append(
                    create_var(block, name, {name: np_value}, var_proto))
            var_dict[var_name] = var_list
        else:
            var_dict[var_name] = create_var(block, var_name, np_list, var_proto)

    return var_dict


212
class OpTest(unittest.TestCase):
213 214 215 216 217 218 219 220 221 222 223 224 225 226 227
    @classmethod
    def setUpClass(cls):
        '''Fix random seeds to remove randomness from tests'''
        cls._np_rand_state = np.random.get_state()
        cls._py_rand_state = random.getstate()

        np.random.seed(123)
        random.seed(124)

    @classmethod
    def tearDownClass(cls):
        '''Restore random seeds'''
        np.random.set_state(cls._np_rand_state)
        random.setstate(cls._py_rand_state)

Y
Yang Yang(Tony) 已提交
228 229 230 231 232 233
    def feed_var(self, input_vars, place):
        feed_map = {}
        for var_name in input_vars:
            if isinstance(input_vars[var_name], list):
                for name, np_value in self.inputs[var_name]:
                    tensor = core.LoDTensor()
234 235 236 237 238
                    if isinstance(np_value, tuple):
                        tensor.set(np_value[0], place)
                        tensor.set_lod(np_value[1])
                    else:
                        tensor.set(np_value, place)
Y
Yang Yang(Tony) 已提交
239 240 241 242 243 244 245 246 247 248 249 250
                    feed_map[name] = tensor
            else:
                tensor = core.LoDTensor()
                if isinstance(self.inputs[var_name], tuple):
                    tensor.set(self.inputs[var_name][0], place)
                    tensor.set_lod(self.inputs[var_name][1])
                else:
                    tensor.set(self.inputs[var_name], place)
                feed_map[var_name] = tensor

        return feed_map

251 252 253 254 255
    def calc_output(self, place):
        outs, _ = self._calc_output(place)
        return outs

    def _calc_output(self, place):
Y
Yang Yang(Tony) 已提交
256 257 258 259 260 261 262 263 264 265 266 267
        op_proto = OpProtoHolder.instance().get_op_proto(self.op_type)

        program = Program()
        block = program.global_block()

        inputs = append_input_output(block, op_proto, self.inputs, True)
        outputs = append_input_output(block, op_proto, self.outputs, False)
        op = block.append_op(
            type=self.op_type,
            inputs=inputs,
            outputs=outputs,
            attrs=self.attrs if hasattr(self, "attrs") else dict())
Q
QI JUN 已提交
268 269 270
        # infer variable type and infer shape in compile-time
        op.desc.infer_var_type(block.desc)
        op.desc.infer_shape(block.desc)
Y
Yang Yang(Tony) 已提交
271 272 273 274 275 276 277 278 279 280 281 282 283

        fetch_list = []
        for var_name, var in outputs.iteritems():
            if var_name in self.outputs:
                if isinstance(var, list):
                    for v in var:
                        fetch_list.append(v)
                else:
                    fetch_list.append(var)

        feed_map = self.feed_var(inputs, place)

        exe = Executor(place)
D
dzhwinter 已提交
284 285 286 287
        outs = exe.run(program,
                       feed=feed_map,
                       fetch_list=fetch_list,
                       return_numpy=False)
288
        return outs, fetch_list
Y
Yang Yang(Tony) 已提交
289

290 291
    def check_output_with_place(self, place, atol):
        outs, fetch_list = self._calc_output(place)
Y
Yang Yang(Tony) 已提交
292
        for out_name, out_dup in Operator.get_op_outputs(self.op_type):
293 294 295
            if out_name not in self.outputs:
                continue

Y
Yang Yang(Tony) 已提交
296 297 298 299 300 301 302 303 304 305
            def find_actual(target_name, fetch_list):
                found = [
                    i for i, var in enumerate(fetch_list)
                    if var.name == target_name
                ]
                self.assertTrue(
                    len(found) == 1, "Found {} {}".format(
                        len(found), target_name))
                return found[0]

306 307
            if out_dup:
                sub_out = self.outputs[out_name]
Y
Yancey 已提交
308 309 310
                if not isinstance(sub_out, list):
                    raise AssertionError("sub_out type %s is not list",
                                         type(sub_out))
311 312
                for item in sub_out:
                    sub_out_name, expect = item[0], item[1]
Y
Yang Yang(Tony) 已提交
313
                    idx = find_actual(sub_out_name, fetch_list)
Q
QI JUN 已提交
314 315
                    actual = outs[idx]
                    actual_t = np.array(actual)
316 317
                    expect_t = expect[0] \
                        if isinstance(expect, tuple) else expect
318 319
                    self.assertTrue(
                        np.allclose(
320
                            actual_t, expect_t, atol=atol),
Y
Yang Yang(Tony) 已提交
321 322
                        "Output (" + sub_out_name + ") has diff at " +
                        str(place))
323 324
                    if isinstance(expect, tuple):
                        self.assertListEqual(
Q
QI JUN 已提交
325 326
                            actual.lod(), expect[1], "Output (" + sub_out_name +
                            ") has different lod at " + str(place))
327
            else:
Y
Yang Yang(Tony) 已提交
328
                idx = find_actual(out_name, fetch_list)
Q
QI JUN 已提交
329 330
                actual = outs[idx]
                actual_t = np.array(actual)
331
                expect = self.outputs[out_name]
332
                expect_t = expect[0] if isinstance(expect, tuple) else expect
333 334
                self.assertTrue(
                    np.allclose(
335
                        actual_t, expect_t, atol=atol),
E
emailweixu 已提交
336 337
                    "Output (" + out_name + ") has diff at " + str(place) +
                    str(actual_t) + str(expect_t))
338
                if isinstance(expect, tuple):
Q
QI JUN 已提交
339
                    self.assertListEqual(actual.lod(), expect[1],
340 341
                                         "Output (" + out_name +
                                         ") has different lod at " + str(place))
342

343
    def check_output(self, atol=1e-5):
Q
qijun 已提交
344
        places = [core.CPUPlace()]
345
        if core.is_compiled_with_cuda() and core.op_support_gpu(self.op_type):
D
dzhwinter 已提交
346
            places.append(core.CUDAPlace(0))
Q
qijun 已提交
347
        for place in places:
348
            self.check_output_with_place(place, atol)
Q
qijun 已提交
349

350 351 352 353 354 355 356 357 358
    def check_output_customized(self, checker):
        places = [core.CPUPlace()]
        if core.is_compiled_with_cuda() and core.op_support_gpu(self.op_type):
            places.append(core.CUDAPlace(0))
        for place in places:
            outs = self.calc_output(place)
            outs = [np.array(out) for out in outs]
            checker(outs)

359 360 361 362 363 364 365 366 367 368 369 370
    def __assert_is_close(self, numeric_grads, analytic_grads, names,
                          max_relative_error, msg_prefix):

        for a, b, name in itertools.izip(numeric_grads, analytic_grads, names):
            abs_a = np.abs(a)
            abs_a[abs_a < 1e-3] = 1

            diff_mat = np.abs(a - b) / abs_a
            max_diff = np.max(diff_mat)

            def err_msg():
                offset = np.argmax(diff_mat > max_relative_error)
371
                return ("%s Variable %s max gradient diff %f over limit %f, "
372
                        "the first error element is %d, %f, %f") % (
373
                            msg_prefix, name, max_diff, max_relative_error,
374
                            offset, a.flatten()[offset], b.flatten()[offset])
375 376 377 378 379

            self.assertLessEqual(max_diff, max_relative_error, err_msg())

    def check_grad(self,
                   inputs_to_check,
Y
Yancey 已提交
380
                   output_names,
381
                   no_grad_set=None,
382
                   numeric_grad_delta=0.005,
383
                   in_place=False,
Q
Qiao Longfei 已提交
384 385
                   max_relative_error=0.005,
                   user_defined_grads=None):
386
        places = [core.CPUPlace()]
387
        if core.is_compiled_with_cuda() and core.op_support_gpu(self.op_type):
388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403
            places.append(core.CUDAPlace(0))
        for place in places:
            self.check_grad_with_place(place, inputs_to_check, output_names,
                                       no_grad_set, numeric_grad_delta,
                                       in_place, max_relative_error,
                                       user_defined_grads)

    def check_grad_with_place(self,
                              place,
                              inputs_to_check,
                              output_names,
                              no_grad_set=None,
                              numeric_grad_delta=0.005,
                              in_place=False,
                              max_relative_error=0.005,
                              user_defined_grads=None):
404
        self.scope = core.Scope()
Q
qijun 已提交
405
        op_inputs = self.inputs if hasattr(self, "inputs") else dict()
406
        op_outputs = self.outputs if hasattr(self, "outputs") else dict()
Q
qijun 已提交
407
        op_attrs = self.attrs if hasattr(self, "attrs") else dict()
408
        self.op = create_op(self.scope, self.op_type, op_inputs, op_outputs,
Q
qijun 已提交
409
                            op_attrs)
Y
Yu Yang 已提交
410

411 412 413
        if no_grad_set is None:
            no_grad_set = set()

Y
Yancey 已提交
414 415 416
        if not type(output_names) is list:
            output_names = [output_names]

Q
Qiao Longfei 已提交
417
        numeric_grads = user_defined_grads or [
418
            get_numeric_gradient(
419
                place,
420 421 422 423
                self.scope,
                self.op,
                self.inputs,
                input_to_check,
Y
Yancey 已提交
424
                output_names,
425
                delta=numeric_grad_delta,
426 427
                in_place=in_place) for input_to_check in inputs_to_check
        ]
428 429 430 431 432 433
        analytic_grads = self._get_gradient(inputs_to_check, place,
                                            output_names, no_grad_set)

        self.__assert_is_close(numeric_grads, analytic_grads, inputs_to_check,
                               max_relative_error,
                               "Gradient Check On %s" % str(place))
Q
qijun 已提交
434

Y
Yu Yang 已提交
435 436 437 438 439 440 441 442 443 444 445 446 447 448
    @staticmethod
    def _create_var_descs_(block, var_dict):
        # FIXME: Try unify with `append_input_output`
        for param_name in var_dict:
            var = var_dict[param_name]
            if not isinstance(var, list) and not isinstance(var, tuple):
                var = [(param_name, var, None)]
            if not isinstance(var[0], list) and not isinstance(var[0], tuple):
                var = [(param_name, var[0], var[1])]

            for i, item in enumerate(var):
                if not isinstance(item[0], basestring):
                    item = [[param_name] + list(item)]
                if len(item) == 2:
449 450 451 452 453
                    if isinstance(item[1], tuple):
                        var[i] = [item[0], item[1][0], item[1][1]]
                    else:
                        # only set var name and value, set lod to None
                        var[i] = list(item) + [None]
Y
Yu Yang 已提交
454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493
            var_descs = [(block.create_var(
                name=name, shape=each.shape, dtype=each.dtype), each, lod)
                         for name, each, lod in var]

            yield param_name, var_descs

    @staticmethod
    def _merge_list(iterable):
        return reduce(lambda a, b: list(a) + list(b), iterable, [])

    @staticmethod
    def _numpy_to_lod_tensor(np_value, lod, place):
        tensor = core.LoDTensor()
        tensor.set(np_value, place)
        if lod is not None:
            tensor.set_lod(lod)
        return tensor

    def _get_gradient(self, input_to_check, place, output_names, no_grad_set):
        prog = Program()
        block = prog.global_block()
        inputs_with_np = {
            key: value
            for (key, value) in OpTest._create_var_descs_(
                block, getattr(self, 'inputs', {}))
        }
        outputs_with_np = {
            key: val
            for (key, val) in OpTest._create_var_descs_(
                block, getattr(self, 'outputs', {}))
        }
        inputs = {
            k: [item[0] for item in inputs_with_np[k]]
            for k in inputs_with_np
        }
        outputs = {
            k: [item[0] for item in outputs_with_np[k]]
            for k in outputs_with_np
        }

Q
QI JUN 已提交
494
        op = block.append_op(
Y
Yu Yang 已提交
495 496 497 498 499
            type=self.op_type,
            inputs=inputs,
            outputs=outputs,
            attrs=getattr(self, 'attrs', {}))

Q
QI JUN 已提交
500 501 502 503
        # infer variable type and infer shape in compile-time
        op.desc.infer_var_type(block.desc)
        op.desc.infer_shape(block.desc)

Y
Yu Yang 已提交
504 505 506
        mean_inputs = map(block.var, output_names)

        if len(mean_inputs) == 1:
F
fengjiayi 已提交
507
            loss = block.create_var(dtype=mean_inputs[0].dtype, shape=[1])
Q
QI JUN 已提交
508
            op = block.append_op(
Y
Yu Yang 已提交
509
                inputs={"X": mean_inputs}, outputs={"Out": loss}, type='mean')
Q
QI JUN 已提交
510 511
            op.desc.infer_var_type(block.desc)
            op.desc.infer_shape(block.desc)
Y
Yu Yang 已提交
512 513 514
        else:
            avg_sum = []
            for cur_loss in mean_inputs:
F
fengjiayi 已提交
515
                cur_avg_loss = block.create_var(dtype=cur_loss.dtype, shape=[1])
Q
QI JUN 已提交
516
                op = block.append_op(
Y
Yu Yang 已提交
517 518 519
                    inputs={"X": [cur_loss]},
                    outputs={"Out": [cur_avg_loss]},
                    type="mean")
Q
QI JUN 已提交
520 521
                op.desc.infer_var_type(block.desc)
                op.desc.infer_shape(block.desc)
Y
Yu Yang 已提交
522 523
                avg_sum.append(cur_avg_loss)

F
fengjiayi 已提交
524
            loss_sum = block.create_var(dtype=avg_sum[0].dtype, shape=[1])
Q
QI JUN 已提交
525
            op_sum = block.append_op(
Y
Yu Yang 已提交
526
                inputs={"X": avg_sum}, outputs={"Out": loss_sum}, type='sum')
Q
QI JUN 已提交
527 528
            op_sum.desc.infer_var_type(block.desc)
            op_sum.desc.infer_shape(block.desc)
Y
Yu Yang 已提交
529

F
fengjiayi 已提交
530
            loss = block.create_var(dtype=loss_sum.dtype, shape=[1])
Q
QI JUN 已提交
531
            op_loss = block.append_op(
Y
Yu Yang 已提交
532 533 534 535
                inputs={"X": loss_sum},
                outputs={"Out": loss},
                type='scale',
                attrs={'scale': 1.0 / float(len(avg_sum))})
Q
QI JUN 已提交
536 537
            op_loss.desc.infer_var_type(block.desc)
            op_loss.desc.infer_shape(block.desc)
Y
Yu Yang 已提交
538

F
fengjiayi 已提交
539
        param_grad_list = append_backward(
Y
Yu Yang 已提交
540 541 542 543 544 545 546 547 548
            loss=loss, parameter_list=input_to_check, no_grad_set=no_grad_set)

        feed_dict = {
            item[0].name: OpTest._numpy_to_lod_tensor(item[1], item[2], place)
            for p_name in inputs_with_np for item in inputs_with_np[p_name]
        }

        fetch_list = [g for p, g in param_grad_list]
        executor = Executor(place)
D
dzhwinter 已提交
549 550 551
        return map(
            np.array,
            executor.run(prog, feed_dict, fetch_list, return_numpy=False))