op_test.py 11.9 KB
Newer Older
1 2
import unittest
import numpy as np
3
import random
4 5 6 7 8 9 10 11 12
import itertools
import paddle.v2.framework.core as core
from paddle.v2.framework.op import Operator


def grad_var_name(var_name):
    return var_name + "@GRAD"


Q
qijun 已提交
13
def create_op(scope, op_type, inputs, outputs, attrs):
14 15
    kwargs = dict()

Y
Yu Yang 已提交
16
    def __create_var__(name, var_name):
D
dongzhihong 已提交
17
        scope.get_or_create(var_name)
Y
Yu Yang 已提交
18 19
        kwargs[name].append(var_name)

Q
qijun 已提交
20
    for in_name, in_dup in Operator.get_op_inputs(op_type):
21 22 23 24
        if in_name in inputs:
            kwargs[in_name] = []
            if in_dup:
                sub_in = inputs[in_name]
Q
qijun 已提交
25
                for sub_in_name, _ in sub_in:
Y
Yu Yang 已提交
26
                    __create_var__(in_name, sub_in_name)
27
            else:
Y
Yu Yang 已提交
28
                __create_var__(in_name, in_name)
29

Q
qijun 已提交
30
    for out_name, out_dup in Operator.get_op_outputs(op_type):
31 32 33
        if out_name in outputs:
            kwargs[out_name] = []
            if out_dup:
34 35
                sub_out = outputs[out_name]
                for sub_out_name, _ in sub_out:
Y
Yu Yang 已提交
36
                    __create_var__(out_name, sub_out_name)
37
            else:
Y
Yu Yang 已提交
38
                __create_var__(out_name, out_name)
39

Q
qijun 已提交
40
    for attr_name in Operator.get_op_attr_names(op_type):
Q
qijun 已提交
41 42
        if attr_name in attrs:
            kwargs[attr_name] = attrs[attr_name]
43

44 45 46 47
    return Operator(op_type, **kwargs)


def set_input(scope, op, inputs, place):
Y
Yu Yang 已提交
48
    def __set_input__(var_name, var):
49 50 51 52 53 54 55 56 57 58 59
        if isinstance(var, tuple) or isinstance(var, np.ndarray):
            tensor = scope.find_var(var_name).get_tensor()
            if isinstance(var, tuple):
                tensor.set_lod(var[1])
                var = var[0]
            tensor.set_dims(var.shape)
            tensor.set(var, place)
        elif isinstance(var, float):
            scope.find_var(var_name).set_float(var)
        elif isinstance(var, int):
            scope.find_var(var_name).set_int(var)
Y
Yu Yang 已提交
60

Q
qijun 已提交
61
    for in_name, in_dup in Operator.get_op_inputs(op.type()):
62 63 64
        if in_name in inputs:
            if in_dup:
                sub_in = inputs[in_name]
65
                for sub_in_name, sub_in_val in sub_in:
Y
Yu Yang 已提交
66
                    __set_input__(sub_in_name, sub_in_val)
67
            else:
Y
Yu Yang 已提交
68
                __set_input__(in_name, inputs[in_name])
69 70 71


def set_output_grad(scope, op, outputs, place):
72 73
    def __set_tensor__(name):
        out_tensor = scope.find_var(name).get_tensor()
D
dongzhihong 已提交
74
        grad_tensor = scope.get_or_create(grad_var_name(name)).get_tensor()
75 76 77 78 79 80 81 82 83 84
        out_dtype = out_tensor.dtype()
        if out_dtype == core.DataType.FP64:
            data = np.ones(out_tensor.shape(), dtype=np.float64)
        elif out_dtype == core.DataType.FP32:
            data = np.ones(out_tensor.shape(), dtype=np.float32)
        else:
            raise ValueError("Not supported data type " + str(out_dtype))

        grad_tensor.set(data, place)

Q
qijun 已提交
85
    for out_name, out_dup in Operator.get_op_outputs(op.type()):
86 87 88
        if out_name in outputs:
            if out_dup:
                sub_out = outputs[out_name]
89
                for sub_out_name, _ in sub_out:
90
                    __set_tensor__(sub_out_name)
91
            else:
92
                __set_tensor__(out_name)
93 94 95 96 97 98


def get_numeric_gradient(scope,
                         op,
                         inputs,
                         input_to_check,
Y
Yancey 已提交
99
                         output_names,
100 101 102 103 104 105 106 107 108 109 110 111
                         delta=0.005,
                         in_place=False):
    set_input(scope, op, inputs, core.CPUPlace())

    tensor_to_check = scope.find_var(input_to_check).get_tensor()

    def product(dim):
        return reduce(lambda a, b: a * b, dim, 1)

    ctx = core.DeviceContext.create(core.CPUPlace())

    def get_output():
Y
Yancey 已提交
112 113 114 115 116
        sum = 0.0
        for output_name in output_names:
            op.run(scope, ctx)
            sum += np.array(scope.find_var(output_name).get_tensor()).sum()
        return sum
117 118 119

    tensor_to_check = scope.find_var(input_to_check).get_tensor()
    tensor_size = product(tensor_to_check.get_dims())
120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142
    tensor_to_check_dtype = tensor_to_check.dtype()
    if tensor_to_check_dtype == core.DataType.FP32:
        tensor_to_check_dtype = np.float32
    elif tensor_to_check_dtype == core.DataType.FP64:
        tensor_to_check_dtype = np.float64
    else:
        raise ValueError("Not supported data type " + str(
            tensor_to_check_dtype))

    gradient_flat = np.zeros(shape=(tensor_size, ), dtype=tensor_to_check_dtype)

    def __get_elem__(tensor, i):
        if tensor_to_check_dtype == np.float32:
            return tensor.get_float_element(i)
        else:
            return tensor.get_double_element(i)

    def __set_elem__(tensor, i, e):
        if tensor_to_check_dtype == np.float32:
            tensor.set_float_element(i, e)
        else:
            tensor.set_double_element(i, e)

143 144 145 146
    # we only compute gradient of one element each time.
    # we use a for loop to compute the gradient of every element.
    for i in xrange(tensor_size):
        if in_place:
Q
qijun 已提交
147
            set_input(scope, op, inputs, core.CPUPlace())
148 149

        # get one input element throw it's index i.
150
        origin = __get_elem__(tensor_to_check, i)
151 152
        # add delta to it, run op and then get the sum of the result tensor.
        x_pos = origin + delta
153
        __set_elem__(tensor_to_check, i, x_pos)
154 155 156
        y_pos = get_output()

        if in_place:
Q
qijun 已提交
157
            set_input(scope, op, inputs, core.CPUPlace())
158 159

        x_neg = origin - delta
160
        __set_elem__(tensor_to_check, i, x_neg)
161 162
        y_neg = get_output()

163
        __set_elem__(tensor_to_check, i, origin)
164 165 166 167 168 169 170
        gradient_flat[i] = (y_pos - y_neg) / delta / 2

    return gradient_flat.reshape(tensor_to_check.get_dims())


def get_backward_op(scope, op, no_grad_set):
    backward_op = core.Operator.backward(op, no_grad_set)
Q
qijun 已提交
171
    for input in backward_op.input_vars():
D
dongzhihong 已提交
172
        var = scope.get_or_create(input)
173
        var.get_tensor()
Q
qijun 已提交
174
    for output in backward_op.output_vars():
D
dongzhihong 已提交
175
        var = scope.get_or_create(output)
176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200
        var.get_tensor()
    return backward_op


def get_gradient(scope, op, inputs, outputs, grad_name, place,
                 no_grad_set=None):
    ctx = core.DeviceContext.create(place)

    set_input(scope, op, inputs, place)

    op.run(scope, ctx)

    if no_grad_set is None:
        no_grad_set = set()

    backward_op = get_backward_op(scope, op, no_grad_set)
    set_output_grad(scope, op, outputs, place)

    backward_op.run(scope, ctx)

    out = np.array(scope.find_var(grad_name).get_tensor())
    return out


class OpTest(unittest.TestCase):
201 202 203 204 205 206 207 208 209 210 211 212 213 214 215
    @classmethod
    def setUpClass(cls):
        '''Fix random seeds to remove randomness from tests'''
        cls._np_rand_state = np.random.get_state()
        cls._py_rand_state = random.getstate()

        np.random.seed(123)
        random.seed(124)

    @classmethod
    def tearDownClass(cls):
        '''Restore random seeds'''
        np.random.set_state(cls._np_rand_state)
        random.setstate(cls._py_rand_state)

216
    def check_output_with_place(self, place, atol):
217
        self.scope = core.Scope()
Q
qijun 已提交
218
        op_inputs = self.inputs if hasattr(self, "inputs") else dict()
219
        op_outputs = self.outputs if hasattr(self, "outputs") else dict()
Q
qijun 已提交
220
        op_attrs = self.attrs if hasattr(self, "attrs") else dict()
221
        self.op = create_op(self.scope, self.op_type, op_inputs, op_outputs,
Q
qijun 已提交
222
                            op_attrs)
223 224 225 226 227 228
        if isinstance(place, core.GPUPlace) and not self.op.support_gpu():
            return
        set_input(self.scope, self.op, self.inputs, place)
        ctx = core.DeviceContext.create(place)
        self.op.run(self.scope, ctx)

Q
qijun 已提交
229
        for out_name, out_dup in Operator.get_op_outputs(self.op.type()):
230 231 232
            if out_name not in self.outputs:
                continue

233 234
            if out_dup:
                sub_out = self.outputs[out_name]
Y
Yancey 已提交
235 236 237 238 239
                if not isinstance(sub_out, list):
                    raise AssertionError("sub_out type %s is not list",
                                         type(sub_out))

                for sub_out_name, expect in sub_out:
240 241 242 243
                    actual = np.array(
                        self.scope.find_var(sub_out_name).get_tensor())
                    self.assertTrue(
                        np.allclose(
244 245
                            actual, expect, atol=atol),
                        "output name: " + out_name + " has diff.")
246 247 248
            else:
                actual = np.array(self.scope.find_var(out_name).get_tensor())
                expect = self.outputs[out_name]
249

250 251
                self.assertTrue(
                    np.allclose(
252 253
                        actual, expect, atol=atol),
                    "output name: " + out_name + " has diff.")
254

255
    def check_output(self, atol=1e-5):
Q
qijun 已提交
256
        places = [core.CPUPlace()]
Q
qijun 已提交
257
        if core.is_compile_gpu():
Q
qijun 已提交
258 259
            places.append(core.GPUPlace(0))
        for place in places:
260
            self.check_output_with_place(place, atol)
Q
qijun 已提交
261

262 263 264 265 266 267 268 269 270 271 272 273
    def __assert_is_close(self, numeric_grads, analytic_grads, names,
                          max_relative_error, msg_prefix):

        for a, b, name in itertools.izip(numeric_grads, analytic_grads, names):
            abs_a = np.abs(a)
            abs_a[abs_a < 1e-3] = 1

            diff_mat = np.abs(a - b) / abs_a
            max_diff = np.max(diff_mat)

            def err_msg():
                offset = np.argmax(diff_mat > max_relative_error)
274 275 276 277
                return ("%s Variable %s max gradient diff %f over limit %f, "
                        "the first error element is %d") % (
                            msg_prefix, name, max_diff, max_relative_error,
                            offset)
278 279 280 281 282

            self.assertLessEqual(max_diff, max_relative_error, err_msg())

    def check_grad(self,
                   inputs_to_check,
Y
Yancey 已提交
283
                   output_names,
284 285 286 287
                   no_grad_set=None,
                   in_place=False,
                   max_relative_error=0.005):
        self.scope = core.Scope()
Q
qijun 已提交
288
        op_inputs = self.inputs if hasattr(self, "inputs") else dict()
289
        op_outputs = self.outputs if hasattr(self, "outputs") else dict()
Q
qijun 已提交
290
        op_attrs = self.attrs if hasattr(self, "attrs") else dict()
291
        self.op = create_op(self.scope, self.op_type, op_inputs, op_outputs,
Q
qijun 已提交
292
                            op_attrs)
293 294 295
        if no_grad_set is None:
            no_grad_set = set()

Y
Yancey 已提交
296 297 298
        if not type(output_names) is list:
            output_names = [output_names]

299 300 301 302 303 304
        numeric_grads = [
            get_numeric_gradient(
                self.scope,
                self.op,
                self.inputs,
                input_to_check,
Y
Yancey 已提交
305
                output_names,
306 307 308 309 310 311
                in_place=in_place) for input_to_check in inputs_to_check
        ]
        grad_names = [
            grad_var_name(input_to_check) for input_to_check in inputs_to_check
        ]

Q
qijun 已提交
312 313 314 315 316 317
        cpu_place = core.CPUPlace()
        cpu_analytic_grads = [
            get_gradient(self.scope, self.op, self.inputs, self.outputs,
                         grad_name, cpu_place, no_grad_set)
            for grad_name in grad_names
        ]
318

Q
qijun 已提交
319 320 321 322 323 324 325
        self.__assert_is_close(numeric_grads, cpu_analytic_grads, grad_names,
                               max_relative_error,
                               "Gradient Check On %s" % str(cpu_place))

        if core.is_compile_gpu() and self.op.support_gpu():
            gpu_place = core.GPUPlace(0)
            gpu_analytic_grads = [
326
                get_gradient(self.scope, self.op, self.inputs, self.outputs,
Q
qijun 已提交
327
                             grad_name, gpu_place, no_grad_set)
328 329 330
                for grad_name in grad_names
            ]

Q
qijun 已提交
331 332 333 334 335 336 337
            self.__assert_is_close(numeric_grads, gpu_analytic_grads,
                                   grad_names, max_relative_error,
                                   "Gradient Check On %s" % str(gpu_place))

            for c_grad, g_grad, name in itertools.izip(
                    cpu_analytic_grads, gpu_analytic_grads, grad_names):
                self.assertTrue(
Q
qijun 已提交
338
                    np.allclose(
Q
qijun 已提交
339 340
                        c_grad, g_grad, atol=1e-4),
                    "output name: " + name + " has diff")