op_test.py 9.8 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14
import unittest
import numpy as np
import itertools
import paddle.v2.framework.core as core
from paddle.v2.framework.op import Operator


def grad_var_name(var_name):
    return var_name + "@GRAD"


def create_op(scope, op_type, inputs, outputs, attrs=None):
    kwargs = dict()

Q
qijun 已提交
15
    for in_name, in_dup in Operator.get_op_inputs(op_type):
16 17 18 19 20 21 22 23 24 25 26
        if in_name in inputs:
            kwargs[in_name] = []
            if in_dup:
                sub_in = inputs[in_name]
                for sub_in_name in sub_in:
                    var = scope.new_var(sub_in_name)
                    kwargs[in_name].append(sub_in_name)
            else:
                var = scope.new_var(in_name)
                kwargs[in_name].append(in_name)

Q
qijun 已提交
27
    for out_name, out_dup in Operator.get_op_outputs(op_type):
28 29 30 31 32 33 34 35 36 37 38
        if out_name in outputs:
            kwargs[out_name] = []
            if out_dup:
                sub_in = outputs[out_name]
                for sun_in_name in sub_in:
                    var = scope.new_var(sun_in_name)
                    kwargs[out_name].append(sun_in_name)
            else:
                var = scope.new_var(out_name)
                kwargs[out_name].append(out_name)

Q
qijun 已提交
39 40
    for attr_name in Operator.get_op_attr_names(op_type):
        kwargs[attr_name] = attrs[attr_name]
41 42 43 44
    return Operator(op_type, **kwargs)


def set_input(scope, op, inputs, place):
Q
qijun 已提交
45
    for in_name, in_dup in Operator.get_op_inputs(op.type()):
46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
        if in_name in inputs:
            if in_dup:
                sub_in = inputs[in_name]
                for sub_in_name in sub_in:
                    var = scope.find_var(sub_in_name)
                    tensor = var.get_tensor()
                    arr = sub_in[sub_in_name]
                    tensor.set_dims(arr.shape)
                    tensor.set(arr, place)
            else:
                var = scope.find_var(in_name)
                tensor = var.get_tensor()
                arr = inputs[in_name]
                tensor.set_dims(arr.shape)
                tensor.set(arr, place)


def set_output_grad(scope, op, outputs, place):
Q
qijun 已提交
64
    for out_name, out_dup in Operator.get_op_outputs(op.type()):
65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136
        if out_name in outputs:
            if out_dup:
                sub_out = outputs[out_name]
                for sub_out_name in sub_out:
                    out_tensor = scope.find_var(sub_out_name).get_tensor()
                    grad_tensor = scope.new_var(grad_var_name(
                        sub_out_name)).get_tensor()
                    grad_tensor.set_dims(out_tensor.shape())
                    data = np.ones(out_tensor.shape(), dtype=np.float32)
                    grad_tensor.set(data, place)
            else:
                out_tensor = scope.find_var(out_name).get_tensor()
                grad_tensor = scope.new_var(grad_var_name(out_name)).get_tensor(
                )
                grad_tensor.set_dims(out_tensor.shape())
                data = np.ones(out_tensor.shape(), dtype=np.float32)
                grad_tensor.set(data, place)


def get_numeric_gradient(scope,
                         op,
                         inputs,
                         input_to_check,
                         output_name,
                         delta=0.005,
                         in_place=False):

    set_input(scope, op, inputs, core.CPUPlace())
    op.infer_shape(scope)

    tensor_to_check = scope.find_var(input_to_check).get_tensor()

    def product(dim):
        return reduce(lambda a, b: a * b, dim, 1)

    ctx = core.DeviceContext.create(core.CPUPlace())

    def get_output():
        op.run(scope, ctx)
        return np.array(scope.find_var(output_name).get_tensor()).sum()

    tensor_to_check = scope.find_var(input_to_check).get_tensor()
    tensor_size = product(tensor_to_check.get_dims())
    gradient_flat = np.zeros(shape=(tensor_size, ), dtype='float32')
    # we only compute gradient of one element each time.
    # we use a for loop to compute the gradient of every element.
    for i in xrange(tensor_size):
        if in_place:
            set_input(op, inputs, core.CPUPlace())

        # get one input element throw it's index i.
        origin = tensor_to_check.get_float_element(i)
        # add delta to it, run op and then get the sum of the result tensor.
        x_pos = origin + delta
        tensor_to_check.set_float_element(i, x_pos)
        y_pos = get_output()

        if in_place:
            set_input(op, inputs, core.CPUPlace())

        x_neg = origin - delta
        tensor_to_check.set_float_element(i, x_neg)
        y_neg = get_output()

        tensor_to_check.set_float_element(i, origin)
        gradient_flat[i] = (y_pos - y_neg) / delta / 2

    return gradient_flat.reshape(tensor_to_check.get_dims())


def get_backward_op(scope, op, no_grad_set):
    backward_op = core.Operator.backward(op, no_grad_set)
Q
qijun 已提交
137
    for input in backward_op.input_vars():
138 139
        var = scope.new_var(input)
        var.get_tensor()
Q
qijun 已提交
140
    for output in backward_op.output_vars():
141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168
        var = scope.new_var(output)
        var.get_tensor()
    return backward_op


def get_gradient(scope, op, inputs, outputs, grad_name, place,
                 no_grad_set=None):
    ctx = core.DeviceContext.create(place)

    set_input(scope, op, inputs, place)

    op.infer_shape(scope)
    op.run(scope, ctx)

    if no_grad_set is None:
        no_grad_set = set()

    backward_op = get_backward_op(scope, op, no_grad_set)
    set_output_grad(scope, op, outputs, place)

    backward_op.infer_shape(scope)
    backward_op.run(scope, ctx)

    out = np.array(scope.find_var(grad_name).get_tensor())
    return out


class OpTest(unittest.TestCase):
Q
qijun 已提交
169
    def check_output_with_place(self, place):
170 171 172 173 174 175 176 177 178
        self.scope = core.Scope()
        self.op = create_op(self.scope, self.op_type, self.inputs, self.outputs)
        if isinstance(place, core.GPUPlace) and not self.op.support_gpu():
            return
        set_input(self.scope, self.op, self.inputs, place)
        self.op.infer_shape(self.scope)
        ctx = core.DeviceContext.create(place)
        self.op.run(self.scope, ctx)

Q
qijun 已提交
179
        for out_name, out_dup in Operator.get_op_outputs(self.op.type()):
180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197
            if out_dup:
                sub_out = self.outputs[out_name]
                for sub_out_name in sub_out:
                    actual = np.array(
                        self.scope.find_var(sub_out_name).get_tensor())
                    expect = sub_out[sub_out_name]
                    self.assertTrue(
                        np.allclose(
                            actual, expect, atol=1e-05),
                        "output name: " + out_name + "has diff")
            else:
                actual = np.array(self.scope.find_var(out_name).get_tensor())
                expect = self.outputs[out_name]
                self.assertTrue(
                    np.allclose(
                        actual, expect, atol=1e-05),
                    "output name: " + out_name + "has diff")

Q
qijun 已提交
198 199
    def check_output(self):
        places = [core.CPUPlace()]
Q
qijun 已提交
200
        if core.is_compile_gpu():
Q
qijun 已提交
201 202 203 204
            places.append(core.GPUPlace(0))
        for place in places:
            self.check_output_with_place(place)

205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246
    def __assert_is_close(self, numeric_grads, analytic_grads, names,
                          max_relative_error, msg_prefix):

        for a, b, name in itertools.izip(numeric_grads, analytic_grads, names):
            abs_a = np.abs(a)
            abs_a[abs_a < 1e-3] = 1

            diff_mat = np.abs(a - b) / abs_a
            max_diff = np.max(diff_mat)

            def err_msg():
                offset = np.argmax(diff_mat > max_relative_error)
                return "%s Variable %s max gradient diff %f over limit %f, the first " \
                  "error element is %d" % (
                   msg_prefix, name, max_diff, max_relative_error, offset)

            self.assertLessEqual(max_diff, max_relative_error, err_msg())

    def check_grad(self,
                   inputs_to_check,
                   output_name,
                   no_grad_set=None,
                   in_place=False,
                   max_relative_error=0.005):
        self.scope = core.Scope()
        self.op = create_op(self.scope, self.op_type, self.inputs, self.outputs)
        if no_grad_set is None:
            no_grad_set = set()

        numeric_grads = [
            get_numeric_gradient(
                self.scope,
                self.op,
                self.inputs,
                input_to_check,
                output_name,
                in_place=in_place) for input_to_check in inputs_to_check
        ]
        grad_names = [
            grad_var_name(input_to_check) for input_to_check in inputs_to_check
        ]

Q
qijun 已提交
247 248 249 250 251 252
        cpu_place = core.CPUPlace()
        cpu_analytic_grads = [
            get_gradient(self.scope, self.op, self.inputs, self.outputs,
                         grad_name, cpu_place, no_grad_set)
            for grad_name in grad_names
        ]
253

Q
qijun 已提交
254 255 256 257 258 259 260
        self.__assert_is_close(numeric_grads, cpu_analytic_grads, grad_names,
                               max_relative_error,
                               "Gradient Check On %s" % str(cpu_place))

        if core.is_compile_gpu() and self.op.support_gpu():
            gpu_place = core.GPUPlace(0)
            gpu_analytic_grads = [
261
                get_gradient(self.scope, self.op, self.inputs, self.outputs,
Q
qijun 已提交
262
                             grad_name, gpu_place, no_grad_set)
263 264 265
                for grad_name in grad_names
            ]

Q
qijun 已提交
266 267 268 269 270 271 272
            self.__assert_is_close(numeric_grads, gpu_analytic_grads,
                                   grad_names, max_relative_error,
                                   "Gradient Check On %s" % str(gpu_place))

            for c_grad, g_grad, name in itertools.izip(
                    cpu_analytic_grads, gpu_analytic_grads, grad_names):
                self.assertTrue(
Q
qijun 已提交
273
                    np.allclose(
Q
qijun 已提交
274 275
                        c_grad, g_grad, atol=1e-4),
                    "output name: " + name + " has diff")