From d3f52bccd9f419b28ac449bd20fe3476c09f8da8 Mon Sep 17 00:00:00 2001 From: Charles-hit <56987902+Charles-hit@users.noreply.github.com> Date: Thu, 8 Sep 2022 16:12:47 +0800 Subject: [PATCH] backward refuse foward part1 (#45868) * support more op for high level * add unit test for high level op * remove unnecessary comments --- .../fluid/tests/unittests/gradient_checker.py | 12 ++- .../fluid/tests/unittests/test_mean_op.py | 77 +++++++++++++++++ .../fluid/tests/unittests/test_scale_op.py | 77 +++++++++++++++++ .../fluid/tests/unittests/test_squeeze_op.py | 77 +++++++++++++++++ .../fluid/tests/unittests/test_sum_op.py | 83 +++++++++++++++++++ .../tests/unittests/test_transpose_op.py | 77 +++++++++++++++++ .../tests/unittests/test_unsqueeze_op.py | 77 +++++++++++++++++ 7 files changed, 477 insertions(+), 3 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/gradient_checker.py b/python/paddle/fluid/tests/unittests/gradient_checker.py index be1fa92f088..e261fd8165d 100644 --- a/python/paddle/fluid/tests/unittests/gradient_checker.py +++ b/python/paddle/fluid/tests/unittests/gradient_checker.py @@ -692,7 +692,8 @@ def get_eager_double_grad(func, allow_unused=True) if return_mid_result: - return dd_inputs, inputs + ddys + return [dd_input for dd_input in dd_inputs + if dd_input is not None], inputs + ddys else: return [ dd_input.numpy() for dd_input in dd_inputs if dd_input is not None @@ -857,8 +858,13 @@ def get_eager_triple_grad(func, dddy = paddle.ones(shape=dd_yi.shape, dtype=dd_yi.dtype) dddy.stop_gradient = False dddys.append(dddy) - ddd_inputs = paddle.grad(outputs=dd_y, inputs=dd_x, grad_outputs=dddys) - return [ddd_input.numpy() for ddd_input in ddd_inputs] + ddd_inputs = paddle.grad(outputs=dd_y, + inputs=dd_x, + grad_outputs=dddys, + allow_unused=True) + return [ + ddd_input.numpy() for ddd_input in ddd_inputs if ddd_input is not None + ] def triple_grad_check_for_dygraph(func, diff --git a/python/paddle/fluid/tests/unittests/test_mean_op.py b/python/paddle/fluid/tests/unittests/test_mean_op.py index a4ed6dc3b57..206e984fefb 100644 --- a/python/paddle/fluid/tests/unittests/test_mean_op.py +++ b/python/paddle/fluid/tests/unittests/test_mean_op.py @@ -23,6 +23,9 @@ import paddle.fluid as fluid from paddle.fluid import Program, program_guard from paddle.fluid.framework import _test_eager_guard from test_sum_op import TestReduceOPTensorAxisBase +import gradient_checker +from decorator_helper import prog_scope +import paddle.fluid.layers as layers np.random.seed(10) @@ -433,6 +436,80 @@ class TestMeanWithTensorAxis2(TestReduceOPTensorAxisBase): ] +class TestMeanDoubleGradCheck(unittest.TestCase): + + def mean_wrapper(self, x): + return paddle.mean(x[0]) + + @prog_scope() + def func(self, place): + # the shape of input variable should be clearly specified, not inlcude -1. + eps = 0.005 + dtype = np.float32 + + data = layers.data('data', [3, 4, 5], False, dtype) + data.persistable = True + out = paddle.mean(data) + data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) + + gradient_checker.double_grad_check([data], + out, + x_init=[data_arr], + place=place, + eps=eps) + fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) + gradient_checker.double_grad_check_for_dygraph(self.mean_wrapper, + [data], + out, + x_init=[data_arr], + place=place) + + def test_grad(self): + paddle.enable_static() + places = [fluid.CPUPlace()] + if core.is_compiled_with_cuda(): + places.append(fluid.CUDAPlace(0)) + for p in places: + self.func(p) + + +class TestMeanTripleGradCheck(unittest.TestCase): + + def mean_wrapper(self, x): + return paddle.mean(x[0]) + + @prog_scope() + def func(self, place): + # the shape of input variable should be clearly specified, not inlcude -1. + eps = 0.005 + dtype = np.float32 + + data = layers.data('data', [3, 4, 5], False, dtype) + data.persistable = True + out = paddle.mean(data) + data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) + + gradient_checker.triple_grad_check([data], + out, + x_init=[data_arr], + place=place, + eps=eps) + fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) + gradient_checker.triple_grad_check_for_dygraph(self.mean_wrapper, + [data], + out, + x_init=[data_arr], + place=place) + + def test_grad(self): + paddle.enable_static() + places = [fluid.CPUPlace()] + if core.is_compiled_with_cuda(): + places.append(fluid.CUDAPlace(0)) + for p in places: + self.func(p) + + if __name__ == "__main__": paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_scale_op.py b/python/paddle/fluid/tests/unittests/test_scale_op.py index 0beee7b0a1c..65876c7ceae 100644 --- a/python/paddle/fluid/tests/unittests/test_scale_op.py +++ b/python/paddle/fluid/tests/unittests/test_scale_op.py @@ -22,6 +22,9 @@ import paddle.fluid as fluid import paddle.fluid.core as core from paddle.fluid.op import Operator from paddle.static import Program, program_guard +import gradient_checker +from decorator_helper import prog_scope +import paddle.fluid.layers as layers class TestScaleOp(OpTest): @@ -244,5 +247,79 @@ class TestScaleInplaceApiDygraph(TestScaleApiDygraph): return x.scale_(scale, bias) +class TestScaleDoubleGradCheck(unittest.TestCase): + + def scale_wrapper(self, x): + return paddle.scale(x[0], scale=2.0) + + @prog_scope() + def func(self, place): + # the shape of input variable should be clearly specified, not inlcude -1. + eps = 0.005 + dtype = np.float32 + + data = layers.data('data', [2, 3], False, dtype) + data.persistable = True + out = paddle.scale(data, 2.0) + data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) + + gradient_checker.double_grad_check([data], + out, + x_init=[data_arr], + place=place, + eps=eps) + fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) + gradient_checker.double_grad_check_for_dygraph(self.scale_wrapper, + [data], + out, + x_init=[data_arr], + place=place) + + def test_grad(self): + paddle.enable_static() + places = [fluid.CPUPlace()] + if core.is_compiled_with_cuda(): + places.append(fluid.CUDAPlace(0)) + for p in places: + self.func(p) + + +class TestScaleTripleGradCheck(unittest.TestCase): + + def scale_wrapper(self, x): + return paddle.scale(x[0], scale=2.0) + + @prog_scope() + def func(self, place): + # the shape of input variable should be clearly specified, not inlcude -1. + eps = 0.005 + dtype = np.float32 + + data = layers.data('data', [2, 3], False, dtype) + data.persistable = True + out = paddle.scale(data, 2.0) + data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) + + gradient_checker.triple_grad_check([data], + out, + x_init=[data_arr], + place=place, + eps=eps) + fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) + gradient_checker.triple_grad_check_for_dygraph(self.scale_wrapper, + [data], + out, + x_init=[data_arr], + place=place) + + def test_grad(self): + paddle.enable_static() + places = [fluid.CPUPlace()] + if core.is_compiled_with_cuda(): + places.append(fluid.CUDAPlace(0)) + for p in places: + self.func(p) + + if __name__ == "__main__": unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_squeeze_op.py b/python/paddle/fluid/tests/unittests/test_squeeze_op.py index d64cfcaa8d2..9de95259328 100755 --- a/python/paddle/fluid/tests/unittests/test_squeeze_op.py +++ b/python/paddle/fluid/tests/unittests/test_squeeze_op.py @@ -22,6 +22,9 @@ import paddle.fluid as fluid from paddle.fluid import compiler, Program, program_guard from op_test import OpTest, convert_float_to_uint16 import paddle.fluid.core as core +import gradient_checker +from decorator_helper import prog_scope +import paddle.fluid.layers as layers paddle.enable_static() @@ -224,5 +227,79 @@ class API_TestDygraphSqueezeInplace(API_TestDygraphSqueeze): self.squeeze = paddle.squeeze_ +class TestSqueezeDoubleGradCheck(unittest.TestCase): + + def squeeze_wrapper(self, x): + return paddle.squeeze(x[0]) + + @prog_scope() + def func(self, place): + # the shape of input variable should be clearly specified, not inlcude -1. + eps = 0.005 + dtype = np.float32 + + data = layers.data('data', [2, 3], False, dtype) + data.persistable = True + out = paddle.squeeze(data) + data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) + + gradient_checker.double_grad_check([data], + out, + x_init=[data_arr], + place=place, + eps=eps) + fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) + gradient_checker.double_grad_check_for_dygraph(self.squeeze_wrapper, + [data], + out, + x_init=[data_arr], + place=place) + + def test_grad(self): + paddle.enable_static() + places = [fluid.CPUPlace()] + if core.is_compiled_with_cuda(): + places.append(fluid.CUDAPlace(0)) + for p in places: + self.func(p) + + +class TestSqueezeTripleGradCheck(unittest.TestCase): + + def squeeze_wrapper(self, x): + return paddle.squeeze(x[0]) + + @prog_scope() + def func(self, place): + # the shape of input variable should be clearly specified, not inlcude -1. + eps = 0.005 + dtype = np.float32 + + data = layers.data('data', [2, 3], False, dtype) + data.persistable = True + out = paddle.squeeze(data) + data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) + + gradient_checker.triple_grad_check([data], + out, + x_init=[data_arr], + place=place, + eps=eps) + fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) + gradient_checker.triple_grad_check_for_dygraph(self.squeeze_wrapper, + [data], + out, + x_init=[data_arr], + place=place) + + def test_grad(self): + paddle.enable_static() + places = [fluid.CPUPlace()] + if core.is_compiled_with_cuda(): + places.append(fluid.CUDAPlace(0)) + for p in places: + self.func(p) + + if __name__ == "__main__": unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_sum_op.py b/python/paddle/fluid/tests/unittests/test_sum_op.py index b143af2ac50..ebf5631fea9 100644 --- a/python/paddle/fluid/tests/unittests/test_sum_op.py +++ b/python/paddle/fluid/tests/unittests/test_sum_op.py @@ -30,6 +30,9 @@ from paddle.fluid.tests.unittests.op_test import (OpTest, from paddle import _C_ops, _legacy_C_ops from paddle.fluid.framework import _test_eager_guard import paddle.inference as paddle_infer +import gradient_checker +from decorator_helper import prog_scope +import paddle.fluid.layers as layers class TestSumOp(OpTest): @@ -580,6 +583,86 @@ class TestSumWithTensorAxis1(TestReduceOPTensorAxisBase): ] +class TestAddNDoubleGradCheck(unittest.TestCase): + + def add_n_wrapper(self, x): + return paddle.add_n(x) + + @prog_scope() + def func(self, place): + # the shape of input variable should be clearly specified, not inlcude -1. + eps = 0.005 + dtype = np.float32 + + data1 = layers.data('data1', [3, 4, 5], False, dtype) + data1.persistable = True + data2 = layers.data('data2', [3, 4, 5], False, dtype) + data2.persistable = True + out = paddle.add_n([data1, data2]) + data1_arr = np.random.uniform(-1, 1, data1.shape).astype(dtype) + data2_arr = np.random.uniform(-1, 1, data1.shape).astype(dtype) + + gradient_checker.double_grad_check([data1, data2], + out, + x_init=[data1_arr, data2_arr], + place=place, + eps=eps) + fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) + gradient_checker.double_grad_check_for_dygraph( + self.add_n_wrapper, [data1, data2], + out, + x_init=[data1_arr, data2_arr], + place=place) + + def test_grad(self): + paddle.enable_static() + places = [fluid.CPUPlace()] + if core.is_compiled_with_cuda(): + places.append(fluid.CUDAPlace(0)) + for p in places: + self.func(p) + + +class TestAddNTripleGradCheck(unittest.TestCase): + + def add_n_wrapper(self, x): + return paddle.add_n(x) + + @prog_scope() + def func(self, place): + # the shape of input variable should be clearly specified, not inlcude -1. + eps = 0.005 + dtype = np.float32 + + data1 = layers.data('data1', [3, 4, 5], False, dtype) + data1.persistable = True + data2 = layers.data('data2', [3, 4, 5], False, dtype) + data2.persistable = True + out = paddle.add_n([data1, data2]) + data1_arr = np.random.uniform(-1, 1, data1.shape).astype(dtype) + data2_arr = np.random.uniform(-1, 1, data1.shape).astype(dtype) + + gradient_checker.triple_grad_check([data1, data2], + out, + x_init=[data1_arr, data2_arr], + place=place, + eps=eps) + fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) + gradient_checker.triple_grad_check_for_dygraph( + self.add_n_wrapper, [data1, data2], + out, + x_init=[data1_arr, data2_arr], + place=place) + + def test_grad(self): + paddle.enable_static() + places = [fluid.CPUPlace()] + if core.is_compiled_with_cuda(): + places.append(fluid.CUDAPlace(0)) + for p in places: + self.func(p) + + if __name__ == "__main__": enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_transpose_op.py b/python/paddle/fluid/tests/unittests/test_transpose_op.py index 7f1794c39fc..adef1b25aaa 100644 --- a/python/paddle/fluid/tests/unittests/test_transpose_op.py +++ b/python/paddle/fluid/tests/unittests/test_transpose_op.py @@ -21,6 +21,9 @@ import paddle import paddle.fluid as fluid from paddle.fluid import Program, program_guard import paddle.fluid.core as core +import gradient_checker +from decorator_helper import prog_scope +import paddle.fluid.layers as layers paddle.enable_static() @@ -526,6 +529,80 @@ class TestMoveAxis(unittest.TestCase): paddle.moveaxis(x, [2, 1], [10, 3]) +class TestTransposeDoubleGradCheck(unittest.TestCase): + + def transpose_wrapper(self, x): + return paddle.transpose(x[0], [1, 0, 2]) + + @prog_scope() + def func(self, place): + # the shape of input variable should be clearly specified, not inlcude -1. + eps = 0.005 + dtype = np.float32 + + data = layers.data('data', [2, 3, 4], False, dtype) + data.persistable = True + out = paddle.transpose(data, [1, 0, 2]) + data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) + + gradient_checker.double_grad_check([data], + out, + x_init=[data_arr], + place=place, + eps=eps) + fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) + gradient_checker.double_grad_check_for_dygraph(self.transpose_wrapper, + [data], + out, + x_init=[data_arr], + place=place) + + def test_grad(self): + paddle.enable_static() + places = [fluid.CPUPlace()] + if core.is_compiled_with_cuda(): + places.append(fluid.CUDAPlace(0)) + for p in places: + self.func(p) + + +class TestTransposeTripleGradCheck(unittest.TestCase): + + def transpose_wrapper(self, x): + return paddle.transpose(x[0], [1, 0, 2]) + + @prog_scope() + def func(self, place): + # the shape of input variable should be clearly specified, not inlcude -1. + eps = 0.005 + dtype = np.float32 + + data = layers.data('data', [2, 3, 4], False, dtype) + data.persistable = True + out = paddle.transpose(data, [1, 0, 2]) + data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) + + gradient_checker.triple_grad_check([data], + out, + x_init=[data_arr], + place=place, + eps=eps) + fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) + gradient_checker.triple_grad_check_for_dygraph(self.transpose_wrapper, + [data], + out, + x_init=[data_arr], + place=place) + + def test_grad(self): + paddle.enable_static() + places = [fluid.CPUPlace()] + if core.is_compiled_with_cuda(): + places.append(fluid.CUDAPlace(0)) + for p in places: + self.func(p) + + if __name__ == '__main__': paddle.enable_static() unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_unsqueeze_op.py b/python/paddle/fluid/tests/unittests/test_unsqueeze_op.py index aeb0dac695c..29f5e37cd0c 100755 --- a/python/paddle/fluid/tests/unittests/test_unsqueeze_op.py +++ b/python/paddle/fluid/tests/unittests/test_unsqueeze_op.py @@ -21,6 +21,9 @@ import paddle import paddle.fluid as fluid from op_test import OpTest, convert_float_to_uint16 import paddle.fluid.core as core +import gradient_checker +from decorator_helper import prog_scope +import paddle.fluid.layers as layers paddle.enable_static() @@ -309,5 +312,79 @@ class API_TestDygraphUnSqueezeInplace(API_TestDygraphUnSqueeze): self.unsqueeze = paddle.unsqueeze_ +class TestUnsqueezeDoubleGradCheck(unittest.TestCase): + + def unsqueeze_wrapper(self, x): + return paddle.unsqueeze(x[0], [0, 2]) + + @prog_scope() + def func(self, place): + # the shape of input variable should be clearly specified, not inlcude -1. + eps = 0.005 + dtype = np.float32 + + data = layers.data('data', [2, 3, 4], False, dtype) + data.persistable = True + out = paddle.unsqueeze(data, [0, 2]) + data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) + + gradient_checker.double_grad_check([data], + out, + x_init=[data_arr], + place=place, + eps=eps) + fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) + gradient_checker.double_grad_check_for_dygraph(self.unsqueeze_wrapper, + [data], + out, + x_init=[data_arr], + place=place) + + def test_grad(self): + paddle.enable_static() + places = [fluid.CPUPlace()] + if core.is_compiled_with_cuda(): + places.append(fluid.CUDAPlace(0)) + for p in places: + self.func(p) + + +class TestUnsqueezeTripleGradCheck(unittest.TestCase): + + def unsqueeze_wrapper(self, x): + return paddle.unsqueeze(x[0], [0, 2]) + + @prog_scope() + def func(self, place): + # the shape of input variable should be clearly specified, not inlcude -1. + eps = 0.005 + dtype = np.float32 + + data = layers.data('data', [2, 3, 4], False, dtype) + data.persistable = True + out = paddle.unsqueeze(data, [0, 2]) + data_arr = np.random.uniform(-1, 1, data.shape).astype(dtype) + + gradient_checker.triple_grad_check([data], + out, + x_init=[data_arr], + place=place, + eps=eps) + fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) + gradient_checker.triple_grad_check_for_dygraph(self.unsqueeze_wrapper, + [data], + out, + x_init=[data_arr], + place=place) + + def test_grad(self): + paddle.enable_static() + places = [fluid.CPUPlace()] + if core.is_compiled_with_cuda(): + places.append(fluid.CUDAPlace(0)) + for p in places: + self.func(p) + + if __name__ == "__main__": unittest.main() -- GitLab