From 267b218f5b41c66692579818204d2f278ddfd27c Mon Sep 17 00:00:00 2001 From: cyber-pioneer <116002591+cyber-pioneer@users.noreply.github.com> Date: Wed, 9 Nov 2022 12:00:55 +0800 Subject: [PATCH] add sin triple grad operator (#47753) --- .../generator/codegen_utils.py | 2 + paddle/phi/api/yaml/backward.yaml | 12 ++++ paddle/phi/api/yaml/op_compat.yaml | 2 +- paddle/phi/kernels/activation_grad_kernel.h | 11 ++++ .../phi/kernels/cpu/activation_grad_kernel.cc | 9 +++ paddle/phi/kernels/funcs/activation_functor.h | 58 +++++++++++++++++++ .../phi/kernels/gpu/activation_grad_kernel.cu | 10 ++++ .../phi/kernels/impl/activation_grad_impl.h | 34 +++++++++++ .../unittests/test_activation_nn_grad.py | 32 ++++++++++ 9 files changed, 169 insertions(+), 1 deletion(-) diff --git a/paddle/fluid/eager/auto_code_generator/generator/codegen_utils.py b/paddle/fluid/eager/auto_code_generator/generator/codegen_utils.py index 587dfb571b..b2f91c53a9 100644 --- a/paddle/fluid/eager/auto_code_generator/generator/codegen_utils.py +++ b/paddle/fluid/eager/auto_code_generator/generator/codegen_utils.py @@ -36,6 +36,8 @@ ops_to_fill_zero_for_empty_grads = set( "batch_norm_double_grad", "tanh_double_grad", "tanh_triple_grad", + "sin_double_grad", + "sin_triple_grad", "subtract_double_grad", "divide_double_grad", "log_double_grad", diff --git a/paddle/phi/api/yaml/backward.yaml b/paddle/phi/api/yaml/backward.yaml index 7aee6e9f89..110650c2cb 100644 --- a/paddle/phi/api/yaml/backward.yaml +++ b/paddle/phi/api/yaml/backward.yaml @@ -661,6 +661,7 @@ param : [x, x] kernel : func : sin_double_grad + backward : sin_triple_grad inplace : (grad_x_grad -> grad_out_grad) - backward_op : sin_grad @@ -675,6 +676,17 @@ backward : sin_double_grad inplace : (out_grad -> x_grad) +- backward_op : sin_triple_grad + forward : sin_double_grad (Tensor x, Tensor grad_out_forward, Tensor grad_x_grad_forward) -> Tensor(grad_x), Tensor(grad_out_grad) + args : (Tensor x, Tensor grad_out_forward, Tensor grad_x_grad_forward, Tensor grad_x_grad, Tensor grad_out_grad_grad) + output : Tensor(x_grad), Tensor(grad_out_forward_grad), Tensor(grad_x_grad_forward_grad) + infer_meta : + func : GeneralTernaryGradInferMeta + param : [x, x, grad_x_grad_forward] + kernel : + func : sin_triple_grad + inplace : (grad_x_grad_forward -> grad_out_forward_grad) + - backward_op : sinh_grad forward : sinh (Tensor x) -> Tensor(out) args : (Tensor x, Tensor out_grad) diff --git a/paddle/phi/api/yaml/op_compat.yaml b/paddle/phi/api/yaml/op_compat.yaml index a67d4721c1..31e506b461 100644 --- a/paddle/phi/api/yaml/op_compat.yaml +++ b/paddle/phi/api/yaml/op_compat.yaml @@ -831,7 +831,7 @@ attrs : [bool use_mkldnn = false, bool use_cudnn = false] - op : sin - backward : sin_grad, sin_double_grad + backward : sin_grad, sin_double_grad, sin_triple_grad inputs : x : X outputs : diff --git a/paddle/phi/kernels/activation_grad_kernel.h b/paddle/phi/kernels/activation_grad_kernel.h index fa220e1c51..edd6212446 100644 --- a/paddle/phi/kernels/activation_grad_kernel.h +++ b/paddle/phi/kernels/activation_grad_kernel.h @@ -107,6 +107,17 @@ void TanhTripleGradKernel(const Context& dev_ctx, DenseTensor* d_dout, DenseTensor* d_ddx); +template +void SinTripleGradKernel(const Context& dev_ctx, + const DenseTensor& x, + const DenseTensor& dout, + const DenseTensor& ddx, + const DenseTensor& d_dx_new, + const DenseTensor& d_ddout, + DenseTensor* d_x_new, + DenseTensor* d_dout, + DenseTensor* d_ddx); + template void LeakyReluDoubleGradKernel(const Context& dev_ctx, const DenseTensor& x, diff --git a/paddle/phi/kernels/cpu/activation_grad_kernel.cc b/paddle/phi/kernels/cpu/activation_grad_kernel.cc index 5db837726c..74dc1f3b5c 100644 --- a/paddle/phi/kernels/cpu/activation_grad_kernel.cc +++ b/paddle/phi/kernels/cpu/activation_grad_kernel.cc @@ -345,6 +345,15 @@ PD_REGISTER_KERNEL(sin_double_grad, phi::dtype::float16, int, int64_t) {} +PD_REGISTER_KERNEL(sin_triple_grad, + CPU, + ALL_LAYOUT, + phi::SinTripleGradKernel, + float, + double, + phi::dtype::float16, + int, + int64_t) {} PD_REGISTER_ACTIVATION_GRAD_KERNEL(softsign_grad, SoftsignGradKernel) PD_REGISTER_ACTIVATION_GRAD_KERNEL(sigmoid_grad, SigmoidGradKernel) PD_REGISTER_ACTIVATION_GRAD_KERNEL(sigmoid_double_grad, SigmoidDoubleGradKernel) diff --git a/paddle/phi/kernels/funcs/activation_functor.h b/paddle/phi/kernels/funcs/activation_functor.h index 3e5a1e3bf6..460e6300c4 100644 --- a/paddle/phi/kernels/funcs/activation_functor.h +++ b/paddle/phi/kernels/funcs/activation_functor.h @@ -138,6 +138,64 @@ struct SinDoubleGradFunctor : public BaseActivationFunctor { static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepX; } }; +// 1st reverse grad +// y = sin(x) +// x --> y +// d1x = d1y * cos(x) +// +// 2nd reverse grad +// x, d1y --> d1x +// d2x = -sin(x) * d1y * d2d1x +// d2d1y = cos(x) * d2d1x +// +// 3rd reverse grad +// x, d1y, d2d1x --> d2x, d2d1y +// d3x = -cos(x) * d1y * d2d1x * d3d2x - sin(x) * d2d1x * d3d2d1y +// d3d1y = -sin(x) * d2d1x * d3d2x +// d3d2d1x = -sin(x) * d1y * d3d2x + cos(x) * d3d2d1y +template +struct SinTripleGradFunctor : public BaseActivationFunctor { + template + void operator()(const Device& dev, + const DenseTensor* X, + const DenseTensor* ddX, + const DenseTensor* dOut, + const DenseTensor* d_DDOut, + const DenseTensor* d_dx_New, + DenseTensor* d_d_Out, + DenseTensor* d_x_New, + DenseTensor* d_DDx) const { + auto* d = dev.eigen_device(); + auto x = EigenVector::Flatten( + GET_DATA_SAFELY(X, "Input", "x", "SinTripleGrad")); + auto d2d1x = EigenVector::Flatten( + GET_DATA_SAFELY(ddX, "Input", "d2d1x", "SinTripleGrad")); + auto d1y = EigenVector::Flatten( + GET_DATA_SAFELY(dOut, "Input", "d1y", "SinTripleGrad")); + auto d3d2d1y = EigenVector::Flatten( + GET_DATA_SAFELY(d_DDOut, "Input", "d3d2d1y", "SinTripleGrad")); + auto d3d2x = EigenVector::Flatten( + GET_DATA_SAFELY(d_dx_New, "Input", "d3d2x", "SinTripleGrad")); + + auto d3x = EigenVector::Flatten( + GET_DATA_SAFELY(d_x_New, "Output", "d3x", "SinTripleGrad")); + d3x.device(*d) = -x.unaryExpr(Cosine()) * d1y * d2d1x * d3d2x - + x.unaryExpr(Sine()) * d2d1x * d3d2d1y; + + auto d3d1y = EigenVector::Flatten( + GET_DATA_SAFELY(d_d_Out, "Output", "d3d1y", "SinTripleGrad")); + d3d1y.device(*d) = -x.unaryExpr(Sine()) * d2d1x * d3d2x; + + auto d3d2d1x = EigenVector::Flatten( + GET_DATA_SAFELY(d_DDx, "Output", "d3d2d1x", "SinTripleGrad")); + d3d2d1x.device(*d) = -x.unaryExpr(Sine()) * d1y * d3d2x + + x.unaryExpr(Cosine()) * d3d2d1y; + } + static constexpr ActBwdOpFwdDeps FwdDeps() { + return ActBwdOpFwdDeps::kDepOut; + } +}; + // reciprocal(x) = 1 / x template struct ReciprocalFunctor : public BaseActivationFunctor { diff --git a/paddle/phi/kernels/gpu/activation_grad_kernel.cu b/paddle/phi/kernels/gpu/activation_grad_kernel.cu index 3f24910a0c..a21c6511b0 100644 --- a/paddle/phi/kernels/gpu/activation_grad_kernel.cu +++ b/paddle/phi/kernels/gpu/activation_grad_kernel.cu @@ -427,6 +427,16 @@ PD_REGISTER_KERNEL(sin_double_grad, int64_t, phi::dtype::float16) {} +PD_REGISTER_KERNEL(sin_triple_grad, + GPU, + ALL_LAYOUT, + phi::SinTripleGradKernel, + float, + double, + int, + int64_t, + phi::dtype::float16) {} + PD_REGISTER_ACTIVATION_GRAD_KERNEL(softsign_grad, SoftsignGradKernel) PD_REGISTER_ACTIVATION_GRAD_KERNEL(sigmoid_grad, SigmoidGradKernel) PD_REGISTER_ACTIVATION_GRAD_KERNEL(sigmoid_double_grad, SigmoidDoubleGradKernel) diff --git a/paddle/phi/kernels/impl/activation_grad_impl.h b/paddle/phi/kernels/impl/activation_grad_impl.h index efd6eba6ae..5c15564f52 100644 --- a/paddle/phi/kernels/impl/activation_grad_impl.h +++ b/paddle/phi/kernels/impl/activation_grad_impl.h @@ -467,4 +467,38 @@ void SinDoubleGradKernel(const Context& dev_ctx, functor(dev_ctx, &x, &dout, &ddx, dx, ddout); } +template +void SinTripleGradKernel(const Context& dev_ctx, + const DenseTensor& x, + const DenseTensor& dout, + const DenseTensor& ddx, + const DenseTensor& d_dx_new, + const DenseTensor& d_ddout, + DenseTensor* d_x_new, + DenseTensor* d_dout, + DenseTensor* d_ddx) { + if (d_dout) { + d_dout->Resize(x.dims()); + dev_ctx.template Alloc(d_dout); + } + if (d_x_new) { + d_dout->Resize(x.dims()); + dev_ctx.template Alloc(d_x_new); + } + if (d_ddx) { + d_dout->Resize(ddx.dims()); + dev_ctx.template Alloc(d_ddx); + } + funcs::SinTripleGradFunctor functor; + functor(dev_ctx, + &x, + &ddx, + &dout, + &d_ddout, + &d_dx_new, // input + d_dout, + d_x_new, + d_ddx); // output +} + } // namespace phi diff --git a/python/paddle/fluid/tests/unittests/test_activation_nn_grad.py b/python/paddle/fluid/tests/unittests/test_activation_nn_grad.py index 0106791076..ea9ac9fa9d 100644 --- a/python/paddle/fluid/tests/unittests/test_activation_nn_grad.py +++ b/python/paddle/fluid/tests/unittests/test_activation_nn_grad.py @@ -565,5 +565,37 @@ class TestPowDoubleGradCheck2(unittest.TestCase): self.func(p) +class TestSinTripleGradCheck(unittest.TestCase): + def sin_wrapper(self, x): + return paddle.sin(x[0]) + + @prog_scope() + def func(self, place): + shape = [2, 3, 7, 9] + eps = 0.0005 + dtype = np.float64 + x = layers.data('x', shape, False, dtype=dtype) + x.persistable = True + y = layers.sin(x) + x_arr = np.random.random(shape).astype(dtype) + x_arr[np.abs(x_arr) < 0.005] = 0.002 + gradient_checker.triple_grad_check( + [x], y, x_init=x_arr, place=place, eps=eps + ) + fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": True}) + gradient_checker.triple_grad_check_for_dygraph( + self.sin_wrapper, [x], y, x_init=x_arr, place=place + ) + fluid.set_flags({"FLAGS_retain_grad_for_all_tensor": False}) + + def test_grad(self): + paddle.enable_static() + places = [fluid.CPUPlace()] + if core.is_compiled_with_cuda(): + places.append(fluid.CUDAPlace(0)) + for p in places: + self.func(p) + + if __name__ == "__main__": unittest.main() -- GitLab