diff --git a/paddle/fluid/operators/temporal_shift_op.cu b/paddle/fluid/operators/temporal_shift_op.cu
index b62b4703e2cf4e8c825e2802387aba9a3ba3cd0a..b555c08c223b2852d66129780e67c9cdd3deccfd 100644
--- a/paddle/fluid/operators/temporal_shift_op.cu
+++ b/paddle/fluid/operators/temporal_shift_op.cu
@@ -129,6 +129,9 @@ class TemporalShiftGradOpCUDAKernel : public framework::OpKernel<T> {
 
     const T* output_grad_data = output_grad->data<T>();
     T* input_grad_data = input_grad->mutable_data<T>({nt, c, h, w}, ctx.GetPlace());
+    math::SetConstant<platform::CUDADeviceContext, T>()(
+        ctx.template device_context<platform::CUDADeviceContext>(), input_grad,
+        static_cast<T>(0));
 
     int pixelNum = nt * chw;
     int grid_dim = (pixelNum + 512 - 1) / 512;
diff --git a/paddle/fluid/operators/temporal_shift_op.h b/paddle/fluid/operators/temporal_shift_op.h
index 9b96def3c728ac7d14a43b15ed0ecebfd667d0de..3342a8b4a1bed3c818c42dd0f2d5dd6e7102ff15 100644
--- a/paddle/fluid/operators/temporal_shift_op.h
+++ b/paddle/fluid/operators/temporal_shift_op.h
@@ -88,6 +88,7 @@ class TemporalShiftGradKernel : public framework::OpKernel<T> {
 
     const T* output_grad_data = output_grad->data<T>();
     T* input_grad_data = input_grad->mutable_data<T>({nt, c, h, w}, ctx.GetPlace());
+    memset(input_grad_data, 0, input_grad->numel() * sizeof(T));
 
     int src_it = 0;
     for (int i = 0; i < output_grad->numel(); i++) {