diff --git a/paddle/operators/gru_op.h b/paddle/operators/gru_op.h
index b2cf358994f1bdf3597eefe67bee4e77599a9b6b..9fb60e20d143b7e81bf9fa6af6a9286adefec360 100644
--- a/paddle/operators/gru_op.h
+++ b/paddle/operators/gru_op.h
@@ -154,6 +154,7 @@ class GRUGradKernel : public framework::OpKernel<T> {
     }
     if (h0_grad) {
       ordered_h0_grad.mutable_data<T>(h0_grad->dims(), context.GetPlace());
+      zero(context.device_context(), &ordered_h0_grad, static_cast<T>(0.0));
     }
 
     bool is_reverse = context.Attr<bool>("is_reverse");
diff --git a/python/paddle/v2/framework/tests/test_gru_op.py b/python/paddle/v2/framework/tests/test_gru_op.py
index 2bb78d10e0e08b9916e12f733b2fe4dfc8e4bae5..fa2c5a53ec4a01b6545e25f773c11277a4d24706 100644
--- a/python/paddle/v2/framework/tests/test_gru_op.py
+++ b/python/paddle/v2/framework/tests/test_gru_op.py
@@ -149,7 +149,7 @@ class TestGRUOpReverse(TestGRUOp):
         self.is_reverse = True
         self.attrs = {
             'activation': 'tanh',
-            'gate_activation': 'tanh',
+            'gate_activation': 'sigmoid',
             'is_reverse': self.is_reverse
         }