diff --git a/paddle/fluid/imperative/dygraph_grad_maker.h b/paddle/fluid/imperative/dygraph_grad_maker.h
index f1eb8aa62c9271b194d5159883392372f4cbd4f3..f5a4b0fa315768128537333259169975dd73ac0e 100644
--- a/paddle/fluid/imperative/dygraph_grad_maker.h
+++ b/paddle/fluid/imperative/dygraph_grad_maker.h
@@ -347,15 +347,27 @@ class TracedGradOp {
     //  Use original var_wrapper if its inplace_version is not
     //  changed. Otherwise, it will affect the accuracy of the model
     //  results and affect double grad.
-    if (!var_wrapper->MutableVar()->IsInitialized() ||
-        var_wrapper->InplaceVersionSnapshot() ==
-            var_wrapper->MutableVar()->CurrentInplaceVersion()) {
+    if (!var_wrapper->MutableVar()->IsInitialized()) {
       return var_wrapper;
-    } else {
-      VariableWrapper new_var_wrapper = *var_wrapper.get();
-      new_var_wrapper.ResetInplaceVersion();
-      return std::make_shared<VariableWrapper>(new_var_wrapper);
+    } else if (var_wrapper->InplaceVersionSnapshot() ==
+               var_wrapper->MutableVar()->CurrentInplaceVersion()) {
+      return var_wrapper;
+    } else if (var_wrapper->MutableVar()->IsType<framework::LoDTensor>() ||
+               var_wrapper->MutableVar()->IsType<framework::SelectedRows>()) {
+      auto* tensor =
+          var_wrapper->MutableVar()->IsType<framework::LoDTensor>()
+              ? var_wrapper->MutableVar()->GetMutable<framework::LoDTensor>()
+              : var_wrapper->MutableVar()
+                    ->GetMutable<framework::SelectedRows>()
+                    ->mutable_value();
+      if (!tensor->IsInitialized()) {
+        return var_wrapper;
+      }
     }
+
+    VariableWrapper new_var_wrapper = *var_wrapper.get();
+    new_var_wrapper.ResetInplaceVersion();
+    return std::make_shared<VariableWrapper>(new_var_wrapper);
   }
 
  private:
diff --git a/python/paddle/fluid/tests/unittests/test_inplace_and_clear_gradient.py b/python/paddle/fluid/tests/unittests/test_inplace_and_clear_gradient.py
new file mode 100644
index 0000000000000000000000000000000000000000..b685900eadfa3e0bf216e6cf318703d76a11328d
--- /dev/null
+++ b/python/paddle/fluid/tests/unittests/test_inplace_and_clear_gradient.py
@@ -0,0 +1,50 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+import paddle
+import paddle.fluid as fluid
+from paddle import _C_ops
+import unittest
+
+paddle.disable_static()
+
+
+def clear_grad(w, a):
+    @paddle.no_grad()
+    def warp(*_):
+        assert w.grad is not None
+        _C_ops.scale_(w.grad, 'scale', 0.5)
+        w.clear_gradient(False)
+
+    return warp
+
+
+class TestInplaceAndClearGradient(unittest.TestCase):
+    def test(self):
+        paddle.set_device('cpu')
+
+        input_data = np.ones([2, 2]).astype('float32')
+        w = paddle.to_tensor(input_data, 'float32', stop_gradient=False)
+
+        _clear_grad = clear_grad(w, a="1")
+        w._register_backward_hook(_clear_grad)
+
+        for i in range(10):
+            out = _C_ops.scale(w, 'scale', 0.1)
+            out.backward()
+
+
+if __name__ == '__main__':
+    unittest.main()