diff --git a/paddle/fluid/imperative/layer.cc b/paddle/fluid/imperative/layer.cc
index 2458b954488a9163f5acf680418b00c47bd235f7..9698d250c99d3294a2e868635a48806163aa2a26 100644
--- a/paddle/fluid/imperative/layer.cc
+++ b/paddle/fluid/imperative/layer.cc
@@ -118,7 +118,7 @@ void AddGradBySort(BackwardSumMap* bck_map, VarBase* target) {
     VLOG(2) << "added grad: " << var_pair.second->Name()
             << " trace id is: " << var_pair.first;
     AddTo(grad_to_add, origin_grad, current.first);
-    delete grad_to_add;
+    delete var_pair.second;
     var_pair.second = nullptr;
   }
 }
@@ -230,16 +230,14 @@ std::unique_ptr<VarBase> VarBase::NewVarBase(const platform::Place& dst_place,
       new_var->var_->GetMutable<framework::LoDTensor>();
   tensor->set_lod(var_->Get<framework::LoDTensor>().lod());
 
+  const auto& src_tensor = var_->Get<framework::LoDTensor>();
+  framework::TensorCopy(src_tensor, dst_place, tensor);
   if (blocking) {
-    platform::DeviceContext* dev_ctx =
-        platform::DeviceContextPool::Instance().Get(dst_place);
-
-    framework::TensorCopySync(var_->Get<framework::LoDTensor>(), dst_place,
-                              tensor);
-
-    dev_ctx->Wait();
-  } else {
-    framework::TensorCopy(var_->Get<framework::LoDTensor>(), dst_place, tensor);
+    platform::DeviceContextPool::Instance().Get(dst_place)->Wait();
+    auto src_place = src_tensor.place();
+    if (!(src_place == dst_place)) {
+      platform::DeviceContextPool::Instance().Get(src_place)->Wait();
+    }
   }
 
   if (platform::is_gpu_place(dst_place)) {
@@ -402,7 +400,7 @@ std::map<std::string, std::vector<VarBase*>> OpBase::ApplyGrad(
                   << origin_outputs[i]->name_ << " Grad to be added is "
                   << outputs[i]->name_;
           AddTo(grad, orig_grad, place_);
-          delete grad;
+          delete outputs[i];
         }
       }
     }
diff --git a/paddle/fluid/platform/temporary_allocator.cc b/paddle/fluid/platform/temporary_allocator.cc
index d489ed5368ed95a1a0a8b0d6759310501cd49fcd..03b5814840f4268f132ffe71441ebc33b67f251c 100644
--- a/paddle/fluid/platform/temporary_allocator.cc
+++ b/paddle/fluid/platform/temporary_allocator.cc
@@ -77,8 +77,8 @@ void TemporaryAllocator::Free(alloc::Allocation *allocation) {
                << "wait_delete_mem: " << wait_delete_mem;
     }
 
-    if (FLAGS_limit_of_tmp_allocation > 0 &&
-        wait_delete_mem > static_cast<size_t>(FLAGS_limit_of_tmp_allocation)) {
+    if (FLAGS_limit_of_tmp_allocation >= 0 &&
+        wait_delete_mem >= static_cast<size_t>(FLAGS_limit_of_tmp_allocation)) {
       PADDLE_ENFORCE(callback_ != nullptr, "The callback is non-initialized.");
       Release(callback_);
     }
diff --git a/python/paddle/fluid/dygraph/tracer.py b/python/paddle/fluid/dygraph/tracer.py
index 9d2cbb4f03fdc807e1609f46eac44a0bb92af785..4248e3c310f56325594b1f5a7bbfade11f34c56e 100644
--- a/python/paddle/fluid/dygraph/tracer.py
+++ b/python/paddle/fluid/dygraph/tracer.py
@@ -49,6 +49,10 @@ class Tracer(core.Tracer):
         return list((item for name, item in six.iteritems(self._vars)
                      if isinstance(item, framework.Parameter)))
 
+    def _clear_ops(self):
+        self._ops = defaultdict()
+        self._trace_id = 0
+
     def trace_op(self, op, inputs, outputs, stop_gradient=False):
         # TODO(minqiyang): remove this line after we take apart all
         # backward grads and forward variables
diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py
index 08b5789f0b30224869ae9dbe47d2b0a973a27017..58d5ebfd5ee7d37c7bf4d0fb83615ca61f4c4c08 100644
--- a/python/paddle/fluid/framework.py
+++ b/python/paddle/fluid/framework.py
@@ -531,15 +531,12 @@ class Variable(object):
 
     def backward(self, backward_strategy=None):
         from .dygraph import BackwardStrategy
-        if isinstance(backward_strategy, BackwardStrategy):
-            self._ivar._run_backward(backward_strategy)
-        elif backward_strategy is not None:
-            raise TypeError(
-                "only BackwardStrategy type should be passed in backward")
-        else:
+        if backward_strategy is None:
             backward_strategy = BackwardStrategy()
             backward_strategy.sort_sum_gradient = False
-            self._ivar._run_backward(backward_strategy)
+
+        self._ivar._run_backward(backward_strategy)
+        _dygraph_tracer()._clear_ops()
 
     def gradient(self):
         new_ivar = self._ivar._grad_ivar()._copy_to(core.CPUPlace(), True)