diff --git a/paddle/fluid/framework/executor.cc b/paddle/fluid/framework/executor.cc
index c163f0edf16238ef7467c3563c417b26e2bf0923..81983746dbfa68643647f45b7d96ca4a9d77be59 100644
--- a/paddle/fluid/framework/executor.cc
+++ b/paddle/fluid/framework/executor.cc
@@ -479,21 +479,35 @@ void Executor::RunPartialPreparedContext(ExecutorPrepareContext* ctx,
     }
   }
 
-  platform::DeviceContextPool::Instance().Get(place_)->Wait();
+  auto callback = [scope, local_scope, keep_kids]() {
+    if (local_scope != scope) {
+      VLOG(4) << "Delete scope: " << local_scope;
+      scope->DeleteScope(local_scope);
+    } else {
+      if (!keep_kids) {
+        VLOG(4) << "Drop kids: " << scope;
+        // By default, we should delete all kid scopes after run executor
+        // because
+        // some operators may create local scope when running, such as while_op.
+        // But when while_op also create a local executor to run it's sub block,
+        // the sub scopes it created should not be dropped immediately, because
+        // while_grad_op will use some variables created during while_op run, so
+        // we need to keep the kids and wait for the outer executor to drop
+        // them.
+
+        scope->DropKids();
+      }
+      VLOG(4) << "Keep kids: " << scope;
+    }
+  };
 
-  if (local_scope != scope) {
-    scope->DeleteScope(local_scope);
+  if (gc) {
+    VLOG(4) << "Async deleting scope";
+    gc->DirectClearCallback(callback);
   } else {
-    if (!keep_kids) {
-      // By default, we should delete all kid scopes after run executor because
-      // some operators may create local scope when running, such as while_op.
-      // But when while_op also create a local executor to run it's sub block,
-      // the sub scopes it created should not be dropped immediately, because
-      // while_grad_op will use some variables created during while_op run, so
-      // we need to keep the kids and wait for the outer executor to drop them.
-
-      scope->DropKids();
-    }
+    VLOG(4) << "Sync deleting scope";
+    platform::DeviceContextPool::Instance().Get(place_)->Wait();
+    callback();
   }
 }
 
diff --git a/paddle/fluid/framework/garbage_collector.h b/paddle/fluid/framework/garbage_collector.h
index 884d230816b2ecae61c332f024fa8a4c9b0f2070..0b5fdc4745c24875a360e715ba30f23343d6f41b 100644
--- a/paddle/fluid/framework/garbage_collector.h
+++ b/paddle/fluid/framework/garbage_collector.h
@@ -48,6 +48,10 @@ class GarbageCollector {
   template <typename Container, typename Callback>
   void Add(Container &&objs, Callback &&callback);
 
+  void DirectClearCallback(const std::function<void()> &callback) {
+    ClearCallback(callback);
+  }
+
  protected:
   virtual void ClearCallback(const std::function<void()> &callback) = 0;