diff --git a/paddle/fluid/framework/details/scope_buffered_ssa_graph_executor.cc b/paddle/fluid/framework/details/scope_buffered_ssa_graph_executor.cc
index ea783c609081cd4a3b2deb83737939e250492361..22bf0d308b2debe3b3c064b68c92f75cb8ab9c12 100644
--- a/paddle/fluid/framework/details/scope_buffered_ssa_graph_executor.cc
+++ b/paddle/fluid/framework/details/scope_buffered_ssa_graph_executor.cc
@@ -74,12 +74,18 @@ FeedFetchList ScopeBufferedSSAGraphExecutor::Run(
   }
 
   if (drop_scope_counter_ == strategy_.num_iteration_per_drop_scope_) {
-    drop_scope_counter_ = 0;
+    // Wait All computational streams
+    for (auto p : places_) {
+      platform::DeviceContextPool::Instance().Get(p)->Wait();
+    }
+
     for (auto &scope : local_scopes_) {
       auto &local_scope =
           *scope->Var(details::kLocalExecScopeName)->GetMutable<Scope *>();
       scope->DeleteScope(local_scope);
     }
+
+    drop_scope_counter_ = 0;
   }
 
   if (eptr) {
diff --git a/paddle/fluid/framework/rw_lock.h b/paddle/fluid/framework/rw_lock.h
index 75e6bef9bf33d438b7b0ee7f6df8d3fa6732fcfa..f8aa87519a2fc1a14765887e95c96883d7b4589f 100644
--- a/paddle/fluid/framework/rw_lock.h
+++ b/paddle/fluid/framework/rw_lock.h
@@ -69,12 +69,13 @@ class AutoWRLock {
  public:
   explicit AutoWRLock(RWLock* rw_lock) : lock_(rw_lock) { Lock(); }
 
+  ~AutoWRLock() { UnLock(); }
+
+ private:
   inline void Lock() { lock_->WRLock(); }
 
   inline void UnLock() { lock_->UNLock(); }
 
-  ~AutoWRLock() { UnLock(); }
-
  private:
   RWLock* lock_;
 };
@@ -83,12 +84,13 @@ class AutoRDLock {
  public:
   explicit AutoRDLock(RWLock* rw_lock) : lock_(rw_lock) { Lock(); }
 
+  ~AutoRDLock() { UnLock(); }
+
+ private:
   inline void Lock() { lock_->RDLock(); }
 
   inline void UnLock() { lock_->UNLock(); }
 
-  ~AutoRDLock() { UnLock(); }
-
  private:
   RWLock* lock_;
 };
diff --git a/paddle/fluid/framework/spin_lock.h b/paddle/fluid/framework/spin_lock.h
deleted file mode 100644
index 11a763d655abb0f422cb2647a820c80e79f33f1b..0000000000000000000000000000000000000000
--- a/paddle/fluid/framework/spin_lock.h
+++ /dev/null
@@ -1,71 +0,0 @@
-/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License. */
-
-#pragma once
-
-#if !defined(_WIN32)
-#include <pthread.h>
-#else
-#include <mutex>  // NOLINT
-#endif            // !_WIN32
-
-#include "paddle/fluid/platform/enforce.h"
-
-namespace paddle {
-namespace framework {
-
-#if !defined(_WIN32)
-struct SpinLock {
-  SpinLock() { pthread_spin_init(&lock_, PTHREAD_PROCESS_PRIVATE); }
-
-  ~SpinLock() { pthread_spin_destroy(&lock_); }
-
-  void Lock() {
-    PADDLE_ENFORCE_EQ(pthread_spin_lock(&lock_), 0, "acquire spin lock failed");
-  }
-
-  void Unlock() {
-    PADDLE_ENFORCE_EQ(pthread_spin_unlock(&lock_), 0,
-                      "release spin lock failed");
-  }
-
- private:
-  pthread_spinlock_t lock_;
-};
-#else
-// FIXME(minqiyang): use mutex here to do fake spin lock
-struct SpinLock {
-  void Lock() { mutex_.lock(); }
-
-  void Unlock() { mutex_.lock(); }
-
- private:
-  std::mutex mutex_;
-};
-#endif
-
-class AutoSpinLock {
- public:
-  explicit SpinLockGuard(SpinLock* spin_lock) : lock_(spin_lock) {
-    lock_->Lock();
-  }
-
-  ~SpinLockGuard() { lock_->Unlock(); }
-
- private:
-  SpinLock* lock_;
-};
-
-}  // namespace framework
-}  // namespace paddle