diff --git a/paddle/fluid/framework/details/op_handle_base.cc b/paddle/fluid/framework/details/op_handle_base.cc
index 63affb705424f847d531fc8c2f20d132c92784d8..07a4b8921753a0b3cb8e62397e0a0ab2385a094a 100644
--- a/paddle/fluid/framework/details/op_handle_base.cc
+++ b/paddle/fluid/framework/details/op_handle_base.cc
@@ -33,6 +33,9 @@ std::string OpHandleBase::DebugString() const {
 
 OpHandleBase::~OpHandleBase() {
 #ifdef PADDLE_WITH_CUDA
+  for (auto &ctx : dev_ctx_) {
+    ctx.second->Wait();
+  }
   for (auto &ev : events_) {
     PADDLE_ENFORCE(cudaEventDestroy(ev.second));
   }