diff --git a/paddle/fluid/framework/parallel_executor.cc b/paddle/fluid/framework/parallel_executor.cc
index 121302880cbe4decb4bbbc09c26a4091eba50293..2a1652f749d87418f115172a0e5aee4671d790ee 100644
--- a/paddle/fluid/framework/parallel_executor.cc
+++ b/paddle/fluid/framework/parallel_executor.cc
@@ -108,14 +108,13 @@ struct OpHandle {
   }
 
   virtual void Wait(platform::DeviceContext *waited_dev) {
-    if (platform::is_cpu_place(waited_dev->GetPlace()) && events_.empty()) {
+    if (platform::is_cpu_place(waited_dev->GetPlace()) || events_.empty()) {
       for (auto &dev_ctx : dev_ctx_) {
         dev_ctx.second->Wait();
       }
     } else {
       auto stream =
           static_cast<platform::CUDADeviceContext *>(waited_dev)->stream();
-
       for (auto &ev : events_) {
         PADDLE_ENFORCE(cudaStreamWaitEvent(stream, ev.second, 0));
       }