diff --git a/paddle/fluid/framework/parallel_executor.cc b/paddle/fluid/framework/parallel_executor.cc
index c42101e21a58659b04ce2f171e89c5a82adbe8e8..1782430927b90f05cf2f33f3472f2f42f7e653c9 100644
--- a/paddle/fluid/framework/parallel_executor.cc
+++ b/paddle/fluid/framework/parallel_executor.cc
@@ -345,8 +345,9 @@ struct NCCLAllReduceOpHandle : public OpHandle {
   }
 
   void Wait(platform::DeviceContext *waited_dev) override {
-    VLOG(3) << "Wait NCCL AllReduce";
-    this->dev_ctx_.at(waited_dev->GetPlace())->Wait();
+    for (auto &pair : member_->communication_streams_) {
+      pair.second.ctx_->Wait();
+    }
   }
 };