From 9b1f4d5d621d2d0d24f884c4afde8e974fd9ed9c Mon Sep 17 00:00:00 2001
From: Yu Yang <yuyang18@baidu.com>
Date: Tue, 20 Mar 2018 15:31:57 +0800
Subject: [PATCH] After nccl add event

---
 paddle/fluid/framework/parallel_executor.cc | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/paddle/fluid/framework/parallel_executor.cc b/paddle/fluid/framework/parallel_executor.cc
index f9fc35d8c..21a19cb5b 100644
--- a/paddle/fluid/framework/parallel_executor.cc
+++ b/paddle/fluid/framework/parallel_executor.cc
@@ -402,10 +402,13 @@ struct NCCLAllReduceOpHandle : public OpHandle {
         platform::dynload::ncclAllReduce(
             buffer, buffer, numel, static_cast<ncclDataType_t>(dtype), ncclSum,
             nccl_ctx.comm, nccl_ctx.stream());
-        PADDLE_ENFORCE(cudaEventRecord(events_[dev_id], nccl_ctx.stream()));
       }
-
       platform::dynload::ncclGroupEnd();
+
+      for (auto &ev : events_) {
+        PADDLE_ENFORCE(cudaEventRecord(
+            ev.second, member_->communication_streams_.at(ev.first).stream()));
+      }
     }
   }
 
-- 
GitLab