From 65bc7d17d52741cd124a00444bf063195e4f9c5d Mon Sep 17 00:00:00 2001
From: Yu Yang <yuyang18@baidu.com>
Date: Tue, 20 Mar 2018 18:46:20 +0800
Subject: [PATCH] Add mtx to ncclAllReduce

---
 paddle/fluid/framework/parallel_executor.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/paddle/fluid/framework/parallel_executor.cc b/paddle/fluid/framework/parallel_executor.cc
index bbfaac7339d..d61f1438a61 100644
--- a/paddle/fluid/framework/parallel_executor.cc
+++ b/paddle/fluid/framework/parallel_executor.cc
@@ -340,6 +340,8 @@ ncclDataType_t ToNCCLDataType(std::type_index type) {
   }
 }
 
+static std::mutex g_nccl_mtx_;
+
 struct NCCLAllReduceOpHandle : public OpHandle {
   ParallelExecutorPrivate *member_;
 
@@ -361,6 +363,8 @@ struct NCCLAllReduceOpHandle : public OpHandle {
       int dtype = -1;
       size_t numel = 0;
 
+      std::lock_guard<std::mutex> g(g_nccl_mtx_);
+
       platform::dynload::ncclGroupStart();
 
       for (size_t i = 0; i < member_->local_scopes_.size(); ++i) {
-- 
GitLab