From 87f4311a88694284d95e81460a5adeda47e4367d Mon Sep 17 00:00:00 2001
From: "Yang Yang(Tony)" <yangyang62@baidu.com>
Date: Tue, 13 Feb 2018 14:05:33 -0800
Subject: [PATCH] compile with nccl2 (#8411)

* compile with nccl2

* add ncclGroup; it is necessary in nccl2

* add back libnccl-dev
---
 CMakeLists.txt                               | 1 -
 paddle/fluid/platform/CMakeLists.txt         | 2 +-
 paddle/fluid/platform/dynload/CMakeLists.txt | 2 +-
 paddle/fluid/platform/nccl_test.cu           | 2 ++
 paddle/scripts/docker/build.sh               | 5 ++++-
 5 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index fb91e3b369c..5db5c228be2 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -142,7 +142,6 @@ include(external/boost)     # download boost
 include(external/any)       # download libn::any
 include(external/eigen)     # download eigen3
 include(external/pybind11)  # download pybind11
-include(external/nccl)
 include(external/cares)
 include(external/grpc)
 
diff --git a/paddle/fluid/platform/CMakeLists.txt b/paddle/fluid/platform/CMakeLists.txt
index 32e768fdf44..0d0cee21d14 100644
--- a/paddle/fluid/platform/CMakeLists.txt
+++ b/paddle/fluid/platform/CMakeLists.txt
@@ -1,5 +1,5 @@
 if(WITH_GPU)
-  cc_library(enforce SRCS enforce.cc DEPS nccl)
+  cc_library(enforce SRCS enforce.cc DEPS)
 else()
   cc_library(enforce SRCS enforce.cc)
 endif()
diff --git a/paddle/fluid/platform/dynload/CMakeLists.txt b/paddle/fluid/platform/dynload/CMakeLists.txt
index cf2081b4349..264b4ebf2c0 100644
--- a/paddle/fluid/platform/dynload/CMakeLists.txt
+++ b/paddle/fluid/platform/dynload/CMakeLists.txt
@@ -1,4 +1,4 @@
 cc_library(dynamic_loader SRCS dynamic_loader.cc DEPS glog gflags enforce)
 nv_library(dynload_cuda SRCS cublas.cc cudnn.cc curand.cc nccl.cc
-        DEPS dynamic_loader nccl)
+        DEPS dynamic_loader)
 cc_library(dynload_warpctc SRCS warpctc.cc DEPS dynamic_loader warpctc)
diff --git a/paddle/fluid/platform/nccl_test.cu b/paddle/fluid/platform/nccl_test.cu
index 71230353633..212ea8517e8 100644
--- a/paddle/fluid/platform/nccl_test.cu
+++ b/paddle/fluid/platform/nccl_test.cu
@@ -89,6 +89,7 @@ TEST(NCCL, all_reduce) {
 
   VLOG(1) << "Invoking ncclAllReduce";
 
+  dynload::ncclGroupStart();
   for (int i = 0; i < dev_count; ++i) {
     VLOG(1) << "Invoking ncclAllReduce with device " << i;
     SetDeviceId(i);
@@ -97,6 +98,7 @@ TEST(NCCL, all_reduce) {
         ncclSum, comms[i], data[i]->dev_ctx.stream()));
     VLOG(1) << "Invoked ncclAllReduce for device " << i;
   }
+  dynload::ncclGroupEnd();
 
   VLOG(1) << "Invoked ncclAllReduce";
 
diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh
index 442a7ea8830..56fa1387861 100644
--- a/paddle/scripts/docker/build.sh
+++ b/paddle/scripts/docker/build.sh
@@ -34,6 +34,7 @@ function cmake_gen() {
     Configuring cmake in /paddle/build ...
         -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release}
         ${PYTHON_FLAGS}
+        -DWITH_DSO=ON
         -DWITH_DOC=OFF
         -DWITH_GPU=${WITH_GPU:-OFF}
         -DWITH_DISTRIBUTE=${WITH_DISTRIBUTE:-OFF}
@@ -57,6 +58,7 @@ EOF
     cmake .. \
         -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release} \
         ${PYTHON_FLAGS} \
+        -DWITH_DSO=ON \
         -DWITH_DOC=OFF \
         -DWITH_GPU=${WITH_GPU:-OFF} \
         -DWITH_DISTRIBUTE=${WITH_DISTRIBUTE:-OFF} \
@@ -171,7 +173,7 @@ EOF
     if [[ ${WITH_GPU} == "ON"  ]]; then
         NCCL_DEPS="apt-get install -y libnccl-dev &&"
     else
-        NCCL_DEPS="" 
+        NCCL_DEPS=""
     fi
 
     cat >> /paddle/build/Dockerfile <<EOF
@@ -187,6 +189,7 @@ EOF
         ldconfig
     ${DOCKERFILE_CUDNN_DSO}
     ${DOCKERFILE_GPU_ENV}
+    ENV NCCL_LAUNCH_MODE PARALLEL
     ADD go/cmd/pserver/pserver /usr/bin/
     ADD go/cmd/master/master /usr/bin/
     # default command shows the paddle version and exit
-- 
GitLab