diff --git a/CMakeLists.txt b/CMakeLists.txt
index 49334279f6dc88c0d35fec43daf80e3cbe65760c..3b4c7e65c67e1015b78a7f7e92bb70f197be9592 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -141,7 +141,6 @@ include(external/boost)     # download boost
 include(external/any)       # download libn::any
 include(external/eigen)     # download eigen3
 include(external/pybind11)  # download pybind11
-include(external/nccl)
 include(external/cares)
 include(external/grpc)
 
diff --git a/Dockerfile b/Dockerfile
index 6ac9901ac6cea12e97047efdfb6272c957f166ae..ed559ca5c432d76c6d2e503e3c5cafd112d146a0 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -22,7 +22,7 @@ COPY ./paddle/scripts/docker/root/ /root/
 
 RUN apt-get update && \
     apt-get install -y \
-    git python-pip python-dev openssh-server bison libnccl-dev \
+    git python-pip python-dev openssh-server bison \
     wget unzip unrar tar xz-utils bzip2 gzip coreutils ntp \
     curl sed grep graphviz libjpeg-dev zlib1g-dev  \
     python-matplotlib gcc-4.8 g++-4.8 \
diff --git a/paddle/platform/CMakeLists.txt b/paddle/platform/CMakeLists.txt
index d68caea99719b37816391f9bddcc5cac051025b2..83164f07aaa1abd07320a47b9d617a7c00fb686b 100644
--- a/paddle/platform/CMakeLists.txt
+++ b/paddle/platform/CMakeLists.txt
@@ -1,5 +1,5 @@
 if(WITH_GPU)
-  cc_library(enforce SRCS enforce.cc DEPS nccl)
+  cc_library(enforce SRCS enforce.cc DEPS)
 else()
   cc_library(enforce SRCS enforce.cc)
 endif()
diff --git a/paddle/platform/dynload/CMakeLists.txt b/paddle/platform/dynload/CMakeLists.txt
index cf2081b434961c17c1b65509909699788d2b9ad9..264b4ebf2c06d9e688a32a223dff3ec079333fd9 100644
--- a/paddle/platform/dynload/CMakeLists.txt
+++ b/paddle/platform/dynload/CMakeLists.txt
@@ -1,4 +1,4 @@
 cc_library(dynamic_loader SRCS dynamic_loader.cc DEPS glog gflags enforce)
 nv_library(dynload_cuda SRCS cublas.cc cudnn.cc curand.cc nccl.cc
-        DEPS dynamic_loader nccl)
+        DEPS dynamic_loader)
 cc_library(dynload_warpctc SRCS warpctc.cc DEPS dynamic_loader warpctc)
diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh
index ba496db5f834efe767bfe446a46877932faa81a0..26ecb128eb82a69c03484846e1aab3fa5ef914de 100644
--- a/paddle/scripts/docker/build.sh
+++ b/paddle/scripts/docker/build.sh
@@ -34,6 +34,7 @@ function cmake_gen() {
     Configuring cmake in /paddle/build ...
         -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release}
         ${PYTHON_FLAGS}
+        -DWITH_DSO=ON
         -DWITH_DOC=OFF
         -DWITH_GPU=${WITH_GPU:-OFF}
         -DWITH_DISTRIBUTE=${WITH_DISTRIBUTE:-OFF}
@@ -57,6 +58,7 @@ EOF
     cmake .. \
         -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release} \
         ${PYTHON_FLAGS} \
+        -DWITH_DSO=ON \
         -DWITH_DOC=OFF \
         -DWITH_GPU=${WITH_GPU:-OFF} \
         -DWITH_DISTRIBUTE=${WITH_DISTRIBUTE:-OFF} \
@@ -173,7 +175,7 @@ EOF
     if [[ ${WITH_GPU} == "ON"  ]]; then
         NCCL_DEPS="apt-get install -y libnccl-dev &&"
     else
-        NCCL_DEPS="" 
+        NCCL_DEPS=""
     fi
 
     cat >> /paddle/build/Dockerfile <<EOF
@@ -189,6 +191,7 @@ EOF
         ldconfig
     ${DOCKERFILE_CUDNN_DSO}
     ${DOCKERFILE_GPU_ENV}
+    ENV NCCL_LAUNCH_MODE PARALLEL
     ADD go/cmd/pserver/pserver /usr/bin/
     ADD go/cmd/master/master /usr/bin/
     # default command shows the paddle version and exit