diff --git a/CMakeLists.txt b/CMakeLists.txt
index 0cc4e4768280d9e0f368b15857b3952fbe312aaf..264420ad830ed39b38f1918951d8d66c84fd5ee9 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -127,9 +127,9 @@ include(external/warpctc)   # download, build, install warpctc
 include(external/any)       # download libn::any
 include(external/eigen)     # download eigen3
 include(external/pybind11)    # download pybind11
+include(external/nccl)
 
 include(cudnn)              # set cudnn libraries, must before configure
-include(nccl)               # set nccl libraries
 include(configure)          # add paddle env configuration
 include(generic)            # simplify cmake module
 include(package)            # set paddle packages
diff --git a/cmake/configure.cmake b/cmake/configure.cmake
index 00dc33514133c9f5a789a4db765b33fb525421ec..24ddb24399dabeec9b8e5faf36be3eb21f420111 100644
--- a/cmake/configure.cmake
+++ b/cmake/configure.cmake
@@ -68,13 +68,6 @@ else()
     if(NOT CUDNN_FOUND)
         message(FATAL_ERROR "Paddle needs cudnn to compile")
     endif()
-    if (NOT NCCL_INCLUDE_DIR)
-        message(FATAL_ERROR "Paddle needs nccl header to compile")
-    endif()
-    if (NOT WITH_DSO AND NOT NCCL_LIBRARY)
-        message(FATAL_ERROR "Paddle needs nccl libraries when WITH_DSO=OFF")
-    endif()
-
 
     set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} "-Xcompiler ${SIMD_FLAG}")
 
diff --git a/cmake/external/nccl.cmake b/cmake/external/nccl.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..10e8e838095d5f4ddd4161d26a188ab8166d10fa
--- /dev/null
+++ b/cmake/external/nccl.cmake
@@ -0,0 +1,50 @@
+INCLUDE(ExternalProject)
+
+SET(NCCL_SOURCE_DIR ${THIRD_PARTY_PATH}/nccl)
+
+INCLUDE_DIRECTORIES(${NCCL_SOURCE_DIR}/src/extern_nccl)
+
+
+if(WITH_DSO)
+  # If we use DSO, we do not build nccl, just download the dependencies
+  set(NCCL_BUILD_COMMAND "")
+  set(NCCL_INSTALL_COMMAND "")
+  set(NCCL_INSTALL_DIR "")
+else()
+  # otherwise, we build nccl and link it.
+  set(NCCL_BUILD_COMMAND "make -j 8")
+  set(NCCL_INSTALL_COMMAND  "make install")
+  SET(NCCL_INSTALL_DIR ${THIRD_PARTY_PATH}/install/nccl)
+endif()
+
+ExternalProject_Add(
+        extern_nccl
+        ${EXTERNAL_PROJECT_LOG_ARGS}
+        GIT_REPOSITORY  "https://github.com/NVIDIA/nccl.git"
+        GIT_TAG         "v1.3.4-1"
+        PREFIX          "${NCCL_SOURCE_DIR}"
+        UPDATE_COMMAND  ""
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND     "${NCCL_BUILD_COMMAND}"
+        INSTALL_COMMAND   "${NCCL_INSTALL_COMMAND}"
+        INSTALL_DIR       "${NCCL_INSTALL_DIR}"
+        TEST_COMMAND      ""
+)
+
+if (WITH_DSO)
+  if (${CMAKE_VERSION} VERSION_LESS "3.3.0")
+    set(dummyfile ${CMAKE_CURRENT_BINARY_DIR}/lib_any_dummy.c)
+    file(WRITE ${dummyfile} "const char * dummy_any = \"${dummyfile}\";")
+    add_library(nccl STATIC ${dummyfile})
+  else()
+    add_library(nccl INTERFACE)
+  endif()
+else()
+  ADD_LIBRARY(nccl STATIC IMPORTED GLOBAL)
+  SET_PROPERTY(TARGET nccl PROPERTY IMPORTED_LOCATION
+          ${NCCL_INSTALL_DIR}/lib/libnccl.a)
+endif()
+
+add_dependencies(nccl extern_nccl)
+
+LIST(APPEND external_project_dependencies nccl)
diff --git a/cmake/nccl.cmake b/cmake/nccl.cmake
deleted file mode 100644
index 872b4d56fb2a0a30809b937ecd7bae1656a518d6..0000000000000000000000000000000000000000
--- a/cmake/nccl.cmake
+++ /dev/null
@@ -1,30 +0,0 @@
-if (NOT WITH_GPU)
-  return ()
-endif()
-
-set(NCCL_ROOT "/usr" CACHE PATH "CUDNN ROOT")
-find_path(NCCL_INCLUDE_DIR nccl.h PATHS
-        ${NCCL_ROOT} ${NCCL_ROOT}/include
-        $ENV{NCCL_ROOT} $ENV{NCCL_ROOT}/include ${CUDA_TOOLKIT_INCLUDE}
-        NO_DEFAULT_PATH)
-
-get_filename_component(__libpath_hist ${CUDA_CUDART_LIBRARY} PATH)
-
-set(TARGET_ARCH "x86_64")
-if(NOT ${CMAKE_SYSTEM_PROCESSOR})
-  set(TARGET_ARCH ${CMAKE_SYSTEM_PROCESSOR})
-endif()
-
-list(APPEND NCCL_CHECK_LIBRARY_DIRS
-        ${NCCL_ROOT}
-        ${NCCL_ROOT}/lib64
-        ${NCCL_ROOT}/lib
-        ${NCCL_ROOT}/lib/${TARGET_ARCH}-linux-gnu
-        $ENV{NCCL_ROOT}
-        $ENV{NCCL_ROOT}/lib64
-        $ENV{NCCL_ROOT}/lib
-        /usr/lib)
-find_library(NCCL_LIBRARY NAMES libnccl.so libnccl.dylib # libcudnn_static.a
-        PATHS ${NCCL_CHECK_LIBRARY_DIRS} ${NCCL_INCLUDE_DIR} ${__libpath_hist}
-        NO_DEFAULT_PATH
-        DOC "Path to nccl library.")
diff --git a/paddle/platform/dynload/CMakeLists.txt b/paddle/platform/dynload/CMakeLists.txt
index 4c8be33480e80e586a4bbbdb2664eb4cb2db8f0b..bb3fec1be9e811c26cc6851314e960e96fc366b3 100644
--- a/paddle/platform/dynload/CMakeLists.txt
+++ b/paddle/platform/dynload/CMakeLists.txt
@@ -1,2 +1,3 @@
 cc_library(dynamic_loader SRCS dynamic_loader.cc DEPS glog gflags)
-nv_library(dynload_cuda SRCS cublas.cc cudnn.cc curand.cc nccl.cc DEPS dynamic_loader)
+nv_library(dynload_cuda SRCS cublas.cc cudnn.cc curand.cc nccl.cc
+        DEPS dynamic_loader nccl)