diff --git a/cmake/external/dgc.cmake b/cmake/external/dgc.cmake
index edd46aca80a9f158d188877c067aa2be5f680fb4..8dd3fc94c18734fd1d825b201473e433c0ca3374 100644
--- a/cmake/external/dgc.cmake
+++ b/cmake/external/dgc.cmake
@@ -12,8 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+if(WIN32)
+  message(SEND_ERROR "The current dgc support linux only")
+  return()
+endif()
+
 include(ExternalProject)
 
+set(DGC_DOWNLOAD_DIR ${PADDLE_SOURCE_DIR}/third_party/dgc/${CMAKE_SYSTEM_NAME})
 set(DGC_PREFIX_DIR "${THIRD_PARTY_PATH}/dgc")
 set(DGC_SOURCES_DIR "${THIRD_PARTY_PATH}/dgc/src/extern_dgc")
 set(DGC_INSTALL_DIR "${THIRD_PARTY_PATH}/install/dgc")
@@ -25,15 +31,47 @@ set(DGC_LIBRARIES
     CACHE FILEPATH "dgc library." FORCE)
 set(DGC_URL "https://fleet.bj.bcebos.com/dgc/collective_7369ff.tgz")
 include_directories(${DGC_INCLUDE_DIR})
+set(DGC_CACHE_FILENAME "collective_7369ff.tgz")
+set(DGC_URL_MD5 ede459281a0f979da8d84f81287369ff)
+
+function(download_dgc)
+  message(
+    STATUS "Downloading ${DGC_URL} to ${DGC_DOWNLOAD_DIR}/${DGC_CACHE_FILENAME}"
+  )
+  # NOTE: If the version is updated, consider emptying the folder; maybe add timeout
+  file(
+    DOWNLOAD ${DGC_URL} ${DGC_DOWNLOAD_DIR}/${DGC_CACHE_FILENAME}
+    EXPECTED_MD5 ${DGC_URL_MD5}
+    STATUS ERR)
+  if(ERR EQUAL 0)
+    message(STATUS "Download ${DGC_CACHE_FILENAME} success")
+  else()
+    message(
+      FATAL_ERROR
+        "Download failed, error: ${ERR}\n You can try downloading ${DGC_CACHE_FILENAME} again"
+    )
+  endif()
+endfunction()
+
+if(EXISTS ${DGC_DOWNLOAD_DIR}/${DGC_CACHE_FILENAME})
+  file(MD5 ${DGC_DOWNLOAD_DIR}/${DGC_CACHE_FILENAME} DGC_MD5)
+  if(NOT DGC_MD5 STREQUAL DGC_URL_MD5)
+    download_dgc()
+  endif()
+else()
+  download_dgc()
+endif()
 
 ExternalProject_Add(
   extern_dgc
   ${EXTERNAL_PROJECT_LOG_ARGS}
-  URL ${DGC_URL}
-  URL_MD5 "ede459281a0f979da8d84f81287369ff"
+  URL ${DGC_DOWNLOAD_DIR}/${DGC_CACHE_FILENAME}
+  URL_MD5 ${DGC_URL_MD5}
   PREFIX "${DGC_PREFIX_DIR}"
   CONFIGURE_COMMAND ""
   BUILD_COMMAND make -j${NPROC}
+  DOWNLOAD_DIR ${DGC_DOWNLOAD_DIR}
+  SOURCE_DIR ${DGC_SOURCES_DIR}
   INSTALL_COMMAND
     mkdir -p ${DGC_INSTALL_DIR}/lib/ ${DGC_INCLUDE_DIR}/dgc && cp
     ${DGC_SOURCES_DIR}/build/lib/libdgc.a ${DGC_LIBRARIES} && cp
diff --git a/cmake/external/jemalloc.cmake b/cmake/external/jemalloc.cmake
index 8a2dc0e60a2ef78f131a3a4796c0ab110d85a114..1fc2d508fb7356d81dba0082366a61256fed69fb 100644
--- a/cmake/external/jemalloc.cmake
+++ b/cmake/external/jemalloc.cmake
@@ -1,31 +1,65 @@
 include(ExternalProject)
 
+set(JEMALLOC_DOWNLOAD_DIR
+    ${PADDLE_SOURCE_DIR}/third_party/jemalloc/${CMAKE_SYSTEM_NAME})
 set(JEMALLOC_PROJECT "extern_jemalloc")
+set(JEMALLOC_BUILD ${THIRD_PARTY_PATH}/jemalloc/src/extern_jemalloc)
+set(JEMALLOC_PREFIX_DIR ${THIRD_PARTY_PATH}/jemalloc)
 set(JEMALLOC_URL
     https://github.com/jemalloc/jemalloc/releases/download/5.1.0/jemalloc-5.1.0.tar.bz2
 )
-set(JEMALLOC_BUILD ${THIRD_PARTY_PATH}/jemalloc/src/extern_jemalloc)
-set(JEMALLOC_SOURCE_DIR "${THIRD_PARTY_PATH}/jemalloc")
 set(JEMALLOC_INSTALL ${THIRD_PARTY_PATH}/install/jemalloc)
 set(JEMALLOC_INCLUDE_DIR ${JEMALLOC_INSTALL}/include)
-set(JEMALLOC_DOWNLOAD_DIR "${JEMALLOC_SOURCE_DIR}/src/${JEMALLOC_PROJECT}")
 
 set(JEMALLOC_STATIC_LIBRARIES
     ${THIRD_PARTY_PATH}/install/jemalloc/lib/libjemalloc_pic.a)
 set(JEMALLOC_LIBRARIES
     ${THIRD_PARTY_PATH}/install/jemalloc/lib/libjemalloc_pic.a)
+set(JEMALLOC_CACHE_FILENAME "jemalloc-5.1.0.tar.bz2")
+set(JEMALLOC_URL_MD5 1f47a5aff2d323c317dfa4cf23be1ce4)
+
+function(download_jemalloc)
+  message(
+    STATUS
+      "Downloading ${JEMALLOC_URL} to ${JEMALLOC_DOWNLOAD_DIR}/${JEMALLOC_CACHE_FILENAME}"
+  )
+  # NOTE: If the version is updated, consider emptying the folder; maybe add timeout
+  file(
+    DOWNLOAD ${JEMALLOC_URL} ${JEMALLOC_DOWNLOAD_DIR}/${JEMALLOC_CACHE_FILENAME}
+    EXPECTED_MD5 ${JEMALLOC_URL_MD5}
+    STATUS ERR)
+  if(ERR EQUAL 0)
+    message(STATUS "Download ${JEMALLOC_CACHE_FILENAME} success")
+  else()
+    message(
+      FATAL_ERROR
+        "Download failed, error: ${ERR}\n You can try downloading ${JEMALLOC_CACHE_FILENAME} again"
+    )
+  endif()
+endfunction()
+
+if(EXISTS ${JEMALLOC_DOWNLOAD_DIR}/${JEMALLOC_CACHE_FILENAME})
+  file(MD5 ${JEMALLOC_DOWNLOAD_DIR}/${JEMALLOC_CACHE_FILENAME} JEMALLOC_MD5)
+  if(NOT JEMALLOC_MD5 STREQUAL JEMALLOC_URL_MD5)
+    download_jemalloc()
+  endif()
+else()
+  download_jemalloc()
+endif()
 
 ExternalProject_Add(
   extern_jemalloc
-  PREFIX ${JEMALLOC_SOURCE_DIR}
-  URL ${JEMALLOC_URL}
-  INSTALL_DIR ${JEMALLOC_INSTALL}
-  DOWNLOAD_DIR "${JEMALLOC_DOWNLOAD_DIR}"
-  BUILD_COMMAND make
+  PREFIX ${JEMALLOC_PREFIX_DIR}
+  URL ${JEMALLOC_DOWNLOAD_DIR}/${JEMALLOC_CACHE_FILENAME}
+  URL_MD5 ${JEMALLOC_URL_MD5}
+  #INSTALL_DIR ${JEMALLOC_INSTALL}
+  DOWNLOAD_DIR ${JEMALLOC_DOWNLOAD_DIR}
+  SOURCE_DIR ${JEMALLOC_BUILD}
   BUILD_IN_SOURCE 1
+  BUILD_COMMAND make
   INSTALL_COMMAND make install
-  CONFIGURE_COMMAND "${JEMALLOC_DOWNLOAD_DIR}/configure"
-                    --prefix=${JEMALLOC_INSTALL} --disable-initial-exec-tls)
+  CONFIGURE_COMMAND "${JEMALLOC_BUILD}/configure" --prefix=${JEMALLOC_INSTALL}
+                    --disable-initial-exec-tls)
 
 add_library(jemalloc STATIC IMPORTED GLOBAL)
 set_property(TARGET jemalloc PROPERTY IMPORTED_LOCATION