Supports offline compilation of Paddle third-party libraries (#53744)

* optimize logsumexp in small data scale * fix * fix * add #pragma once * compile protobuf offline * add submodlu gflags * check_submodules * check_submodules * add_submodule protobuf * add_submodule_protobuf * add_submodule * add .gitmodules * add_submodules * fix_compiler error * support offline compile * support offline compile * support offline_compile * remove cub * remove brpc * support offline compile * support offline compile * canning patching on cryptopp * modify .gitigonre of cryptopp * test * offline compile * add_submodule zlib * modify .gitmodules * modify .gitmodules * fix setup.py bug * delete submodule cryptopp * fix windows compile bug * fix xxhash compile problem --------- Co-authored-by: Asthestarsfalll <1186454801@qq.com> Co-authored-by: N Asthestarsfalll <72954905+Asthestarsfalll@users.noreply.github.com>

Supports offline compilation of Paddle third-party libraries (#53744)
* optimize logsumexp in small data scale * fix * fix * add #pragma once * compile protobuf offline * add submodlu gflags * check_submodules * check_submodules * add_submodule protobuf * add_submodule_protobuf * add_submodule * add .gitmodules * add_submodules * fix_compiler error * support offline compile * support offline compile * support offline_compile * remove cub * remove brpc * support offline compile * support offline compile * canning patching on cryptopp * modify .gitigonre of cryptopp * test * offline compile * add_submodule zlib * modify .gitmodules * modify .gitmodules * fix setup.py bug * delete submodule cryptopp * fix windows compile bug * fix xxhash compile problem --------- Co-authored-by: Asthestarsfalll <1186454801@qq.com> Co-authored-by: N Asthestarsfalll <72954905+Asthestarsfalll@users.noreply.github.com>
734dc448 · risemeup1 · GitHub · 2cb28014 · 734dc448 · 734dc448
29 changed file
--- a/.gitignore
+++ b/.gitignore
@@ -52,11 +52,11 @@ CMakeSettings.json
 Makefile
 .test_env/
 .cache/
-third_party/
+build/third_party/

 *~
 bazel-*
-third_party/
+build/third_party/

 build_*
 # clion workspace.

--- a/.gitmodules
+++ b/.gitmodules
+[submodule "third_party/protobuf"]
+	path = third_party/protobuf
+	url = https://github.com/protocolbuffers/protobuf.git
+[submodule "third_party/gflags"]
+	path = third_party/gflags
+	url = https://github.com/gflags/gflags.git
+[submodule "third_party/gloo"]
+	path = third_party/gloo
+	url = https://github.com/ziyoujiyi/gloo.git
+[submodule "third_party/dlpack"]
+	path = third_party/dlpack
+	url = https://github.com/dmlc/dlpack.git
+[submodule "third_party/utf8proc"]
+	path = third_party/utf8proc
+	url = https://github.com/JuliaStrings/utf8proc.git
+[submodule "third_party/warpctc"]
+	path = third_party/warpctc
+	url = https://github.com/baidu-research/warp-ctc.git
+[submodule "third_party/warprnnt"]
+	path = third_party/warprnnt
+	url = https://github.com/PaddlePaddle/warp-transducer.git
+[submodule "third_party/xxhash"]
+	path = third_party/xxhash
+	url = https://github.com/Cyan4973/xxHash.git
+[submodule "third_party/eigen3"]
+	path = third_party/eigen3
+	url = https://gitlab.com/libeigen/eigen.git
+[submodule "third_party/leveldb"]
+	path = third_party/leveldb
+	url = https://github.com/google/leveldb
+[submodule "third_party/threadpool"]
+	path = third_party/threadpool
+	url = https://github.com/progschj/ThreadPool.git
+[submodule "third_party/zlib"]
+	path = third_party/zlib
+	url = https://github.com/madler/zlib.git
+[submodule "third_party/glog"]
+	path = third_party/glog
+	url = https://github.com/google/glog.git
--- a/cmake/external/dlpack.cmake
+++ b/cmake/external/dlpack.cmake
@@ -15,19 +15,15 @@
 include(ExternalProject)

 set(DLPACK_PREFIX_DIR ${THIRD_PARTY_PATH}/dlpack)
-
-set(DLPACK_REPOSITORY ${GIT_URL}/dmlc/dlpack.git)
-
 set(DLPACK_TAG v0.4)
-
 set(DLPACK_INCLUDE_DIR ${THIRD_PARTY_PATH}/dlpack/src/extern_dlpack/include)
-include_directories(${DLPACK_INCLUDE_DIR})
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/dlpack)
+include_directories(${SOURCE_DIR}/include)

 ExternalProject_Add(
  extern_dlpack
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${DLPACK_REPOSITORY}
-  GIT_TAG ${DLPACK_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
  PREFIX ${DLPACK_PREFIX_DIR}
  UPDATE_COMMAND ""
  CONFIGURE_COMMAND ""

--- a/cmake/external/eigen.cmake
+++ b/cmake/external/eigen.cmake
@@ -17,8 +17,8 @@ include(ExternalProject)
 # update eigen to the commit id f612df27 on 03/16/2021
 set(EIGEN_PREFIX_DIR ${THIRD_PARTY_PATH}/eigen3)
 set(EIGEN_SOURCE_DIR ${THIRD_PARTY_PATH}/eigen3/src/extern_eigen3)
-set(EIGEN_REPOSITORY https://gitlab.com/libeigen/eigen.git)
 set(EIGEN_TAG f612df273689a19d25b45ca4f8269463207c4fee)
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/eigen3)

 if(WIN32)
  add_definitions(-DEIGEN_STRONG_INLINE=inline)
@@ -28,14 +28,12 @@ elseif(LINUX)
    # which will cause compiler error of using __host__ funciont
    # in __host__ __device__
    file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/eigen/Meta.h native_src)
-    file(TO_NATIVE_PATH ${EIGEN_SOURCE_DIR}/Eigen/src/Core/util/Meta.h
-         native_dst)
+    file(TO_NATIVE_PATH ${SOURCE_DIR}/Eigen/src/Core/util/Meta.h native_dst)
    file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/eigen/TensorReductionGpu.h
         native_src1)
-    file(
-      TO_NATIVE_PATH
-      ${EIGEN_SOURCE_DIR}/unsupported/Eigen/CXX11/src/Tensor/TensorReductionGpu.h
-      native_dst1)
+    file(TO_NATIVE_PATH
+         ${SOURCE_DIR}/unsupported/Eigen/CXX11/src/Tensor/TensorReductionGpu.h
+         native_dst1)
    set(EIGEN_PATCH_COMMAND cp ${native_src} ${native_dst} && cp ${native_src1}
                            ${native_dst1})
  endif()
@@ -54,18 +52,17 @@ if(CMAKE_COMPILER_IS_GNUCC)
    # See: [Why calling some `git` commands before `patch`?]
    set(EIGEN_PATCH_COMMAND
        git checkout -- . && git checkout ${EIGEN_TAG} && patch -Nd
-        ${EIGEN_SOURCE_DIR}/Eigen/src/Core/arch/SSE/ < ${complex_header})
+        ${SOURCE_DIR}/Eigen/src/Core/arch/SSE/ < ${complex_header})
  endif()
 endif()

-set(EIGEN_INCLUDE_DIR ${EIGEN_SOURCE_DIR})
+set(EIGEN_INCLUDE_DIR ${SOURCE_DIR})
 include_directories(${EIGEN_INCLUDE_DIR})

 ExternalProject_Add(
  extern_eigen3
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${EIGEN_REPOSITORY}
-  GIT_TAG ${EIGEN_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
  PREFIX ${EIGEN_PREFIX_DIR}
  UPDATE_COMMAND ""
  PATCH_COMMAND ${EIGEN_PATCH_COMMAND}

--- a/cmake/external/gflags.cmake
+++ b/cmake/external/gflags.cmake
@@ -19,9 +19,8 @@ set(GFLAGS_PREFIX_DIR ${THIRD_PARTY_PATH}/gflags)
 set(GFLAGS_INCLUDE_DIR
    "${GFLAGS_INSTALL_DIR}/include"
    CACHE PATH "gflags include directory." FORCE)
-set(GFLAGS_REPOSITORY ${GIT_URL}/gflags/gflags.git)
 set(GFLAGS_TAG "v2.2.2")
-
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/gflags)
 if(WIN32)
  set(GFLAGS_LIBRARIES
      "${GFLAGS_INSTALL_DIR}/lib/gflags_static.lib"
@@ -64,10 +63,9 @@ if(WITH_ARM_BRPC)
 else()
  ExternalProject_Add(
    extern_gflags
-    ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-    GIT_REPOSITORY ${GFLAGS_REPOSITORY}
-    GIT_TAG ${GFLAGS_TAG}
+    ${EXTERNAL_PROJECT_LOG_ARGS}
    PREFIX ${GFLAGS_PREFIX_DIR}
+    SOURCE_DIR ${SOURCE_DIR}
    UPDATE_COMMAND ""
    BUILD_COMMAND ${BUILD_COMMAND}
    INSTALL_COMMAND ${INSTALL_COMMAND}

--- a/cmake/external/glog.cmake
+++ b/cmake/external/glog.cmake
@@ -19,9 +19,8 @@ set(GLOG_INSTALL_DIR ${THIRD_PARTY_PATH}/install/glog)
 set(GLOG_INCLUDE_DIR
    "${GLOG_INSTALL_DIR}/include"
    CACHE PATH "glog include directory." FORCE)
-set(GLOG_REPOSITORY ${GIT_URL}/google/glog.git)
 set(GLOG_TAG v0.4.0)
-
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/glog)
 if(WIN32)
  set(GLOG_LIBRARIES
      "${GLOG_INSTALL_DIR}/lib/glog.lib"
@@ -67,8 +66,7 @@ else()
  ExternalProject_Add(
    extern_glog
    ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-    GIT_REPOSITORY ${GLOG_REPOSITORY}
-    GIT_TAG ${GLOG_TAG}
+    SOURCE_DIR ${SOURCE_DIR}
    DEPENDS gflags
    PREFIX ${GLOG_PREFIX_DIR}
    UPDATE_COMMAND ""

--- a/cmake/external/gloo.cmake
+++ b/cmake/external/gloo.cmake
@@ -25,12 +25,11 @@ set(GLOO_LIBRARY_DIR
    "${GLOO_INSTALL_DIR}/lib"
    CACHE PATH "gloo library directory." FORCE)
 # As we add extra features for gloo, we use the non-official repo
-set(GLOO_REPOSITORY ${GIT_URL}/ziyoujiyi/gloo.git)
 set(GLOO_TAG v0.0.3)
 set(GLOO_LIBRARIES
    "${GLOO_INSTALL_DIR}/lib/libgloo.a"
    CACHE FILEPATH "gloo library." FORCE)
-
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/gloo)
 set(GLOO_PATCH_COMMAND "")
 if(WITH_GPU)
  if(${CMAKE_CUDA_COMPILER_VERSION} LESS 12.0 AND ${CMAKE_CXX_COMPILER_VERSION}
@@ -39,7 +38,7 @@ if(WITH_GPU)
         native_dst)
    set(GLOO_PATCH_COMMAND
        git checkout -- . && git checkout ${GLOO_TAG} &&patch -Nd
-        ${GLOO_SOURCE_DIR}/gloo/transport/tcp < ${native_dst})
+        ${SOURCE_DIR}/gloo/transport/tcp < ${native_dst})
  endif()
 endif()

@@ -58,28 +57,27 @@ if(CMAKE_COMPILER_IS_GNUCC)
    # See: [Why calling some `git` commands before `patch`?]
    set(GLOO_PATCH_COMMAND
        git checkout -- . && git checkout ${GLOO_TAG} && patch -Nd
-        ${GLOO_SOURCE_DIR}/gloo/transport/tcp < ${native_dst} && patch -Nd
-        ${GLOO_SOURCE_DIR}/gloo/ < ${types_header})
+        ${SOURCE_DIR}/gloo/transport/tcp < ${native_dst} && patch -Nd
+        ${SOURCE_DIR}/gloo/ < ${types_header})
  endif()
 endif()
 include_directories(${GLOO_INCLUDE_DIR})

 ExternalProject_Add(
  ${GLOO_PROJECT}
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${GLOO_REPOSITORY}
-  GIT_TAG ${GLOO_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
  PREFIX "${GLOO_PREFIX_DIR}"
  UPDATE_COMMAND ""
  PATCH_COMMAND ${GLOO_PATCH_COMMAND}
  CONFIGURE_COMMAND ""
  BUILD_COMMAND
-    mkdir -p ${GLOO_SOURCE_DIR}/build && cd ${GLOO_SOURCE_DIR}/build && cmake ..
-    -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} && ${CMAKE_COMMAND} --build . && mkdir
-    -p ${GLOO_LIBRARY_DIR} ${GLOO_INCLUDE_DIR}/glo
+    mkdir -p ${GLOO_SOURCE_DIR}/build && cd ${GLOO_SOURCE_DIR}/build && cmake
+    ${SOURCE_DIR} -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} && ${CMAKE_COMMAND}
+    --build . && mkdir -p ${GLOO_LIBRARY_DIR} ${GLOO_INCLUDE_DIR}/glo
  INSTALL_COMMAND ${CMAKE_COMMAND} -E copy
                  ${GLOO_SOURCE_DIR}/build/gloo/libgloo.a ${GLOO_LIBRARY_DIR}
-  COMMAND ${CMAKE_COMMAND} -E copy_directory "${GLOO_SOURCE_DIR}/gloo/"
+  COMMAND ${CMAKE_COMMAND} -E copy_directory "${SOURCE_DIR}/gloo/"
          "${GLOO_INCLUDE_DIR}/gloo"
  BUILD_BYPRODUCTS ${GLOO_LIBRARIES})


--- a/cmake/external/protobuf.cmake
+++ b/cmake/external/protobuf.cmake
@@ -188,6 +188,7 @@ function(build_protobuf TARGET_NAME BUILD_FOR_HOST)
  set(PROTOBUF_SOURCE_DIR
      ${THIRD_PARTY_PATH}/${TARGET_DIR_NAME}/src/${TARGET_NAME})
  set(PROTOBUF_INSTALL_DIR ${THIRD_PARTY_PATH}/install/${TARGET_DIR_NAME})
+  set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/${TARGET_DIR_NAME})

  set(${TARGET_NAME}_INCLUDE_DIR
      "${PROTOBUF_INSTALL_DIR}/include"
@@ -289,14 +290,13 @@ function(build_protobuf TARGET_NAME BUILD_FOR_HOST)
  else()
    ExternalProject_Add(
      ${TARGET_NAME}
-      ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-      GIT_REPOSITORY ${PROTOBUF_REPOSITORY}
-      GIT_TAG ${PROTOBUF_TAG}
+      ${EXTERNAL_PROJECT_LOG_ARGS}
      PREFIX ${PROTOBUF_PREFIX_DIR}
+      SOURCE_DIR ${SOURCE_DIR}
      UPDATE_COMMAND ""
      DEPENDS zlib
      CONFIGURE_COMMAND
-        ${CMAKE_COMMAND} ${PROTOBUF_SOURCE_DIR}/cmake ${OPTIONAL_ARGS}
+        ${CMAKE_COMMAND} ${SOURCE_DIR}/cmake ${OPTIONAL_ARGS}
        -G${CMAKE_GENERATOR} -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_SKIP_RPATH=ON
        -DCMAKE_POSITION_INDEPENDENT_CODE=ON
        -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}

--- a/cmake/external/threadpool.cmake
+++ b/cmake/external/threadpool.cmake
@@ -15,17 +15,15 @@
 include(ExternalProject)

 set(THREADPOOL_PREFIX_DIR ${THIRD_PARTY_PATH}/threadpool)
-set(THREADPOOL_REPOSITORY ${GIT_URL}/progschj/ThreadPool.git)
 set(THREADPOOL_TAG 9a42ec1329f259a5f4881a291db1dcb8f2ad9040)
-
-set(THREADPOOL_INCLUDE_DIR ${THIRD_PARTY_PATH}/threadpool/src/extern_threadpool)
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/threadpool)
+set(THREADPOOL_INCLUDE_DIR ${SOURCE_DIR})
 include_directories(${THREADPOOL_INCLUDE_DIR})

 ExternalProject_Add(
  extern_threadpool
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${THREADPOOL_REPOSITORY}
-  GIT_TAG ${THREADPOOL_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
  PREFIX ${THREADPOOL_PREFIX_DIR}
  UPDATE_COMMAND ""
  CONFIGURE_COMMAND ""

--- a/cmake/external/utf8proc.cmake
+++ b/cmake/external/utf8proc.cmake
@@ -17,9 +17,8 @@ include(ExternalProject)
 set(UTF8PROC_PREFIX_DIR ${THIRD_PARTY_PATH}/utf8proc)
 set(UTF8PROC_INSTALL_DIR ${THIRD_PARTY_PATH}/install/utf8proc)
 # As we add extra features for utf8proc, we use the non-official repo
-set(UTF8PROC_REPOSITORY ${GIT_URL}/JuliaStrings/utf8proc.git)
 set(UTF8PROC_TAG v2.6.1)
-
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/utf8proc)
 if(WIN32)
  set(UTF8PROC_LIBRARIES "${UTF8PROC_INSTALL_DIR}/lib/utf8proc_static.lib")
  add_definitions(-DUTF8PROC_STATIC)
@@ -31,9 +30,8 @@ include_directories(${UTF8PROC_INSTALL_DIR}/include)

 ExternalProject_Add(
  extern_utf8proc
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${UTF8PROC_REPOSITORY}
-  GIT_TAG ${UTF8PROC_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
  PREFIX ${UTF8PROC_PREFIX_DIR}
  UPDATE_COMMAND ""
  CMAKE_ARGS -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}

--- a/cmake/external/warpctc.cmake
+++ b/cmake/external/warpctc.cmake
@@ -22,9 +22,8 @@ set(WARPCTC_PREFIX_DIR ${THIRD_PARTY_PATH}/warpctc)
 set(WARPCTC_INSTALL_DIR ${THIRD_PARTY_PATH}/install/warpctc)
 # in case of low internet speed
 #set(WARPCTC_REPOSITORY  https://gitee.com/tianjianhe/warp-ctc.git)
-set(WARPCTC_REPOSITORY ${GIT_URL}/baidu-research/warp-ctc.git)
 set(WARPCTC_TAG bdc2b4550453e0ef2d3b5190f9c6103a84eff184)
-
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/warpctc)
 set(WARPCTC_SOURCE_DIR ${THIRD_PARTY_PATH}/warpctc/src/extern_warpctc)
 set(WARPCTC_PATCH_COMMAND "")
 set(WARPCTC_CCBIN_OPTION "")
@@ -85,9 +84,8 @@ endif()

 ExternalProject_Add(
  extern_warpctc
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${WARPCTC_REPOSITORY}
-  GIT_TAG ${WARPCTC_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
  PREFIX ${WARPCTC_PREFIX_DIR}
  UPDATE_COMMAND ""
  PATCH_COMMAND ${WARPCTC_PATCH_COMMAND}

--- a/cmake/external/warprnnt.cmake
+++ b/cmake/external/warprnnt.cmake
@@ -20,9 +20,8 @@ endif()

 set(WARPRNNT_PREFIX_DIR ${THIRD_PARTY_PATH}/warprnnt)
 set(WARPRNNT_INSTALL_DIR ${THIRD_PARTY_PATH}/install/warprnnt)
-set(WARPRNNT_REPOSITORY ${GIT_URL}/PaddlePaddle/warp-transducer.git)
 set(WARPRNNT_TAG 7ea6bfe748779c245a0fcaa5dd9383826273eff2)
-
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/warprnnt)
 set(WARPRNNT_INCLUDE_DIR
    "${WARPRNNT_INSTALL_DIR}/include"
    CACHE PATH "Warp-rnnt Directory" FORCE)
@@ -70,9 +69,8 @@ else()
 endif()
 ExternalProject_Add(
  extern_warprnnt
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${WARPRNNT_REPOSITORY}
-  GIT_TAG ${WARPRNNT_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
  PREFIX ${WARPRNNT_PREFIX_DIR}
  UPDATE_COMMAND ""
  PATCH_COMMAND ""

--- a/cmake/external/xxhash.cmake
+++ b/cmake/external/xxhash.cmake
@@ -18,8 +18,8 @@ set(XXHASH_PREFIX_DIR ${THIRD_PARTY_PATH}/xxhash)
 set(XXHASH_SOURCE_DIR ${THIRD_PARTY_PATH}/xxhash/src/extern_xxhash)
 set(XXHASH_INSTALL_DIR ${THIRD_PARTY_PATH}/install/xxhash)
 set(XXHASH_INCLUDE_DIR "${XXHASH_INSTALL_DIR}/include")
-set(XXHASH_REPOSITORY ${GIT_URL}/Cyan4973/xxHash.git)
 set(XXHASH_TAG v0.6.5)
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/xxhash)

 include_directories(${XXHASH_INCLUDE_DIR})

@@ -29,7 +29,7 @@ if(APPLE)
      -i
      \"\"
      "s/-Wstrict-prototypes -Wundef/-Wstrict-prototypes -Wundef -fPIC/g"
-      ${XXHASH_SOURCE_DIR}/Makefile
+      ${SOURCE_DIR}/Makefile
      &&
      make
      lib)
@@ -38,7 +38,7 @@ elseif(UNIX)
      sed
      -i
      "s/-Wstrict-prototypes -Wundef/-Wstrict-prototypes -Wundef -fPIC/g"
-      ${XXHASH_SOURCE_DIR}/Makefile
+      ${SOURCE_DIR}/Makefile
      &&
      make
      lib)
@@ -57,14 +57,13 @@ endif()
 if(WIN32)
  ExternalProject_Add(
    extern_xxhash
-    ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-    GIT_REPOSITORY ${XXHASH_REPOSITORY}
-    GIT_TAG ${XXHASH_TAG}
+    ${EXTERNAL_PROJECT_LOG_ARGS}
+    SOURCE_DIR ${SOURCE_DIR}
    PREFIX ${XXHASH_PREFIX_DIR}
    UPDATE_COMMAND ""
    PATCH_COMMAND ""
    CONFIGURE_COMMAND
-      ${CMAKE_COMMAND} ${XXHASH_SOURCE_DIR}/cmake_unofficial
+      ${CMAKE_COMMAND} ${SOURCE_DIR}/cmake_unofficial
      -DCMAKE_INSTALL_PREFIX:PATH=${XXHASH_INSTALL_DIR}
      -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
      -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF
@@ -83,8 +82,7 @@ else()
  ExternalProject_Add(
    extern_xxhash
    ${EXTERNAL_PROJECT_LOG_ARGS}
-    GIT_REPOSITORY ${XXHASH_REPOSITORY}
-    GIT_TAG ${XXHASH_TAG}
+    SOURCE_DIR ${SOURCE_DIR}
    PREFIX ${XXHASH_PREFIX_DIR}
    UPDATE_COMMAND ""
    CONFIGURE_COMMAND ""

--- a/cmake/external/zlib.cmake
+++ b/cmake/external/zlib.cmake
@@ -22,9 +22,8 @@ set(ZLIB_ROOT
 set(ZLIB_INCLUDE_DIR
    "${ZLIB_INSTALL_DIR}/include"
    CACHE PATH "zlib include directory." FORCE)
-set(ZLIB_REPOSITORY ${GIT_URL}/madler/zlib.git)
 set(ZLIB_TAG v1.2.8)
-
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/zlib)
 include_directories(${ZLIB_INCLUDE_DIR}
 )# For zlib code to include its own headers.
 include_directories(${THIRD_PARTY_PATH}/install
@@ -42,9 +41,8 @@ endif()

 ExternalProject_Add(
  extern_zlib
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${ZLIB_REPOSITORY}
-  GIT_TAG ${ZLIB_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
  PREFIX ${ZLIB_PREFIX_DIR}
  UPDATE_COMMAND ""
  CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}

--- a/cmake/third_party.cmake
+++ b/cmake/third_party.cmake
@@ -29,7 +29,9 @@ set(third_party_deps)

 include(ProcessorCount)
 ProcessorCount(NPROC)
-
+if(NOT WITH_SETUP_INSTALL)
+  execute_process(COMMAND git submodule update --init --recursive)
+endif()
 # cache funciton to avoid repeat download code of third_party.
 # This function has 4 parameters, URL / REPOSITOR / TAG / DIR:
 # 1. URL:           specify download url of 3rd party

--- a/setup.py
+++ b/setup.py
@@ -22,6 +22,7 @@ import re
 import shutil
 import subprocess
 import sys
+import time
 from contextlib import contextmanager
 from distutils.spawn import find_executable
 from subprocess import CalledProcessError
@@ -1597,13 +1598,52 @@ def install_cpp_dist_and_build_test(install_dir, lib_test_dir, headers, libs):
    subprocess.check_call([CMAKE, "--build", lib_test_dir])


+def check_submodules():
+    def get_submodule_folder():
+        git_submodules_path = os.path.join(TOP_DIR, ".gitmodules")
+        with open(git_submodules_path) as f:
+            return [
+                os.path.join(TOP_DIR, line.split("=", 1)[1].strip())
+                for line in f.readlines()
+                if line.strip().startswith("path")
+            ]
+
+    def submodules_not_exists_or_empty(folder):
+        return not os.path.exists(folder) or (
+            os.path.isdir(folder) and len(os.listdir(folder)) == 0
+        )
+
+    submodule_folders = get_submodule_folder()
+    # f none of the submodule folders exists, try to initialize them
+    if any(
+        submodules_not_exists_or_empty(folder) for folder in submodule_folders
+    ):
+        try:
+            print(' --- Trying to initialize submodules')
+            start = time.time()
+            subprocess.check_call(
+                ["git", "submodule", "update", "--init", "--recursive"],
+                cwd=TOP_DIR,
+            )
+            end = time.time()
+            print(
+                ' --- Submodule initialization took {:.2f} sec'.format(
+                    end - start
+                )
+            )
+        except Exception:
+            print(' --- Submodule initalization failed')
+            print('Please run:\n\tgit submodule update --init --recursive')
+            sys.exit(1)
+
+
 def main():
    # Parse the command line and check arguments before we proceed with building steps and setup
    parse_input_command(filter_args_list)

    # check build dependency
    check_build_dependency()
-
+    check_submodules()
    # Execute the build process,cmake and make
    if cmake_and_build:
        build_steps()

--- a/dlpack @ a07f962d
+++ b/dlpack @ a07f962d
+Subproject commit a07f962d446b577adf4baef2b347a0f3a2a20617
--- a/eigen3 @ f612df27
+++ b/eigen3 @ f612df27
+Subproject commit f612df273689a19d25b45ca4f8269463207c4fee
--- a/gflags @ e171aa2d
+++ b/gflags @ e171aa2d
+Subproject commit e171aa2d15ed9eb17054558e0b3a6a413bb01067
--- a/glog @ 96a2f23d
+++ b/glog @ 96a2f23d
+Subproject commit 96a2f23dca4cc7180821ca5f32e526314395d26a
--- a/gloo @ 8b6b61df
+++ b/gloo @ 8b6b61df
+Subproject commit 8b6b61dfa0dca02b226a01262bfcf0484382048f
--- a/leveldb @ 803d6920
+++ b/leveldb @ 803d6920
+Subproject commit 803d69203a62faf50f1b77897310a3a1fcae712b
--- a/protobuf @ f0dc78d7
+++ b/protobuf @ f0dc78d7
+Subproject commit f0dc78d7e6e331b8c6bb2d5283e06aa26883ca7c
--- a/threadpool @ 9a42ec13
+++ b/threadpool @ 9a42ec13
+Subproject commit 9a42ec1329f259a5f4881a291db1dcb8f2ad9040
--- a/utf8proc @ 3203baa7
+++ b/utf8proc @ 3203baa7
+Subproject commit 3203baa7374d67132384e2830b2183c92351bffc
--- a/warpctc @ bdc2b455
+++ b/warpctc @ bdc2b455
+Subproject commit bdc2b4550453e0ef2d3b5190f9c6103a84eff184
--- a/warprnnt @ 7ea6bfe7
+++ b/warprnnt @ 7ea6bfe7
+Subproject commit 7ea6bfe748779c245a0fcaa5dd9383826273eff2
--- a/xxhash @ 7cc96396
+++ b/xxhash @ 7cc96396
+Subproject commit 7cc9639699f64b750c0b82333dced9ea77e8436e
--- a/zlib @ 50893291
+++ b/zlib @ 50893291
+Subproject commit 50893291621658f355bc5b4d450a8d06a563053d