Supports offline compilation of Paddle third-party libraries (#53744)

* optimize logsumexp in small data scale

* fix

* fix

* add #pragma once

* compile protobuf offline

* add submodlu gflags

* check_submodules

* check_submodules

* add_submodule protobuf

* add_submodule_protobuf

* add_submodule

* add .gitmodules

* add_submodules

* fix_compiler error

* support offline compile

* support offline compile

* support offline_compile

* remove cub

* remove brpc

* support offline compile

* support offline compile

* canning patching on cryptopp

* modify .gitigonre of cryptopp

* test

* offline compile

* add_submodule zlib

* modify .gitmodules

* modify .gitmodules

* fix setup.py bug

* delete submodule cryptopp

* fix windows compile bug

* fix xxhash compile problem

---------
Co-authored-by: Asthestarsfalll <1186454801@qq.com>
Co-authored-by: NAsthestarsfalll <72954905+Asthestarsfalll@users.noreply.github.com>

.gitignore

@@ -52,11 +52,11 @@ CMakeSettings.json
 Makefile
 .test_env/
 .cache/
-third_party/
+build/third_party/
 
 *~
 bazel-*
-third_party/
+build/third_party/
 
 build_*
 # clion workspace.

.gitmodules

+[submodule "third_party/protobuf"]
+	path = third_party/protobuf
+	url = https://github.com/protocolbuffers/protobuf.git
+[submodule "third_party/gflags"]
+	path = third_party/gflags
+	url = https://github.com/gflags/gflags.git
+[submodule "third_party/gloo"]
+	path = third_party/gloo
+	url = https://github.com/ziyoujiyi/gloo.git
+[submodule "third_party/dlpack"]
+	path = third_party/dlpack
+	url = https://github.com/dmlc/dlpack.git
+[submodule "third_party/utf8proc"]
+	path = third_party/utf8proc
+	url = https://github.com/JuliaStrings/utf8proc.git
+[submodule "third_party/warpctc"]
+	path = third_party/warpctc
+	url = https://github.com/baidu-research/warp-ctc.git
+[submodule "third_party/warprnnt"]
+	path = third_party/warprnnt
+	url = https://github.com/PaddlePaddle/warp-transducer.git
+[submodule "third_party/xxhash"]
+	path = third_party/xxhash
+	url = https://github.com/Cyan4973/xxHash.git
+[submodule "third_party/eigen3"]
+	path = third_party/eigen3
+	url = https://gitlab.com/libeigen/eigen.git
+[submodule "third_party/leveldb"]
+	path = third_party/leveldb
+	url = https://github.com/google/leveldb
+[submodule "third_party/threadpool"]
+	path = third_party/threadpool
+	url = https://github.com/progschj/ThreadPool.git
+[submodule "third_party/zlib"]
+	path = third_party/zlib
+	url = https://github.com/madler/zlib.git
+[submodule "third_party/glog"]
+	path = third_party/glog
+	url = https://github.com/google/glog.git

cmake/external/dlpack.cmake

@@ -15,19 +15,15 @@
 include(ExternalProject)
 
 set(DLPACK_PREFIX_DIR ${THIRD_PARTY_PATH}/dlpack)
-
-set(DLPACK_REPOSITORY ${GIT_URL}/dmlc/dlpack.git)
-
 set(DLPACK_TAG v0.4)
-
 set(DLPACK_INCLUDE_DIR ${THIRD_PARTY_PATH}/dlpack/src/extern_dlpack/include)
-include_directories(${DLPACK_INCLUDE_DIR})
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/dlpack)
+include_directories(${SOURCE_DIR}/include)
 
 ExternalProject_Add(
   extern_dlpack
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${DLPACK_REPOSITORY}
-  GIT_TAG ${DLPACK_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
   PREFIX ${DLPACK_PREFIX_DIR}
   UPDATE_COMMAND ""
   CONFIGURE_COMMAND ""

cmake/external/eigen.cmake

@@ -17,8 +17,8 @@ include(ExternalProject)
 # update eigen to the commit id f612df27 on 03/16/2021
 set(EIGEN_PREFIX_DIR ${THIRD_PARTY_PATH}/eigen3)
 set(EIGEN_SOURCE_DIR ${THIRD_PARTY_PATH}/eigen3/src/extern_eigen3)
-set(EIGEN_REPOSITORY https://gitlab.com/libeigen/eigen.git)
 set(EIGEN_TAG f612df273689a19d25b45ca4f8269463207c4fee)
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/eigen3)
 
 if(WIN32)
   add_definitions(-DEIGEN_STRONG_INLINE=inline)
@@ -28,13 +28,11 @@ elseif(LINUX)
     # which will cause compiler error of using __host__ funciont
     # in __host__ __device__
     file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/eigen/Meta.h native_src)
-    file(TO_NATIVE_PATH ${EIGEN_SOURCE_DIR}/Eigen/src/Core/util/Meta.h
-         native_dst)
+    file(TO_NATIVE_PATH ${SOURCE_DIR}/Eigen/src/Core/util/Meta.h native_dst)
     file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/eigen/TensorReductionGpu.h
          native_src1)
-    file(
-      TO_NATIVE_PATH
-      ${EIGEN_SOURCE_DIR}/unsupported/Eigen/CXX11/src/Tensor/TensorReductionGpu.h
+    file(TO_NATIVE_PATH
+         ${SOURCE_DIR}/unsupported/Eigen/CXX11/src/Tensor/TensorReductionGpu.h
          native_dst1)
     set(EIGEN_PATCH_COMMAND cp ${native_src} ${native_dst} && cp ${native_src1}
                             ${native_dst1})
@@ -54,18 +52,17 @@ if(CMAKE_COMPILER_IS_GNUCC)
     # See: [Why calling some `git` commands before `patch`?]
     set(EIGEN_PATCH_COMMAND
         git checkout -- . && git checkout ${EIGEN_TAG} && patch -Nd
-        ${EIGEN_SOURCE_DIR}/Eigen/src/Core/arch/SSE/ < ${complex_header})
+        ${SOURCE_DIR}/Eigen/src/Core/arch/SSE/ < ${complex_header})
   endif()
 endif()
 
-set(EIGEN_INCLUDE_DIR ${EIGEN_SOURCE_DIR})
+set(EIGEN_INCLUDE_DIR ${SOURCE_DIR})
 include_directories(${EIGEN_INCLUDE_DIR})
 
 ExternalProject_Add(
   extern_eigen3
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${EIGEN_REPOSITORY}
-  GIT_TAG ${EIGEN_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
   PREFIX ${EIGEN_PREFIX_DIR}
   UPDATE_COMMAND ""
   PATCH_COMMAND ${EIGEN_PATCH_COMMAND}

cmake/external/gflags.cmake

@@ -19,9 +19,8 @@ set(GFLAGS_PREFIX_DIR ${THIRD_PARTY_PATH}/gflags)
 set(GFLAGS_INCLUDE_DIR
     "${GFLAGS_INSTALL_DIR}/include"
     CACHE PATH "gflags include directory." FORCE)
-set(GFLAGS_REPOSITORY ${GIT_URL}/gflags/gflags.git)
 set(GFLAGS_TAG "v2.2.2")
-
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/gflags)
 if(WIN32)
   set(GFLAGS_LIBRARIES
       "${GFLAGS_INSTALL_DIR}/lib/gflags_static.lib"
@@ -64,10 +63,9 @@ if(WITH_ARM_BRPC)
 else()
   ExternalProject_Add(
     extern_gflags
-    ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-    GIT_REPOSITORY ${GFLAGS_REPOSITORY}
-    GIT_TAG ${GFLAGS_TAG}
+    ${EXTERNAL_PROJECT_LOG_ARGS}
     PREFIX ${GFLAGS_PREFIX_DIR}
+    SOURCE_DIR ${SOURCE_DIR}
     UPDATE_COMMAND ""
     BUILD_COMMAND ${BUILD_COMMAND}
     INSTALL_COMMAND ${INSTALL_COMMAND}

cmake/external/glog.cmake

@@ -19,9 +19,8 @@ set(GLOG_INSTALL_DIR ${THIRD_PARTY_PATH}/install/glog)
 set(GLOG_INCLUDE_DIR
     "${GLOG_INSTALL_DIR}/include"
     CACHE PATH "glog include directory." FORCE)
-set(GLOG_REPOSITORY ${GIT_URL}/google/glog.git)
 set(GLOG_TAG v0.4.0)
-
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/glog)
 if(WIN32)
   set(GLOG_LIBRARIES
       "${GLOG_INSTALL_DIR}/lib/glog.lib"
@@ -67,8 +66,7 @@ else()
   ExternalProject_Add(
     extern_glog
     ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-    GIT_REPOSITORY ${GLOG_REPOSITORY}
-    GIT_TAG ${GLOG_TAG}
+    SOURCE_DIR ${SOURCE_DIR}
     DEPENDS gflags
     PREFIX ${GLOG_PREFIX_DIR}
     UPDATE_COMMAND ""

cmake/external/gloo.cmake

@@ -25,12 +25,11 @@ set(GLOO_LIBRARY_DIR
     "${GLOO_INSTALL_DIR}/lib"
     CACHE PATH "gloo library directory." FORCE)
 # As we add extra features for gloo, we use the non-official repo
-set(GLOO_REPOSITORY ${GIT_URL}/ziyoujiyi/gloo.git)
 set(GLOO_TAG v0.0.3)
 set(GLOO_LIBRARIES
     "${GLOO_INSTALL_DIR}/lib/libgloo.a"
     CACHE FILEPATH "gloo library." FORCE)
-
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/gloo)
 set(GLOO_PATCH_COMMAND "")
 if(WITH_GPU)
   if(${CMAKE_CUDA_COMPILER_VERSION} LESS 12.0 AND ${CMAKE_CXX_COMPILER_VERSION}
@@ -39,7 +38,7 @@ if(WITH_GPU)
          native_dst)
     set(GLOO_PATCH_COMMAND
         git checkout -- . && git checkout ${GLOO_TAG} &&patch -Nd
-        ${GLOO_SOURCE_DIR}/gloo/transport/tcp < ${native_dst})
+        ${SOURCE_DIR}/gloo/transport/tcp < ${native_dst})
   endif()
 endif()
 
@@ -58,28 +57,27 @@ if(CMAKE_COMPILER_IS_GNUCC)
     # See: [Why calling some `git` commands before `patch`?]
     set(GLOO_PATCH_COMMAND
         git checkout -- . && git checkout ${GLOO_TAG} && patch -Nd
-        ${GLOO_SOURCE_DIR}/gloo/transport/tcp < ${native_dst} && patch -Nd
-        ${GLOO_SOURCE_DIR}/gloo/ < ${types_header})
+        ${SOURCE_DIR}/gloo/transport/tcp < ${native_dst} && patch -Nd
+        ${SOURCE_DIR}/gloo/ < ${types_header})
   endif()
 endif()
 include_directories(${GLOO_INCLUDE_DIR})
 
 ExternalProject_Add(
   ${GLOO_PROJECT}
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${GLOO_REPOSITORY}
-  GIT_TAG ${GLOO_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
   PREFIX "${GLOO_PREFIX_DIR}"
   UPDATE_COMMAND ""
   PATCH_COMMAND ${GLOO_PATCH_COMMAND}
   CONFIGURE_COMMAND ""
   BUILD_COMMAND
-    mkdir -p ${GLOO_SOURCE_DIR}/build && cd ${GLOO_SOURCE_DIR}/build && cmake ..
-    -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} && ${CMAKE_COMMAND} --build . && mkdir
-    -p ${GLOO_LIBRARY_DIR} ${GLOO_INCLUDE_DIR}/glo
+    mkdir -p ${GLOO_SOURCE_DIR}/build && cd ${GLOO_SOURCE_DIR}/build && cmake
+    ${SOURCE_DIR} -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} && ${CMAKE_COMMAND}
+    --build . && mkdir -p ${GLOO_LIBRARY_DIR} ${GLOO_INCLUDE_DIR}/glo
   INSTALL_COMMAND ${CMAKE_COMMAND} -E copy
                   ${GLOO_SOURCE_DIR}/build/gloo/libgloo.a ${GLOO_LIBRARY_DIR}
-  COMMAND ${CMAKE_COMMAND} -E copy_directory "${GLOO_SOURCE_DIR}/gloo/"
+  COMMAND ${CMAKE_COMMAND} -E copy_directory "${SOURCE_DIR}/gloo/"
           "${GLOO_INCLUDE_DIR}/gloo"
   BUILD_BYPRODUCTS ${GLOO_LIBRARIES})
 

cmake/external/protobuf.cmake

@@ -188,6 +188,7 @@ function(build_protobuf TARGET_NAME BUILD_FOR_HOST)
   set(PROTOBUF_SOURCE_DIR
       ${THIRD_PARTY_PATH}/${TARGET_DIR_NAME}/src/${TARGET_NAME})
   set(PROTOBUF_INSTALL_DIR ${THIRD_PARTY_PATH}/install/${TARGET_DIR_NAME})
+  set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/${TARGET_DIR_NAME})
 
   set(${TARGET_NAME}_INCLUDE_DIR
       "${PROTOBUF_INSTALL_DIR}/include"
@@ -289,14 +290,13 @@ function(build_protobuf TARGET_NAME BUILD_FOR_HOST)
   else()
     ExternalProject_Add(
       ${TARGET_NAME}
-      ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-      GIT_REPOSITORY ${PROTOBUF_REPOSITORY}
-      GIT_TAG ${PROTOBUF_TAG}
+      ${EXTERNAL_PROJECT_LOG_ARGS}
       PREFIX ${PROTOBUF_PREFIX_DIR}
+      SOURCE_DIR ${SOURCE_DIR}
       UPDATE_COMMAND ""
       DEPENDS zlib
       CONFIGURE_COMMAND
-        ${CMAKE_COMMAND} ${PROTOBUF_SOURCE_DIR}/cmake ${OPTIONAL_ARGS}
+        ${CMAKE_COMMAND} ${SOURCE_DIR}/cmake ${OPTIONAL_ARGS}
         -G${CMAKE_GENERATOR} -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_SKIP_RPATH=ON
         -DCMAKE_POSITION_INDEPENDENT_CODE=ON
         -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}

cmake/external/threadpool.cmake

@@ -15,17 +15,15 @@
 include(ExternalProject)
 
 set(THREADPOOL_PREFIX_DIR ${THIRD_PARTY_PATH}/threadpool)
-set(THREADPOOL_REPOSITORY ${GIT_URL}/progschj/ThreadPool.git)
 set(THREADPOOL_TAG 9a42ec1329f259a5f4881a291db1dcb8f2ad9040)
-
-set(THREADPOOL_INCLUDE_DIR ${THIRD_PARTY_PATH}/threadpool/src/extern_threadpool)
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/threadpool)
+set(THREADPOOL_INCLUDE_DIR ${SOURCE_DIR})
 include_directories(${THREADPOOL_INCLUDE_DIR})
 
 ExternalProject_Add(
   extern_threadpool
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${THREADPOOL_REPOSITORY}
-  GIT_TAG ${THREADPOOL_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
   PREFIX ${THREADPOOL_PREFIX_DIR}
   UPDATE_COMMAND ""
   CONFIGURE_COMMAND ""

cmake/external/utf8proc.cmake

@@ -17,9 +17,8 @@ include(ExternalProject)
 set(UTF8PROC_PREFIX_DIR ${THIRD_PARTY_PATH}/utf8proc)
 set(UTF8PROC_INSTALL_DIR ${THIRD_PARTY_PATH}/install/utf8proc)
 # As we add extra features for utf8proc, we use the non-official repo
-set(UTF8PROC_REPOSITORY ${GIT_URL}/JuliaStrings/utf8proc.git)
 set(UTF8PROC_TAG v2.6.1)
-
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/utf8proc)
 if(WIN32)
   set(UTF8PROC_LIBRARIES "${UTF8PROC_INSTALL_DIR}/lib/utf8proc_static.lib")
   add_definitions(-DUTF8PROC_STATIC)
@@ -31,9 +30,8 @@ include_directories(${UTF8PROC_INSTALL_DIR}/include)
 
 ExternalProject_Add(
   extern_utf8proc
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${UTF8PROC_REPOSITORY}
-  GIT_TAG ${UTF8PROC_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
   PREFIX ${UTF8PROC_PREFIX_DIR}
   UPDATE_COMMAND ""
   CMAKE_ARGS -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}

cmake/external/warpctc.cmake

@@ -22,9 +22,8 @@ set(WARPCTC_PREFIX_DIR ${THIRD_PARTY_PATH}/warpctc)
 set(WARPCTC_INSTALL_DIR ${THIRD_PARTY_PATH}/install/warpctc)
 # in case of low internet speed
 #set(WARPCTC_REPOSITORY  https://gitee.com/tianjianhe/warp-ctc.git)
-set(WARPCTC_REPOSITORY ${GIT_URL}/baidu-research/warp-ctc.git)
 set(WARPCTC_TAG bdc2b4550453e0ef2d3b5190f9c6103a84eff184)
-
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/warpctc)
 set(WARPCTC_SOURCE_DIR ${THIRD_PARTY_PATH}/warpctc/src/extern_warpctc)
 set(WARPCTC_PATCH_COMMAND "")
 set(WARPCTC_CCBIN_OPTION "")
@@ -85,9 +84,8 @@ endif()
 
 ExternalProject_Add(
   extern_warpctc
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${WARPCTC_REPOSITORY}
-  GIT_TAG ${WARPCTC_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
   PREFIX ${WARPCTC_PREFIX_DIR}
   UPDATE_COMMAND ""
   PATCH_COMMAND ${WARPCTC_PATCH_COMMAND}

cmake/external/warprnnt.cmake

@@ -20,9 +20,8 @@ endif()
 
 set(WARPRNNT_PREFIX_DIR ${THIRD_PARTY_PATH}/warprnnt)
 set(WARPRNNT_INSTALL_DIR ${THIRD_PARTY_PATH}/install/warprnnt)
-set(WARPRNNT_REPOSITORY ${GIT_URL}/PaddlePaddle/warp-transducer.git)
 set(WARPRNNT_TAG 7ea6bfe748779c245a0fcaa5dd9383826273eff2)
-
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/warprnnt)
 set(WARPRNNT_INCLUDE_DIR
     "${WARPRNNT_INSTALL_DIR}/include"
     CACHE PATH "Warp-rnnt Directory" FORCE)
@@ -70,9 +69,8 @@ else()
 endif()
 ExternalProject_Add(
   extern_warprnnt
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${WARPRNNT_REPOSITORY}
-  GIT_TAG ${WARPRNNT_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
   PREFIX ${WARPRNNT_PREFIX_DIR}
   UPDATE_COMMAND ""
   PATCH_COMMAND ""

cmake/external/xxhash.cmake

@@ -18,8 +18,8 @@ set(XXHASH_PREFIX_DIR ${THIRD_PARTY_PATH}/xxhash)
 set(XXHASH_SOURCE_DIR ${THIRD_PARTY_PATH}/xxhash/src/extern_xxhash)
 set(XXHASH_INSTALL_DIR ${THIRD_PARTY_PATH}/install/xxhash)
 set(XXHASH_INCLUDE_DIR "${XXHASH_INSTALL_DIR}/include")
-set(XXHASH_REPOSITORY ${GIT_URL}/Cyan4973/xxHash.git)
 set(XXHASH_TAG v0.6.5)
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/xxhash)
 
 include_directories(${XXHASH_INCLUDE_DIR})
 
@@ -29,7 +29,7 @@ if(APPLE)
       -i
       \"\"
       "s/-Wstrict-prototypes -Wundef/-Wstrict-prototypes -Wundef -fPIC/g"
-      ${XXHASH_SOURCE_DIR}/Makefile
+      ${SOURCE_DIR}/Makefile
       &&
       make
       lib)
@@ -38,7 +38,7 @@ elseif(UNIX)
       sed
       -i
       "s/-Wstrict-prototypes -Wundef/-Wstrict-prototypes -Wundef -fPIC/g"
-      ${XXHASH_SOURCE_DIR}/Makefile
+      ${SOURCE_DIR}/Makefile
       &&
       make
       lib)
@@ -57,14 +57,13 @@ endif()
 if(WIN32)
   ExternalProject_Add(
     extern_xxhash
-    ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-    GIT_REPOSITORY ${XXHASH_REPOSITORY}
-    GIT_TAG ${XXHASH_TAG}
+    ${EXTERNAL_PROJECT_LOG_ARGS}
+    SOURCE_DIR ${SOURCE_DIR}
     PREFIX ${XXHASH_PREFIX_DIR}
     UPDATE_COMMAND ""
     PATCH_COMMAND ""
     CONFIGURE_COMMAND
-      ${CMAKE_COMMAND} ${XXHASH_SOURCE_DIR}/cmake_unofficial
+      ${CMAKE_COMMAND} ${SOURCE_DIR}/cmake_unofficial
       -DCMAKE_INSTALL_PREFIX:PATH=${XXHASH_INSTALL_DIR}
       -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
       -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF
@@ -83,8 +82,7 @@ else()
   ExternalProject_Add(
     extern_xxhash
     ${EXTERNAL_PROJECT_LOG_ARGS}
-    GIT_REPOSITORY ${XXHASH_REPOSITORY}
-    GIT_TAG ${XXHASH_TAG}
+    SOURCE_DIR ${SOURCE_DIR}
     PREFIX ${XXHASH_PREFIX_DIR}
     UPDATE_COMMAND ""
     CONFIGURE_COMMAND ""

cmake/external/zlib.cmake

@@ -22,9 +22,8 @@ set(ZLIB_ROOT
 set(ZLIB_INCLUDE_DIR
     "${ZLIB_INSTALL_DIR}/include"
     CACHE PATH "zlib include directory." FORCE)
-set(ZLIB_REPOSITORY ${GIT_URL}/madler/zlib.git)
 set(ZLIB_TAG v1.2.8)
-
+set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/zlib)
 include_directories(${ZLIB_INCLUDE_DIR}
 )# For zlib code to include its own headers.
 include_directories(${THIRD_PARTY_PATH}/install
@@ -42,9 +41,8 @@ endif()
 
 ExternalProject_Add(
   extern_zlib
-  ${EXTERNAL_PROJECT_LOG_ARGS} ${SHALLOW_CLONE}
-  GIT_REPOSITORY ${ZLIB_REPOSITORY}
-  GIT_TAG ${ZLIB_TAG}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  SOURCE_DIR ${SOURCE_DIR}
   PREFIX ${ZLIB_PREFIX_DIR}
   UPDATE_COMMAND ""
   CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}

cmake/third_party.cmake

@@ -29,7 +29,9 @@ set(third_party_deps)
 
 include(ProcessorCount)
 ProcessorCount(NPROC)
-
+if(NOT WITH_SETUP_INSTALL)
+  execute_process(COMMAND git submodule update --init --recursive)
+endif()
 # cache funciton to avoid repeat download code of third_party.
 # This function has 4 parameters, URL / REPOSITOR / TAG / DIR:
 # 1. URL:           specify download url of 3rd party

setup.py

@@ -22,6 +22,7 @@ import re
 import shutil
 import subprocess
 import sys
+import time
 from contextlib import contextmanager
 from distutils.spawn import find_executable
 from subprocess import CalledProcessError
@@ -1597,13 +1598,52 @@ def install_cpp_dist_and_build_test(install_dir, lib_test_dir, headers, libs):
     subprocess.check_call([CMAKE, "--build", lib_test_dir])
 
 
+def check_submodules():
+    def get_submodule_folder():
+        git_submodules_path = os.path.join(TOP_DIR, ".gitmodules")
+        with open(git_submodules_path) as f:
+            return [
+                os.path.join(TOP_DIR, line.split("=", 1)[1].strip())
+                for line in f.readlines()
+                if line.strip().startswith("path")
+            ]
+
+    def submodules_not_exists_or_empty(folder):
+        return not os.path.exists(folder) or (
+            os.path.isdir(folder) and len(os.listdir(folder)) == 0
+        )
+
+    submodule_folders = get_submodule_folder()
+    # f none of the submodule folders exists, try to initialize them
+    if any(
+        submodules_not_exists_or_empty(folder) for folder in submodule_folders
+    ):
+        try:
+            print(' --- Trying to initialize submodules')
+            start = time.time()
+            subprocess.check_call(
+                ["git", "submodule", "update", "--init", "--recursive"],
+                cwd=TOP_DIR,
+            )
+            end = time.time()
+            print(
+                ' --- Submodule initialization took {:.2f} sec'.format(
+                    end - start
+                )
+            )
+        except Exception:
+            print(' --- Submodule initalization failed')
+            print('Please run:\n\tgit submodule update --init --recursive')
+            sys.exit(1)
+
+
 def main():
     # Parse the command line and check arguments before we proceed with building steps and setup
     parse_input_command(filter_args_list)
 
     # check build dependency
     check_build_dependency()
-
+    check_submodules()
     # Execute the build process,cmake and make
     if cmake_and_build:
         build_steps()

dlpack @ a07f962d

+Subproject commit a07f962d446b577adf4baef2b347a0f3a2a20617

eigen3 @ f612df27

+Subproject commit f612df273689a19d25b45ca4f8269463207c4fee

gflags @ e171aa2d

+Subproject commit e171aa2d15ed9eb17054558e0b3a6a413bb01067

glog @ 96a2f23d

+Subproject commit 96a2f23dca4cc7180821ca5f32e526314395d26a

gloo @ 8b6b61df

+Subproject commit 8b6b61dfa0dca02b226a01262bfcf0484382048f

leveldb @ 803d6920

+Subproject commit 803d69203a62faf50f1b77897310a3a1fcae712b

protobuf @ f0dc78d7

+Subproject commit f0dc78d7e6e331b8c6bb2d5283e06aa26883ca7c

threadpool @ 9a42ec13

+Subproject commit 9a42ec1329f259a5f4881a291db1dcb8f2ad9040

utf8proc @ 3203baa7

+Subproject commit 3203baa7374d67132384e2830b2183c92351bffc

warpctc @ bdc2b455

+Subproject commit bdc2b4550453e0ef2d3b5190f9c6103a84eff184

warprnnt @ 7ea6bfe7

+Subproject commit 7ea6bfe748779c245a0fcaa5dd9383826273eff2

xxhash @ 7cc96396

+Subproject commit 7cc9639699f64b750c0b82333dced9ea77e8436e

zlib @ 50893291

+Subproject commit 50893291621658f355bc5b4d450a8d06a563053d