tweak the interface of cache_third_party function - expose the SOURCE_DIR for...

tweak the interface of cache_third_party function - expose the SOURCE_DIR for each external library (#21899)

cmake/external/boost.cmake

@@ -29,9 +29,10 @@ set(BOOST_URL   "http://paddlepaddledeps.bj.bcebos.com/${BOOST_TAR}.tar.gz" CACH
 MESSAGE(STATUS "BOOST_VERSION: ${BOOST_VER}, BOOST_URL: ${BOOST_URL}")
 
 set(BOOST_PREFIX_DIR ${THIRD_PARTY_PATH}/boost)
+set(BOOST_SOURCE_DIR ${THIRD_PARTY_PATH}/boost/src/extern_boost)
 cache_third_party(${BOOST_PROJECT}
         URL       ${BOOST_URL}
-        DIR       ${BOOST_PREFIX_DIR})
+        DIR       BOOST_SOURCE_DIR)
 
 set(BOOST_INCLUDE_DIR "${BOOST_SOURCE_DIR}" CACHE PATH "boost include directory." FORCE)
 set_directory_properties(PROPERTIES CLEAN_NO_CUSTOM 1)

cmake/external/cub.cmake

@@ -15,15 +15,16 @@
 include(ExternalProject)
 
 set(CUB_PREFIX_DIR ${THIRD_PARTY_PATH}/cub)
+set(CUB_SOURCE_DIR ${THIRD_PARTY_PATH}/cub/src/extern_cub)
 set(CUB_REPOSITORY https://github.com/NVlabs/cub.git)
 set(CUB_TAG        v1.8.0)
 
 cache_third_party(extern_cub
     REPOSITORY    ${CUB_REPOSITORY}
     TAG           ${CUB_TAG}
-    DIR           ${CUB_PREFIX_DIR})
+    DIR           CUB_SOURCE_DIR)
 
-SET(CUB_INCLUDE_DIR ${CUB_SOURCE_DIR})
+SET(CUB_INCLUDE_DIR   ${CUB_SOURCE_DIR})
 include_directories(${CUB_INCLUDE_DIR})
 
 ExternalProject_Add(

cmake/external/dlpack.cmake

@@ -15,6 +15,7 @@
 include(ExternalProject)
 
 set(DLPACK_PREFIX_DIR ${THIRD_PARTY_PATH}/dlpack)
+set(DLPACK_SOURCE_DIR ${THIRD_PARTY_PATH}/dlpack/src/extern_dlpack)
 
 set(DLPACK_REPOSITORY https://github.com/dmlc/dlpack.git)
 set(DLPACK_TAG        v0.2)
@@ -22,9 +23,9 @@ set(DLPACK_TAG        v0.2)
 cache_third_party(extern_dlpack
     REPOSITORY    ${DLPACK_REPOSITORY}
     TAG           ${DLPACK_TAG}
-    DIR           ${DLPACK_PREFIX_DIR})
+    DIR           DLPACK_SOURCE_DIR)
 
-set(DLPACK_INCLUDE_DIR ${DLPACK_SOURCE_DIR}/include)
+set(DLPACK_INCLUDE_DIR  ${DLPACK_SOURCE_DIR}/include)
 include_directories(${DLPACK_INCLUDE_DIR})
 
 ExternalProject_Add(

cmake/external/eigen.cmake

@@ -28,7 +28,8 @@ endif()
 
 cache_third_party(extern_eigen3
     REPOSITORY    ${EIGEN_REPOSITORY}
-    TAG           ${EIGEN_TAG})
+    TAG           ${EIGEN_TAG}
+    DIR           EIGEN_SOURCE_DIR)
 
 if(WIN32)
     file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/eigen/Half.h native_src)

cmake/external/gflags.cmake

@@ -14,7 +14,8 @@
 
 INCLUDE(ExternalProject)
 
-SET(GFLAGS_PREFIX_DIR ${THIRD_PARTY_PATH}/gflags)
+SET(GFLAGS_PREFIX_DIR  ${THIRD_PARTY_PATH}/gflags)
+SET(GFLAGS_SOURCE_DIR  ${THIRD_PARTY_PATH}/gflags/src/extern_gflags)
 SET(GFLAGS_INSTALL_DIR ${THIRD_PARTY_PATH}/install/gflags)
 SET(GFLAGS_INCLUDE_DIR "${GFLAGS_INSTALL_DIR}/include" CACHE PATH "gflags include directory." FORCE)
 set(GFLAGS_REPOSITORY https://github.com/gflags/gflags.git)
@@ -31,7 +32,8 @@ INCLUDE_DIRECTORIES(${GFLAGS_INCLUDE_DIR})
 
 cache_third_party(extern_gflags
     REPOSITORY   ${GFLAGS_REPOSITORY}
-    TAG          ${GFLAGS_TAG})
+    TAG          ${GFLAGS_TAG}
+    DIR          GFLAGS_SOURCE_DIR)
 
 ExternalProject_Add(
     extern_gflags

cmake/external/glog.cmake

@@ -15,6 +15,7 @@
 INCLUDE(ExternalProject)
 
 SET(GLOG_PREFIX_DIR  ${THIRD_PARTY_PATH}/glog)
+SET(GLOG_SOURCE_DIR  ${THIRD_PARTY_PATH}/glog/src/extern_glog)
 SET(GLOG_INSTALL_DIR ${THIRD_PARTY_PATH}/install/glog)
 SET(GLOG_INCLUDE_DIR "${GLOG_INSTALL_DIR}/include" CACHE PATH "glog include directory." FORCE)
 SET(GLOG_REPOSITORY https://github.com/google/glog.git)
@@ -32,7 +33,8 @@ INCLUDE_DIRECTORIES(${GLOG_INCLUDE_DIR})
 
 cache_third_party(extern_glog
     REPOSITORY   ${GLOG_REPOSITORY}
-    TAG          ${GLOG_TAG})
+    TAG          ${GLOG_TAG}
+    DIR          GLOG_SOURCE_DIR)
 
 ExternalProject_Add(
     extern_glog

cmake/external/gtest.cmake

@@ -22,6 +22,7 @@ INCLUDE(GNUInstallDirs)
 INCLUDE(ExternalProject)
 
 SET(GTEST_PREFIX_DIR    ${THIRD_PARTY_PATH}/gtest)
+SET(GTEST_SOURCE_DIR    ${THIRD_PARTY_PATH}/gtest/src/extern_gtest)
 SET(GTEST_INSTALL_DIR   ${THIRD_PARTY_PATH}/install/gtest)
 SET(GTEST_INCLUDE_DIR   "${GTEST_INSTALL_DIR}/include" CACHE PATH "gtest include directory." FORCE)
 set(GTEST_REPOSITORY     https://github.com/google/googletest.git)
@@ -48,7 +49,8 @@ ENDIF()
 
 cache_third_party(extern_gtest
     REPOSITORY       ${GTEST_REPOSITORY}
-    TAG              ${GTEST_TAG})
+    TAG              ${GTEST_TAG}
+    DIR              GTEST_SOURCE_DIR)
 
 ExternalProject_Add(
     extern_gtest

cmake/external/mkldnn.cmake

@@ -15,7 +15,8 @@
 INCLUDE(ExternalProject)
 
 SET(MKLDNN_PROJECT        "extern_mkldnn")
-SET(MKLDNN_PREFIX_DIR    ${THIRD_PARTY_PATH}/mkldnn)
+SET(MKLDNN_PREFIX_DIR     ${THIRD_PARTY_PATH}/mkldnn)
+SET(MKLDNN_SOURCE_DIR     ${THIRD_PARTY_PATH}/mkldnn/src/extern_mkldnn)
 SET(MKLDNN_INSTALL_DIR    ${THIRD_PARTY_PATH}/install/mkldnn)
 SET(MKLDNN_INC_DIR        "${MKLDNN_INSTALL_DIR}/include" CACHE PATH "mkldnn include directory." FORCE)
 SET(MKLDNN_REPOSITORY     https://github.com/intel/mkl-dnn.git)
@@ -46,7 +47,8 @@ ENDIF(NOT WIN32)
 
 cache_third_party(${MKLDNN_PROJECT}
     REPOSITORY    ${MKLDNN_REPOSITORY}
-    TAG           ${MKLDNN_TAG})
+    TAG           ${MKLDNN_TAG}
+    DIR           MKLDNN_SOURCE_DIR)
 
 ExternalProject_Add(
     ${MKLDNN_PROJECT}

cmake/external/mklml.cmake

@@ -41,13 +41,14 @@ ELSE()
     SET(MKLML_SHARED_IOMP_LIB     ${MKLML_LIB_DIR}/libiomp5.so)
 ENDIF()
 
-SET(MKLML_PROJECT       "extern_mklml")
-MESSAGE(STATUS "MKLML_VER: ${MKLML_VER}, MKLML_URL: ${MKLML_URL}")
-SET(MKLML_PREFIX_DIR    "${THIRD_PARTY_PATH}/mklml")
+SET(MKLML_PROJECT           "extern_mklml")
+MESSAGE(STATUS      "MKLML_VER: ${MKLML_VER}, MKLML_URL: ${MKLML_URL}")
+SET(MKLML_PREFIX_DIR        ${THIRD_PARTY_PATH}/mklml)
+SET(MKLML_SOURCE_DIR        ${THIRD_PARTY_PATH}/mklml/src/extern_mklml)
 
 cache_third_party(${MKLML_PROJECT}
     URL           ${MKLML_URL}
-    DIR           ${MKLML_PREFIX_DIR})
+    DIR           MKLML_SOURCE_DIR)
 
 ExternalProject_Add(
     ${MKLML_PROJECT}

cmake/external/protobuf.cmake

@@ -157,6 +157,7 @@ endif()
 FUNCTION(build_protobuf TARGET_NAME BUILD_FOR_HOST)
     STRING(REPLACE "extern_" "" TARGET_DIR_NAME "${TARGET_NAME}")
     SET(PROTOBUF_PREFIX_DIR  ${THIRD_PARTY_PATH}/${TARGET_DIR_NAME})
+    SET(PROTOBUF_SOURCE_DIR  ${THIRD_PARTY_PATH}/${TARGET_DIR_NAME}/src/${TARGET_NAME})
     SET(PROTOBUF_INSTALL_DIR ${THIRD_PARTY_PATH}/install/${TARGET_DIR_NAME})
 
     SET(${TARGET_NAME}_INCLUDE_DIR "${PROTOBUF_INSTALL_DIR}/include" PARENT_SCOPE)
@@ -205,7 +206,7 @@ FUNCTION(build_protobuf TARGET_NAME BUILD_FOR_HOST)
     cache_third_party(${TARGET_NAME}
         REPOSITORY    ${PROTOBUF_REPOSITORY}
         TAG           ${PROTOBUF_TAG}
-        DIR           ${PROTOBUF_PREFIX_DIR})
+        DIR           PROTOBUF_SOURCE_DIR)
 
     ExternalProject_Add(
         ${TARGET_NAME}

cmake/external/pybind11.cmake

@@ -14,14 +14,15 @@
 
 include(ExternalProject)
 
-set(PYBIND_PREFIX_DIR    ${THIRD_PARTY_PATH}/pybind)
+set(PYBIND_PREFIX_DIR     ${THIRD_PARTY_PATH}/pybind)
+set(PYBIND_SOURCE_DIR     ${THIRD_PARTY_PATH}/pybind/src/extern_pybind)
 SET(PYBIND_REPOSITORY     https://github.com/pybind/pybind11.git)
 SET(PYBIND_TAG            v2.2.4)
 
 cache_third_party(extern_pybind
     REPOSITORY    ${PYBIND_REPOSITORY}
     TAG           ${PYBIND_TAG}
-    DIR           ${PYBIND_PREFIX_DIR})
+    DIR           PYBIND_SOURCE_DIR)
 
 set(PYBIND_INCLUDE_DIR ${PYBIND_SOURCE_DIR}/include)
 include_directories(${PYBIND_INCLUDE_DIR})

cmake/external/threadpool.cmake

@@ -15,13 +15,14 @@
 INCLUDE(ExternalProject)
 
 SET(THREADPOOL_PREFIX_DIR ${THIRD_PARTY_PATH}/threadpool)
+SET(THREADPOOL_SOURCE_DIR ${THIRD_PARTY_PATH}/threadpool/src/extern_threadpool)
 SET(THREADPOOL_REPOSITORY https://github.com/progschj/ThreadPool.git)
 SET(THREADPOOL_TAG        9a42ec1329f259a5f4881a291db1dcb8f2ad9040)
 
 cache_third_party(extern_threadpool
     REPOSITORY   ${THREADPOOL_REPOSITORY}
     TAG          ${THREADPOOL_TAG}
-    DIR          ${THREADPOOL_PREFIX_DIR})
+    DIR          THREADPOOL_SOURCE_DIR)
 
 SET(THREADPOOL_INCLUDE_DIR ${THREADPOOL_SOURCE_DIR})
 INCLUDE_DIRECTORIES(${THREADPOOL_INCLUDE_DIR})

cmake/external/warpctc.cmake

@@ -34,7 +34,8 @@ ENDIF()
 
 cache_third_party(extern_warpctc
     REPOSITORY   ${WARPCTC_REPOSITORY}
-    TAG          ${WARPCTC_TAG})
+    TAG          ${WARPCTC_TAG}
+    DIR          WARPCTC_SOURCE_DIR)
 
 ExternalProject_Add(
     extern_warpctc

cmake/external/xxhash.cmake

@@ -15,6 +15,7 @@
 INCLUDE(ExternalProject)
 
 set(XXHASH_PREFIX_DIR ${THIRD_PARTY_PATH}/xxhash)
+set(XXHASH_SOURCE_DIR ${THIRD_PARTY_PATH}/xxhash/src/extern_xxhash)
 set(XXHASH_INSTALL_DIR ${THIRD_PARTY_PATH}/install/xxhash)
 set(XXHASH_INCLUDE_DIR "${XXHASH_INSTALL_DIR}/include")
 set(XXHASH_REPOSITORY  https://github.com/Cyan4973/xxHash)
@@ -23,7 +24,7 @@ set(XXHASH_TAG         v0.6.5)
 cache_third_party(extern_xxhash
     REPOSITORY    ${XXHASH_REPOSITORY}
     TAG           ${XXHASH_TAG}
-    DIR           ${XXHASH_PREFIX_DIR})
+    DIR           XXHASH_SOURCE_DIR)
 
 IF(WITH_STATIC_LIB)
   SET(BUILD_CMD make lib)

cmake/external/zlib.cmake

@@ -15,6 +15,7 @@
 INCLUDE(ExternalProject)
 
 SET(ZLIB_PREFIX_DIR ${THIRD_PARTY_PATH}/zlib)
+SET(ZLIB_SOURCE_DIR ${THIRD_PARTY_PATH}/zlib/src/extern_zlib)
 SET(ZLIB_INSTALL_DIR ${THIRD_PARTY_PATH}/install/zlib)
 SET(ZLIB_ROOT ${ZLIB_INSTALL_DIR} CACHE FILEPATH "zlib root directory." FORCE)
 SET(ZLIB_INCLUDE_DIR "${ZLIB_INSTALL_DIR}/include" CACHE PATH "zlib include directory." FORCE)
@@ -26,7 +27,8 @@ INCLUDE_DIRECTORIES(${THIRD_PARTY_PATH}/install) # For Paddle code to include zl
 
 cache_third_party(extern_zlib
     REPOSITORY    ${ZLIB_REPOSITORY}
-    TAG           ${ZLIB_TAG})
+    TAG           ${ZLIB_TAG}
+    DIR           ZLIB_SOURCE_DIR)
 
 ExternalProject_Add(
     extern_zlib

cmake/third_party.cmake

@@ -23,19 +23,20 @@ set(THIRD_PARTY_CACHE_PATH     "${CMAKE_SOURCE_DIR}"    CACHE STRING
 set(THIRD_PARTY_BUILD_TYPE Release)
 
 # cache funciton to avoid repeat download code of third_party.
-# This function has 4 parameters, URL/REPOSITOR/TAG/DIR:
-# 1. URL: specify download url of 3rd party
-# 2. REPOSITORY and TAG: specify git REPOSITORY and tag/branch/commitID of 3rd party
-# 3. DIR(optional): Unify the source dir in cached and uncached mode to "${TARGET}_SOURCE_DIR". 
-#
-# The function Return 2 PARENT_SCOPE variables:
-# 1. ${TARGET}_DOWNLOAD_DIR: Simply place "${TARGET}_DOWNLOAD_DIR" in ExternalProject_Add, 
+# This function has 4 parameters, URL / REPOSITOR / TAG / DIR:
+# 1. URL:           specify download url of 3rd party
+# 2. REPOSITORY:    specify git REPOSITORY of 3rd party
+# 3. TAG:           specify git tag/branch/commitID of 3rd party
+# 4. DIR:           overwrite the original SOURCE_DIR when cache directory
+# 
+# The function Return 1 PARENT_SCOPE variables:
+#  - ${TARGET}_DOWNLOAD_CMD: Simply place "${TARGET}_DOWNLOAD_CMD" in ExternalProject_Add, 
 #                            and you no longer need to set any donwnload steps in ExternalProject_Add.
-# 2. ${TARGET}_SOURCE_DIR: Value of argument: SOURCE_DIR of ExternalProject_Add.
 # For example:
-#    Cache_third_party (${TARGET}
+#    Cache_third_party(${TARGET}
 #            REPOSITORY ${TARGET_REPOSITORY}
-#            TAG ${TARGET_TAG})
+#            TAG        ${TARGET_TAG}
+#            DIR        ${TARGET_SOURCE_DIR})
 FUNCTION(cache_third_party TARGET)
     SET(options "")
     SET(oneValueArgs URL REPOSITORY TAG DIR)
@@ -56,17 +57,12 @@ FUNCTION(cache_third_party TARGET)
         SET(${TARGET_NAME}_DOWNLOAD_CMD
                 URL             ${cache_third_party_URL})
     ELSE()
-        MESSAGE(FATAL_ERROR "Download link (Git repo or URL) must be specified for cache!")
+        MESSAGE(FATAL_ERROR    "Download link (Git repo or URL) must be specified for cache!")
     ENDIF()
-    IF(NOT WITH_TP_CACHE)
-        # The uniform argument "{TAGGET}_SOURCE_DIR" must exists for these targets even if not cache
-        SET(LISTS xxhash protobuf boost mklml cub dlpack eigen pybind threadpool)
-        FOREACH(tmp ${LISTS})
-            IF(${TARGET} MATCHES ${tmp} AND cache_third_party_DIR)
-                SET(${TARGET_NAME}_SOURCE_DIR "${cache_third_party_DIR}/src/${TARGET}")
-            ENDIF()
-        ENDFOREACH()
-    ELSE()
+    IF(WITH_TP_CACHE)
+	IF(NOT cache_third_party_DIR)
+	    MESSAGE(FATAL_ERROR   "Please input the ${TARGET_NAME}_SOURCE_DIR for overwriting when -DWITH_TP_CACHE=ON")
+	ENDIF()
         # Generate and verify cache dir for third_party source code
         SET(cache_third_party_REPOSITORY ${cache_third_party_REPOSITORY} ${cache_third_party_URL})
         IF(cache_third_party_REPOSITORY AND cache_third_party_TAG)
@@ -75,25 +71,27 @@ FUNCTION(cache_third_party TARGET)
             STRING(SUBSTRING ${HASH_REPO} 0 8 HASH_REPO)
             STRING(SUBSTRING ${HASH_GIT} 0 8 HASH_GIT)
             STRING(CONCAT HASH ${HASH_REPO} ${HASH_GIT})
-            SET(${TARGET_NAME}_SOURCE_DIR ${THIRD_PARTY_CACHE_PATH}/third_party/${TARGET}_${HASH})
+            # overwrite the original SOURCE_DIR when cache directory
+            SET(${cache_third_party_DIR} ${THIRD_PARTY_CACHE_PATH}/third_party/${TARGET}_${HASH})
         ELSEIF(cache_third_party_REPOSITORY)
             STRING(MD5 HASH_REPO ${cache_third_party_REPOSITORY})
             STRING(SUBSTRING ${HASH_REPO} 0 16 HASH)
-            SET(${TARGET_NAME}_SOURCE_DIR ${THIRD_PARTY_CACHE_PATH}/third_party/${TARGET}_${HASH})
+            # overwrite the original SOURCE_DIR when cache directory
+            SET(${cache_third_party_DIR} ${THIRD_PARTY_CACHE_PATH}/third_party/${TARGET}_${HASH})
         ENDIF()
 
-        IF(EXISTS ${${TARGET_NAME}_SOURCE_DIR})
+        IF(EXISTS ${${cache_third_party_DIR}})
             # judge whether the cache dir is empty
-            FILE(GLOB files ${${TARGET_NAME}_SOURCE_DIR}/*)
+            FILE(GLOB files ${${cache_third_party_DIR}}/*)
             LIST(LENGTH files files_len)
             IF(files_len GREATER 0)
                 list(APPEND ${TARGET_NAME}_DOWNLOAD_CMD DOWNLOAD_COMMAND "")
             ENDIF()
+            SET(${cache_third_party_DIR} ${${cache_third_party_DIR}} PARENT_SCOPE)
         ENDIF()
     ENDIF()
 
-    # Pass the variable to parent scope, the double quotation marks can't be removed
-    SET(${TARGET_NAME}_SOURCE_DIR "${${TARGET_NAME}_SOURCE_DIR}" PARENT_SCOPE)
+    # Pass ${TARGET_NAME}_DOWNLOAD_CMD to parent scope, the double quotation marks can't be removed
     SET(${TARGET_NAME}_DOWNLOAD_CMD "${${TARGET_NAME}_DOWNLOAD_CMD}" PARENT_SCOPE)
 ENDFUNCTION()