From 4f7a2bd0d19a7c8c37b2642f6cf1a78ffefebedf Mon Sep 17 00:00:00 2001 From: zhouwei25 <52485244+zhouwei25@users.noreply.github.com> Date: Thu, 9 Jan 2020 10:58:26 +0800 Subject: [PATCH] tweak the interface of cache_third_party function - expose the SOURCE_DIR for each external library (#21899) --- cmake/external/boost.cmake | 3 +- cmake/external/cub.cmake | 5 ++-- cmake/external/dlpack.cmake | 5 ++-- cmake/external/eigen.cmake | 3 +- cmake/external/gflags.cmake | 6 ++-- cmake/external/glog.cmake | 4 ++- cmake/external/gtest.cmake | 4 ++- cmake/external/mkldnn.cmake | 6 ++-- cmake/external/mklml.cmake | 9 +++--- cmake/external/protobuf.cmake | 3 +- cmake/external/pybind11.cmake | 5 ++-- cmake/external/threadpool.cmake | 3 +- cmake/external/warpctc.cmake | 3 +- cmake/external/xxhash.cmake | 3 +- cmake/external/zlib.cmake | 4 ++- cmake/third_party.cmake | 50 ++++++++++++++++----------------- 16 files changed, 67 insertions(+), 49 deletions(-) diff --git a/cmake/external/boost.cmake b/cmake/external/boost.cmake index c8854b3894d..c95f7689058 100644 --- a/cmake/external/boost.cmake +++ b/cmake/external/boost.cmake @@ -29,9 +29,10 @@ set(BOOST_URL "http://paddlepaddledeps.bj.bcebos.com/${BOOST_TAR}.tar.gz" CACH MESSAGE(STATUS "BOOST_VERSION: ${BOOST_VER}, BOOST_URL: ${BOOST_URL}") set(BOOST_PREFIX_DIR ${THIRD_PARTY_PATH}/boost) +set(BOOST_SOURCE_DIR ${THIRD_PARTY_PATH}/boost/src/extern_boost) cache_third_party(${BOOST_PROJECT} URL ${BOOST_URL} - DIR ${BOOST_PREFIX_DIR}) + DIR BOOST_SOURCE_DIR) set(BOOST_INCLUDE_DIR "${BOOST_SOURCE_DIR}" CACHE PATH "boost include directory." FORCE) set_directory_properties(PROPERTIES CLEAN_NO_CUSTOM 1) diff --git a/cmake/external/cub.cmake b/cmake/external/cub.cmake index b6e69a5d606..76054812454 100644 --- a/cmake/external/cub.cmake +++ b/cmake/external/cub.cmake @@ -15,15 +15,16 @@ include(ExternalProject) set(CUB_PREFIX_DIR ${THIRD_PARTY_PATH}/cub) +set(CUB_SOURCE_DIR ${THIRD_PARTY_PATH}/cub/src/extern_cub) set(CUB_REPOSITORY https://github.com/NVlabs/cub.git) set(CUB_TAG v1.8.0) cache_third_party(extern_cub REPOSITORY ${CUB_REPOSITORY} TAG ${CUB_TAG} - DIR ${CUB_PREFIX_DIR}) + DIR CUB_SOURCE_DIR) -SET(CUB_INCLUDE_DIR ${CUB_SOURCE_DIR}) +SET(CUB_INCLUDE_DIR ${CUB_SOURCE_DIR}) include_directories(${CUB_INCLUDE_DIR}) ExternalProject_Add( diff --git a/cmake/external/dlpack.cmake b/cmake/external/dlpack.cmake index 7710708cef9..831f1d5a682 100644 --- a/cmake/external/dlpack.cmake +++ b/cmake/external/dlpack.cmake @@ -15,6 +15,7 @@ include(ExternalProject) set(DLPACK_PREFIX_DIR ${THIRD_PARTY_PATH}/dlpack) +set(DLPACK_SOURCE_DIR ${THIRD_PARTY_PATH}/dlpack/src/extern_dlpack) set(DLPACK_REPOSITORY https://github.com/dmlc/dlpack.git) set(DLPACK_TAG v0.2) @@ -22,9 +23,9 @@ set(DLPACK_TAG v0.2) cache_third_party(extern_dlpack REPOSITORY ${DLPACK_REPOSITORY} TAG ${DLPACK_TAG} - DIR ${DLPACK_PREFIX_DIR}) + DIR DLPACK_SOURCE_DIR) -set(DLPACK_INCLUDE_DIR ${DLPACK_SOURCE_DIR}/include) +set(DLPACK_INCLUDE_DIR ${DLPACK_SOURCE_DIR}/include) include_directories(${DLPACK_INCLUDE_DIR}) ExternalProject_Add( diff --git a/cmake/external/eigen.cmake b/cmake/external/eigen.cmake index 43cd12439ce..56c1e89195d 100644 --- a/cmake/external/eigen.cmake +++ b/cmake/external/eigen.cmake @@ -28,7 +28,8 @@ endif() cache_third_party(extern_eigen3 REPOSITORY ${EIGEN_REPOSITORY} - TAG ${EIGEN_TAG}) + TAG ${EIGEN_TAG} + DIR EIGEN_SOURCE_DIR) if(WIN32) file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/eigen/Half.h native_src) diff --git a/cmake/external/gflags.cmake b/cmake/external/gflags.cmake index 15669c6c2a0..2842d73081b 100644 --- a/cmake/external/gflags.cmake +++ b/cmake/external/gflags.cmake @@ -14,7 +14,8 @@ INCLUDE(ExternalProject) -SET(GFLAGS_PREFIX_DIR ${THIRD_PARTY_PATH}/gflags) +SET(GFLAGS_PREFIX_DIR ${THIRD_PARTY_PATH}/gflags) +SET(GFLAGS_SOURCE_DIR ${THIRD_PARTY_PATH}/gflags/src/extern_gflags) SET(GFLAGS_INSTALL_DIR ${THIRD_PARTY_PATH}/install/gflags) SET(GFLAGS_INCLUDE_DIR "${GFLAGS_INSTALL_DIR}/include" CACHE PATH "gflags include directory." FORCE) set(GFLAGS_REPOSITORY https://github.com/gflags/gflags.git) @@ -31,7 +32,8 @@ INCLUDE_DIRECTORIES(${GFLAGS_INCLUDE_DIR}) cache_third_party(extern_gflags REPOSITORY ${GFLAGS_REPOSITORY} - TAG ${GFLAGS_TAG}) + TAG ${GFLAGS_TAG} + DIR GFLAGS_SOURCE_DIR) ExternalProject_Add( extern_gflags diff --git a/cmake/external/glog.cmake b/cmake/external/glog.cmake index 093f6086e3a..f66ecce18a0 100644 --- a/cmake/external/glog.cmake +++ b/cmake/external/glog.cmake @@ -15,6 +15,7 @@ INCLUDE(ExternalProject) SET(GLOG_PREFIX_DIR ${THIRD_PARTY_PATH}/glog) +SET(GLOG_SOURCE_DIR ${THIRD_PARTY_PATH}/glog/src/extern_glog) SET(GLOG_INSTALL_DIR ${THIRD_PARTY_PATH}/install/glog) SET(GLOG_INCLUDE_DIR "${GLOG_INSTALL_DIR}/include" CACHE PATH "glog include directory." FORCE) SET(GLOG_REPOSITORY https://github.com/google/glog.git) @@ -32,7 +33,8 @@ INCLUDE_DIRECTORIES(${GLOG_INCLUDE_DIR}) cache_third_party(extern_glog REPOSITORY ${GLOG_REPOSITORY} - TAG ${GLOG_TAG}) + TAG ${GLOG_TAG} + DIR GLOG_SOURCE_DIR) ExternalProject_Add( extern_glog diff --git a/cmake/external/gtest.cmake b/cmake/external/gtest.cmake index fccbc1f9b1b..60061c44b78 100644 --- a/cmake/external/gtest.cmake +++ b/cmake/external/gtest.cmake @@ -22,6 +22,7 @@ INCLUDE(GNUInstallDirs) INCLUDE(ExternalProject) SET(GTEST_PREFIX_DIR ${THIRD_PARTY_PATH}/gtest) +SET(GTEST_SOURCE_DIR ${THIRD_PARTY_PATH}/gtest/src/extern_gtest) SET(GTEST_INSTALL_DIR ${THIRD_PARTY_PATH}/install/gtest) SET(GTEST_INCLUDE_DIR "${GTEST_INSTALL_DIR}/include" CACHE PATH "gtest include directory." FORCE) set(GTEST_REPOSITORY https://github.com/google/googletest.git) @@ -48,7 +49,8 @@ ENDIF() cache_third_party(extern_gtest REPOSITORY ${GTEST_REPOSITORY} - TAG ${GTEST_TAG}) + TAG ${GTEST_TAG} + DIR GTEST_SOURCE_DIR) ExternalProject_Add( extern_gtest diff --git a/cmake/external/mkldnn.cmake b/cmake/external/mkldnn.cmake index 3a59ce6a341..761b8d87b22 100644 --- a/cmake/external/mkldnn.cmake +++ b/cmake/external/mkldnn.cmake @@ -15,7 +15,8 @@ INCLUDE(ExternalProject) SET(MKLDNN_PROJECT "extern_mkldnn") -SET(MKLDNN_PREFIX_DIR ${THIRD_PARTY_PATH}/mkldnn) +SET(MKLDNN_PREFIX_DIR ${THIRD_PARTY_PATH}/mkldnn) +SET(MKLDNN_SOURCE_DIR ${THIRD_PARTY_PATH}/mkldnn/src/extern_mkldnn) SET(MKLDNN_INSTALL_DIR ${THIRD_PARTY_PATH}/install/mkldnn) SET(MKLDNN_INC_DIR "${MKLDNN_INSTALL_DIR}/include" CACHE PATH "mkldnn include directory." FORCE) SET(MKLDNN_REPOSITORY https://github.com/intel/mkl-dnn.git) @@ -46,7 +47,8 @@ ENDIF(NOT WIN32) cache_third_party(${MKLDNN_PROJECT} REPOSITORY ${MKLDNN_REPOSITORY} - TAG ${MKLDNN_TAG}) + TAG ${MKLDNN_TAG} + DIR MKLDNN_SOURCE_DIR) ExternalProject_Add( ${MKLDNN_PROJECT} diff --git a/cmake/external/mklml.cmake b/cmake/external/mklml.cmake index d67a543fa6f..bbc14313cb2 100644 --- a/cmake/external/mklml.cmake +++ b/cmake/external/mklml.cmake @@ -41,13 +41,14 @@ ELSE() SET(MKLML_SHARED_IOMP_LIB ${MKLML_LIB_DIR}/libiomp5.so) ENDIF() -SET(MKLML_PROJECT "extern_mklml") -MESSAGE(STATUS "MKLML_VER: ${MKLML_VER}, MKLML_URL: ${MKLML_URL}") -SET(MKLML_PREFIX_DIR "${THIRD_PARTY_PATH}/mklml") +SET(MKLML_PROJECT "extern_mklml") +MESSAGE(STATUS "MKLML_VER: ${MKLML_VER}, MKLML_URL: ${MKLML_URL}") +SET(MKLML_PREFIX_DIR ${THIRD_PARTY_PATH}/mklml) +SET(MKLML_SOURCE_DIR ${THIRD_PARTY_PATH}/mklml/src/extern_mklml) cache_third_party(${MKLML_PROJECT} URL ${MKLML_URL} - DIR ${MKLML_PREFIX_DIR}) + DIR MKLML_SOURCE_DIR) ExternalProject_Add( ${MKLML_PROJECT} diff --git a/cmake/external/protobuf.cmake b/cmake/external/protobuf.cmake index c6475c1e8d1..121e9e64d9e 100644 --- a/cmake/external/protobuf.cmake +++ b/cmake/external/protobuf.cmake @@ -157,6 +157,7 @@ endif() FUNCTION(build_protobuf TARGET_NAME BUILD_FOR_HOST) STRING(REPLACE "extern_" "" TARGET_DIR_NAME "${TARGET_NAME}") SET(PROTOBUF_PREFIX_DIR ${THIRD_PARTY_PATH}/${TARGET_DIR_NAME}) + SET(PROTOBUF_SOURCE_DIR ${THIRD_PARTY_PATH}/${TARGET_DIR_NAME}/src/${TARGET_NAME}) SET(PROTOBUF_INSTALL_DIR ${THIRD_PARTY_PATH}/install/${TARGET_DIR_NAME}) SET(${TARGET_NAME}_INCLUDE_DIR "${PROTOBUF_INSTALL_DIR}/include" PARENT_SCOPE) @@ -205,7 +206,7 @@ FUNCTION(build_protobuf TARGET_NAME BUILD_FOR_HOST) cache_third_party(${TARGET_NAME} REPOSITORY ${PROTOBUF_REPOSITORY} TAG ${PROTOBUF_TAG} - DIR ${PROTOBUF_PREFIX_DIR}) + DIR PROTOBUF_SOURCE_DIR) ExternalProject_Add( ${TARGET_NAME} diff --git a/cmake/external/pybind11.cmake b/cmake/external/pybind11.cmake index d079e5c99f1..72a4bf65244 100644 --- a/cmake/external/pybind11.cmake +++ b/cmake/external/pybind11.cmake @@ -14,14 +14,15 @@ include(ExternalProject) -set(PYBIND_PREFIX_DIR ${THIRD_PARTY_PATH}/pybind) +set(PYBIND_PREFIX_DIR ${THIRD_PARTY_PATH}/pybind) +set(PYBIND_SOURCE_DIR ${THIRD_PARTY_PATH}/pybind/src/extern_pybind) SET(PYBIND_REPOSITORY https://github.com/pybind/pybind11.git) SET(PYBIND_TAG v2.2.4) cache_third_party(extern_pybind REPOSITORY ${PYBIND_REPOSITORY} TAG ${PYBIND_TAG} - DIR ${PYBIND_PREFIX_DIR}) + DIR PYBIND_SOURCE_DIR) set(PYBIND_INCLUDE_DIR ${PYBIND_SOURCE_DIR}/include) include_directories(${PYBIND_INCLUDE_DIR}) diff --git a/cmake/external/threadpool.cmake b/cmake/external/threadpool.cmake index cb29be3ed48..20d605183a5 100644 --- a/cmake/external/threadpool.cmake +++ b/cmake/external/threadpool.cmake @@ -15,13 +15,14 @@ INCLUDE(ExternalProject) SET(THREADPOOL_PREFIX_DIR ${THIRD_PARTY_PATH}/threadpool) +SET(THREADPOOL_SOURCE_DIR ${THIRD_PARTY_PATH}/threadpool/src/extern_threadpool) SET(THREADPOOL_REPOSITORY https://github.com/progschj/ThreadPool.git) SET(THREADPOOL_TAG 9a42ec1329f259a5f4881a291db1dcb8f2ad9040) cache_third_party(extern_threadpool REPOSITORY ${THREADPOOL_REPOSITORY} TAG ${THREADPOOL_TAG} - DIR ${THREADPOOL_PREFIX_DIR}) + DIR THREADPOOL_SOURCE_DIR) SET(THREADPOOL_INCLUDE_DIR ${THREADPOOL_SOURCE_DIR}) INCLUDE_DIRECTORIES(${THREADPOOL_INCLUDE_DIR}) diff --git a/cmake/external/warpctc.cmake b/cmake/external/warpctc.cmake index 603e55ee16e..2a9600dec51 100644 --- a/cmake/external/warpctc.cmake +++ b/cmake/external/warpctc.cmake @@ -34,7 +34,8 @@ ENDIF() cache_third_party(extern_warpctc REPOSITORY ${WARPCTC_REPOSITORY} - TAG ${WARPCTC_TAG}) + TAG ${WARPCTC_TAG} + DIR WARPCTC_SOURCE_DIR) ExternalProject_Add( extern_warpctc diff --git a/cmake/external/xxhash.cmake b/cmake/external/xxhash.cmake index 27726dbd497..c84091ff6e0 100644 --- a/cmake/external/xxhash.cmake +++ b/cmake/external/xxhash.cmake @@ -15,6 +15,7 @@ INCLUDE(ExternalProject) set(XXHASH_PREFIX_DIR ${THIRD_PARTY_PATH}/xxhash) +set(XXHASH_SOURCE_DIR ${THIRD_PARTY_PATH}/xxhash/src/extern_xxhash) set(XXHASH_INSTALL_DIR ${THIRD_PARTY_PATH}/install/xxhash) set(XXHASH_INCLUDE_DIR "${XXHASH_INSTALL_DIR}/include") set(XXHASH_REPOSITORY https://github.com/Cyan4973/xxHash) @@ -23,7 +24,7 @@ set(XXHASH_TAG v0.6.5) cache_third_party(extern_xxhash REPOSITORY ${XXHASH_REPOSITORY} TAG ${XXHASH_TAG} - DIR ${XXHASH_PREFIX_DIR}) + DIR XXHASH_SOURCE_DIR) IF(WITH_STATIC_LIB) SET(BUILD_CMD make lib) diff --git a/cmake/external/zlib.cmake b/cmake/external/zlib.cmake index ba7fe845aad..2f2a6b74701 100644 --- a/cmake/external/zlib.cmake +++ b/cmake/external/zlib.cmake @@ -15,6 +15,7 @@ INCLUDE(ExternalProject) SET(ZLIB_PREFIX_DIR ${THIRD_PARTY_PATH}/zlib) +SET(ZLIB_SOURCE_DIR ${THIRD_PARTY_PATH}/zlib/src/extern_zlib) SET(ZLIB_INSTALL_DIR ${THIRD_PARTY_PATH}/install/zlib) SET(ZLIB_ROOT ${ZLIB_INSTALL_DIR} CACHE FILEPATH "zlib root directory." FORCE) SET(ZLIB_INCLUDE_DIR "${ZLIB_INSTALL_DIR}/include" CACHE PATH "zlib include directory." FORCE) @@ -26,7 +27,8 @@ INCLUDE_DIRECTORIES(${THIRD_PARTY_PATH}/install) # For Paddle code to include zl cache_third_party(extern_zlib REPOSITORY ${ZLIB_REPOSITORY} - TAG ${ZLIB_TAG}) + TAG ${ZLIB_TAG} + DIR ZLIB_SOURCE_DIR) ExternalProject_Add( extern_zlib diff --git a/cmake/third_party.cmake b/cmake/third_party.cmake index 43c24ed7e18..ed870aebf98 100644 --- a/cmake/third_party.cmake +++ b/cmake/third_party.cmake @@ -23,19 +23,20 @@ set(THIRD_PARTY_CACHE_PATH "${CMAKE_SOURCE_DIR}" CACHE STRING set(THIRD_PARTY_BUILD_TYPE Release) # cache funciton to avoid repeat download code of third_party. -# This function has 4 parameters, URL/REPOSITOR/TAG/DIR: -# 1. URL: specify download url of 3rd party -# 2. REPOSITORY and TAG: specify git REPOSITORY and tag/branch/commitID of 3rd party -# 3. DIR(optional): Unify the source dir in cached and uncached mode to "${TARGET}_SOURCE_DIR". -# -# The function Return 2 PARENT_SCOPE variables: -# 1. ${TARGET}_DOWNLOAD_DIR: Simply place "${TARGET}_DOWNLOAD_DIR" in ExternalProject_Add, +# This function has 4 parameters, URL / REPOSITOR / TAG / DIR: +# 1. URL: specify download url of 3rd party +# 2. REPOSITORY: specify git REPOSITORY of 3rd party +# 3. TAG: specify git tag/branch/commitID of 3rd party +# 4. DIR: overwrite the original SOURCE_DIR when cache directory +# +# The function Return 1 PARENT_SCOPE variables: +# - ${TARGET}_DOWNLOAD_CMD: Simply place "${TARGET}_DOWNLOAD_CMD" in ExternalProject_Add, # and you no longer need to set any donwnload steps in ExternalProject_Add. -# 2. ${TARGET}_SOURCE_DIR: Value of argument: SOURCE_DIR of ExternalProject_Add. # For example: -# Cache_third_party (${TARGET} +# Cache_third_party(${TARGET} # REPOSITORY ${TARGET_REPOSITORY} -# TAG ${TARGET_TAG}) +# TAG ${TARGET_TAG} +# DIR ${TARGET_SOURCE_DIR}) FUNCTION(cache_third_party TARGET) SET(options "") SET(oneValueArgs URL REPOSITORY TAG DIR) @@ -56,17 +57,12 @@ FUNCTION(cache_third_party TARGET) SET(${TARGET_NAME}_DOWNLOAD_CMD URL ${cache_third_party_URL}) ELSE() - MESSAGE(FATAL_ERROR "Download link (Git repo or URL) must be specified for cache!") + MESSAGE(FATAL_ERROR "Download link (Git repo or URL) must be specified for cache!") ENDIF() - IF(NOT WITH_TP_CACHE) - # The uniform argument "{TAGGET}_SOURCE_DIR" must exists for these targets even if not cache - SET(LISTS xxhash protobuf boost mklml cub dlpack eigen pybind threadpool) - FOREACH(tmp ${LISTS}) - IF(${TARGET} MATCHES ${tmp} AND cache_third_party_DIR) - SET(${TARGET_NAME}_SOURCE_DIR "${cache_third_party_DIR}/src/${TARGET}") - ENDIF() - ENDFOREACH() - ELSE() + IF(WITH_TP_CACHE) + IF(NOT cache_third_party_DIR) + MESSAGE(FATAL_ERROR "Please input the ${TARGET_NAME}_SOURCE_DIR for overwriting when -DWITH_TP_CACHE=ON") + ENDIF() # Generate and verify cache dir for third_party source code SET(cache_third_party_REPOSITORY ${cache_third_party_REPOSITORY} ${cache_third_party_URL}) IF(cache_third_party_REPOSITORY AND cache_third_party_TAG) @@ -75,25 +71,27 @@ FUNCTION(cache_third_party TARGET) STRING(SUBSTRING ${HASH_REPO} 0 8 HASH_REPO) STRING(SUBSTRING ${HASH_GIT} 0 8 HASH_GIT) STRING(CONCAT HASH ${HASH_REPO} ${HASH_GIT}) - SET(${TARGET_NAME}_SOURCE_DIR ${THIRD_PARTY_CACHE_PATH}/third_party/${TARGET}_${HASH}) + # overwrite the original SOURCE_DIR when cache directory + SET(${cache_third_party_DIR} ${THIRD_PARTY_CACHE_PATH}/third_party/${TARGET}_${HASH}) ELSEIF(cache_third_party_REPOSITORY) STRING(MD5 HASH_REPO ${cache_third_party_REPOSITORY}) STRING(SUBSTRING ${HASH_REPO} 0 16 HASH) - SET(${TARGET_NAME}_SOURCE_DIR ${THIRD_PARTY_CACHE_PATH}/third_party/${TARGET}_${HASH}) + # overwrite the original SOURCE_DIR when cache directory + SET(${cache_third_party_DIR} ${THIRD_PARTY_CACHE_PATH}/third_party/${TARGET}_${HASH}) ENDIF() - IF(EXISTS ${${TARGET_NAME}_SOURCE_DIR}) + IF(EXISTS ${${cache_third_party_DIR}}) # judge whether the cache dir is empty - FILE(GLOB files ${${TARGET_NAME}_SOURCE_DIR}/*) + FILE(GLOB files ${${cache_third_party_DIR}}/*) LIST(LENGTH files files_len) IF(files_len GREATER 0) list(APPEND ${TARGET_NAME}_DOWNLOAD_CMD DOWNLOAD_COMMAND "") ENDIF() + SET(${cache_third_party_DIR} ${${cache_third_party_DIR}} PARENT_SCOPE) ENDIF() ENDIF() - # Pass the variable to parent scope, the double quotation marks can't be removed - SET(${TARGET_NAME}_SOURCE_DIR "${${TARGET_NAME}_SOURCE_DIR}" PARENT_SCOPE) + # Pass ${TARGET_NAME}_DOWNLOAD_CMD to parent scope, the double quotation marks can't be removed SET(${TARGET_NAME}_DOWNLOAD_CMD "${${TARGET_NAME}_DOWNLOAD_CMD}" PARENT_SCOPE) ENDFUNCTION() -- GitLab