diff --git a/cmake/external/lapack.cmake b/cmake/external/lapack.cmake index 43305223fe280baafdba048a419be48a3ab51a8e..78c2659a58504e91a3d65f36b39c9a501bd9133f 100644 --- a/cmake/external/lapack.cmake +++ b/cmake/external/lapack.cmake @@ -15,18 +15,19 @@ include(ExternalProject) set(LAPACK_PREFIX_DIR ${THIRD_PARTY_PATH}/lapack) -set(LAPACK_SOURCE_DIR ${THIRD_PARTY_PATH}/lapack/src/extern_lapack) +set(LAPACK_DOWNLOAD_DIR + ${PADDLE_SOURCE_DIR}/third_party/lapack/${CMAKE_SYSTEM_NAME}) set(LAPACK_INSTALL_DIR ${THIRD_PARTY_PATH}/install/lapack) set(LAPACK_LIB_DIR ${LAPACK_INSTALL_DIR}/lib) # Note(zhouwei): lapack need fortan compiler which many machines don't have, so use precompiled library. # use lapack tag v3.10.0 on 06/28/2021 https://github.com/Reference-LAPACK/lapack if(LINUX) - set(LAPACK_VER - "lapack_lnx_v3.10.0.20210628" + set(LAPACK_FILE + "lapack_lnx_v3.10.0.20210628.tar.gz" CACHE STRING "" FORCE) set(LAPACK_URL - "https://paddlepaddledeps.bj.bcebos.com/${LAPACK_VER}.tar.gz" + "https://paddlepaddledeps.bj.bcebos.com/${LAPACK_FILE}" CACHE STRING "" FORCE) set(LAPACK_URL_MD5 71f8cc8237a8571692f3e07f9a4f25f6) set(GNU_RT_LIB_1 "${LAPACK_LIB_DIR}/libquadmath.so.0") @@ -35,11 +36,11 @@ if(LINUX) set(LAPACK_LIB "${LAPACK_LIB_DIR}/liblapack.so.3") elseif(WIN32) # Refer to [lapack-for-windows] http://icl.cs.utk.edu/lapack-for-windows/lapack/#lapacke - set(LAPACK_VER - "lapack_win_v3.10.0.20210628" + set(LAPACK_FILE + "lapack_win_v3.10.0.20210628.zip" CACHE STRING "" FORCE) set(LAPACK_URL - "https://paddlepaddledeps.bj.bcebos.com/${LAPACK_VER}.zip" + "https://paddlepaddledeps.bj.bcebos.com/${LAPACK_FILE}" CACHE STRING "" FORCE) set(LAPACK_URL_MD5 590d080392dcd5abbd5dca767a50b63a) set(GNU_RT_LIB_1 "${LAPACK_LIB_DIR}/libquadmath-0.dll") @@ -48,11 +49,11 @@ elseif(WIN32) set(BLAS_LIB "${LAPACK_LIB_DIR}/libblas.dll") set(LAPACK_LIB "${LAPACK_LIB_DIR}/liblapack.dll") else() - set(LAPACK_VER - "lapack_mac_v3.10.0.20210628" + set(LAPACK_FILE + "lapack_mac_v3.10.0.20210628.tar.gz" CACHE STRING "" FORCE) set(LAPACK_URL - "https://paddlepaddledeps.bj.bcebos.com/${LAPACK_VER}.tar.gz" + "https://paddlepaddledeps.bj.bcebos.com/${LAPACK_FILE}" CACHE STRING "" FORCE) set(LAPACK_URL_MD5 427aecf8dee8523de3566ca8e47944d7) set(GNU_RT_LIB_1 "${LAPACK_LIB_DIR}/libquadmath.0.dylib") @@ -62,18 +63,53 @@ else() set(LAPACK_LIB "${LAPACK_LIB_DIR}/liblapack.3.dylib") endif() +function(download_lapack) + message( + STATUS "Downloading ${LAPACK_URL} to ${LAPACK_DOWNLOAD_DIR}/${LAPACK_FILE}") + # NOTE: If the version is updated, consider emptying the folder; maybe add timeout + file( + DOWNLOAD ${LAPACK_URL} ${LAPACK_DOWNLOAD_DIR}/${LAPACK_FILE} + EXPECTED_MD5 ${LAPACK_URL_MD5} + STATUS ERR) + if(ERR EQUAL 0) + message(STATUS "Download ${LAPACK_FILE} success") + else() + message( + FATAL_ERROR + "Download failed, error: ${ERR}\n You can try downloading ${LAPACK_FILE} again" + ) + endif() +endfunction() + +find_file( + LOCAL_LAPACK_LIB_ZIP + NAMES ${LAPACK_FILE} + PATHS ${LAPACK_DOWNLOAD_DIR} + NO_DEFAULT_PATH) + +# Download and check lapack. +if(LOCAL_LAPACK_LIB_ZIP) + file(MD5 ${LAPACK_DOWNLOAD_DIR}/${LAPACK_FILE} LAPACK_MD5) + if(NOT LAPACK_MD5 EQUAL LAPACK_URL_MD5) + download_lapack() + endif() +else() + download_lapack() +endif() + ExternalProject_Add( extern_lapack ${EXTERNAL_PROJECT_LOG_ARGS} - URL ${LAPACK_URL} + URL ${LAPACK_DOWNLOAD_DIR}/${LAPACK_FILE} URL_MD5 ${LAPACK_URL_MD5} + DOWNLOAD_DIR ${LAPACK_DOWNLOAD_DIR} + SOURCE_DIR ${LAPACK_LIB_DIR} PREFIX ${LAPACK_PREFIX_DIR} DOWNLOAD_NO_PROGRESS 1 PATCH_COMMAND "" UPDATE_COMMAND "" CONFIGURE_COMMAND "" BUILD_COMMAND "" - INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory ${LAPACK_SOURCE_DIR} - ${LAPACK_LIB_DIR} + INSTALL_COMMAND "" BUILD_BYPRODUCTS ${BLAS_LIB} BUILD_BYPRODUCTS ${LAPACK_LIB}) diff --git a/cmake/external/libmct.cmake b/cmake/external/libmct.cmake index 28bf083f7791ed0206794e7e5617a7cb63bc74d6..15ee59642094a817fd4af51a9874b6ac24e0db91 100644 --- a/cmake/external/libmct.cmake +++ b/cmake/external/libmct.cmake @@ -23,13 +23,17 @@ if((NOT DEFINED LIBMCT_VER) OR (NOT DEFINED LIBMCT_URL)) set(LIBMCT_NAME "libmct" CACHE STRING "" FORCE) + set(LIBMCT_DOWNLOAD_FILE + "${LIBMCT_NAME}.tar.gz" + CACHE STRING "" FORCE) set(LIBMCT_URL - "https://pslib.bj.bcebos.com/libmct/libmct.tar.gz" + "https://pslib.bj.bcebos.com/libmct/${LIBMCT_DOWNLOAD_FILE}" CACHE STRING "" FORCE) endif() message(STATUS "LIBMCT_NAME: ${LIBMCT_NAME}, LIBMCT_URL: ${LIBMCT_URL}") set(LIBMCT_PREFIX_DIR "${THIRD_PARTY_PATH}/libmct") -set(LIBMCT_DOWNLOAD_DIR "${LIBMCT_PREFIX_DIR}/src/${LIBMCT_PROJECT}") +set(LIBMCT_DOWNLOAD_DIR + ${PADDLE_SOURCE_DIR}/third_party/libmct/${CMAKE_SYSTEM_NAME}) set(LIBMCT_DST_DIR "libmct") set(LIBMCT_INSTALL_ROOT "${THIRD_PARTY_PATH}/install") set(LIBMCT_INSTALL_DIR ${LIBMCT_INSTALL_ROOT}/${LIBMCT_DST_DIR}) @@ -39,22 +43,27 @@ set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}" "${LIBMCT_ROOT}/lib") include_directories(${LIBMCT_INC_DIR}) +file( + DOWNLOAD ${LIBMCT_URL} ${LIBMCT_DOWNLOAD_DIR}/${LIBMCT_DOWNLOAD_FILE} + TLS_VERIFY OFF + STATUS ERR) + file( WRITE ${LIBMCT_DOWNLOAD_DIR}/CMakeLists.txt "PROJECT(LIBMCT)\n" "cmake_minimum_required(VERSION 3.0)\n" - "install(DIRECTORY ${LIBMCT_NAME}/include ${LIBMCT_NAME}/lib \n" + "install(DIRECTORY ./include ./lib \n" " DESTINATION ${LIBMCT_DST_DIR})\n") ExternalProject_Add( ${LIBMCT_PROJECT} ${EXTERNAL_PROJECT_LOG_ARGS} + URL ${LIBMCT_DOWNLOAD_DIR}/${LIBMCT_DOWNLOAD_FILE} PREFIX ${LIBMCT_PREFIX_DIR} DOWNLOAD_DIR ${LIBMCT_DOWNLOAD_DIR} - DOWNLOAD_COMMAND - wget --no-check-certificate ${LIBMCT_URL} -c -q -O ${LIBMCT_NAME}.tar.gz && - tar --no-same-owner -zxvf ${LIBMCT_NAME}.tar.gz - DOWNLOAD_NO_PROGRESS 1 + SOURCE_DIR ${LIBMCT_INSTALL_DIR} UPDATE_COMMAND "" + COMMAND ${CMAKE_COMMAND} -E copy ${LIBMCT_DOWNLOAD_DIR}/CMakeLists.txt + ${LIBMCT_INSTALL_DIR} CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${LIBMCT_INSTALL_ROOT} -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE} CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${LIBMCT_INSTALL_ROOT} diff --git a/cmake/external/mklml.cmake b/cmake/external/mklml.cmake index 14a829879079914ade05093a72824571403ad49a..801cab62facbb764029bd3b37d3e4d33c183f4ad 100644 --- a/cmake/external/mklml.cmake +++ b/cmake/external/mklml.cmake @@ -17,13 +17,15 @@ set(MKLML_INSTALL_DIR ${THIRD_PARTY_PATH}/install/mklml) set(MKLML_INC_DIR ${MKLML_INSTALL_DIR}/include) set(MKLML_LIB_DIR ${MKLML_INSTALL_DIR}/lib) set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}" "${MKLML_LIB_DIR}") +set(MKLML_DOWNLOAD_DIR + ${PADDLE_SOURCE_DIR}/third_party/mklml/${CMAKE_SYSTEM_NAME}) if(WIN32) - set(MKLML_VER - "mklml_win_2019.0.5.20190502" + set(MKLML_FILE + "mklml_win_2019.0.5.20190502.zip" CACHE STRING "" FORCE) set(MKLML_URL - "https://paddlepaddledeps.bj.bcebos.com/${MKLML_VER}.zip" + "https://paddlepaddledeps.bj.bcebos.com/${MKLML_FILE}" CACHE STRING "" FORCE) set(MKLML_URL_MD5 ff8c5237570f03eea37377ccfc95a08a) set(MKLML_LIB ${MKLML_LIB_DIR}/mklml.lib) @@ -34,11 +36,11 @@ else() #TODO(intel-huying): # Now enable csrmm function in mklml library temporarily, # it will be updated as offical version later. - set(MKLML_VER - "csrmm_mklml_lnx_2019.0.5" + set(MKLML_FILE + "csrmm_mklml_lnx_2019.0.5.tgz" CACHE STRING "" FORCE) set(MKLML_URL - "http://paddlepaddledeps.bj.bcebos.com/${MKLML_VER}.tgz" + "http://paddlepaddledeps.bj.bcebos.com/${MKLML_FILE}" CACHE STRING "" FORCE) set(MKLML_URL_MD5 bc6a7faea6a2a9ad31752386f3ae87da) set(MKLML_LIB ${MKLML_LIB_DIR}/libmklml_intel.so) @@ -48,9 +50,42 @@ else() endif() set(MKLML_PROJECT "extern_mklml") -message(STATUS "MKLML_VER: ${MKLML_VER}, MKLML_URL: ${MKLML_URL}") +message(STATUS "MKLML_FILE: ${MKLML_FILE}, MKLML_URL: ${MKLML_URL}") set(MKLML_PREFIX_DIR ${THIRD_PARTY_PATH}/mklml) -set(MKLML_SOURCE_DIR ${THIRD_PARTY_PATH}/mklml/src/extern_mklml) + +function(download_mklml) + message( + STATUS "Downloading ${MKLML_URL} to ${MKLML_DOWNLOAD_DIR}/${MKLML_FILE}") + # NOTE: If the version is updated, consider emptying the folder; maybe add timeout + file( + DOWNLOAD ${MKLML_URL} ${MKLML_DOWNLOAD_DIR}/${MKLML_FILE} + EXPECTED_MD5 ${MKLML_URL_MD5} + STATUS ERR) + if(ERR EQUAL 0) + message(STATUS "Download ${MKLML_FILE} success") + else() + message( + FATAL_ERROR + "Download failed, error: ${ERR}\n You can try downloading ${MKLML_FILE} again" + ) + endif() +endfunction() + +find_file( + LOCAL_MKLML_LIB_ZIP + NAMES ${MKLML_FILE} + PATHS ${MKLML_DOWNLOAD_DIR} + NO_DEFAULT_PATH) + +# Download and check mklml. +if(LOCAL_MKLML_LIB_ZIP) + file(MD5 ${MKLML_DOWNLOAD_DIR}/${MKLML_FILE} MKLML_MD5) + if(NOT MKLML_MD5 EQUAL MKLML_URL_MD5) + download_mklml() + endif() +else() + download_mklml() +endif() # Ninja Generator can not establish the correct dependency relationship # between the imported library with target, the product file @@ -60,17 +95,15 @@ set(MKLML_SOURCE_DIR ${THIRD_PARTY_PATH}/mklml/src/extern_mklml) ExternalProject_Add( ${MKLML_PROJECT} ${EXTERNAL_PROJECT_LOG_ARGS} - URL ${MKLML_URL} + URL ${MKLML_DOWNLOAD_DIR}/${MKLML_FILE} URL_MD5 ${MKLML_URL_MD5} + DOWNLOAD_DIR ${MKLML_DOWNLOAD_DIR} + SOURCE_DIR ${MKLML_INSTALL_DIR} PREFIX ${MKLML_PREFIX_DIR} - DOWNLOAD_NO_PROGRESS 1 CONFIGURE_COMMAND "" BUILD_COMMAND "" UPDATE_COMMAND "" - INSTALL_COMMAND - ${CMAKE_COMMAND} -E copy_directory ${MKLML_SOURCE_DIR}/include - ${MKLML_INC_DIR} && ${CMAKE_COMMAND} -E copy_directory - ${MKLML_SOURCE_DIR}/lib ${MKLML_LIB_DIR} + INSTALL_COMMAND "" BUILD_BYPRODUCTS ${MKLML_LIB} BUILD_BYPRODUCTS ${MKLML_IOMP_LIB})