third_party.cmake 15.8 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
include(ExternalProject)
16
# Creat a target named "third_party", which can compile external dependencies on all platform(windows/linux/mac)
17

18 19 20 21
set(THIRD_PARTY_PATH  "${CMAKE_BINARY_DIR}/third_party" CACHE STRING
    "A path setting third party libraries download & build directories.")
set(THIRD_PARTY_CACHE_PATH     "${CMAKE_SOURCE_DIR}"    CACHE STRING
    "A path cache third party source code to avoid repeated download.")
22 23

set(THIRD_PARTY_BUILD_TYPE Release)
24
set(third_party_deps)
25

26
# cache funciton to avoid repeat download code of third_party.
27 28 29 30 31
# This function has 4 parameters, URL / REPOSITOR / TAG / DIR:
# 1. URL:           specify download url of 3rd party
# 2. REPOSITORY:    specify git REPOSITORY of 3rd party
# 3. TAG:           specify git tag/branch/commitID of 3rd party
# 4. DIR:           overwrite the original SOURCE_DIR when cache directory
32
#
33
# The function Return 1 PARENT_SCOPE variables:
34
#  - ${TARGET}_DOWNLOAD_CMD: Simply place "${TARGET}_DOWNLOAD_CMD" in ExternalProject_Add,
35 36
#                            and you no longer need to set any donwnload steps in ExternalProject_Add.
# For example:
37
#    Cache_third_party(${TARGET}
38
#            REPOSITORY ${TARGET_REPOSITORY}
39 40
#            TAG        ${TARGET_TAG}
#            DIR        ${TARGET_SOURCE_DIR})
41

42 43 44 45 46 47 48 49 50 51 52 53 54
FUNCTION(cache_third_party TARGET)
    SET(options "")
    SET(oneValueArgs URL REPOSITORY TAG DIR)
    SET(multiValueArgs "")
    cmake_parse_arguments(cache_third_party "${optionps}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})

    STRING(REPLACE "extern_" "" TARGET_NAME ${TARGET})
    STRING(REGEX REPLACE "[0-9]+" "" TARGET_NAME ${TARGET_NAME})
    STRING(TOUPPER ${TARGET_NAME} TARGET_NAME)
    IF(cache_third_party_REPOSITORY)
        SET(${TARGET_NAME}_DOWNLOAD_CMD
                GIT_REPOSITORY  ${cache_third_party_REPOSITORY})
        IF(cache_third_party_TAG)
55
            LIST(APPEND   ${TARGET_NAME}_DOWNLOAD_CMD
56 57 58 59 60 61
                    GIT_TAG     ${cache_third_party_TAG})
        ENDIF()
    ELSEIF(cache_third_party_URL)
        SET(${TARGET_NAME}_DOWNLOAD_CMD
                URL             ${cache_third_party_URL})
    ELSE()
62
        MESSAGE(FATAL_ERROR    "Download link (Git repo or URL) must be specified for cache!")
63
    ENDIF()
64
    IF(WITH_TP_CACHE)
65 66 67
        IF(NOT cache_third_party_DIR)
            MESSAGE(FATAL_ERROR   "Please input the ${TARGET_NAME}_SOURCE_DIR for overwriting when -DWITH_TP_CACHE=ON")
        ENDIF()
68 69 70 71 72 73 74 75
        # Generate and verify cache dir for third_party source code
        SET(cache_third_party_REPOSITORY ${cache_third_party_REPOSITORY} ${cache_third_party_URL})
        IF(cache_third_party_REPOSITORY AND cache_third_party_TAG)
            STRING(MD5 HASH_REPO ${cache_third_party_REPOSITORY})
            STRING(MD5 HASH_GIT ${cache_third_party_TAG})
            STRING(SUBSTRING ${HASH_REPO} 0 8 HASH_REPO)
            STRING(SUBSTRING ${HASH_GIT} 0 8 HASH_GIT)
            STRING(CONCAT HASH ${HASH_REPO} ${HASH_GIT})
76 77
            # overwrite the original SOURCE_DIR when cache directory
            SET(${cache_third_party_DIR} ${THIRD_PARTY_CACHE_PATH}/third_party/${TARGET}_${HASH})
78 79 80
        ELSEIF(cache_third_party_REPOSITORY)
            STRING(MD5 HASH_REPO ${cache_third_party_REPOSITORY})
            STRING(SUBSTRING ${HASH_REPO} 0 16 HASH)
81 82
            # overwrite the original SOURCE_DIR when cache directory
            SET(${cache_third_party_DIR} ${THIRD_PARTY_CACHE_PATH}/third_party/${TARGET}_${HASH})
83 84
        ENDIF()

85
        IF(EXISTS ${${cache_third_party_DIR}})
86
            # judge whether the cache dir is empty
87
            FILE(GLOB files ${${cache_third_party_DIR}}/*)
88 89 90 91 92
            LIST(LENGTH files files_len)
            IF(files_len GREATER 0)
                list(APPEND ${TARGET_NAME}_DOWNLOAD_CMD DOWNLOAD_COMMAND "")
            ENDIF()
        ENDIF()
93
        SET(${cache_third_party_DIR} ${${cache_third_party_DIR}} PARENT_SCOPE)
94 95
    ENDIF()

96
    # Pass ${TARGET_NAME}_DOWNLOAD_CMD to parent scope, the double quotation marks can't be removed
97 98 99 100 101 102 103 104
    SET(${TARGET_NAME}_DOWNLOAD_CMD "${${TARGET_NAME}_DOWNLOAD_CMD}" PARENT_SCOPE)
ENDFUNCTION()

MACRO(UNSET_VAR VAR_NAME)
    UNSET(${VAR_NAME} CACHE)
    UNSET(${VAR_NAME})
ENDMACRO()

105 106 107 108 109
# Funciton to Download the dependencies during compilation
# This function has 2 parameters, URL / DIRNAME:
# 1. URL:           The download url of 3rd dependencies
# 2. NAME:          The name of file, that determin the dirname
#
110
FUNCTION(file_download_and_uncompress URL NAME)
111 112 113 114 115
  set(options "")
  set(oneValueArgs MD5)
  set(multiValueArgs "")
  cmake_parse_arguments(URL "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
  MESSAGE(STATUS "Download dependence[${NAME}] from ${URL}, MD5: ${URL_MD5}")
116
  SET(${NAME}_INCLUDE_DIR ${THIRD_PARTY_PATH}/${NAME}/data PARENT_SCOPE)
117
  ExternalProject_Add(
118
      download_${NAME}
119 120 121
      ${EXTERNAL_PROJECT_LOG_ARGS}
      PREFIX                ${THIRD_PARTY_PATH}/${NAME}
      URL                   ${URL}
122
      URL_MD5               ${URL_MD5}
123
      TIMEOUT               120
124 125 126 127 128 129 130 131
      DOWNLOAD_DIR          ${THIRD_PARTY_PATH}/${NAME}/data/
      SOURCE_DIR            ${THIRD_PARTY_PATH}/${NAME}/data/
      DOWNLOAD_NO_PROGRESS  1
      CONFIGURE_COMMAND     ""
      BUILD_COMMAND         ""
      UPDATE_COMMAND        ""
      INSTALL_COMMAND       ""
    )
132
  set(third_party_deps ${third_party_deps} download_${NAME} PARENT_SCOPE)
133
ENDFUNCTION()
134 135


136 137 138
# Correction of flags on different Platform(WIN/MAC) and Print Warning Message
if (APPLE)
    if(WITH_MKL)
139
        MESSAGE(WARNING
140 141 142 143 144 145
            "Mac is not supported with MKL in Paddle yet. Force WITH_MKL=OFF.")
        set(WITH_MKL OFF CACHE STRING "Disable MKL for building on mac" FORCE)
    endif()
endif()

if(WIN32 OR APPLE)
146 147 148
    MESSAGE(STATUS "Disable XBYAK in Windows and MacOS")
    SET(WITH_XBYAK OFF CACHE STRING "Disable XBYAK in Windows and MacOS" FORCE)

149
    if(WITH_LIBXSMM)
150
        MESSAGE(WARNING
151 152 153 154 155 156 157 158 159 160 161
            "Windows, Mac are not supported with libxsmm in Paddle yet."
            "Force WITH_LIBXSMM=OFF")
        SET(WITH_LIBXSMM OFF CACHE STRING "Disable LIBXSMM in Windows and MacOS" FORCE)
    endif()

    if(WITH_BOX_PS)
        MESSAGE(WARNING
            "Windows or Mac is not supported with BOX_PS in Paddle yet."
            "Force WITH_BOX_PS=OFF")
        SET(WITH_BOX_PS OFF CACHE STRING "Disable BOX_PS package in Windows and MacOS" FORCE)
    endif()
162 163 164 165 166 167 168 169

    if(WITH_PSLIB)
        MESSAGE(WARNING
            "Windows or Mac is not supported with PSLIB in Paddle yet."
            "Force WITH_PSLIB=OFF")
        SET(WITH_PSLIB OFF CACHE STRING "Disable PSLIB package in Windows and MacOS" FORCE)
    endif()

170 171 172 173 174 175 176
    if(WITH_ARM_BRPC)
        MESSAGE(WARNING
            "Windows or Mac is not supported with ARM_BRPC in Paddle yet."
            "Force WITH_ARM_BRPC=OFF")
        SET(WITH_ARM_BRPC OFF CACHE STRING "Disable ARM_BRPC package in Windows and MacOS" FORCE)
    endif()

177 178 179 180 181 182 183 184 185 186 187 188 189
    if(WITH_LIBMCT)
        MESSAGE(WARNING
            "Windows or Mac is not supported with LIBMCT in Paddle yet."
            "Force WITH_LIBMCT=OFF")
        SET(WITH_LIBMCT OFF CACHE STRING "Disable LIBMCT package in Windows and MacOS" FORCE)
    endif()

    if(WITH_PSLIB_BRPC)
        MESSAGE(WARNING
            "Windows or Mac is not supported with PSLIB_BRPC in Paddle yet."
            "Force WITH_PSLIB_BRPC=OFF")
        SET(WITH_PSLIB_BRPC OFF CACHE STRING "Disable PSLIB_BRPC package in Windows and MacOS" FORCE)
    endif()
190 191
endif()

192
set(WITH_MKLML ${WITH_MKL})
193 194
if(NOT DEFINED WITH_MKLDNN)
    if(WITH_MKL AND AVX2_FOUND)
195 196 197 198 199 200 201
        set(WITH_MKLDNN ON)
    else()
        message(STATUS "Do not have AVX2 intrinsics and disabled MKL-DNN")
        set(WITH_MKLDNN OFF)
    endif()
endif()

202 203 204 205
if(WIN32 OR APPLE OR NOT WITH_GPU OR ON_INFER)
    set(WITH_DGC OFF)
endif()

206 207 208 209
if(${CMAKE_VERSION} VERSION_GREATER "3.5.2")
    set(SHALLOW_CLONE "GIT_SHALLOW TRUE") # adds --depth=1 arg to git clone of External_Projects
endif()

T
tianshuo78520a 已提交
210
########################### include third_party according to flags ###############################
211 212 213 214 215 216 217 218 219
include(external/zlib)      # download, build, install zlib
include(external/gflags)    # download, build, install gflags
include(external/glog)      # download, build, install glog
include(external/boost)     # download boost
include(external/eigen)     # download eigen3
include(external/threadpool)# download threadpool
include(external/dlpack)    # download dlpack
include(external/xxhash)    # download, build, install xxhash
include(external/warpctc)   # download, build, install warpctc
S
Steffy-zxf 已提交
220 221 222 223
include(external/utf8proc)   # download, build, install utf8proc

list(APPEND third_party_deps extern_eigen3 extern_gflags extern_glog extern_boost extern_xxhash)
list(APPEND third_party_deps extern_zlib extern_dlpack extern_warpctc extern_threadpool extern_utf8proc)
224
include(external/lapack)    # download, build, install lapack
225

226
list(APPEND third_party_deps extern_eigen3 extern_gflags extern_glog extern_boost extern_xxhash)
227
list(APPEND third_party_deps extern_zlib extern_dlpack extern_warpctc extern_threadpool extern_lapack)
228

229
include(cblas)              	# find first, then download, build, install openblas
230 231

message(STATUS "CBLAS_PROVIDER: ${CBLAS_PROVIDER}")
232
if(${CBLAS_PROVIDER} STREQUAL MKLML)
233
    list(APPEND third_party_deps extern_mklml)
234
elseif(${CBLAS_PROVIDER} STREQUAL EXTERN_OPENBLAS)
235 236 237
    list(APPEND third_party_deps extern_openblas)
endif()

238

239 240
if(WITH_MKLDNN)
    include(external/mkldnn)    # download, build, install mkldnn
241
    list(APPEND third_party_deps extern_mkldnn)
242 243
endif()

244
include(external/protobuf)  	# find first, then download, build, install protobuf
245
if(TARGET extern_protobuf)
246 247 248 249 250 251
    list(APPEND third_party_deps extern_protobuf)
endif()

if(WITH_PYTHON)
    include(external/python)    # find python and python_module
    include(external/pybind11)  # download pybind11
252
    list(APPEND third_party_deps extern_pybind)
253 254
endif()

T
tangwei12 已提交
255
IF(WITH_TESTING OR WITH_DISTRIBUTE)
256 257 258 259
    include(external/gtest)     # download, build, install gtest
    list(APPEND third_party_deps extern_gtest)
ENDIF()

260 261 262 263 264 265
if(WITH_ONNXRUNTIME)
    include(external/onnxruntime)            # download, build, install onnxruntime、paddle2onnx
    include(external/paddle2onnx)          
    list(APPEND third_party_deps extern_onnxruntime extern_paddle2onnx)
endif()

266
if(WITH_GPU)
267 268 269 270
    if (${CMAKE_CUDA_COMPILER_VERSION} LESS 11.0)
        include(external/cub)       # download cub
        list(APPEND third_party_deps extern_cub)
    endif()
271 272
    set(URL  "https://paddlepaddledeps.bj.bcebos.com/externalErrorMsg_20210928.tar.gz" CACHE STRING "" FORCE)
    file_download_and_uncompress(${URL} "externalError" MD5 a712a49384e77ca216ad866712f7cafa)   # download file externalErrorMsg.tar.gz
273
    if(WITH_TESTING)
274
        # copy externalErrorMsg.pb, just for unittest can get error message correctly.
275 276
        set(SRC_DIR ${THIRD_PARTY_PATH}/externalError/data)
        if(WIN32 AND (NOT "${CMAKE_GENERATOR}" STREQUAL "Ninja"))
277
            set(DST_DIR1 ${CMAKE_BINARY_DIR}/paddle/fluid/third_party/externalError/data)
278
        else()
279
            set(DST_DIR1 ${CMAKE_BINARY_DIR}/paddle/third_party/externalError/data)
280
        endif()
281
        set(DST_DIR2 ${CMAKE_BINARY_DIR}/python/paddle/include/third_party/externalError/data)
282
        add_custom_command(TARGET download_externalError POST_BUILD
283 284
            COMMAND ${CMAKE_COMMAND} -E copy_directory ${SRC_DIR} ${DST_DIR1}
            COMMAND ${CMAKE_COMMAND} -E copy_directory ${SRC_DIR} ${DST_DIR2}
285 286
            COMMENT "copy_directory from ${SRC_DIR} to ${DST_DIR}")
    endif()
287 288
endif(WITH_GPU)

289 290 291 292 293
if(WITH_XPU)
    include(external/xpu)          # download, build, install xpu
    list(APPEND third_party_deps extern_xpu)
endif(WITH_XPU)

F
fwenguang 已提交
294 295 296 297 298
if(WITH_MLU)
    include(external/concurrentqueue) # download, build, install concurrentqueue
    list(APPEND third_party_deps extern_concurrentqueue)
endif(WITH_MLU)

299
if(WITH_PSLIB)
300 301 302 303 304 305 306 307 308
    include(external/pslib)          # download, build, install pslib
    list(APPEND third_party_deps extern_pslib)
    if(WITH_LIBMCT)
        include(external/libmct)     # download, build, install libmct
        list(APPEND third_party_deps extern_libxsmm)
    endif()
    if(WITH_PSLIB_BRPC)
        include(external/pslib_brpc) # download, build, install pslib_brpc
        list(APPEND third_party_deps extern_pslib_brpc)
T
Thunderbrook 已提交
309 310 311 312 313 314
    else()    
        include(external/snappy)
        list(APPEND third_party_deps extern_snappy)

        include(external/leveldb)
        list(APPEND third_party_deps extern_leveldb)
T
Thunderbrook 已提交
315 316 317 318
        if(NOT WITH_HETERPS)
            include(external/brpc)
            list(APPEND third_party_deps extern_brpc)
        endif()
319
    endif()
320 321
endif(WITH_PSLIB)

T
Thunderbrook 已提交
322 323 324 325
if(NOT WIN32 AND NOT APPLE)
    include(external/gloo)
    list(APPEND third_party_deps extern_gloo)
endif()
326

327 328
if(WITH_BOX_PS)
    include(external/box_ps)
329
    list(APPEND third_party_deps extern_box_ps)
330 331
endif(WITH_BOX_PS)

332
if(WITH_ASCEND OR WITH_ASCEND_CL)
333
    include(external/ascend)
334
    if(WITH_ASCEND OR WITH_ASCEND_CL)
335 336 337 338 339 340
        list(APPEND third_party_deps extern_ascend)
    endif()
    if(WITH_ASCEND_CL)
        list(APPEND third_party_deps extern_ascend_cl)
    endif()
endif ()
341

T
tangwei12 已提交
342
if (WITH_PSCORE)
T
tangwei12 已提交
343 344
    include(external/snappy)
    list(APPEND third_party_deps extern_snappy)
345

T
tangwei12 已提交
346 347
    include(external/leveldb)
    list(APPEND third_party_deps extern_leveldb)
348 349 350 351 352 353 354 355
    
    if (WITH_ARM_BRPC)
        include(external/arm_brpc)
        list(APPEND third_party_deps extern_arm_brpc)
    else()
        include(external/brpc)
        list(APPEND third_party_deps extern_brpc)
    endif()
T
tangwei12 已提交
356 357 358

    include(external/libmct)     # download, build, install libmct
    list(APPEND third_party_deps extern_libmct)
359

T
Thunderbrook 已提交
360 361 362 363
    if (WITH_HETERPS)
        include(external/rocksdb)     # download, build, install libmct
        list(APPEND third_party_deps extern_rocksdb)
    endif()
364 365 366
endif()

if(WITH_XBYAK)
367
    include(external/xbyak)         # download, build, install xbyak
368
    list(APPEND third_party_deps extern_xbyak)
369 370 371
endif()

if(WITH_LIBXSMM)
372
    include(external/libxsmm)       # download, build, install libxsmm
373
    list(APPEND third_party_deps extern_libxsmm)
374 375 376 377
endif()

if(WITH_DGC)
    message(STATUS "add dgc lib.")
378
    include(external/dgc)           # download, build, install dgc
379
    add_definitions(-DPADDLE_WITH_DGC)
380
    list(APPEND third_party_deps extern_dgc)
381 382
endif()

石晓伟 已提交
383
if (WITH_LITE)
W
Wilber 已提交
384
    message(STATUS "Compile Paddle with Lite Engine.")
石晓伟 已提交
385 386 387
    include(external/lite)
endif (WITH_LITE)

388 389 390
if (WITH_CINN)
    message(STATUS "Compile Paddle with CINN.")
    include(external/cinn)
391
    add_definitions(-DPADDLE_WITH_CINN)
Z
Zhen Wang 已提交
392 393 394 395 396 397 398 399
    if (WITH_GPU)
        add_definitions(-DCINN_WITH_CUDA)
        add_definitions(-DCINN_WITH_CUDNN)
    endif (WITH_GPU)
    if (WITH_MKL)
        add_definitions(-DCINN_WITH_MKL_CBLAS)
        add_definitions(-DCINN_WITH_MKLDNN)
    endif (WITH_MKL)
400 401
endif (WITH_CINN)

Y
Yanghello 已提交
402 403
if (WITH_CRYPTO)
    include(external/cryptopp)   # download, build, install cryptopp
404
    list(APPEND third_party_deps extern_cryptopp)
405
    add_definitions(-DPADDLE_WITH_CRYPTO)
Y
Yanghello 已提交
406 407
endif (WITH_CRYPTO)

408 409 410 411 412 413
if (WITH_POCKETFFT)
    include(external/pocketfft)
    list(APPEND third_party_deps extern_pocketfft)
    add_definitions(-DPADDLE_WITH_POCKETFFT)
endif (WITH_POCKETFFT)

414 415 416 417 418
if (WIN32)
    include(external/dirent)
    list(APPEND third_party_deps extern_dirent)
endif (WIN32)

Y
Yan Chunwei 已提交
419 420
if (WITH_INFRT)
    include(external/llvm)
421
    list(APPEND third_party_deps ${llvm_libs})
Y
Yan Chunwei 已提交
422 423
endif()

J
jianghaicheng 已提交
424 425 426 427 428
if (WITH_IPU)
    include(external/poplar)
    list(APPEND third_party_deps extern_poplar)
endif()

429
add_custom_target(third_party ALL DEPENDS ${third_party_deps})