From b7417610987d545072c5a8e9f15d950f21385a80 Mon Sep 17 00:00:00 2001 From: zhouwei25 <52485244+zhouwei25@users.noreply.github.com> Date: Thu, 31 Oct 2019 12:18:16 +0800 Subject: [PATCH] Integration of third_party compilation structure (#20887) --- CMakeLists.txt | 124 ++++++------------------ cmake/external/box_ps.cmake | 12 --- cmake/external/gtest.cmake | 118 +++++++++++------------ cmake/external/libxsmm.cmake | 13 --- cmake/external/mkldnn.cmake | 12 --- cmake/external/mklml.cmake | 10 -- cmake/external/ngraph.cmake | 21 +--- cmake/external/protobuf.cmake | 2 +- cmake/external/pybind11.cmake | 4 - cmake/external/python.cmake | 4 - cmake/inference_lib.cmake | 30 ++---- cmake/third_party.cmake | 176 ++++++++++++++++++++++++++++++++++ 12 files changed, 273 insertions(+), 253 deletions(-) mode change 100644 => 100755 CMakeLists.txt create mode 100755 cmake/third_party.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt old mode 100644 new mode 100755 index 831448d1bf..215cba1925 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -72,6 +72,7 @@ option(WITH_AMD_GPU "Compile PaddlePaddle with AMD GPU" OFF) option(WITH_NGRAPH "Compile PaddlePaddle with nGraph support." OFF) option(WITH_PROFILER "Compile PaddlePaddle with GPU profiler and gperftools" OFF) option(WITH_COVERAGE "Compile PaddlePaddle with code coverage" OFF) +OPTION(WITH_LIBXSMM "Compile with libxsmm" OFF) option(COVERALLS_UPLOAD "Package code coverage data to coveralls" OFF) option(WITH_PSLIB "Compile with pslib support" OFF) option(WITH_BOX_PS "Compile with box_ps support" OFF) @@ -97,88 +98,43 @@ if(NOT CMAKE_BUILD_TYPE) FORCE) endif() -if (APPLE) - set(WITH_MKL OFF CACHE STRING - "Disable MKL for building on mac" FORCE) -endif() - -if (WIN32) - set(WITH_DISTRIBUTE OFF CACHE STRING - "Disable DISTRIBUTE when compiling for Windows" FORCE) -endif() - -set(THIRD_PARTY_PATH "${CMAKE_BINARY_DIR}/third_party" CACHE STRING - "A path setting third party libraries download & build directories.") - -set(FLUID_INSTALL_DIR "${CMAKE_BINARY_DIR}/fluid_install_dir" CACHE STRING - "A path setting fluid shared and static libraries") - -set(FLUID_INFERENCE_INSTALL_DIR "${CMAKE_BINARY_DIR}/fluid_inference_install_dir" CACHE STRING - "A path setting fluid inference shared and static libraries") - -set(THIRD_PARTY_BUILD_TYPE Release) - -set(WITH_MKLML ${WITH_MKL}) -if (NOT DEFINED WITH_MKLDNN) - if (WITH_MKL AND AVX2_FOUND) - set(WITH_MKLDNN ON) - else() - message(STATUS "Do not have AVX2 intrinsics and disabled MKL-DNN") - set(WITH_MKLDNN OFF) - endif() -endif() - -if (REPLACE_ENFORCE_GLOG) +# Replace PADDLE_ENFORCE with glog/CHECK for better debug +if(REPLACE_ENFORCE_GLOG) add_definitions("-DREPLACE_ENFORCE_GLOG") endif() -if (SANITIZER_TYPE AND NOT "${SANITIZER_TYPE}" MATCHES "^(Address|Leak|Memory|Thread|Undefined)$") +# the type of sanitizer, options are: Address, Leak, Memory, Thread, Undefined. Default: OFF +if(SANITIZER_TYPE AND NOT "${SANITIZER_TYPE}" MATCHES "^(Address|Leak|Memory|Thread|Undefined)$") message("Choose the correct type of sanitizer") return() endif() -######################################################################################## - -include(external/mklml) # download mklml package -include(external/xbyak) # download xbyak package -include(external/libxsmm) # download, build, install libxsmm -include(external/zlib) # download, build, install zlib -include(external/gflags) # download, build, install gflags -include(external/glog) # download, build, install glog -include(external/gtest) # download, build, install gtest -include(external/protobuf) # download, build, install protobuf -include(external/python) # download, build, install python -include(external/openblas) # download, build, install openblas -include(external/mkldnn) # download, build, install mkldnn -include(external/ngraph) # download, build, install nGraph -include(external/boost) # download boost -include(external/eigen) # download eigen3 -include(external/pybind11) # download pybind11 -include(external/cares) -include(external/cub) -include(external/rocprim) -include(external/xxhash) # download xxhash -include(external/dlpack) -include(external/warpctc) # download, build, install warpctc +if(WIN32) + if(WITH_DISTRIBUTE) + MESSAGE(WARNING + "Disable DISTRIBUTE when compiling for Windows. Force WITH_DISTRIBUTE=OFF.") + set(WITH_DISTRIBUTE OFF CACHE STRING + "Disable DISTRIBUTE when compiling for Windows" FORCE) + endif() +endif() -if (NOT WIN32) -# there is no official support of nccl, cupti in windows -include(cupti) -endif (NOT WIN32) +if(WITH_BRPC_RDMA) + message(STATUS "Use brpc with rdma.") + if(WITH_GRPC) + message(FATAL_ERROR "Can't use grpc with brpc rdma.") + endif() + if(NOT WITH_DISTRIBUTE) + message(FATAL_ERROR "Can't use brpc rdma in no distribute env.") + endif() +endif() -if(WITH_PSLIB) - include(external/libmct) - include(external/pslib_brpc) - include(external/pslib) -endif(WITH_PSLIB) -if(WITH_BOX_PS) - include(external/box_ps) -endif(WITH_BOX_PS) +include(third_party) # download, build, install third_party if(WITH_DISTRIBUTE) + include(external/cares) if(WITH_GRPC) - include(external/grpc) message(STATUS "Use grpc framework.") + include(external/grpc) else() message(STATUS "Use brpc framework.") include(external/leveldb) @@ -186,39 +142,21 @@ if(WITH_DISTRIBUTE) endif() endif() -if(WITH_BRPC_RDMA) - message(STATUS "Use brpc with rdma.") - if(WITH_GRPC) - message(FATAL_ERROR "Can't use grpc with brpc rdma.") - endif() - if(NOT WITH_DISTRIBUTE) - message(FATAL_ERROR "Can't use brpc rdma in no distribute env.") - endif() +# there is no official support of nccl, cupti in windows +if(NOT WIN32) + include(cupti) endif() include(anakin_subgraph) - -include(external/threadpool) include(flags) # set paddle compile flags include(cudnn) # set cudnn libraries, must before configure -include(configure) # add paddle env configuration if(WITH_GPU) include(cuda) include(tensorrt) endif() -if(WIN32 OR APPLE OR NOT WITH_GPU OR ON_INFER) - set(WITH_DGC OFF) -endif() - -if(WITH_DGC) - message(STATUS "add dgc lib.") - include(external/dgc) - add_definitions(-DPADDLE_WITH_DGC) -endif() - -if (WITH_PROFILER) +if(WITH_PROFILER) find_package(Gperftools REQUIRED) include_directories(${GPERFTOOLS_INCLUDE_DIR}) add_definitions(-DWITH_GPERFTOOLS) @@ -230,7 +168,7 @@ include(util) # set unittest and link libs include(version) # set PADDLE_VERSION include(coveralls) # set code coverage include(inference_lib) # add paddle fluid inference libraries - +include(configure) # add paddle env configuration include_directories("${PADDLE_SOURCE_DIR}") @@ -244,7 +182,7 @@ set(PADDLE_PYTHON_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/python/build") set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -g -DNDEBUG") set(CMAKE_C_FLAGS_RELWITHDEBINFO "-O3 -g -DNDEBUG") -if (ON_INFER) +if(ON_INFER) message(STATUS "On inference mode, will take place some specific optimization.") add_definitions(-DPADDLE_ON_INFERENCE) else() diff --git a/cmake/external/box_ps.cmake b/cmake/external/box_ps.cmake index ddb4c82e1d..c6716d13f1 100644 --- a/cmake/external/box_ps.cmake +++ b/cmake/external/box_ps.cmake @@ -12,18 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -IF(NOT ${WITH_BOX_PS}) - return() -ENDIF(NOT ${WITH_BOX_PS}) - -IF(WIN32 OR APPLE) - MESSAGE(WARNING - "Windows or Mac is not supported with BOX_PS in Paddle yet." - "Force WITH_BOX_PS=OFF") - SET(WITH_BOX_PS OFF CACHE STRING "Disable BOX_PS package in Windows and MacOS" FORCE) - return() -ENDIF() - INCLUDE(ExternalProject) SET(BOX_PS_PROJECT "extern_box_ps") diff --git a/cmake/external/gtest.cmake b/cmake/external/gtest.cmake index 04189c4fa1..99cdca5455 100644 --- a/cmake/external/gtest.cmake +++ b/cmake/external/gtest.cmake @@ -14,72 +14,68 @@ #FIXME:(gongwb) Move brpc's gtest dependency. -include(GNUInstallDirs) +IF(WITH_TESTING) + ENABLE_TESTING() +ENDIF() -IF(WITH_TESTING OR (WITH_DISTRIBUTE AND NOT WITH_GRPC)) - IF(WITH_TESTING) - ENABLE_TESTING() - ENDIF(WITH_TESTING) +INCLUDE(GNUInstallDirs) +INCLUDE(ExternalProject) - INCLUDE(ExternalProject) +SET(GTEST_SOURCES_DIR ${THIRD_PARTY_PATH}/gtest) +SET(GTEST_INSTALL_DIR ${THIRD_PARTY_PATH}/install/gtest) +SET(GTEST_INCLUDE_DIR "${GTEST_INSTALL_DIR}/include" CACHE PATH "gtest include directory." FORCE) - SET(GTEST_SOURCES_DIR ${THIRD_PARTY_PATH}/gtest) - SET(GTEST_INSTALL_DIR ${THIRD_PARTY_PATH}/install/gtest) - SET(GTEST_INCLUDE_DIR "${GTEST_INSTALL_DIR}/include" CACHE PATH "gtest include directory." FORCE) +INCLUDE_DIRECTORIES(${GTEST_INCLUDE_DIR}) - INCLUDE_DIRECTORIES(${GTEST_INCLUDE_DIR}) +IF(WIN32) + set(GTEST_LIBRARIES + "${GTEST_INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/gtest.lib" CACHE FILEPATH "gtest libraries." FORCE) + set(GTEST_MAIN_LIBRARIES + "${GTEST_INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/gtest_main.lib" CACHE FILEPATH "gtest main libraries." FORCE) +ELSE(WIN32) + set(GTEST_LIBRARIES + "${GTEST_INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/libgtest.a" CACHE FILEPATH "gtest libraries." FORCE) + set(GTEST_MAIN_LIBRARIES + "${GTEST_INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/libgtest_main.a" CACHE FILEPATH "gtest main libraries." FORCE) +ENDIF(WIN32) - IF(WIN32) - set(GTEST_LIBRARIES - "${GTEST_INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/gtest.lib" CACHE FILEPATH "gtest libraries." FORCE) - set(GTEST_MAIN_LIBRARIES - "${GTEST_INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/gtest_main.lib" CACHE FILEPATH "gtest main libraries." FORCE) - ELSE(WIN32) - set(GTEST_LIBRARIES - "${GTEST_INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/libgtest.a" CACHE FILEPATH "gtest libraries." FORCE) - set(GTEST_MAIN_LIBRARIES - "${GTEST_INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/libgtest_main.a" CACHE FILEPATH "gtest main libraries." FORCE) - ENDIF(WIN32) +IF(WITH_MKLML) + # wait for mklml downloading completed + SET(GTEST_DEPENDS ${MKLML_PROJECT}) +ENDIF() - IF(WITH_MKLML) - # wait for mklml downloading completed - SET(GTEST_DEPENDS ${MKLML_PROJECT}) - ENDIF() +ExternalProject_Add( + extern_gtest + ${EXTERNAL_PROJECT_LOG_ARGS} + DEPENDS ${GTEST_DEPENDS} + GIT_REPOSITORY "https://github.com/google/googletest.git" + GIT_TAG "release-1.8.1" + PREFIX ${GTEST_SOURCES_DIR} + UPDATE_COMMAND "" + CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} + -DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE} + -DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG} + -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} + -DCMAKE_C_FLAGS_DEBUG=${CMAKE_C_FLAGS_DEBUG} + -DCMAKE_C_FLAGS_RELEASE=${CMAKE_C_FLAGS_RELEASE} + -DCMAKE_INSTALL_PREFIX=${GTEST_INSTALL_DIR} + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DBUILD_GMOCK=ON + -Dgtest_disable_pthreads=ON + -Dgtest_force_shared_crt=ON + -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE} + ${EXTERNAL_OPTIONAL_ARGS} + CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GTEST_INSTALL_DIR} + -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON + -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE} +) - ExternalProject_Add( - extern_gtest - ${EXTERNAL_PROJECT_LOG_ARGS} - DEPENDS ${GTEST_DEPENDS} - GIT_REPOSITORY "https://github.com/google/googletest.git" - GIT_TAG "release-1.8.1" - PREFIX ${GTEST_SOURCES_DIR} - UPDATE_COMMAND "" - CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} - -DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE} - -DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG} - -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} - -DCMAKE_C_FLAGS_DEBUG=${CMAKE_C_FLAGS_DEBUG} - -DCMAKE_C_FLAGS_RELEASE=${CMAKE_C_FLAGS_RELEASE} - -DCMAKE_INSTALL_PREFIX=${GTEST_INSTALL_DIR} - -DCMAKE_POSITION_INDEPENDENT_CODE=ON - -DBUILD_GMOCK=ON - -Dgtest_disable_pthreads=ON - -Dgtest_force_shared_crt=ON - -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE} - ${EXTERNAL_OPTIONAL_ARGS} - CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GTEST_INSTALL_DIR} - -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON - -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE} - ) +ADD_LIBRARY(gtest STATIC IMPORTED GLOBAL) +SET_PROPERTY(TARGET gtest PROPERTY IMPORTED_LOCATION ${GTEST_LIBRARIES}) +ADD_DEPENDENCIES(gtest extern_gtest) - ADD_LIBRARY(gtest STATIC IMPORTED GLOBAL) - SET_PROPERTY(TARGET gtest PROPERTY IMPORTED_LOCATION ${GTEST_LIBRARIES}) - ADD_DEPENDENCIES(gtest extern_gtest) - - ADD_LIBRARY(gtest_main STATIC IMPORTED GLOBAL) - SET_PROPERTY(TARGET gtest_main PROPERTY IMPORTED_LOCATION ${GTEST_MAIN_LIBRARIES}) - ADD_DEPENDENCIES(gtest_main extern_gtest) - -ENDIF(WITH_TESTING OR (WITH_DISTRIBUTE AND NOT WITH_GRPC)) +ADD_LIBRARY(gtest_main STATIC IMPORTED GLOBAL) +SET_PROPERTY(TARGET gtest_main PROPERTY IMPORTED_LOCATION ${GTEST_MAIN_LIBRARIES}) +ADD_DEPENDENCIES(gtest_main extern_gtest) diff --git a/cmake/external/libxsmm.cmake b/cmake/external/libxsmm.cmake index 69cdba7c59..5a7ee04655 100644 --- a/cmake/external/libxsmm.cmake +++ b/cmake/external/libxsmm.cmake @@ -11,19 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# - -OPTION(WITH_LIBXSMM "Compile with libxsmm" OFF) - -IF(NOT WITH_LIBXSMM) - return() -ENDIF() - -IF(WIN32 OR APPLE) - MESSAGE(WARNING "Windows, Mac are not supported with libxsmm in Paddle yet.") - SET(WITH_LIBXSMM OFF CACHE STRING "Disable LIBXSMM" FORCE) - return() -ENDIF() INCLUDE (ExternalProject) diff --git a/cmake/external/mkldnn.cmake b/cmake/external/mkldnn.cmake index 0ca37f506f..03d59889d7 100644 --- a/cmake/external/mkldnn.cmake +++ b/cmake/external/mkldnn.cmake @@ -12,10 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -IF(NOT ${WITH_MKLDNN}) - return() -ENDIF(NOT ${WITH_MKLDNN}) - INCLUDE(ExternalProject) SET(MKLDNN_PROJECT "extern_mkldnn") @@ -23,14 +19,6 @@ SET(MKLDNN_SOURCES_DIR ${THIRD_PARTY_PATH}/mkldnn) SET(MKLDNN_INSTALL_DIR ${THIRD_PARTY_PATH}/install/mkldnn) SET(MKLDNN_INC_DIR "${MKLDNN_INSTALL_DIR}/include" CACHE PATH "mkldnn include directory." FORCE) -IF(APPLE) - MESSAGE(WARNING - "Mac is not supported with MKLDNN in Paddle yet." - "Force WITH_MKLDNN=OFF") - SET(WITH_MKLDNN OFF CACHE STRING "Disable MKLDNN in MacOS" FORCE) - return() -ENDIF() - # Introduce variables: # * CMAKE_INSTALL_LIBDIR INCLUDE(GNUInstallDirs) diff --git a/cmake/external/mklml.cmake b/cmake/external/mklml.cmake index 17556afec8..d511784d70 100644 --- a/cmake/external/mklml.cmake +++ b/cmake/external/mklml.cmake @@ -12,16 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -IF(NOT ${WITH_MKLML}) - return() -ENDIF(NOT ${WITH_MKLML}) - -IF(APPLE) - MESSAGE(WARNING "Mac is not supported with MKLML in Paddle yet. Force WITH_MKLML=OFF.") - SET(WITH_MKLML OFF CACHE STRING "Disable MKLML package in MacOS" FORCE) - return() -ENDIF() - INCLUDE(ExternalProject) SET(MKLML_DST_DIR "mklml") SET(MKLML_INSTALL_ROOT "${THIRD_PARTY_PATH}/install") diff --git a/cmake/external/ngraph.cmake b/cmake/external/ngraph.cmake index d28cc1c373..fb752a22b3 100644 --- a/cmake/external/ngraph.cmake +++ b/cmake/external/ngraph.cmake @@ -12,26 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -add_library(ngraph INTERFACE) - -IF(WIN32 OR APPLE) - MESSAGE(WARNING - "Windows or Mac is not supported with nGraph in Paddle yet." - "Force WITH_NGRAPH=OFF") - SET(WITH_NGRAPH OFF CACHE STRING "Disable nGraph in Windows and MacOS" FORCE) -ENDIF() - -IF(${WITH_NGRAPH} AND NOT ${WITH_MKLDNN}) - MESSAGE(WARNING - "nGraph needs mkl-dnn to be enabled." - "Force WITH_NGRAPH=OFF") - SET(WITH_NGRAPH OFF CACHE STRING "Disable nGraph if mkl-dnn is disabled" FORCE) -ENDIF() - -IF(NOT ${WITH_NGRAPH}) - return() -ENDIF() - INCLUDE(GNUInstallDirs) INCLUDE(ExternalProject) @@ -79,6 +59,7 @@ ExternalProject_Add( CMAKE_ARGS -NGRAPH_USE_LEGACY_MKLDNN=TRUE ) +add_library(ngraph INTERFACE) add_dependencies(ngraph ${NGRAPH_PROJECT}) target_compile_definitions(ngraph INTERFACE -DPADDLE_WITH_NGRAPH) target_include_directories(ngraph INTERFACE ${NGRAPH_INC_DIR}) diff --git a/cmake/external/protobuf.cmake b/cmake/external/protobuf.cmake index e167f2dc42..b437e151c7 100644 --- a/cmake/external/protobuf.cmake +++ b/cmake/external/protobuf.cmake @@ -15,7 +15,7 @@ INCLUDE(ExternalProject) # Always invoke `FIND_PACKAGE(Protobuf)` for importing function protobuf_generate_cpp IF(NOT WIN32) -FIND_PACKAGE(Protobuf QUIET) + FIND_PACKAGE(Protobuf QUIET) ENDIF(NOT WIN32) macro(UNSET_VAR VAR_NAME) UNSET(${VAR_NAME} CACHE) diff --git a/cmake/external/pybind11.cmake b/cmake/external/pybind11.cmake index 3a10ea945d..a2dbf4cdb7 100644 --- a/cmake/external/pybind11.cmake +++ b/cmake/external/pybind11.cmake @@ -12,10 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -if(NOT WITH_PYTHON) - return() -endif() - include(ExternalProject) set(PYBIND_SOURCE_DIR ${THIRD_PARTY_PATH}/pybind) diff --git a/cmake/external/python.cmake b/cmake/external/python.cmake index 623c53f4f7..4828d9ae90 100644 --- a/cmake/external/python.cmake +++ b/cmake/external/python.cmake @@ -12,10 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -IF(NOT WITH_PYTHON) - return() -ENDIF() - INCLUDE(python_module) FIND_PACKAGE(PythonInterp ${PY_VERSION} REQUIRED) diff --git a/cmake/inference_lib.cmake b/cmake/inference_lib.cmake index 9dd65438d5..161bb29c81 100644 --- a/cmake/inference_lib.cmake +++ b/cmake/inference_lib.cmake @@ -14,6 +14,12 @@ # make package for paddle fluid shared and static library +set(FLUID_INSTALL_DIR "${CMAKE_BINARY_DIR}/fluid_install_dir" CACHE STRING + "A path setting fluid shared and static libraries") + +set(FLUID_INFERENCE_INSTALL_DIR "${CMAKE_BINARY_DIR}/fluid_inference_install_dir" CACHE STRING + "A path setting fluid inference shared and static libraries") + if(WIN32) if(NOT PYTHON_EXECUTABLE) FIND_PACKAGE(PythonInterp REQUIRED) @@ -50,29 +56,7 @@ function(copy TARGET) endforeach () endfunction() -# third party -set(third_party_deps eigen3 gflags glog boost xxhash zlib dlpack) -if(NOT PROTOBUF_FOUND OR WIN32) - list(APPEND third_party_deps extern_protobuf) -endif () - -if (WITH_MKLML) - list(APPEND third_party_deps mklml) -elseif (NOT CBLAS_FOUND OR WIN32) - list(APPEND third_party_deps extern_openblas) -endif () - -if (WITH_MKLDNN) - list(APPEND third_party_deps mkldnn_shared_lib) -endif () - -if (WITH_NGRAPH) - list(APPEND third_party_deps ngraph) -endif () - -add_custom_target(third_party DEPENDS ${third_party_deps}) - -# inference-only library +# inference library for only inference set(inference_lib_deps third_party paddle_fluid paddle_fluid_shared) add_custom_target(inference_lib_dist DEPENDS ${inference_lib_deps}) diff --git a/cmake/third_party.cmake b/cmake/third_party.cmake new file mode 100755 index 0000000000..1910e4111f --- /dev/null +++ b/cmake/third_party.cmake @@ -0,0 +1,176 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# compile third party for fluid on both windows/linux/mac + +set(THIRD_PARTY_PATH "${CMAKE_BINARY_DIR}/third_party" CACHE STRING + "A path setting third party libraries download & build directories.") + +set(THIRD_PARTY_BUILD_TYPE Release) + +set(WITH_MKLML ${WITH_MKL}) +if (NOT DEFINED WITH_MKLDNN) + if (WITH_MKL AND AVX2_FOUND) + set(WITH_MKLDNN ON) + else() + message(STATUS "Do not have AVX2 intrinsics and disabled MKL-DNN") + set(WITH_MKLDNN OFF) + endif() +endif() + +# Correction of flags on different Platform(WIN/MAC) and Print Warning Message +if (APPLE) + if(WITH_MKL) + MESSAGE(WARNING + "Mac is not supported with MKL in Paddle yet. Force WITH_MKL=OFF.") + set(WITH_MKL OFF CACHE STRING "Disable MKL for building on mac" FORCE) + endif() + + if(WITH_MKLML) + MESSAGE(WARNING + "Mac is not supported with MKLML in Paddle yet. Force WITH_MKLML=OFF.") + set(WITH_MKLML OFF CACHE STRING "Disable MKLML package in MacOS" FORCE) + endif() +endif() + +if(WIN32 OR APPLE) + if(WITH_LIBXSMM) + MESSAGE(WARNING + "Windows, Mac are not supported with libxsmm in Paddle yet." + "Force WITH_LIBXSMM=OFF") + SET(WITH_LIBXSMM OFF CACHE STRING "Disable LIBXSMM in Windows and MacOS" FORCE) + endif() + + if(WITH_NGRAPH) + MESSAGE(WARNING + "Windows or Mac is not supported with nGraph in Paddle yet." + "Force WITH_NGRAPH=OFF") + SET(WITH_NGRAPH OFF CACHE STRING "Disable nGraph in Windows and MacOS" FORCE) + endif() + + if(WITH_BOX_PS) + MESSAGE(WARNING + "Windows or Mac is not supported with BOX_PS in Paddle yet." + "Force WITH_BOX_PS=OFF") + SET(WITH_BOX_PS OFF CACHE STRING "Disable BOX_PS package in Windows and MacOS" FORCE) + endif() +endif() + +if(WIN32 OR APPLE OR NOT WITH_GPU OR ON_INFER) + set(WITH_DGC OFF) +endif() + +########################### include third_party accoring to flags ############################### +include(external/zlib) # download, build, install zlib +include(external/gflags) # download, build, install gflags +include(external/glog) # download, build, install glog +include(external/boost) # download boost +include(external/eigen) # download eigen3 +include(external/threadpool)# download threadpool +include(external/dlpack) # download dlpack +include(external/xxhash) # download, build, install xxhash +include(external/warpctc) # download, build, install warpctc +include(external/rocprim) # download, build, install rocprim +include(external/xbyak) # download, build, install xbyak + +set(third_party_deps eigen3 gflags glog boost xxhash zlib dlpack warpctc simple_threadpool) + +if(WITH_AMD_GPU) + list(APPEND third_party_deps rocprim) +endif() + +if(WITH_MKLML) + include(external/mklml) # download, install mklml package + list(APPEND third_party_deps mklml) +elseif (NOT CBLAS_FOUND OR WIN32) + list(APPEND third_party_deps extern_openblas) +endif() +include(external/openblas) # find first, then download, build, install openblas + +if(WITH_MKLDNN) + include(external/mkldnn) # download, build, install mkldnn + list(APPEND third_party_deps mkldnn_shared_lib) +endif() + +include(external/protobuf) # find first, then download, build, install protobuf +if(NOT PROTOBUF_FOUND OR WIN32) + list(APPEND third_party_deps extern_protobuf) +endif() + +if(WITH_PYTHON) + include(external/python) # find python and python_module + include(external/pybind11) # download pybind11 + list(APPEND third_party_deps pybind) +endif() + +IF(WITH_TESTING OR (WITH_DISTRIBUTE AND NOT WITH_GRPC)) + include(external/gtest) # download, build, install gtest + list(APPEND third_party_deps extern_gtest) +ENDIF() + +if(WITH_GPU) + include(external/cub) # download cub + list(APPEND third_party_deps cub) +endif(WITH_GPU) + +if(WITH_PSLIB) + include(external/libmct) + include(external/pslib) + include(external/pslib_brpc) +endif(WITH_PSLIB) + +if(WITH_BOX_PS) + include(external/box_ps) + list(APPEND third_party_deps box_ps) +endif(WITH_BOX_PS) + +if(WITH_DISTRIBUTE) + list(APPEND third_party_deps cares) + if(WITH_GRPC) + list(APPEND third_party_deps grpc++_unsecure) + else() + list(APPEND third_party_deps leveldb) + list(APPEND third_party_deps brpc) + endif() +endif() + +if(WITH_NGRAPH) + if(WITH_MKLDNN) + include(external/ngraph) # download, build, install nGraph + list(APPEND third_party_deps ngraph) + else() + MESSAGE(WARNING + "nGraph needs mkl-dnn to be enabled." + "Force WITH_NGRAPH=OFF") + SET(WITH_NGRAPH OFF CACHE STRING "Disable nGraph if mkl-dnn is disabled" FORCE) + endif() +endif() + +if(WITH_XBYAK) + list(APPEND third_party_deps xbyak) +endif() + +if(WITH_LIBXSMM) + include(external/libxsmm) # download, build, install libxsmm + list(APPEND third_party_deps libxsmm) +endif() + +if(WITH_DGC) + message(STATUS "add dgc lib.") + include(external/dgc) # download, build, install dgc + add_definitions(-DPADDLE_WITH_DGC) + list(APPEND third_party_deps dgc) +endif() + +add_custom_target(third_party DEPENDS ${third_party_deps}) \ No newline at end of file -- GitLab