From 43c38c67f9f5a6b9f3ace6560dcdc2ef16f5000a Mon Sep 17 00:00:00 2001 From: Zhou Wei Date: Tue, 29 Jun 2021 10:58:52 +0800 Subject: [PATCH] support Ninja, establish dependencies relationship between paddle with third_party (#33140) * support Ninja and establish dependencies relationship between paddle with third_party * fix CI * support Ninja --- cmake/external/box_ps.cmake | 3 ++ cmake/external/brpc.cmake | 33 +++++++++++---------- cmake/external/cryptopp.cmake | 1 + cmake/external/dgc.cmake | 1 + cmake/external/gflags.cmake | 1 + cmake/external/glog.cmake | 1 + cmake/external/gloo.cmake | 4 ++- cmake/external/gtest.cmake | 2 ++ cmake/external/leveldb.cmake | 1 + cmake/external/libmct.cmake | 2 ++ cmake/external/libxsmm.cmake | 10 ++++--- cmake/external/mkldnn.cmake | 8 ++--- cmake/external/mklml.cmake | 8 ++++- cmake/external/protobuf.cmake | 4 +++ cmake/external/pslib.cmake | 3 ++ cmake/external/pslib_brpc.cmake | 3 ++ cmake/external/snappy.cmake | 52 ++++++++++++++++----------------- cmake/external/warpctc.cmake | 21 ++++++------- cmake/external/xbyak.cmake | 2 ++ cmake/external/xpu.cmake | 2 ++ cmake/external/xxhash.cmake | 25 +++++++++------- cmake/external/zlib.cmake | 12 ++++---- paddle/scripts/paddle_build.bat | 20 +++++++++---- 23 files changed, 133 insertions(+), 86 deletions(-) diff --git a/cmake/external/box_ps.cmake b/cmake/external/box_ps.cmake index adfc6dba1f..85e1f94fd2 100644 --- a/cmake/external/box_ps.cmake +++ b/cmake/external/box_ps.cmake @@ -49,7 +49,10 @@ ExternalProject_Add( DOWNLOAD_NO_PROGRESS 1 UPDATE_COMMAND "" CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${BOX_PS_INSTALL_ROOT} + -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE} CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${BOX_PS_INSTALL_ROOT} + -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE} + BUILD_BYPRODUCTS ${BOX_PS_LIB} ) ADD_LIBRARY(box_ps SHARED IMPORTED GLOBAL) SET_PROPERTY(TARGET box_ps PROPERTY IMPORTED_LOCATION ${BOX_PS_LIB}) diff --git a/cmake/external/brpc.cmake b/cmake/external/brpc.cmake index 2d72b6eb56..1a45cfa0a1 100644 --- a/cmake/external/brpc.cmake +++ b/cmake/external/brpc.cmake @@ -45,23 +45,24 @@ ExternalProject_Add( PREFIX ${BRPC_SOURCES_DIR} UPDATE_COMMAND "" CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} - -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} - -DCMAKE_INSTALL_PREFIX=${BRPC_INSTALL_DIR} - -DCMAKE_INSTALL_LIBDIR=${BRPC_INSTALL_DIR}/lib - -DCMAKE_POSITION_INDEPENDENT_CODE=ON - -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE} - -DCMAKE_PREFIX_PATH=${prefix_path} - -DWITH_GLOG=ON - -DIOBUF_WITH_HUGE_BLOCK=ON - -DBRPC_WITH_RDMA=${WITH_BRPC_RDMA} - ${EXTERNAL_OPTIONAL_ARGS} - LIST_SEPARATOR | + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} + -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} + -DCMAKE_INSTALL_PREFIX=${BRPC_INSTALL_DIR} + -DCMAKE_INSTALL_LIBDIR=${BRPC_INSTALL_DIR}/lib + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE} + -DCMAKE_PREFIX_PATH=${prefix_path} + -DWITH_GLOG=ON + -DIOBUF_WITH_HUGE_BLOCK=ON + -DBRPC_WITH_RDMA=${WITH_BRPC_RDMA} + ${EXTERNAL_OPTIONAL_ARGS} + LIST_SEPARATOR | CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${BRPC_INSTALL_DIR} - -DCMAKE_INSTALL_LIBDIR:PATH=${BRPC_INSTALL_DIR}/lib - -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON - -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE} + -DCMAKE_INSTALL_LIBDIR:PATH=${BRPC_INSTALL_DIR}/lib + -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON + -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE} + BUILD_BYPRODUCTS ${BRPC_LIBRARIES} ) # ADD_DEPENDENCIES(extern_brpc protobuf ssl crypto leveldb gflags glog gtest snappy) ADD_DEPENDENCIES(extern_brpc protobuf ssl crypto leveldb gflags glog snappy) diff --git a/cmake/external/cryptopp.cmake b/cmake/external/cryptopp.cmake index a30164ada2..f7f7a9b52e 100644 --- a/cmake/external/cryptopp.cmake +++ b/cmake/external/cryptopp.cmake @@ -72,6 +72,7 @@ ExternalProject_Add( CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${CRYPTOPP_INSTALL_DIR} -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE} + BUILD_BYPRODUCTS ${CRYPTOPP_LIBRARIES} ) ADD_LIBRARY(cryptopp STATIC IMPORTED GLOBAL) diff --git a/cmake/external/dgc.cmake b/cmake/external/dgc.cmake index bc8611f386..3c64e1ea11 100644 --- a/cmake/external/dgc.cmake +++ b/cmake/external/dgc.cmake @@ -39,6 +39,7 @@ ExternalProject_Add( && cp ${DGC_SOURCES_DIR}/build/lib/libdgc.a ${DGC_LIBRARIES} && cp ${DGC_SOURCES_DIR}/build/include/dgc.h ${DGC_INCLUDE_DIR}/dgc/ BUILD_IN_SOURCE 1 + BUILD_BYPRODUCTS ${DGC_LIBRARIES} ) ADD_LIBRARY(dgc STATIC IMPORTED GLOBAL) diff --git a/cmake/external/gflags.cmake b/cmake/external/gflags.cmake index 576598b4ac..8360761de6 100644 --- a/cmake/external/gflags.cmake +++ b/cmake/external/gflags.cmake @@ -61,6 +61,7 @@ ExternalProject_Add( CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GFLAGS_INSTALL_DIR} -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE} + BUILD_BYPRODUCTS ${GFLAGS_LIBRARIES} ) ADD_LIBRARY(gflags STATIC IMPORTED GLOBAL) diff --git a/cmake/external/glog.cmake b/cmake/external/glog.cmake index 05b98e2b56..d2bb1e62e8 100644 --- a/cmake/external/glog.cmake +++ b/cmake/external/glog.cmake @@ -64,6 +64,7 @@ ExternalProject_Add( -DCMAKE_INSTALL_LIBDIR:PATH=${GLOG_INSTALL_DIR}/lib -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE} + BUILD_BYPRODUCTS ${GLOG_LIBRARIES} ) ADD_LIBRARY(glog STATIC IMPORTED GLOBAL) diff --git a/cmake/external/gloo.cmake b/cmake/external/gloo.cmake index e8db13a694..03e45e3e5c 100644 --- a/cmake/external/gloo.cmake +++ b/cmake/external/gloo.cmake @@ -32,7 +32,7 @@ cache_third_party(extern_gloo TAG ${GLOO_TAG} DIR GLOO_SOURCE_DIR) - if(WITH_ASCEND OR WITH_ASCEND_CL) +if(WITH_ASCEND OR WITH_ASCEND_CL) ExternalProject_Add( extern_gloo ${EXTERNAL_PROJECT_LOG_ARGS} @@ -47,6 +47,7 @@ cache_third_party(extern_gloo && mkdir -p ${GLOO_LIBRARY_DIR} ${GLOO_INCLUDE_DIR}/gloo INSTALL_COMMAND ${CMAKE_COMMAND} -E copy ${GLOO_SOURCE_DIR}/build/gloo/libgloo.a ${GLOO_LIBRARY_DIR} COMMAND ${CMAKE_COMMAND} -E copy_directory "${GLOO_SOURCE_DIR}/gloo/" "${GLOO_INCLUDE_DIR}/gloo" + BUILD_BYPRODUCTS ${GLOO_LIBRARIES} ) else() ExternalProject_Add( @@ -63,6 +64,7 @@ else() && mkdir -p ${GLOO_LIBRARY_DIR} ${GLOO_INCLUDE_DIR}/gloo INSTALL_COMMAND ${CMAKE_COMMAND} -E copy ${GLOO_SOURCE_DIR}/build/gloo/libgloo.a ${GLOO_LIBRARY_DIR} COMMAND ${CMAKE_COMMAND} -E copy_directory "${GLOO_SOURCE_DIR}/gloo/" "${GLOO_INCLUDE_DIR}/gloo" + BUILD_BYPRODUCTS ${GLOO_LIBRARIES} ) endif() diff --git a/cmake/external/gtest.cmake b/cmake/external/gtest.cmake index 3db12f084e..e7d4783a95 100644 --- a/cmake/external/gtest.cmake +++ b/cmake/external/gtest.cmake @@ -79,6 +79,8 @@ ExternalProject_Add( CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GTEST_INSTALL_DIR} -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE} + BUILD_BYPRODUCTS ${GTEST_LIBRARIES} + BUILD_BYPRODUCTS ${GTEST_MAIN_LIBRARIES} ) ADD_LIBRARY(gtest STATIC IMPORTED GLOBAL) diff --git a/cmake/external/leveldb.cmake b/cmake/external/leveldb.cmake index 79dc403e67..c36f49d3bd 100644 --- a/cmake/external/leveldb.cmake +++ b/cmake/external/leveldb.cmake @@ -33,6 +33,7 @@ ExternalProject_Add( && cp ${LEVELDB_SOURCES_DIR}/src/extern_leveldb/libleveldb.a ${LEVELDB_LIBRARIES} && cp -r ${LEVELDB_SOURCES_DIR}/src/extern_leveldb/include ${LEVELDB_INSTALL_DIR}/ BUILD_IN_SOURCE 1 + BUILD_BYPRODUCTS ${LEVELDB_LIBRARIES} ) ADD_DEPENDENCIES(extern_leveldb snappy) diff --git a/cmake/external/libmct.cmake b/cmake/external/libmct.cmake index c10a662485..d318bc7d0f 100644 --- a/cmake/external/libmct.cmake +++ b/cmake/external/libmct.cmake @@ -49,7 +49,9 @@ ExternalProject_Add( DOWNLOAD_NO_PROGRESS 1 UPDATE_COMMAND "" CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${LIBMCT_INSTALL_ROOT} + -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE} CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${LIBMCT_INSTALL_ROOT} + -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE} ) add_library(libmct INTERFACE) diff --git a/cmake/external/libxsmm.cmake b/cmake/external/libxsmm.cmake index 0d09576286..fae8154eb1 100644 --- a/cmake/external/libxsmm.cmake +++ b/cmake/external/libxsmm.cmake @@ -18,8 +18,8 @@ SET(LIBXSMM_SOURCES_DIR ${THIRD_PARTY_PATH}/libxsmm) SET(LIBXSMM_INSTALL_DIR ${THIRD_PARTY_PATH}/install/libxsmm) SET(LIBXSMM_INCLUDE_DIR "${LIBXSMM_INSTALL_DIR}/include" CACHE PATH "LIBXSMM include directory." FORCE) SET(LIBXSMM_LIBRARY_DIR "${LIBXSMM_INSTALL_DIR}/lib" CACHE PATH "LIBXSMM library directory." FORCE) -SET(LIBXSMM_LIBS "${LIBXSMM_LIBRARY_DIR}/libxsmm.a" - "${LIBXSMM_LIBRARY_DIR}/libxsmmnoblas.a") +SET(LIBXSMM_LIB "${LIBXSMM_LIBRARY_DIR}/libxsmm.a") +SET(LIBXSMMNOBLAS_LIB "${LIBXSMM_LIBRARY_DIR}/libxsmmnoblas.a") ExternalProject_Add( extern_libxsmm @@ -32,10 +32,12 @@ ExternalProject_Add( BUILD_IN_SOURCE 1 BUILD_COMMAND $(MAKE) --silent PREFIX=${LIBXSMM_INSTALL_DIR} CXX=g++ CC=gcc WARP=0 install INSTALL_COMMAND "" + BUILD_BYPRODUCTS ${LIBXSMM_LIB} + BUILD_BYPRODUCTS ${LIBXSMMNOBLAS_LIB} ) ADD_LIBRARY(libxsmm STATIC IMPORTED GLOBAL) -SET_PROPERTY(TARGET libxsmm PROPERTY IMPORTED_LOCATION "${LIBXSMM_LIBRARY_DIR}/libxsmm.a") -SET_PROPERTY(TARGET libxsmm PROPERTY IMPORTED_LOCATION "${LIBXSMM_LIBRARY_DIR}/libxsmmnoblas.a") +SET_PROPERTY(TARGET libxsmm PROPERTY IMPORTED_LOCATION "${LIBXSMM_LIB}") +SET_PROPERTY(TARGET libxsmm PROPERTY IMPORTED_LOCATION "${LIBXSMMNOBLAS_LIB}") MESSAGE(STATUS "Libxsmm library: ${LIBXSMM_LIBS}") include_directories(${LIBXSMM_INCLUDE_DIR}) diff --git a/cmake/external/mkldnn.cmake b/cmake/external/mkldnn.cmake index 4a5b3f3c5f..69a0511052 100644 --- a/cmake/external/mkldnn.cmake +++ b/cmake/external/mkldnn.cmake @@ -43,8 +43,10 @@ IF(NOT WIN32) SET(MKLDNN_FLAG "${MKLDNN_FLAG} -Wno-unused-result -Wno-unused-value") SET(MKLDNN_CFLAG "${CMAKE_C_FLAGS} ${MKLDNN_FLAG}") SET(MKLDNN_CXXFLAG "${CMAKE_CXX_FLAGS} ${MKLDNN_FLAG}") + SET(MKLDNN_LIB "${MKLDNN_INSTALL_DIR}/${LIBDIR}/libdnnl.so" CACHE FILEPATH "mkldnn library." FORCE) ELSE() SET(MKLDNN_CXXFLAG "${CMAKE_CXX_FLAGS} /EHsc") + SET(MKLDNN_LIB "${MKLDNN_INSTALL_DIR}/bin/mkldnn.lib" CACHE FILEPATH "mkldnn library." FORCE) ENDIF(NOT WIN32) cache_third_party(${MKLDNN_PROJECT} @@ -77,12 +79,8 @@ ExternalProject_Add( -DCMAKE_CXX_FLAGS=${MKLDNN_CXXFLAG} -DDNNL_BUILD_TESTS=OFF -DDNNL_BUILD_EXAMPLES=OFF CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${MKLDNN_INSTALL_DIR} + BUILD_BYPRODUCTS ${MKLDNN_LIB} ) -if(WIN32) - SET(MKLDNN_LIB "${MKLDNN_INSTALL_DIR}/bin/mkldnn.lib" CACHE FILEPATH "mkldnn library." FORCE) -else(WIN32) - SET(MKLDNN_LIB "${MKLDNN_INSTALL_DIR}/${LIBDIR}/libdnnl.so" CACHE FILEPATH "mkldnn library." FORCE) -endif(WIN32) ADD_LIBRARY(shared_mkldnn SHARED IMPORTED GLOBAL) SET_PROPERTY(TARGET shared_mkldnn PROPERTY IMPORTED_LOCATION ${MKLDNN_LIB}) diff --git a/cmake/external/mklml.cmake b/cmake/external/mklml.cmake index d99cb19529..a4df5756ce 100644 --- a/cmake/external/mklml.cmake +++ b/cmake/external/mklml.cmake @@ -50,6 +50,10 @@ cache_third_party(${MKLML_PROJECT} URL ${MKLML_URL} DIR MKLML_SOURCE_DIR) +# Ninja Generator can not establish the correct dependency relationship between the imported library with target, +# the product file in the ExternalProject need to be specified manually, please refer to +# https://stackoverflow.com/questions/54866067/cmake-and-ninja-missing-and-no-known-rule-to-make-it +# It is the same to all other ExternalProject. ExternalProject_Add( ${MKLML_PROJECT} ${EXTERNAL_PROJECT_LOG_ARGS} @@ -63,7 +67,9 @@ ExternalProject_Add( BUILD_COMMAND "" UPDATE_COMMAND "" INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory ${MKLML_SOURCE_DIR}/include ${MKLML_INC_DIR} && - ${CMAKE_COMMAND} -E copy_directory ${MKLML_SOURCE_DIR}/lib ${MKLML_LIB_DIR} + ${CMAKE_COMMAND} -E copy_directory ${MKLML_SOURCE_DIR}/lib ${MKLML_LIB_DIR} + BUILD_BYPRODUCTS ${MKLML_LIB} + BUILD_BYPRODUCTS ${MKLML_IOMP_LIB} ) INCLUDE_DIRECTORIES(${MKLML_INC_DIR}) diff --git a/cmake/external/protobuf.cmake b/cmake/external/protobuf.cmake index c108c05368..a2b6ddadb6 100644 --- a/cmake/external/protobuf.cmake +++ b/cmake/external/protobuf.cmake @@ -239,6 +239,10 @@ endif() -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON ${OPTIONAL_CACHE_ARGS} + BUILD_BYPRODUCTS ${PROTOBUF_INSTALL_DIR}/lib/libprotobuf${CMAKE_STATIC_LIBRARY_SUFFIX} + BUILD_BYPRODUCTS ${PROTOBUF_INSTALL_DIR}/lib/libprotobuf-lite${CMAKE_STATIC_LIBRARY_SUFFIX} + BUILD_BYPRODUCTS ${PROTOBUF_INSTALL_DIR}/lib/libprotoc${CMAKE_STATIC_LIBRARY_SUFFIX} + BUILD_BYPRODUCTS ${PROTOBUF_INSTALL_DIR}/bin/protoc${CMAKE_EXECUTABLE_SUFFIX} ) ENDFUNCTION() diff --git a/cmake/external/pslib.cmake b/cmake/external/pslib.cmake index bdfd335172..40d198b295 100644 --- a/cmake/external/pslib.cmake +++ b/cmake/external/pslib.cmake @@ -53,7 +53,10 @@ ExternalProject_Add( DOWNLOAD_NO_PROGRESS 1 UPDATE_COMMAND "" CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${PSLIB_INSTALL_ROOT} + -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE} CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${PSLIB_INSTALL_ROOT} + -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE} + BUILD_BYPRODUCTS ${PSLIB_LIB} ) ADD_LIBRARY(pslib SHARED IMPORTED GLOBAL) diff --git a/cmake/external/pslib_brpc.cmake b/cmake/external/pslib_brpc.cmake index 7b00474a65..d69c27a197 100644 --- a/cmake/external/pslib_brpc.cmake +++ b/cmake/external/pslib_brpc.cmake @@ -52,7 +52,10 @@ ExternalProject_Add( DOWNLOAD_NO_PROGRESS 1 UPDATE_COMMAND "" CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${PSLIB_BRPC_INSTALL_ROOT} + -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE} CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${PSLIB_BRPC_INSTALL_ROOT} + -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE} + BUILD_BYPRODUCTS ${PSLIB_BRPC_LIB} ) ADD_LIBRARY(pslib_brpc SHARED IMPORTED GLOBAL) diff --git a/cmake/external/snappy.cmake b/cmake/external/snappy.cmake index ab9cb02307..fb4c1c7cc8 100644 --- a/cmake/external/snappy.cmake +++ b/cmake/external/snappy.cmake @@ -22,8 +22,15 @@ set(SNAPPY_INCLUDE_DIR "${SNAPPY_INSTALL_DIR}/include" CACHE PATH "snappy includ if(WIN32) SET(SNAPPY_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4267") + IF(NOT EXISTS "${SNAPPY_INSTALL_DIR}/lib/libsnappy.lib") + add_custom_command(TARGET extern_snappy POST_BUILD + COMMAND cmake -E copy ${SNAPPY_INSTALL_DIR}/lib/snappy.lib ${SNAPPY_INSTALL_DIR}/lib/libsnappy.lib + ) + ENDIF() + set(SNAPPY_LIBRARIES "${SNAPPY_INSTALL_DIR}/lib/libsnappy.lib") else() SET(SNAPPY_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) + set(SNAPPY_LIBRARIES "${SNAPPY_INSTALL_DIR}/lib/libsnappy.a") endif() ExternalProject_Add( @@ -33,35 +40,26 @@ ExternalProject_Add( PREFIX ${SNAPPY_SOURCES_DIR} UPDATE_COMMAND "" CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} - -DCMAKE_C_FLAGS_DEBUG=${CMAKE_C_FLAGS_DEBUG} - -DCMAKE_C_FLAGS_RELEASE=${CMAKE_C_FLAGS_RELEASE} - -DCMAKE_CXX_FLAGS=${SNAPPY_CMAKE_CXX_FLAGS} - -DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE} - -DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG} - -DCMAKE_INSTALL_PREFIX=${SNAPPY_INSTALL_DIR} - -DCMAKE_INSTALL_LIBDIR=${SNAPPY_INSTALL_DIR}/lib - -DCMAKE_POSITION_INDEPENDENT_CODE=ON - -DBUILD_TESTING=OFF - -DSNAPPY_BUILD_TESTS:BOOL=OFF - -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE} - ${EXTERNAL_OPTIONAL_ARGS} + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} + -DCMAKE_C_FLAGS_DEBUG=${CMAKE_C_FLAGS_DEBUG} + -DCMAKE_C_FLAGS_RELEASE=${CMAKE_C_FLAGS_RELEASE} + -DCMAKE_CXX_FLAGS=${SNAPPY_CMAKE_CXX_FLAGS} + -DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE} + -DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG} + -DCMAKE_INSTALL_PREFIX=${SNAPPY_INSTALL_DIR} + -DCMAKE_INSTALL_LIBDIR=${SNAPPY_INSTALL_DIR}/lib + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DBUILD_TESTING=OFF + -DSNAPPY_BUILD_TESTS:BOOL=OFF + -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE} + ${EXTERNAL_OPTIONAL_ARGS} CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${SNAPPY_INSTALL_DIR} - -DCMAKE_INSTALL_LIBDIR:PATH=${SNAPPY_INSTALL_DIR}/lib - -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON - -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE} + -DCMAKE_INSTALL_LIBDIR:PATH=${SNAPPY_INSTALL_DIR}/lib + -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON + -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE} + BUILD_BYPRODUCTS ${SNAPPY_LIBRARIES} ) -IF(WIN32) - IF(NOT EXISTS "${SNAPPY_INSTALL_DIR}/lib/libsnappy.lib") - add_custom_command(TARGET extern_snappy POST_BUILD - COMMAND cmake -E copy ${SNAPPY_INSTALL_DIR}/lib/snappy.lib ${SNAPPY_INSTALL_DIR}/lib/libsnappy.lib - ) - ENDIF() - set(SNAPPY_LIBRARIES "${SNAPPY_INSTALL_DIR}/lib/libsnappy.lib") -else(WIN32) - set(SNAPPY_LIBRARIES "${SNAPPY_INSTALL_DIR}/lib/libsnappy.a") -endif (WIN32) add_library(snappy STATIC IMPORTED GLOBAL) set_property(TARGET snappy PROPERTY IMPORTED_LOCATION ${SNAPPY_LIBRARIES}) diff --git a/cmake/external/warpctc.cmake b/cmake/external/warpctc.cmake index 6597e259aa..532ebaaf5c 100644 --- a/cmake/external/warpctc.cmake +++ b/cmake/external/warpctc.cmake @@ -32,6 +32,14 @@ SET(WARPCTC_INCLUDE_DIR "${WARPCTC_INSTALL_DIR}/include" SET(WARPCTC_LIB_DIR "${WARPCTC_INSTALL_DIR}/lib" CACHE PATH "Warp-ctc Library Directory" FORCE) +IF(WIN32) + SET(WARPCTC_LIBRARIES "${WARPCTC_INSTALL_DIR}/bin/warpctc${CMAKE_SHARED_LIBRARY_SUFFIX}" + CACHE FILEPATH "Warp-ctc Library" FORCE) +else(WIN32) + SET(WARPCTC_LIBRARIES "${WARPCTC_INSTALL_DIR}/lib/libwarpctc${CMAKE_SHARED_LIBRARY_SUFFIX}" + CACHE FILEPATH "Warp-ctc Library" FORCE) +ENDIF(WIN32) + IF(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR WIN32) SET(USE_OMP OFF) ELSE() @@ -59,7 +67,7 @@ if(WITH_ASCEND OR WITH_ASCEND_CL) -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} -DCMAKE_C_FLAGS_DEBUG=${CMAKE_C_FLAGS_DEBUG} -DCMAKE_C_FLAGS_RELEASE=${CMAKE_C_FLAGS_RELEASE} - "-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}" + -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} -DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE} -DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG} -DCMAKE_INSTALL_PREFIX=${WARPCTC_INSTALL_DIR} @@ -76,6 +84,7 @@ if(WITH_ASCEND OR WITH_ASCEND_CL) CMAKE_CACHE_ARGS -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE} -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON -DCMAKE_INSTALL_PREFIX:PATH=${WARPCTC_INSTALL_DIR} + BUILD_BYPRODUCTS ${WARPCTC_LIBRARIES} ) else() if(WIN32) @@ -125,18 +134,10 @@ else() CMAKE_CACHE_ARGS -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE} -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON -DCMAKE_INSTALL_PREFIX:PATH=${WARPCTC_INSTALL_DIR} + BUILD_BYPRODUCTS ${WARPCTC_LIBRARIES} ) endif() - -IF(WIN32) - SET(WARPCTC_LIBRARIES "${WARPCTC_INSTALL_DIR}/bin/warpctc${CMAKE_SHARED_LIBRARY_SUFFIX}" - CACHE FILEPATH "Warp-ctc Library" FORCE) -else(WIN32) - SET(WARPCTC_LIBRARIES "${WARPCTC_INSTALL_DIR}/lib/libwarpctc${CMAKE_SHARED_LIBRARY_SUFFIX}" - CACHE FILEPATH "Warp-ctc Library" FORCE) -ENDIF(WIN32) - MESSAGE(STATUS "warp-ctc library: ${WARPCTC_LIBRARIES}") get_filename_component(WARPCTC_LIBRARY_PATH ${WARPCTC_LIBRARIES} DIRECTORY) INCLUDE_DIRECTORIES(${WARPCTC_INCLUDE_DIR}) # For warpctc code to include its headers. diff --git a/cmake/external/xbyak.cmake b/cmake/external/xbyak.cmake index 610a692ef1..eabcabf743 100644 --- a/cmake/external/xbyak.cmake +++ b/cmake/external/xbyak.cmake @@ -46,7 +46,9 @@ ExternalProject_Add( SOURCE_DIR ${XBYAK_SOURCE_DIR} # UPDATE_COMMAND "" CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${XBYAK_INSTALL_ROOT} + -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE} CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${XBYAK_INSTALL_ROOT} + -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE} ) add_library(xbyak INTERFACE) diff --git a/cmake/external/xpu.cmake b/cmake/external/xpu.cmake index a8c33618a6..03b4801e2c 100644 --- a/cmake/external/xpu.cmake +++ b/cmake/external/xpu.cmake @@ -71,6 +71,8 @@ ExternalProject_Add( UPDATE_COMMAND "" CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${XPU_INSTALL_ROOT} CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${XPU_INSTALL_ROOT} + BUILD_BYPRODUCTS ${XPU_API_LIB} + BUILD_BYPRODUCTS ${XPU_RT_LIB} ) INCLUDE_DIRECTORIES(${XPU_INC_DIR}) diff --git a/cmake/external/xxhash.cmake b/cmake/external/xxhash.cmake index bdd7df190f..0279d4e2a8 100644 --- a/cmake/external/xxhash.cmake +++ b/cmake/external/xxhash.cmake @@ -21,10 +21,7 @@ set(XXHASH_INCLUDE_DIR "${XXHASH_INSTALL_DIR}/include") set(XXHASH_REPOSITORY ${GIT_URL}/Cyan4973/xxHash.git) set(XXHASH_TAG v0.6.5) -cache_third_party(extern_xxhash - REPOSITORY ${XXHASH_REPOSITORY} - TAG ${XXHASH_TAG} - DIR XXHASH_SOURCE_DIR) +INCLUDE_DIRECTORIES(${XXHASH_INCLUDE_DIR}) IF(APPLE) SET(BUILD_CMD sed -i \"\" "s/-Wstrict-prototypes -Wundef/-Wstrict-prototypes -Wundef -fPIC/g" ${XXHASH_SOURCE_DIR}/Makefile && make lib) @@ -32,6 +29,17 @@ ELSEIF(UNIX) SET(BUILD_CMD sed -i "s/-Wstrict-prototypes -Wundef/-Wstrict-prototypes -Wundef -fPIC/g" ${XXHASH_SOURCE_DIR}/Makefile && make lib) ENDIF() +if (WIN32) + set(XXHASH_LIBRARIES "${XXHASH_INSTALL_DIR}/lib/xxhash.lib") +else() + set(XXHASH_LIBRARIES "${XXHASH_INSTALL_DIR}/lib/libxxhash.a") +endif () + +cache_third_party(extern_xxhash + REPOSITORY ${XXHASH_REPOSITORY} + TAG ${XXHASH_TAG} + DIR XXHASH_SOURCE_DIR) + if(WIN32) ExternalProject_Add( extern_xxhash @@ -54,6 +62,7 @@ if(WIN32) -DBUILD_SHARED_LIBS=OFF ${OPTIONAL_CACHE_ARGS} TEST_COMMAND "" + BUILD_BYPRODUCTS ${XXHASH_LIBRARIES} ) else() ExternalProject_Add( @@ -68,16 +77,10 @@ else() BUILD_COMMAND ${BUILD_CMD} INSTALL_COMMAND make PREFIX=${XXHASH_INSTALL_DIR} install TEST_COMMAND "" + BUILD_BYPRODUCTS ${XXHASH_LIBRARIES} ) endif() -if (WIN32) - set(XXHASH_LIBRARIES "${XXHASH_INSTALL_DIR}/lib/xxhash.lib") -else() - set(XXHASH_LIBRARIES "${XXHASH_INSTALL_DIR}/lib/libxxhash.a") -endif () -INCLUDE_DIRECTORIES(${XXHASH_INCLUDE_DIR}) - add_library(xxhash STATIC IMPORTED GLOBAL) set_property(TARGET xxhash PROPERTY IMPORTED_LOCATION ${XXHASH_LIBRARIES}) include_directories(${XXHASH_INCLUDE_DIR}) diff --git a/cmake/external/zlib.cmake b/cmake/external/zlib.cmake index 4464787a0c..f1a015f630 100644 --- a/cmake/external/zlib.cmake +++ b/cmake/external/zlib.cmake @@ -25,6 +25,12 @@ set(ZLIB_TAG v1.2.8) INCLUDE_DIRECTORIES(${ZLIB_INCLUDE_DIR}) # For zlib code to include its own headers. INCLUDE_DIRECTORIES(${THIRD_PARTY_PATH}/install) # For Paddle code to include zlib.h. +IF(WIN32) + SET(ZLIB_LIBRARIES "${ZLIB_INSTALL_DIR}/lib/zlibstatic.lib" CACHE FILEPATH "zlib library." FORCE) +ELSE(WIN32) + SET(ZLIB_LIBRARIES "${ZLIB_INSTALL_DIR}/lib/libz.a" CACHE FILEPATH "zlib library." FORCE) +ENDIF(WIN32) + cache_third_party(extern_zlib REPOSITORY ${ZLIB_REPOSITORY} TAG ${ZLIB_TAG} @@ -51,12 +57,8 @@ ExternalProject_Add( CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${ZLIB_INSTALL_DIR} -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE} + BUILD_BYPRODUCTS ${ZLIB_LIBRARIES} ) -IF(WIN32) - SET(ZLIB_LIBRARIES "${ZLIB_INSTALL_DIR}/lib/zlibstatic.lib" CACHE FILEPATH "zlib library." FORCE) -ELSE(WIN32) - SET(ZLIB_LIBRARIES "${ZLIB_INSTALL_DIR}/lib/libz.a" CACHE FILEPATH "zlib library." FORCE) -ENDIF(WIN32) ADD_LIBRARY(zlib STATIC IMPORTED GLOBAL) SET_PROPERTY(TARGET zlib PROPERTY IMPORTED_LOCATION ${ZLIB_LIBRARIES}) diff --git a/paddle/scripts/paddle_build.bat b/paddle/scripts/paddle_build.bat index ede4003bd8..09df6a621f 100644 --- a/paddle/scripts/paddle_build.bat +++ b/paddle/scripts/paddle_build.bat @@ -72,7 +72,7 @@ if not defined INFERENCE_DEMO_INSTALL_DIR set INFERENCE_DEMO_INSTALL_DIR=%cache_ if not defined LOG_LEVEL set LOG_LEVEL=normal if not defined PRECISION_TEST set PRECISION_TEST=OFF if not defined NIGHTLY_MODE set PRECISION_TEST=OFF -if not defined retry_times set retry_times=2 +if not defined retry_times set retry_times=3 if not defined PYTHON_ROOT set PYTHON_ROOT=C:\Python37 rem -------set cache build directory----------- @@ -193,7 +193,7 @@ rem ------Build windows avx whl package------ set WITH_AVX=ON set ON_INFER=OFF set CUDA_ARCH_NAME=All -set retry_times=3 +set retry_times=4 call :cmake || goto cmake_error call :build || goto build_error @@ -205,7 +205,7 @@ rem ------Build windows no-avx whl package------ set WITH_AVX=OFF set ON_INFER=OFF set CUDA_ARCH_NAME=All -set retry_times=3 +set retry_times=4 call :cmake || goto cmake_error call :build || goto build_error @@ -366,18 +366,26 @@ echo ======================================== for /F %%# in ('wmic cpu get NumberOfLogicalProcessors^|findstr [0-9]') do set /a PARALLEL_PROJECT_COUNT=%%#*4/5 echo "PARALLEL PROJECT COUNT is %PARALLEL_PROJECT_COUNT%" + set build_times=1 +rem MSbuild will build third_party first to improve compiler stability. +if NOT %GENERATOR% == "Ninja" ( + goto :build_tp +) else ( + goto :build_paddle +) + :build_tp echo Build third_party the %build_times% time: - if %GENERATOR% == "Ninja" ( ninja third_party ) else ( MSBuild /m /p:PreferredToolArchitecture=x64 /p:Configuration=Release /verbosity:%LOG_LEVEL% third_party.vcxproj ) + if %ERRORLEVEL% NEQ 0 ( set /a build_times=%build_times%+1 - if %build_times% GTR %retry_times% ( + if %build_times% GEQ %retry_times% ( exit /b 7 ) else ( echo Build third_party failed, will retry! @@ -430,7 +438,7 @@ if %GENERATOR% == "Ninja" ( if %ERRORLEVEL% NEQ 0 ( set /a build_times=%build_times%+1 - if %build_times% GTR %retry_times% ( + if %build_times% GEQ %retry_times% ( exit /b 7 ) else ( echo Build Paddle failed, will retry! -- GitLab