From aab4d6e4adbcdfbb94371483d147d6fc44f5d08b Mon Sep 17 00:00:00 2001 From: Zhou Wei <1183042833@qq.com> Date: Mon, 9 Aug 2021 14:23:26 +0800 Subject: [PATCH] Increase the speed of incremental compilation (#34616) --- CMakeLists.txt | 2 +- cmake/external/gflags.cmake | 1 + cmake/external/glog.cmake | 1 + cmake/external/mkldnn.cmake | 55 +++++++++--------- cmake/external/protobuf.cmake | 20 +++---- cmake/external/pybind11.cmake | 1 + paddle/fluid/pybind/CMakeLists.txt | 89 ++++++++++++++---------------- 7 files changed, 81 insertions(+), 88 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4f6ed9de30e..83191254f1a 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -130,7 +130,7 @@ if(WIN32) # NOTE(zhouwei25): GPU compile have too high memory utilization when parallel compiling, # For Visual Studio generators, /MP should be added. # For other generators like Ninja, it is not need to add /MP. - if("${CMAKE_GENERATOR}" STREQUAL "Visual Studio" AND NOT WITH_GPU) + if(CMAKE_GENERATOR MATCHES "Visual Studio" AND NOT WITH_GPU) math(EXPR PROCESS_MAX "${CPU_CORES} * 2 / 3") set(${flag_var} "${${flag_var}} /MP${PROCESS_MAX}") endif() diff --git a/cmake/external/gflags.cmake b/cmake/external/gflags.cmake index 8360761de6f..0f9739014d5 100644 --- a/cmake/external/gflags.cmake +++ b/cmake/external/gflags.cmake @@ -41,6 +41,7 @@ ExternalProject_Add( ${SHALLOW_CLONE} "${GFLAGS_DOWNLOAD_CMD}" PREFIX ${GFLAGS_PREFIX_DIR} + UPDATE_COMMAND "" SOURCE_DIR ${GFLAGS_SOURCE_DIR} BUILD_COMMAND ${BUILD_COMMAND} INSTALL_COMMAND ${INSTALL_COMMAND} diff --git a/cmake/external/glog.cmake b/cmake/external/glog.cmake index d2bb1e62e83..b9dbe90a92e 100644 --- a/cmake/external/glog.cmake +++ b/cmake/external/glog.cmake @@ -45,6 +45,7 @@ ExternalProject_Add( DEPENDS gflags PREFIX ${GLOG_PREFIX_DIR} SOURCE_DIR ${GLOG_SOURCE_DIR} + UPDATE_COMMAND "" CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_FLAGS=${GLOG_CMAKE_CXX_FLAGS} diff --git a/cmake/external/mkldnn.cmake b/cmake/external/mkldnn.cmake index 9963237ff18..0a3b64e5d56 100644 --- a/cmake/external/mkldnn.cmake +++ b/cmake/external/mkldnn.cmake @@ -79,22 +79,10 @@ ExternalProject_Add( -DCMAKE_CXX_FLAGS=${MKLDNN_CXXFLAG} -DDNNL_BUILD_TESTS=OFF -DDNNL_BUILD_EXAMPLES=OFF CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${MKLDNN_INSTALL_DIR} - BUILD_BYPRODUCTS ${MKLDNN_LIB} ) -ADD_LIBRARY(shared_mkldnn SHARED IMPORTED GLOBAL) -SET_PROPERTY(TARGET shared_mkldnn PROPERTY IMPORTED_LOCATION ${MKLDNN_LIB}) -ADD_DEPENDENCIES(shared_mkldnn ${MKLDNN_PROJECT}) MESSAGE(STATUS "MKLDNN library: ${MKLDNN_LIB}") add_definitions(-DPADDLE_WITH_MKLDNN) - -# generate a static dummy target to track mkldnn dependencies -# for cc_library(xxx SRCS xxx.c DEPS mkldnn) -generate_dummy_static_lib(LIB_NAME "mkldnn" GENERATOR "mkldnn.cmake") - -TARGET_LINK_LIBRARIES(mkldnn ${MKLDNN_LIB} ${MKLML_IOMP_LIB}) -ADD_DEPENDENCIES(mkldnn ${MKLDNN_PROJECT}) - # copy the real so.0 lib to install dir # it can be directly contained in wheel or capi if(WIN32) @@ -102,26 +90,33 @@ if(WIN32) file(TO_NATIVE_PATH ${MKLDNN_INSTALL_DIR} NATIVE_MKLDNN_INSTALL_DIR) file(TO_NATIVE_PATH ${MKLDNN_SHARED_LIB} NATIVE_MKLDNN_SHARED_LIB) - ADD_CUSTOM_COMMAND(TARGET ${MKLDNN_PROJECT} POST_BUILD - COMMAND (copy ${NATIVE_MKLDNN_INSTALL_DIR}\\bin\\dnnl.dll ${NATIVE_MKLDNN_SHARED_LIB} /Y)) - add_custom_command(TARGET ${MKLDNN_PROJECT} POST_BUILD VERBATIM - COMMAND dumpbin /exports ${MKLDNN_INSTALL_DIR}/bin/mkldnn.dll > ${MKLDNN_INSTALL_DIR}/bin/exports.txt) - add_custom_command(TARGET ${MKLDNN_PROJECT} POST_BUILD VERBATIM - COMMAND echo LIBRARY mkldnn > ${MKLDNN_INSTALL_DIR}/bin/mkldnn.def) - add_custom_command(TARGET ${MKLDNN_PROJECT} POST_BUILD VERBATIM - COMMAND echo EXPORTS >> ${MKLDNN_INSTALL_DIR}/bin/mkldnn.def) - add_custom_command(TARGET ${MKLDNN_PROJECT} POST_BUILD VERBATIM - COMMAND echo off && (for /f "skip=19 tokens=4" %A in (${MKLDNN_INSTALL_DIR}/bin/exports.txt) do echo %A >> ${MKLDNN_INSTALL_DIR}/bin/mkldnn.def) && echo on) - add_custom_command(TARGET ${MKLDNN_PROJECT} POST_BUILD VERBATIM - COMMAND lib /def:${MKLDNN_INSTALL_DIR}/bin/mkldnn.def /out:${MKLDNN_INSTALL_DIR}/bin/mkldnn.lib /machine:x64) + + ADD_CUSTOM_COMMAND(OUTPUT ${MKLDNN_LIB} + COMMAND (copy ${NATIVE_MKLDNN_INSTALL_DIR}\\bin\\dnnl.dll ${NATIVE_MKLDNN_SHARED_LIB} /Y) + COMMAND dumpbin /exports ${MKLDNN_INSTALL_DIR}/bin/mkldnn.dll > ${MKLDNN_INSTALL_DIR}/bin/exports.txt + COMMAND echo LIBRARY mkldnn > ${MKLDNN_INSTALL_DIR}/bin/mkldnn.def + COMMAND echo EXPORTS >> ${MKLDNN_INSTALL_DIR}/bin/mkldnn.def + COMMAND echo off && (for /f "skip=19 tokens=4" %A in (${MKLDNN_INSTALL_DIR}/bin/exports.txt) do echo %A >> ${MKLDNN_INSTALL_DIR}/bin/mkldnn.def) && echo on + COMMAND lib /def:${MKLDNN_INSTALL_DIR}/bin/mkldnn.def /out:${MKLDNN_LIB} /machine:x64 + COMMENT "Generate mkldnn.lib manually--->" + DEPENDS ${MKLDNN_PROJECT} + VERBATIM) + ADD_CUSTOM_TARGET(mkldnn_cmd ALL DEPENDS ${MKLDNN_LIB}) else(WIN32) SET(MKLDNN_SHARED_LIB ${MKLDNN_INSTALL_DIR}/libmkldnn.so.0) SET(MKLDNN_SHARED_LIB_1 ${MKLDNN_INSTALL_DIR}/libdnnl.so.1) SET(MKLDNN_SHARED_LIB_2 ${MKLDNN_INSTALL_DIR}/libdnnl.so.2) - ADD_CUSTOM_COMMAND(TARGET ${MKLDNN_PROJECT} POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy ${MKLDNN_LIB} ${MKLDNN_SHARED_LIB}) - ADD_CUSTOM_COMMAND(TARGET ${MKLDNN_PROJECT} POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy ${MKLDNN_LIB} ${MKLDNN_SHARED_LIB_1}) - ADD_CUSTOM_COMMAND(TARGET ${MKLDNN_PROJECT} POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy ${MKLDNN_LIB} ${MKLDNN_SHARED_LIB_2}) + ADD_CUSTOM_COMMAND(OUTPUT ${MKLDNN_SHARED_LIB_2} + COMMAND ${CMAKE_COMMAND} -E copy ${MKLDNN_LIB} ${MKLDNN_SHARED_LIB} + COMMAND ${CMAKE_COMMAND} -E copy ${MKLDNN_LIB} ${MKLDNN_SHARED_LIB_1} + COMMAND ${CMAKE_COMMAND} -E copy ${MKLDNN_LIB} ${MKLDNN_SHARED_LIB_2} + DEPENDS ${MKLDNN_PROJECT}) + ADD_CUSTOM_TARGET(mkldnn_cmd ALL DEPENDS ${MKLDNN_SHARED_LIB_2}) endif(WIN32) + +# generate a static dummy target to track mkldnn dependencies +# for cc_library(xxx SRCS xxx.c DEPS mkldnn) +generate_dummy_static_lib(LIB_NAME "mkldnn" GENERATOR "mkldnn.cmake") + +TARGET_LINK_LIBRARIES(mkldnn ${MKLDNN_LIB} ${MKLML_IOMP_LIB}) +ADD_DEPENDENCIES(mkldnn ${MKLDNN_PROJECT} mkldnn_cmd) diff --git a/cmake/external/protobuf.cmake b/cmake/external/protobuf.cmake index a2b6ddadb62..8a9bc6e42c1 100644 --- a/cmake/external/protobuf.cmake +++ b/cmake/external/protobuf.cmake @@ -198,16 +198,16 @@ FUNCTION(build_protobuf TARGET_NAME BUILD_FOR_HOST) "-Dprotobuf_MSVC_STATIC_RUNTIME=${MSVC_STATIC_CRT}") ENDIF() -if(WITH_ASCEND AND NOT WITH_ASCEND_CXX11) - SET(PROTOBUF_REPOSITORY https://gitee.com/tianjianhe/protobuf.git) - SET(PROTOBUF_TAG v3.8.0) -elseif(WITH_ASCEND_CL AND NOT WITH_ASCEND_CXX11) - SET(PROTOBUF_REPOSITORY https://gitee.com/tianjianhe/protobuf.git) - SET(PROTOBUF_TAG v3.8.0) -else() - SET(PROTOBUF_REPOSITORY ${GIT_URL}/protocolbuffers/protobuf.git) - SET(PROTOBUF_TAG 9f75c5aa851cd877fb0d93ccc31b8567a6706546) -endif() + if(WITH_ASCEND AND NOT WITH_ASCEND_CXX11) + SET(PROTOBUF_REPOSITORY https://gitee.com/tianjianhe/protobuf.git) + SET(PROTOBUF_TAG v3.8.0) + elseif(WITH_ASCEND_CL AND NOT WITH_ASCEND_CXX11) + SET(PROTOBUF_REPOSITORY https://gitee.com/tianjianhe/protobuf.git) + SET(PROTOBUF_TAG v3.8.0) + else() + SET(PROTOBUF_REPOSITORY ${GIT_URL}/protocolbuffers/protobuf.git) + SET(PROTOBUF_TAG 9f75c5aa851cd877fb0d93ccc31b8567a6706546) + endif() cache_third_party(${TARGET_NAME} REPOSITORY ${PROTOBUF_REPOSITORY} diff --git a/cmake/external/pybind11.cmake b/cmake/external/pybind11.cmake index 69bd68c2778..353cb5c72fd 100644 --- a/cmake/external/pybind11.cmake +++ b/cmake/external/pybind11.cmake @@ -39,6 +39,7 @@ ExternalProject_Add( # to be modified without triggering incremental compilation, and the # third-party library version changes cannot be incorporated. # reference: https://cmake.org/cmake/help/latest/module/ExternalProject.html + UPDATE_COMMAND "" CONFIGURE_COMMAND "" BUILD_COMMAND "" INSTALL_COMMAND "" diff --git a/paddle/fluid/pybind/CMakeLists.txt b/paddle/fluid/pybind/CMakeLists.txt index f362808a4b9..d6693e8abaa 100644 --- a/paddle/fluid/pybind/CMakeLists.txt +++ b/paddle/fluid/pybind/CMakeLists.txt @@ -124,23 +124,20 @@ if(WITH_PYTHON) set(impl_file ${CMAKE_SOURCE_DIR}/paddle/fluid/pybind/op_function_impl.h) set(tmp_impl_file ${impl_file}.tmp) + set(OP_IMPL_DEPS op_function_generator) if(WIN32) if("${CMAKE_GENERATOR}" STREQUAL "Ninja") - set(op_function_generator_path "${CMAKE_CURRENT_BINARY_DIR}") + set(op_impl_path "${CMAKE_CURRENT_BINARY_DIR}") else() - set(op_function_generator_path "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}") + set(op_impl_path "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}") endif() - file(TO_NATIVE_PATH ${op_function_generator_path} op_function_generator_path) - file(TO_NATIVE_PATH ${impl_file} impl_file) - file(TO_NATIVE_PATH ${tmp_impl_file} tmp_impl_file) file(WRITE ${CMAKE_BINARY_DIR}/paddle/fluid/pybind/op_function_generator_retry.bat "" "set build_times=1\n" ":retry\n" "ECHO op_function_generator run %build_times% time\n" - "if exist ${tmp_impl_file} del ${tmp_impl_file}\n" "taskkill /f /im op_function_generator.exe 2>NUL\n" - "${op_function_generator_path}\\op_function_generator.exe ${tmp_impl_file}\n" + "${op_impl_path}/op_function_generator.exe ${tmp_impl_file}\n" "if %ERRORLEVEL% NEQ 0 (\n" " set /a build_times=%build_times%+1\n" " if %build_times% GEQ 10 (\n" @@ -151,63 +148,61 @@ if(WITH_PYTHON) ")\n" "exit /b 0") - add_custom_command(TARGET op_function_generator POST_BUILD - COMMAND ${CMAKE_BINARY_DIR}/paddle/fluid/pybind/op_function_generator_retry.bat - COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_impl_file} ${impl_file} - COMMENT "copy_if_different ${tmp_impl_file} to ${impl_file}" - ) - if(${CBLAS_PROVIDER} STREQUAL MKLML) - add_custom_command(TARGET op_function_generator - PRE_LINK - COMMAND ${CMAKE_COMMAND} -E copy ${MKLML_SHARED_LIB} ${op_function_generator_path} - COMMAND ${CMAKE_COMMAND} -E copy ${MKLML_SHARED_IOMP_LIB} ${op_function_generator_path} - ) + ADD_CUSTOM_COMMAND(OUTPUT ${op_impl_path}/libiomp5md.dll + COMMAND ${CMAKE_COMMAND} -E copy ${MKLML_SHARED_IOMP_LIB} ${op_impl_path} + DEPENDS mklml) + list(APPEND OP_IMPL_DEPS ${op_impl_path}/libiomp5md.dll) else(${CBLAS_PROVIDER} STREQUAL EXTERN_OPENBLAS) - add_custom_command(TARGET op_function_generator - PRE_LINK - COMMAND ${CMAKE_COMMAND} -E copy ${OPENBLAS_SHARED_LIB} ${op_function_generator_path} - ) + ADD_CUSTOM_COMMAND(OUTPUT ${op_impl_path}/openblas.dll + COMMAND ${CMAKE_COMMAND} -E copy ${OPENBLAS_SHARED_LIB} ${op_impl_path} + DEPENDS extern_openblas) + list(APPEND OP_IMPL_DEPS ${op_impl_path}/openblas.dll) endif() if(WITH_MKLDNN) - add_custom_command(TARGET op_function_generator - PRE_LINK - COMMAND ${CMAKE_COMMAND} -E copy ${MKLDNN_SHARED_LIB} ${op_function_generator_path} - ) + ADD_CUSTOM_COMMAND(OUTPUT ${op_impl_path}/mkldnn.dll + COMMAND ${CMAKE_COMMAND} -E copy ${MKLDNN_SHARED_LIB} ${op_impl_path} + DEPENDS mkldnn) + list(APPEND OP_IMPL_DEPS ${op_impl_path}/mkldnn.dll) endif() + + add_custom_command(OUTPUT ${impl_file} + COMMAND ${CMAKE_BINARY_DIR}/paddle/fluid/pybind/op_function_generator_retry.bat + COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_impl_file} ${impl_file} + COMMENT "copy_if_different ${tmp_impl_file} to ${impl_file}" + DEPENDS ${OP_IMPL_DEPS}) else(WIN32) # If there are no *.so in /usr/lib or LD_LIBRARY_PATH, # copy these *.so to current directory and append current directory to # LD_LIBRARY_PATH. This is different with Windows platformm, which search # *.dll in current directory automatically. - add_custom_command(TARGET op_function_generator - POST_BUILD + if(WITH_MKLML) + ADD_CUSTOM_COMMAND(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/libiomp5.so + COMMAND ${CMAKE_COMMAND} -E copy ${MKLML_SHARED_IOMP_LIB} ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS mklml) + list(APPEND OP_IMPL_DEPS ${CMAKE_CURRENT_BINARY_DIR}/libiomp5.so) + endif() + if(WITH_MKLDNN) + ADD_CUSTOM_COMMAND(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/libdnnl.so.0 + COMMAND ${CMAKE_COMMAND} -E copy ${MKLDNN_SHARED_LIB} ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS mkldnn) + list(APPEND OP_IMPL_DEPS ${CMAKE_CURRENT_BINARY_DIR}/libdnnl.so.0) + endif() + add_custom_command(OUTPUT ${impl_file} COMMAND ${CMAKE_COMMAND} -E env "LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH}:." "${CMAKE_CURRENT_BINARY_DIR}/op_function_generator" "${tmp_impl_file}" COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_impl_file} ${impl_file} COMMENT "copy_if_different ${tmp_impl_file} to ${impl_file}" - VERBATIM - ) - if(WITH_MKL) - add_custom_command(TARGET op_function_generator - PRE_LINK - COMMAND ${CMAKE_COMMAND} -E copy ${MKLML_SHARED_LIB} ${CMAKE_CURRENT_BINARY_DIR} - COMMAND ${CMAKE_COMMAND} -E copy ${MKLML_SHARED_IOMP_LIB} ${CMAKE_CURRENT_BINARY_DIR} - ) - endif(WITH_MKL) - if(WITH_MKLDNN) - add_custom_command(TARGET op_function_generator - PRE_LINK - COMMAND ${CMAKE_COMMAND} -E copy ${MKLDNN_SHARED_LIB} ${CMAKE_CURRENT_BINARY_DIR} - ) - endif(WITH_MKLDNN) + DEPENDS ${OP_IMPL_DEPS} + VERBATIM) endif(WIN32) + add_custom_target(op_function_generator_cmd ALL DEPENDS ${impl_file}) cc_library(paddle_pybind SHARED - SRCS ${PYBIND_SRCS} - DEPS ${PYBIND_DEPS} - ${GLOB_OP_LIB} ${GLOB_OPERATOR_DEPS}) + SRCS ${PYBIND_SRCS} + DEPS ${PYBIND_DEPS} ${GLOB_OP_LIB} ${GLOB_OPERATOR_DEPS}) + if(NOT APPLE AND NOT WIN32) target_link_libraries(paddle_pybind rt) endif(NOT APPLE AND NOT WIN32) @@ -218,5 +213,5 @@ if(WITH_PYTHON) get_property (os_dependency_modules GLOBAL PROPERTY OS_DEPENDENCY_MODULES) target_link_libraries(paddle_pybind ${os_dependency_modules}) - add_dependencies(paddle_pybind op_function_generator) + add_dependencies(paddle_pybind op_function_generator_cmd) endif(WITH_PYTHON) -- GitLab