Increase the speed of incremental compilation (#34616)

aab4d6e4 · zhouweiwei2014 · GitHub · 4c1ba73f · aab4d6e4 · aab4d6e4
7 changed file
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -130,7 +130,7 @@ if(WIN32)
        # NOTE(zhouwei25): GPU compile have too high memory utilization when parallel compiling,
        # For Visual Studio generators, /MP should be added. 
        # For other generators like Ninja, it is not need to add /MP.
-        if("${CMAKE_GENERATOR}" STREQUAL "Visual Studio" AND NOT WITH_GPU)
+        if(CMAKE_GENERATOR MATCHES "Visual Studio" AND NOT WITH_GPU)
            math(EXPR PROCESS_MAX "${CPU_CORES} * 2 / 3")
            set(${flag_var} "${${flag_var}} /MP${PROCESS_MAX}")
        endif()

--- a/cmake/external/gflags.cmake
+++ b/cmake/external/gflags.cmake
@@ -41,6 +41,7 @@ ExternalProject_Add(
    ${SHALLOW_CLONE}
    "${GFLAGS_DOWNLOAD_CMD}"
    PREFIX          ${GFLAGS_PREFIX_DIR}
+    UPDATE_COMMAND  ""
    SOURCE_DIR      ${GFLAGS_SOURCE_DIR}
    BUILD_COMMAND   ${BUILD_COMMAND}
    INSTALL_COMMAND ${INSTALL_COMMAND}

--- a/cmake/external/glog.cmake
+++ b/cmake/external/glog.cmake
@@ -45,6 +45,7 @@ ExternalProject_Add(
    DEPENDS         gflags
    PREFIX          ${GLOG_PREFIX_DIR}
    SOURCE_DIR      ${GLOG_SOURCE_DIR}
+    UPDATE_COMMAND  ""
    CMAKE_ARGS      -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
                    -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
                    -DCMAKE_CXX_FLAGS=${GLOG_CMAKE_CXX_FLAGS}

--- a/cmake/external/mkldnn.cmake
+++ b/cmake/external/mkldnn.cmake
@@ -79,22 +79,10 @@ ExternalProject_Add(
                        -DCMAKE_CXX_FLAGS=${MKLDNN_CXXFLAG}
                        -DDNNL_BUILD_TESTS=OFF -DDNNL_BUILD_EXAMPLES=OFF
    CMAKE_CACHE_ARGS    -DCMAKE_INSTALL_PREFIX:PATH=${MKLDNN_INSTALL_DIR}
-    BUILD_BYPRODUCTS    ${MKLDNN_LIB}
 )

-ADD_LIBRARY(shared_mkldnn SHARED IMPORTED GLOBAL)
-SET_PROPERTY(TARGET shared_mkldnn PROPERTY IMPORTED_LOCATION ${MKLDNN_LIB})
-ADD_DEPENDENCIES(shared_mkldnn ${MKLDNN_PROJECT})
 MESSAGE(STATUS "MKLDNN library: ${MKLDNN_LIB}")
 add_definitions(-DPADDLE_WITH_MKLDNN)
-
-# generate a static dummy target to track mkldnn dependencies
-# for cc_library(xxx SRCS xxx.c DEPS mkldnn)
-generate_dummy_static_lib(LIB_NAME "mkldnn" GENERATOR "mkldnn.cmake")
-
-TARGET_LINK_LIBRARIES(mkldnn ${MKLDNN_LIB} ${MKLML_IOMP_LIB})
-ADD_DEPENDENCIES(mkldnn ${MKLDNN_PROJECT})
-
 # copy the real so.0 lib to install dir
 # it can be directly contained in wheel or capi
 if(WIN32)
@@ -102,26 +90,33 @@ if(WIN32)

    file(TO_NATIVE_PATH ${MKLDNN_INSTALL_DIR} NATIVE_MKLDNN_INSTALL_DIR)
    file(TO_NATIVE_PATH ${MKLDNN_SHARED_LIB} NATIVE_MKLDNN_SHARED_LIB)
-    ADD_CUSTOM_COMMAND(TARGET ${MKLDNN_PROJECT} POST_BUILD
-        COMMAND (copy ${NATIVE_MKLDNN_INSTALL_DIR}\\bin\\dnnl.dll ${NATIVE_MKLDNN_SHARED_LIB} /Y))
-    add_custom_command(TARGET ${MKLDNN_PROJECT} POST_BUILD VERBATIM
-        COMMAND dumpbin /exports ${MKLDNN_INSTALL_DIR}/bin/mkldnn.dll > ${MKLDNN_INSTALL_DIR}/bin/exports.txt)
-    add_custom_command(TARGET ${MKLDNN_PROJECT} POST_BUILD VERBATIM
-        COMMAND echo LIBRARY mkldnn > ${MKLDNN_INSTALL_DIR}/bin/mkldnn.def)
-    add_custom_command(TARGET ${MKLDNN_PROJECT} POST_BUILD VERBATIM
-        COMMAND echo EXPORTS >> ${MKLDNN_INSTALL_DIR}/bin/mkldnn.def)
-    add_custom_command(TARGET ${MKLDNN_PROJECT} POST_BUILD VERBATIM
-        COMMAND echo off && (for /f "skip=19 tokens=4" %A in (${MKLDNN_INSTALL_DIR}/bin/exports.txt) do echo %A >> ${MKLDNN_INSTALL_DIR}/bin/mkldnn.def) && echo on)
-    add_custom_command(TARGET ${MKLDNN_PROJECT} POST_BUILD VERBATIM
-        COMMAND lib /def:${MKLDNN_INSTALL_DIR}/bin/mkldnn.def /out:${MKLDNN_INSTALL_DIR}/bin/mkldnn.lib /machine:x64)
+
+    ADD_CUSTOM_COMMAND(OUTPUT ${MKLDNN_LIB}
+        COMMAND (copy ${NATIVE_MKLDNN_INSTALL_DIR}\\bin\\dnnl.dll ${NATIVE_MKLDNN_SHARED_LIB} /Y)
+        COMMAND dumpbin /exports ${MKLDNN_INSTALL_DIR}/bin/mkldnn.dll > ${MKLDNN_INSTALL_DIR}/bin/exports.txt
+        COMMAND echo LIBRARY mkldnn > ${MKLDNN_INSTALL_DIR}/bin/mkldnn.def
+        COMMAND echo EXPORTS >> ${MKLDNN_INSTALL_DIR}/bin/mkldnn.def
+        COMMAND echo off && (for /f "skip=19 tokens=4" %A in (${MKLDNN_INSTALL_DIR}/bin/exports.txt) do echo %A >> ${MKLDNN_INSTALL_DIR}/bin/mkldnn.def) && echo on
+        COMMAND lib /def:${MKLDNN_INSTALL_DIR}/bin/mkldnn.def /out:${MKLDNN_LIB} /machine:x64
+        COMMENT "Generate mkldnn.lib manually--->"
+        DEPENDS ${MKLDNN_PROJECT}
+        VERBATIM)
+    ADD_CUSTOM_TARGET(mkldnn_cmd ALL DEPENDS ${MKLDNN_LIB})
 else(WIN32)
    SET(MKLDNN_SHARED_LIB ${MKLDNN_INSTALL_DIR}/libmkldnn.so.0)
    SET(MKLDNN_SHARED_LIB_1 ${MKLDNN_INSTALL_DIR}/libdnnl.so.1)
    SET(MKLDNN_SHARED_LIB_2 ${MKLDNN_INSTALL_DIR}/libdnnl.so.2)
-    ADD_CUSTOM_COMMAND(TARGET ${MKLDNN_PROJECT} POST_BUILD
-            COMMAND ${CMAKE_COMMAND} -E copy ${MKLDNN_LIB} ${MKLDNN_SHARED_LIB})
-    ADD_CUSTOM_COMMAND(TARGET ${MKLDNN_PROJECT} POST_BUILD
-            COMMAND ${CMAKE_COMMAND} -E copy ${MKLDNN_LIB} ${MKLDNN_SHARED_LIB_1})
-    ADD_CUSTOM_COMMAND(TARGET ${MKLDNN_PROJECT} POST_BUILD
-            COMMAND ${CMAKE_COMMAND} -E copy ${MKLDNN_LIB} ${MKLDNN_SHARED_LIB_2})
+    ADD_CUSTOM_COMMAND(OUTPUT ${MKLDNN_SHARED_LIB_2}
+        COMMAND ${CMAKE_COMMAND} -E copy ${MKLDNN_LIB} ${MKLDNN_SHARED_LIB}
+        COMMAND ${CMAKE_COMMAND} -E copy ${MKLDNN_LIB} ${MKLDNN_SHARED_LIB_1}
+        COMMAND ${CMAKE_COMMAND} -E copy ${MKLDNN_LIB} ${MKLDNN_SHARED_LIB_2}
+        DEPENDS ${MKLDNN_PROJECT})
+    ADD_CUSTOM_TARGET(mkldnn_cmd ALL DEPENDS ${MKLDNN_SHARED_LIB_2})
 endif(WIN32)
+
+# generate a static dummy target to track mkldnn dependencies
+# for cc_library(xxx SRCS xxx.c DEPS mkldnn)
+generate_dummy_static_lib(LIB_NAME "mkldnn" GENERATOR "mkldnn.cmake")
+
+TARGET_LINK_LIBRARIES(mkldnn ${MKLDNN_LIB} ${MKLML_IOMP_LIB})
+ADD_DEPENDENCIES(mkldnn ${MKLDNN_PROJECT} mkldnn_cmd)
--- a/cmake/external/protobuf.cmake
+++ b/cmake/external/protobuf.cmake
@@ -198,16 +198,16 @@ FUNCTION(build_protobuf TARGET_NAME BUILD_FOR_HOST)
            "-Dprotobuf_MSVC_STATIC_RUNTIME=${MSVC_STATIC_CRT}")
    ENDIF()

-if(WITH_ASCEND AND NOT WITH_ASCEND_CXX11)
-    SET(PROTOBUF_REPOSITORY  https://gitee.com/tianjianhe/protobuf.git)
-    SET(PROTOBUF_TAG         v3.8.0)
-elseif(WITH_ASCEND_CL AND NOT WITH_ASCEND_CXX11)
-    SET(PROTOBUF_REPOSITORY  https://gitee.com/tianjianhe/protobuf.git)
-    SET(PROTOBUF_TAG         v3.8.0)
-else()
-    SET(PROTOBUF_REPOSITORY  ${GIT_URL}/protocolbuffers/protobuf.git)
-    SET(PROTOBUF_TAG         9f75c5aa851cd877fb0d93ccc31b8567a6706546)
-endif()
+    if(WITH_ASCEND AND NOT WITH_ASCEND_CXX11)
+        SET(PROTOBUF_REPOSITORY  https://gitee.com/tianjianhe/protobuf.git)
+        SET(PROTOBUF_TAG         v3.8.0)
+    elseif(WITH_ASCEND_CL AND NOT WITH_ASCEND_CXX11)
+        SET(PROTOBUF_REPOSITORY  https://gitee.com/tianjianhe/protobuf.git)
+        SET(PROTOBUF_TAG         v3.8.0)
+    else()
+        SET(PROTOBUF_REPOSITORY  ${GIT_URL}/protocolbuffers/protobuf.git)
+        SET(PROTOBUF_TAG         9f75c5aa851cd877fb0d93ccc31b8567a6706546)
+    endif()

    cache_third_party(${TARGET_NAME}
        REPOSITORY    ${PROTOBUF_REPOSITORY}

--- a/cmake/external/pybind11.cmake
+++ b/cmake/external/pybind11.cmake
@@ -39,6 +39,7 @@ ExternalProject_Add(
        # to be modified without triggering incremental compilation, and the
        # third-party library version changes cannot be incorporated.
        # reference: https://cmake.org/cmake/help/latest/module/ExternalProject.html
+        UPDATE_COMMAND    ""
        CONFIGURE_COMMAND ""
        BUILD_COMMAND     ""
        INSTALL_COMMAND   ""

--- a/paddle/fluid/pybind/CMakeLists.txt
+++ b/paddle/fluid/pybind/CMakeLists.txt
@@ -124,23 +124,20 @@ if(WITH_PYTHON)
  set(impl_file ${CMAKE_SOURCE_DIR}/paddle/fluid/pybind/op_function_impl.h)
  set(tmp_impl_file ${impl_file}.tmp)

+  set(OP_IMPL_DEPS op_function_generator)
  if(WIN32)
    if("${CMAKE_GENERATOR}" STREQUAL "Ninja")
-      set(op_function_generator_path "${CMAKE_CURRENT_BINARY_DIR}")
+      set(op_impl_path "${CMAKE_CURRENT_BINARY_DIR}")
    else()
-      set(op_function_generator_path "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}")
+      set(op_impl_path "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}")
    endif()
-    file(TO_NATIVE_PATH ${op_function_generator_path} op_function_generator_path)
-    file(TO_NATIVE_PATH ${impl_file} impl_file)
-    file(TO_NATIVE_PATH ${tmp_impl_file} tmp_impl_file)

    file(WRITE ${CMAKE_BINARY_DIR}/paddle/fluid/pybind/op_function_generator_retry.bat ""
    "set build_times=1\n"
    ":retry\n"
    "ECHO op_function_generator run %build_times% time\n"
-    "if exist ${tmp_impl_file} del ${tmp_impl_file}\n"
    "taskkill /f /im op_function_generator.exe 2>NUL\n"
-    "${op_function_generator_path}\\op_function_generator.exe ${tmp_impl_file}\n"
+    "${op_impl_path}/op_function_generator.exe ${tmp_impl_file}\n"
    "if %ERRORLEVEL% NEQ 0 (\n"
    "    set /a build_times=%build_times%+1\n"
    "    if %build_times% GEQ 10 (\n"
@@ -151,63 +148,61 @@ if(WITH_PYTHON)
    ")\n"
    "exit /b 0")

-    add_custom_command(TARGET op_function_generator POST_BUILD
-          COMMAND ${CMAKE_BINARY_DIR}/paddle/fluid/pybind/op_function_generator_retry.bat
-          COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_impl_file} ${impl_file}
-          COMMENT "copy_if_different ${tmp_impl_file} to ${impl_file}"
-    )
-
    if(${CBLAS_PROVIDER} STREQUAL MKLML)
-      add_custom_command(TARGET op_function_generator
-            PRE_LINK
-            COMMAND ${CMAKE_COMMAND} -E copy ${MKLML_SHARED_LIB} ${op_function_generator_path}
-            COMMAND ${CMAKE_COMMAND} -E copy ${MKLML_SHARED_IOMP_LIB} ${op_function_generator_path}
-          )
+      ADD_CUSTOM_COMMAND(OUTPUT ${op_impl_path}/libiomp5md.dll
+        COMMAND ${CMAKE_COMMAND} -E copy ${MKLML_SHARED_IOMP_LIB} ${op_impl_path}
+        DEPENDS mklml)
+      list(APPEND OP_IMPL_DEPS ${op_impl_path}/libiomp5md.dll)
    else(${CBLAS_PROVIDER} STREQUAL EXTERN_OPENBLAS)
-      add_custom_command(TARGET op_function_generator
-            PRE_LINK
-            COMMAND ${CMAKE_COMMAND} -E copy ${OPENBLAS_SHARED_LIB} ${op_function_generator_path}
-          )
+      ADD_CUSTOM_COMMAND(OUTPUT ${op_impl_path}/openblas.dll
+        COMMAND ${CMAKE_COMMAND} -E copy ${OPENBLAS_SHARED_LIB} ${op_impl_path}
+        DEPENDS extern_openblas)
+      list(APPEND OP_IMPL_DEPS ${op_impl_path}/openblas.dll)
    endif()
    if(WITH_MKLDNN)
-      add_custom_command(TARGET op_function_generator
-          PRE_LINK
-          COMMAND ${CMAKE_COMMAND} -E copy ${MKLDNN_SHARED_LIB} ${op_function_generator_path}
-          )
+      ADD_CUSTOM_COMMAND(OUTPUT ${op_impl_path}/mkldnn.dll
+        COMMAND ${CMAKE_COMMAND} -E copy ${MKLDNN_SHARED_LIB} ${op_impl_path}
+        DEPENDS mkldnn)
+        list(APPEND OP_IMPL_DEPS ${op_impl_path}/mkldnn.dll)
    endif()
+
+    add_custom_command(OUTPUT ${impl_file}
+      COMMAND ${CMAKE_BINARY_DIR}/paddle/fluid/pybind/op_function_generator_retry.bat
+      COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_impl_file} ${impl_file}
+      COMMENT "copy_if_different ${tmp_impl_file} to ${impl_file}"
+      DEPENDS ${OP_IMPL_DEPS})
  else(WIN32)
    # If there are no *.so in /usr/lib or LD_LIBRARY_PATH,
    # copy these *.so to current directory and append current directory to
    # LD_LIBRARY_PATH. This is different with Windows platformm, which search
    # *.dll in current directory automatically.
-    add_custom_command(TARGET op_function_generator
-          POST_BUILD
+    if(WITH_MKLML)
+      ADD_CUSTOM_COMMAND(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/libiomp5.so
+        COMMAND ${CMAKE_COMMAND} -E copy ${MKLML_SHARED_IOMP_LIB} ${CMAKE_CURRENT_BINARY_DIR}
+        DEPENDS mklml)
+      list(APPEND OP_IMPL_DEPS ${CMAKE_CURRENT_BINARY_DIR}/libiomp5.so)
+    endif()
+    if(WITH_MKLDNN)
+      ADD_CUSTOM_COMMAND(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/libdnnl.so.0
+        COMMAND ${CMAKE_COMMAND} -E copy ${MKLDNN_SHARED_LIB} ${CMAKE_CURRENT_BINARY_DIR}
+        DEPENDS mkldnn)
+      list(APPEND OP_IMPL_DEPS ${CMAKE_CURRENT_BINARY_DIR}/libdnnl.so.0)
+    endif()
+    add_custom_command(OUTPUT ${impl_file}
          COMMAND ${CMAKE_COMMAND} -E env "LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH}:."
              "${CMAKE_CURRENT_BINARY_DIR}/op_function_generator"
              "${tmp_impl_file}"
          COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_impl_file} ${impl_file}
          COMMENT "copy_if_different ${tmp_impl_file} to ${impl_file}"
-          VERBATIM
-    )
-    if(WITH_MKL)
-      add_custom_command(TARGET op_function_generator
-            PRE_LINK
-            COMMAND ${CMAKE_COMMAND} -E copy ${MKLML_SHARED_LIB} ${CMAKE_CURRENT_BINARY_DIR}
-            COMMAND ${CMAKE_COMMAND} -E copy ${MKLML_SHARED_IOMP_LIB} ${CMAKE_CURRENT_BINARY_DIR}
-            )
-    endif(WITH_MKL)
-    if(WITH_MKLDNN)
-      add_custom_command(TARGET op_function_generator
-          PRE_LINK
-          COMMAND ${CMAKE_COMMAND} -E copy ${MKLDNN_SHARED_LIB} ${CMAKE_CURRENT_BINARY_DIR}
-          )
-    endif(WITH_MKLDNN)
+          DEPENDS ${OP_IMPL_DEPS}
+          VERBATIM)
  endif(WIN32)
+  add_custom_target(op_function_generator_cmd ALL DEPENDS ${impl_file})

  cc_library(paddle_pybind SHARED
-  SRCS ${PYBIND_SRCS}
-  DEPS ${PYBIND_DEPS}
-  ${GLOB_OP_LIB} ${GLOB_OPERATOR_DEPS})
+    SRCS ${PYBIND_SRCS}
+    DEPS ${PYBIND_DEPS} ${GLOB_OP_LIB} ${GLOB_OPERATOR_DEPS})
+
  if(NOT APPLE AND NOT WIN32)
    target_link_libraries(paddle_pybind rt)
  endif(NOT APPLE AND NOT WIN32)
@@ -218,5 +213,5 @@ if(WITH_PYTHON)

  get_property (os_dependency_modules GLOBAL PROPERTY OS_DEPENDENCY_MODULES)
  target_link_libraries(paddle_pybind ${os_dependency_modules})
-  add_dependencies(paddle_pybind op_function_generator)
+  add_dependencies(paddle_pybind op_function_generator_cmd)
 endif(WITH_PYTHON)