From accf284bb6d609b0b00af690b6df262f78a99509 Mon Sep 17 00:00:00 2001 From: Zhou Wei <52485244+zhouwei25@users.noreply.github.com> Date: Wed, 26 May 2021 10:12:57 +0800 Subject: [PATCH] Fix ninja compilation bug and warning on windows (#32987) * fix ninja compilation bug on windows * polish windows ci * polish windows ci --- CMakeLists.txt | 13 ++++---- cmake/cuda.cmake | 27 ++++++--------- cmake/external/mkldnn.cmake | 4 +-- cmake/external/warpctc.cmake | 4 +-- cmake/flags.cmake | 7 +++- cmake/generic.cmake | 4 +-- cmake/init.cmake | 37 ++++++++++++--------- paddle/scripts/paddle_build.bat | 22 ++++++------ tools/parallel_UT_rule.py | 1 - tools/windows/build_compile_environment.bat | 4 +-- 10 files changed, 65 insertions(+), 58 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f30671bd3a..28dc39920c 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,6 +20,13 @@ set(PADDLE_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}) include(system) +# Note(zhouwei): Ninja Generator will set CMAKE_BUILD_TYPE to Debug +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE "Release" CACHE STRING + "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel" + FORCE) +endif() + project(paddle CXX C) # enable language CUDA @@ -213,12 +220,6 @@ if(NOT PY_VERSION) endif() set(PYBIND11_PYTHON_VERSION ${PY_VERSION}) -# CMAKE_BUILD_TYPE -if(NOT CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE "Release" CACHE STRING - "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel" - FORCE) -endif() # the type of sanitizer, options are: Address, Leak, Memory, Thread, Undefined. Default: OFF if(SANITIZER_TYPE AND NOT "${SANITIZER_TYPE}" MATCHES "^(Address|Leak|Memory|Thread|Undefined)$") diff --git a/cmake/cuda.cmake b/cmake/cuda.cmake index 7f2addb02d..033b40622e 100644 --- a/cmake/cuda.cmake +++ b/cmake/cuda.cmake @@ -205,23 +205,16 @@ set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} --expt-extended-lambda") if(WIN32) set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler \"/wd4244 /wd4267 /wd4819 \"") set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler /bigobj") - if(CMAKE_BUILD_TYPE STREQUAL "Debug") - # match the cl's _ITERATOR_DEBUG_LEVEL - set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler \"-g -G -D_DEBUG\"") - if(MSVC_STATIC_CRT) - set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler /MTd") - else() - set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler /MDd") - endif() - elseif(CMAKE_BUILD_TYPE STREQUAL "Release") - set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler \"-DNDEBUG\"") - if(MSVC_STATIC_CRT) - set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler /MT") - else() - set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler /MD") - endif() - else() - message(FATAL "Windows only support Release or Debug build now. Please set visual studio build type to Release/Debug, x64 build.") + if(MSVC_STATIC_CRT) + set(CMAKE_CUDA_FLAGS_DEBUG "${CMAKE_CUDA_FLAGS_DEBUG} -Xcompiler /MTd") + set(CMAKE_CUDA_FLAGS_RELEASE "${CMAKE_CUDA_FLAGS_RELEASE} -Xcompiler /MT") + foreach(flag_var + CMAKE_CUDA_FLAGS CMAKE_CUDA_FLAGS_DEBUG CMAKE_CUDA_FLAGS_RELEASE + CMAKE_CUDA_FLAGS_MINSIZEREL CMAKE_CUDA_FLAGS_RELWITHDEBINFO) + if(${flag_var} MATCHES "-MD") + string(REGEX REPLACE "-MD" "-MT" ${flag_var} "${${flag_var}}") + endif() + endforeach(flag_var) endif() endif() diff --git a/cmake/external/mkldnn.cmake b/cmake/external/mkldnn.cmake index 4e0768fc10..c37e28523f 100644 --- a/cmake/external/mkldnn.cmake +++ b/cmake/external/mkldnn.cmake @@ -60,8 +60,8 @@ ExternalProject_Add( DEPENDS ${MKLDNN_DEPENDS} PREFIX ${MKLDNN_PREFIX_DIR} SOURCE_DIR ${MKLDNN_SOURCE_DIR} - BUILD_ALWAYS 1 - # UPDATE_COMMAND "" + UPDATE_COMMAND "" + #BUILD_ALWAYS 1 CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE} diff --git a/cmake/external/warpctc.cmake b/cmake/external/warpctc.cmake index c591a9391d..b0ea338d20 100644 --- a/cmake/external/warpctc.cmake +++ b/cmake/external/warpctc.cmake @@ -100,9 +100,9 @@ else() "${WARPCTC_DOWNLOAD_CMD}" PREFIX ${WARPCTC_PREFIX_DIR} SOURCE_DIR ${WARPCTC_SOURCE_DIR} - #UPDATE_COMMAND "" + UPDATE_COMMAND "" PATCH_COMMAND "" - BUILD_ALWAYS 1 + #BUILD_ALWAYS 1 CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_C_FLAGS=${WARPCTC_C_FLAGS} diff --git a/cmake/flags.cmake b/cmake/flags.cmake index a2ddad557c..94fd29b905 100644 --- a/cmake/flags.cmake +++ b/cmake/flags.cmake @@ -28,7 +28,12 @@ function(CheckCompilerCXX14Flag) endfunction() CheckCompilerCXX14Flag() -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14") +if(NOT WIN32) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14") +else() + set(CMAKE_CXX_STANDARD 14) +endif() + # safe_set_flag # # Set a compile flag only if compiler is support diff --git a/cmake/generic.cmake b/cmake/generic.cmake index 53dcde616b..cea65f17fb 100644 --- a/cmake/generic.cmake +++ b/cmake/generic.cmake @@ -92,7 +92,7 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR}) # including io directory for inference lib paddle_api.h include_directories("${PADDLE_SOURCE_DIR}/paddle/fluid/framework/io") -if(NOT APPLE) +if(NOT APPLE AND NOT WIN32) find_package(Threads REQUIRED) link_libraries(${CMAKE_THREAD_LIBS_INIT}) if(WITH_PSLIB OR WITH_DISTRIBUTE) @@ -100,7 +100,7 @@ if(NOT APPLE) else() set(CMAKE_CXX_LINK_EXECUTABLE "${CMAKE_CXX_LINK_EXECUTABLE} -pthread -ldl -lrt") endif() -endif(NOT APPLE) +endif() set_property(GLOBAL PROPERTY FLUID_MODULES "") # find all fluid modules is used for paddle fluid static library diff --git a/cmake/init.cmake b/cmake/init.cmake index 4bdcaeb4c5..0ebcdc8cee 100644 --- a/cmake/init.cmake +++ b/cmake/init.cmake @@ -17,16 +17,30 @@ if(NOT WIN32) set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG") set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g -DNDEBUG") set(CMAKE_CXX_FLAGS_MINSIZEREL "-Os -DNDEBUG") + + if(WITH_GPU) + set(CMAKE_CUDA_FLAGS_DEBUG "-g") + set(CMAKE_CUDA_FLAGS_RELEASE "-O3 -DNDEBUG") + set(CMAKE_CUDA_FLAGS_RELWITHDEBINFO "-O2 -g -DNDEBUG") + set(CMAKE_CUDA_FLAGS_MINSIZEREL "-O1 -DNDEBUG") + endif() else() - set(CMAKE_C_FLAGS_DEBUG "/Zi /DEBUG") - set(CMAKE_C_FLAGS_RELEASE "/O2 /DNDEBUG") - set(CMAKE_C_FLAGS_RELWITHDEBINFO "/O2 /DNDEBUG") - set(CMAKE_C_FLAGS_MINSIZEREL "/Os /DNDEBUG") + set(CMAKE_C_FLAGS_DEBUG "/MDd /Zi /Ob0 /Od /RTC1") + set(CMAKE_C_FLAGS_RELEASE "/MD /O2 /Ob2 /DNDEBUG") + set(CMAKE_C_FLAGS_RELWITHDEBINFO "/MD /Zi /O2 /Ob1 /DNDEBUG") + set(CMAKE_C_FLAGS_MINSIZEREL "/MD /O1 /Ob1 /DNDEBUG") - set(CMAKE_CXX_FLAGS_DEBUG "/Zi /DEBUG") - set(CMAKE_CXX_FLAGS_RELEASE "/O2 /DNDEBUG") - set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "/O2 /DNDEBUG") - set(CMAKE_CXX_FLAGS_MINSIZEREL "/Os /DNDEBUG") + set(CMAKE_CXX_FLAGS_DEBUG "/MDd /Zi /Ob0 /Od /RTC1") + set(CMAKE_CXX_FLAGS_RELEASE "/MD /O2 /Ob2 /DNDEBUG") + set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "/MD /Zi /O2 /Ob1 /DNDEBUG") + set(CMAKE_CXX_FLAGS_MINSIZEREL "/MD /O1 /Ob1 /DNDEBUG") + + if(WITH_GPU) + set(CMAKE_CUDA_FLAGS_DEBUG "-Xcompiler=\"-MDd -Zi -Ob0 -Od /RTC1\"") + set(CMAKE_CUDA_FLAGS_RELEASE "-Xcompiler=\"-MD -O2 -Ob2\" -DNDEBUG") + set(CMAKE_CUDA_FLAGS_RELWITHDEBINFO "-Xcompiler=\"-MD -Zi -O2 -Ob1\" -DNDEBUG") + set(CMAKE_CUDA_FLAGS_MINSIZEREL "-Xcompiler=\"-MD -O1 -Ob1\" -DNDEBUG") + endif() # It can specify CUDA compile flag manualy, # its use is to remvoe /Zi to reduce GPU static library size. But it's dangerous @@ -34,10 +48,3 @@ else() # Now, it's only used in VS2015 + CUDA:[10.0, 10.2] set(WIN_PROPS ${CMAKE_SOURCE_DIR}/cmake/paddle_win.props) endif() - -if(WITH_GPU) - set(CMAKE_CUDA_FLAGS_DEBUG "-g") - set(CMAKE_CUDA_FLAGS_RELEASE "-O3 -DNDEBUG") - set(CMAKE_CUDA_FLAGS_RELWITHDEBINFO "-O2 -g -DNDEBUG") - set(CMAKE_CUDA_FLAGS_MINSIZEREL "-O1 -DNDEBUG") -endif() diff --git a/paddle/scripts/paddle_build.bat b/paddle/scripts/paddle_build.bat index 69138a37f4..dd8146aa3a 100644 --- a/paddle/scripts/paddle_build.bat +++ b/paddle/scripts/paddle_build.bat @@ -18,7 +18,7 @@ rem Paddle CI Task On Windows Platform rem ================================================= @ECHO ON -setlocal +setlocal enabledelayedexpansion rem -------clean up environment----------- set work_dir=%cd% @@ -63,7 +63,7 @@ if not defined WITH_PYTHON set WITH_PYTHON=ON if not defined ON_INFER set ON_INFER=ON if not defined WITH_INFERENCE_API_TEST set WITH_INFERENCE_API_TEST=ON if not defined WITH_STATIC_LIB set WITH_STATIC_LIB=ON -if not defined WITH_TPCACHE set WITH_TPCACHE=ON +if not defined WITH_TPCACHE set WITH_TPCACHE=OFF if not defined WITH_CLCACHE set WITH_CLCACHE=OFF if not defined WITH_CACHE set WITH_CACHE=OFF if not defined WITH_UNITY_BUILD set WITH_UNITY_BUILD=OFF @@ -236,6 +236,8 @@ call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary set DISTUTILS_USE_SDK=1 rem Windows 10 Kit bin dir set PATH=C:\Program Files (x86)\Windows Kits\10\bin\10.0.17763.0\x64;%PATH% +rem Use 64-bit ToolSet to compile +set PreferredToolArchitecture=x64 for /F %%# in ('wmic os get localdatetime^|findstr 20') do set start=%%# set start=%start:~4,10% @@ -263,12 +265,12 @@ rem ------initialize the python environment------ @ECHO ON set PYTHON_EXECUTABLE=%PYTHON_ROOT%\python.exe set PATH=%PYTHON_ROOT%;%PYTHON_ROOT%\Scripts;%PATH% -if %WITH_PYTHON% == "ON" ( +if "%WITH_PYTHON%" == "ON" ( where python where pip pip install wheel --user pip install -r %work_dir%\python\requirements.txt --user - if %ERRORLEVEL% NEQ 0 ( + if !ERRORLEVEL! NEQ 0 ( echo pip install requirements.txt failed! exit /b 7 ) @@ -329,14 +331,14 @@ if "%WITH_GPU%"=="ON" ( ) :cmake_impl -echo cmake .. -G %GENERATOR% -T host=x64 -DCMAKE_BUILD_TYPE=Release -DWITH_AVX=%WITH_AVX% -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% ^ +echo cmake .. -G %GENERATOR% -DCMAKE_BUILD_TYPE=Release -DWITH_AVX=%WITH_AVX% -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% ^ -DWITH_TESTING=%WITH_TESTING% -DWITH_PYTHON=%WITH_PYTHON% -DPYTHON_EXECUTABLE=%PYTHON_EXECUTABLE% -DON_INFER=%ON_INFER% ^ -DWITH_INFERENCE_API_TEST=%WITH_INFERENCE_API_TEST% -DTHIRD_PARTY_PATH=%THIRD_PARTY_PATH% ^ -DINFERENCE_DEMO_INSTALL_DIR=%INFERENCE_DEMO_INSTALL_DIR% -DWITH_STATIC_LIB=%WITH_STATIC_LIB% ^ -DWITH_TENSORRT=%WITH_TENSORRT% -DTENSORRT_ROOT="%TENSORRT_ROOT%" -DMSVC_STATIC_CRT=%MSVC_STATIC_CRT% ^ -DWITH_UNITY_BUILD=%WITH_UNITY_BUILD% -DCUDA_ARCH_NAME=%CUDA_ARCH_NAME% -cmake .. -G %GENERATOR% -DCMAKE_BUILD_TYPE=Release -T host=x64 -DWITH_AVX=%WITH_AVX% -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% ^ +cmake .. -G %GENERATOR% -DCMAKE_BUILD_TYPE=Release -DWITH_AVX=%WITH_AVX% -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% ^ -DWITH_TESTING=%WITH_TESTING% -DWITH_PYTHON=%WITH_PYTHON% -DPYTHON_EXECUTABLE=%PYTHON_EXECUTABLE% -DON_INFER=%ON_INFER% ^ -DWITH_INFERENCE_API_TEST=%WITH_INFERENCE_API_TEST% -DTHIRD_PARTY_PATH=%THIRD_PARTY_PATH% ^ -DINFERENCE_DEMO_INSTALL_DIR=%INFERENCE_DEMO_INSTALL_DIR% -DWITH_STATIC_LIB=%WITH_STATIC_LIB% ^ @@ -366,7 +368,7 @@ echo Build third_party the %build_times% time: if %GENERATOR% == "Ninja" ( ninja third_party ) else ( - MSBuild /m /p:PreferredToolArchitecture=x64 /p:Configuration=Release /verbosity:quiet third_party.vcxproj + MSBuild /m /p:PreferredToolArchitecture=x64 /p:Configuration=Release /verbosity:%LOG_LEVEL% third_party.vcxproj ) if %ERRORLEVEL% NEQ 0 ( set /a build_times=%build_times%+1 @@ -412,10 +414,10 @@ if "%WITH_TESTING%"=="ON" ( echo Build Paddle the %build_times% time: if %GENERATOR% == "Ninja" ( - ninja -j %PARALLEL_PROJECT_COUNT% + ninja all ) else ( if "%WITH_CLCACHE%"=="OFF" ( - MSBuild /m:%PARALLEL_PROJECT_COUNT% /p:PreferredToolArchitecture=x64 /p:TrackFileAccess=false /p:Configuration=Release /verbosity:%LOG_LEVEL% ALL_BUILD.vcxproj + MSBuild /m:%PARALLEL_PROJECT_COUNT% /p:PreferredToolArchitecture=x64 /p:Configuration=Release /verbosity:%LOG_LEVEL% ALL_BUILD.vcxproj ) else ( MSBuild /m:%PARALLEL_PROJECT_COUNT% /p:PreferredToolArchitecture=x64 /p:TrackFileAccess=false /p:CLToolExe=clcache.exe /p:CLToolPath=%PYTHON_ROOT%\Scripts /p:Configuration=Release /verbosity:%LOG_LEVEL% ALL_BUILD.vcxproj ) @@ -644,7 +646,7 @@ echo git fetch upstream $BRANCH # develop is not fetched>> check_change_of_ echo fi>> check_change_of_unittest.sh echo git checkout -b origin_pr >> check_change_of_unittest.sh echo git checkout -f $BRANCH >> check_change_of_unittest.sh -echo cmake .. -G %GENERATOR% -T host=x64 -DWITH_AVX=%WITH_AVX% -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% ^ +echo cmake .. -G %GENERATOR% -DWITH_AVX=%WITH_AVX% -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% ^ -DWITH_TESTING=%WITH_TESTING% -DWITH_PYTHON=%WITH_PYTHON% -DPYTHON_EXECUTABLE=%PYTHON_EXECUTABLE% -DON_INFER=%ON_INFER% ^ -DWITH_INFERENCE_API_TEST=%WITH_INFERENCE_API_TEST% -DTHIRD_PARTY_PATH=%THIRD_PARTY_PATH% ^ -DINFERENCE_DEMO_INSTALL_DIR=%INFERENCE_DEMO_INSTALL_DIR% -DWITH_STATIC_LIB=%WITH_STATIC_LIB% ^ diff --git a/tools/parallel_UT_rule.py b/tools/parallel_UT_rule.py index cb0581d671..55b82084f6 100644 --- a/tools/parallel_UT_rule.py +++ b/tools/parallel_UT_rule.py @@ -650,7 +650,6 @@ TETRAD_PARALLEL_JOB = [ 'test_collective_wait', 'test_collective_split_row_linear', 'test_collective_split_embedding', - 'test_custom_attrs_jit', 'float16_gpu_test', 'test_leaky_relu_grad_grad_functor', 'test_complex_simplenet', diff --git a/tools/windows/build_compile_environment.bat b/tools/windows/build_compile_environment.bat index 4a61a99c34..603c9911a4 100644 --- a/tools/windows/build_compile_environment.bat +++ b/tools/windows/build_compile_environment.bat @@ -132,7 +132,7 @@ goto :eof :vs echo ">>>>>>>> step [4/7]: Visual Studio 2017 " cmd /C "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat" > nul 2> nul || call :install_visual_studio -goto :cuda10 +goto :cuda :install_visual_studio echo There is not Visual Studio in this PC, will install VS2017. @@ -153,7 +153,7 @@ goto :eof :: ===== end step 4: Visual Studio 2017 ===== :: ===== start step 5: CUDA 11 ===== -:cuda10 +:cuda echo ">>>>>>>> step [5/7]: CUDA 11.2" cmd /C nvcc --version 2> nul | findstr /C:"11.2" > nul 2> nul || call :install_cuda goto java-jre -- GitLab