From 7a681f0b81d11ee72c95c3b0a455cfe749210c71 Mon Sep 17 00:00:00 2001 From: Zhou Wei <52485244+zhouwei25@users.noreply.github.com> Date: Fri, 23 Apr 2021 14:41:10 +0800 Subject: [PATCH] fix Windows CI MP compile and environment install script and openblas CI (#32378) * fix Windows CI MP compile and environment install script * clear Windows CI environment * clear Windows CI environment * clear Windows CI environment --- CMakeLists.txt | 16 ++-- cmake/init.cmake | 4 +- cmake/paddle_win.props | 1 - paddle/scripts/paddle_build.bat | 42 ++++++++-- tools/parallel_UT_rule.py | 4 - tools/windows/build_compile_environment.bat | 93 ++++++++++----------- 6 files changed, 93 insertions(+), 67 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 580b7111140..d874b21b087 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,9 +22,6 @@ include(system) project(paddle CXX C) -include(init) -include(generic) # simplify cmake module - # enable language CUDA # TODO(Shibo Tao): remove find_package(CUDA) completely. find_package(CUDA QUIET) @@ -38,6 +35,10 @@ option(WITH_ROCM "Compile PaddlePaddle with ROCM platform" OFF) # to develop some acl related functionality on x86 option(WITH_ASCEND_CL "Compile PaddlePaddle with ASCEND CL" ${WITH_ASCEND}) option(WITH_ASCEND_CXX11 "Compile PaddlePaddle with ASCEND and CXX11 ABI" OFF) +# Note(zhouwei): It use option above, so put here +include(init) +include(generic) # simplify cmake module + if (WITH_GPU AND WITH_XPU) message(FATAL_ERROR "Error when compile GPU and XPU at the same time") endif() @@ -104,8 +105,10 @@ if(WIN32) endforeach(flag_var) endif() - # NOTE(Avin0323): Less parallel count result in faster compilation. - math(EXPR PROCESS_MAX "${CPU_CORES} * 2 / 3") + # NOTE(zhouwei25): temporarily change MP to 1 for reducing CPU & memory utilization + set(PROCESS_MAX 1) + #math(EXPR PROCESS_MAX "${CPU_CORES} * 1 / 2") + # windows build turn off warnings, use parallel compiling. foreach(flag_var CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE @@ -133,6 +136,9 @@ if(WIN32) foreach(flag_var CMAKE_SHARED_LINKER_FLAGS CMAKE_STATIC_LINKER_FLAGS CMAKE_EXE_LINKER_FLAGS CMAKE_LINKER_FLAGS) set(${flag_var} "${${flag_var}} /ignore:4049 /ignore:4217 /ignore:4006 /ignore:4221") + if(MSVC_STATIC_CRT) + set(${flag_var} "${${flag_var}} /NODEFAULTLIB:MSVCRT.LIB") + endif() endforeach(flag_var) if (WITH_WIN_DUMP_DBG) diff --git a/cmake/init.cmake b/cmake/init.cmake index 19fdb6c601a..b11156d2e99 100644 --- a/cmake/init.cmake +++ b/cmake/init.cmake @@ -18,10 +18,10 @@ if(NOT WIN32) set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g -DNDEBUG") set(CMAKE_CXX_FLAGS_MINSIZEREL "-Os -DNDEBUG") else() - # It has not been used now, it can specify CUDA compile flag manualy, + # It can specify CUDA compile flag manualy, # its use is to remvoe /Zi to reduce GPU static library size. But it's dangerous # because CUDA will update by nvidia, then error will occur. - # Now, it's used in CUDA:[10.0, 10.2] + # Now, it's only used in VS2015 + CUDA:[10.0, 10.2] set(WIN_PROPS ${CMAKE_SOURCE_DIR}/cmake/paddle_win.props) endif() diff --git a/cmake/paddle_win.props b/cmake/paddle_win.props index 296940dc3f5..3c069bd2981 100644 --- a/cmake/paddle_win.props +++ b/cmake/paddle_win.props @@ -88,4 +88,3 @@ set CUDAFE_FLAGS=--sdk_dir "$(WindowsSdkDir)" - diff --git a/paddle/scripts/paddle_build.bat b/paddle/scripts/paddle_build.bat index 20c8794ba63..cbd97660560 100644 --- a/paddle/scripts/paddle_build.bat +++ b/paddle/scripts/paddle_build.bat @@ -26,9 +26,20 @@ set cache_dir=%work_dir:Paddle=cache% if not exist %cache_dir%\tools ( git clone https://github.com/zhouwei25/tools.git %cache_dir%\tools ) -taskkill /f /im op_function_generator.exe -wmic process where name="op_function_generator.exe" call terminate +taskkill /f /im op_function_generator.exe 2>NUL +taskkill /f /im cmake.exe 2>NUL +taskkill /f /im MSBuild.exe 2>NUL +taskkill /f /im CL.exe 2>NUL +taskkill /f /im Lib.exe 2>NUL +taskkill /f /im link.exe 2>NUL +taskkill /f /im vctip.exe 2>NUL +taskkill /f /im cvtres.exe 2>NUL +taskkill /f /im rc.exe 2>NUL taskkill /f /im python.exe 2>NUL +wmic process where name="op_function_generator.exe" call terminate 2>NUL +wmic process where name="python.exe" call terminate 2>NUL +wmic process where name="cvtres.exe" call terminate 2>NUL +wmic process where name="rc.exe" call terminate 2>NUL rem ------initialize common variable------ @@ -54,6 +65,7 @@ if not defined INFERENCE_DEMO_INSTALL_DIR set INFERENCE_DEMO_INSTALL_DIR=%cache_ if not defined LOG_LEVEL set LOG_LEVEL=normal if not defined PRECISION_TEST set PRECISION_TEST=OFF if not defined NIGHTLY_MODE set PRECISION_TEST=OFF +if not defined retry_times set retry_times=2 rem -------set cache build directory----------- rmdir build\python /s/q @@ -209,6 +221,7 @@ rem ------Build windows avx whl package------ set WITH_AVX=ON set ON_INFER=OFF set CUDA_ARCH_NAME=All +set retry_times=4 call :cmake || goto cmake_error call :build || goto build_error @@ -220,6 +233,7 @@ rem ------Build windows no-avx whl package------ set WITH_AVX=OFF set ON_INFER=OFF set CUDA_ARCH_NAME=All +set retry_times=4 call :cmake || goto cmake_error call :build || goto build_error @@ -249,6 +263,8 @@ echo ======================================== rem Configure the environment for 64-bit builds. 'DISTUTILS_USE_SDK' indicates that the user has selected the compiler. call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat" set DISTUTILS_USE_SDK=1 +rem Windows 10 Kit bin dir +set PATH=C:\Program Files (x86)\Windows Kits\10\bin\10.0.17763.0\x64;%PATH% for /F %%# in ('wmic os get localdatetime^|findstr 20') do set start=%%# set start=%start:~4,10% @@ -329,7 +345,7 @@ echo ======================================== echo Step 2. Buile Paddle ... echo ======================================== -for /F %%# in ('wmic cpu get NumberOfLogicalProcessors^|findstr [0-9]') do set /a PARALLEL_PROJECT_COUNT=%%#*4/5 +for /F %%# in ('wmic cpu get NumberOfLogicalProcessors^|findstr [0-9]') do set /a PARALLEL_PROJECT_COUNT=%%#*3/5 echo "PARALLEL PROJECT COUNT is %PARALLEL_PROJECT_COUNT%" set build_times=1 :build_tp @@ -342,7 +358,7 @@ if %GENERATOR% == "Ninja" ( ) if %ERRORLEVEL% NEQ 0 ( set /a build_times=%build_times%+1 - if %build_times% GTR 2 ( + if %build_times% GTR %retry_times% ( exit /b 7 ) else ( echo Build third_party failed, will retry! @@ -356,6 +372,19 @@ set build_times=1 :: reset clcache zero stats for collect PR's actual hit rate rem clcache.exe -z +rem -------clean up environment again----------- +taskkill /f /im MSBuild.exe 2>NUL +taskkill /f /im cl.exe 2>NUL +taskkill /f /im lib.exe 2>NUL +taskkill /f /im link.exe 2>NUL +taskkill /f /im vctip.exe 2>NUL +taskkill /f /im cvtres.exe 2>NUL +taskkill /f /im rc.exe 2>NUL +taskkill /f /im test_api_impl.exe 2>NUL +wmic process where name="op_function_generator.exe" call terminate 2>NUL +wmic process where name="cvtres.exe" call terminate 2>NUL +wmic process where name="rc.exe" call terminate 2>NUL + echo Build Paddle the %build_times% time: if %GENERATOR% == "Ninja" ( ninja -j %PARALLEL_PROJECT_COUNT% @@ -369,7 +398,7 @@ if %GENERATOR% == "Ninja" ( if %ERRORLEVEL% NEQ 0 ( set /a build_times=%build_times%+1 - if %build_times% GTR 1 ( + if %build_times% GTR %retry_times% ( exit /b 7 ) else ( echo Build Paddle failed, will retry! @@ -706,9 +735,10 @@ taskkill /f /im git-remote-https.exe 2>NUL taskkill /f /im vctip.exe 2>NUL taskkill /f /im cvtres.exe 2>NUL taskkill /f /im rc.exe 2>NUL +taskkill /f /im python.exe 2>NUL wmic process where name="op_function_generator.exe" call terminate 2>NUL wmic process where name="python.exe" call terminate 2>NUL -taskkill /f /im python.exe 2>NUL +wmic process where name="cvtres.exe" call terminate 2>NUL echo Windows CI run successfully! exit /b 0 diff --git a/tools/parallel_UT_rule.py b/tools/parallel_UT_rule.py index 0f745f21207..9354751fdb9 100644 --- a/tools/parallel_UT_rule.py +++ b/tools/parallel_UT_rule.py @@ -518,7 +518,6 @@ TWO_PARALLEL_JOB = [ 'test_softmax_with_cross_entropy_op', 'test_imperative_gan', 'test_simnet', - 'test_instance_norm_op', 'test_amp_check_finite_and_scale_op', 'test_random_seed', 'test_histogram_op', @@ -561,7 +560,6 @@ TWO_PARALLEL_JOB = [ 'test_add_reader_dependency', 'test_is_tensor', 'test_variable', - 'test_unsqueeze_op', 'test_save_model_without_var', 'test_unfold_op', 'test_conv_bn_fuse_pass', @@ -813,7 +811,6 @@ TWO_PARALLEL_JOB = [ 'test_sequence_expand_as', 'test_sequence_reverse', 'test_shape_op', - 'test_lod_tensor', 'test_diag', 'test_strided_slice_op', 'test_switch_case', @@ -831,7 +828,6 @@ TWO_PARALLEL_JOB = [ 'test_arange', 'test_lrn_mkldnn_op', 'test_imperative_gnn', - 'test_eager_deletion_while_op', 'test_dequantize_abs_max_op', 'test_elementwise_mul_op', 'test_tensor_scalar_type_promotion_dynamic', diff --git a/tools/windows/build_compile_environment.bat b/tools/windows/build_compile_environment.bat index 736a19ddf52..4a61a99c34f 100644 --- a/tools/windows/build_compile_environment.bat +++ b/tools/windows/build_compile_environment.bat @@ -22,9 +22,9 @@ :: Include: :: 1. CMake 3.17.0 :: 2. Git 2.28.0 -:: 3. Python 3.7.8 -:: 4. Visual Studio 2015 with update 3 -:: 5. CUDA 10 +:: 3. Python 3.8.3 +:: 4. Visual Studio 2017 Community +:: 5. CUDA 11.2 :: 6. java jre :: 7. xly agent @@ -73,7 +73,6 @@ if %errorlevel% == 0 ( ) else ( echo Error***** Install Cmake-3.17.0 failed, please re-install it manually. ) -del cmake-3.17.0-win64-x64.msi goto :eof :: ===== end step 1: cmake ===== @@ -99,91 +98,87 @@ if %errorlevel% == 0 ( ) else ( echo Error***** Install Git-2.28.0 failed, please re-install it manually. ) -del Git-2.28.0-64-bit.exe goto :eof :: ===== end step 2: Git ===== :: ===== start step 3: Python ===== -:: Download Python-3.7.8 and add in PATH when it not installed. -:: TODO: limit version >= 3.7.8 +:: Download Python-3.8.3 and add in PATH when it not installed. +:: TODO: limit version >= 3.8.3 :python -echo ">>>>>>>> step [3/7]: Python 3.7.8" -python -V 2>&1 | findstr /C:"Python 3.7.8" > nul 2> nul || call :install_python -goto vs2015 +echo ">>>>>>>> step [3/7]: Python 3.8.3" +python -V 2>&1 | findstr /C:"Python 3.8.3" > nul 2> nul || call :install_python +goto vs :install_python -echo There is not Python in this PC, will install Python-3.7.8. -echo Download package from https://npm.taobao.org/mirrors/python/3.7.8/python-3.7.8-amd64.exe ... -wget -O python-3.7.8-amd64.exe https://npm.taobao.org/mirrors/python/3.7.8/python-3.7.8-amd64.exe -echo Install Python-3.7.8 ... +echo There is not Python in this PC, will install Python-3.8.3 +echo Download package from https://paddle-ci.gz.bcebos.com/window_requirement/python-3.8.3-amd64.exe ... +wget -O python-3.8.3-amd64.exe https://paddle-ci.gz.bcebos.com/window_requirement/python-3.8.3-amd64.exe +echo Install Python-3.8.3 ... :: /passive [silent install] :: InstallAllUsers [add path for all users] :: PrependPath [add script/install into PATH] :: TargetDir [install directory] -start /wait python-3.7.8-amd64.exe /passive InstallAllUsers=1 PrependPath=1 TargetDir=C:\Python37 +start /wait python-3.8.3-amd64.exe /passive InstallAllUsers=1 PrependPath=1 TargetDir=C:\Python38 if %errorlevel% == 0 ( - echo Install python-3.7.8 success! + echo Install python-3.8.3 success! ) else ( - echo Error***** Install python-3.7.8 failed, please re-install it manually. + echo Error***** Install python-3.8.3 failed, please re-install it manually. ) -del python-3.7.8-amd64.exe goto :eof :: ===== end step 3: Python ===== -:: ===== start step 4: Visual Studio 2015 ===== -:: Download Visual Studio 2015 when it not installed. -:vs2015 -echo ">>>>>>>> step [4/7]: Visual Studio 2015" -cmd /C "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64 > nul 2> nul || call :install_visual_studio +:: ===== start step 4: Visual Studio 2017 Community ===== +:: Download Visual Studio 2017 when it not installed. +:vs +echo ">>>>>>>> step [4/7]: Visual Studio 2017 " +cmd /C "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat" > nul 2> nul || call :install_visual_studio goto :cuda10 :install_visual_studio -echo There is not Visual Studio in this PC, will install VS2015. -echo Download package from "https://paddle-ci.gz.bcebos.com/window_requirement/en_visual_studio_enterprise_2015_with_update_3_x86_x64_web_installer_8922986.exe" -wget -O vs_installer.exe "https://paddle-ci.gz.bcebos.com/window_requirement/en_visual_studio_enterprise_2015_with_update_3_x86_x64_web_installer_8922986.exe" -echo Install Visual Studio 2015 ... +echo There is not Visual Studio in this PC, will install VS2017. +echo Download package from "https://paddle-ci.gz.bcebos.com/window_requirement/VS2017/vs_Community.exe" +wget -O vs_Community.exe "https://paddle-ci.gz.bcebos.com/window_requirement/VS2017/vs_Community.exe" +echo Install Visual Studio 2017 ... :: /passive [silent install] :: /norestart [no restart] :: /NoRefresh [no refresh] :: /InstallSelectableItems NativeLanguageSupport_Group [select Visual C++ for installing] -start /wait vs_installer.exe /passive /norestart /NoRefresh /InstallSelectableItems NativeLanguageSupport_Group +start /wait vs_Community.exe --passive --add Microsoft.VisualStudio.Workload.NativeDesktop --add Microsoft.VisualStudio.Workload.Universal --includeRecommended if %errorlevel% == 0 ( - echo Install Visual Studio 2015 success! + echo Install Visual Studio 2017 success! ) else ( - echo Error***** Install Visual Studio 2015 failed, please re-install it manually. + echo Error***** Install Visual Studio 2017 failed, please re-install it manually. ) -del vs_installer.exe goto :eof -:: ===== end step 4: Visual Studio 2015 ===== +:: ===== end step 4: Visual Studio 2017 ===== -:: ===== start step 5: CUDA 10 ===== +:: ===== start step 5: CUDA 11 ===== :cuda10 -echo ">>>>>>>> step [5/7]: CUDA 10.2" -cmd /C nvcc --version 2> nul | findstr /C:"10.2" > nul 2> nul || call :install_cuda +echo ">>>>>>>> step [5/7]: CUDA 11.2" +cmd /C nvcc --version 2> nul | findstr /C:"11.2" > nul 2> nul || call :install_cuda goto java-jre :install_cuda -echo There is not CUDA in this PC, will install CUDA-10.2. -echo Download package from "https://paddle-ci.gz.bcebos.com/window_requirement/cuda_10.2.89_441.22_win10.exe" -wget -O cuda_installer.exe "https://paddle-ci.gz.bcebos.com/window_requirement/cuda_10.2.89_441.22_win10.exe" -echo Install CUDA-10.2 ... +echo There is not CUDA in this PC, will install CUDA-11.2. +echo Download package from "https://paddle-ci.gz.bcebos.com/window_requirement/cuda_11.2.0_460.89_win10.exe" +wget -O cuda_installer.exe "https://paddle-ci.gz.bcebos.com/window_requirement/cuda_11.2.0_460.89_win10.exe" +echo Install CUDA-11.2 ... :: -s [silent install] start /wait cuda_installer.exe -s if %errorlevel% == 0 ( - echo Install CUDA-10.2 success! + echo Install CUDA-11.2 success! ) else ( - echo Error***** Install CUDA-10.2 failed, please re-install it manually. + echo Error***** Install CUDA-11.2 failed, please re-install it manually. goto :eof ) del cuda_installer.exe -echo Download cudnn from "https://paddle-ci.gz.bcebos.com/window_requirement/cudnn-10.2-windows10-x64-v7.6.5.32.zip" -wget -O cudnn-10.2-windows10-x64-v7.6.5.32.zip "https://paddle-ci.gz.bcebos.com/window_requirement/cudnn-10.2-windows10-x64-v7.6.5.32.zip" -tar xf cudnn-10.2-windows10-x64-v7.6.5.32.zip -xcopy /E /Y /R "cuda\bin\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.2\bin" -xcopy /E /Y /R "cuda\include\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.2\include" -xcopy /E /Y /R "cuda\lib\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.2\lib" +echo Download cudnn from "https://paddle-ci.gz.bcebos.com/window_requirement/cudnn-11.2-windows-x64-v8.1.0.77.zip" +wget -O cudnn-11.2-windows-x64-v8.1.0.77.zip "https://paddle-ci.gz.bcebos.com/window_requirement/cudnn-11.2-windows-x64-v8.1.0.77.zip" +tar xf cudnn-11.2-windows-x64-v8.1.0.77.zip +xcopy /E /Y /R "cuda\bin\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2\bin" +xcopy /E /Y /R "cuda\include\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2\include" +xcopy /E /Y /R "cuda\lib\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2\lib" rd /s /q cuda -del cudnn-10.2-windows10-x64-v7.6.5.32.zip goto :eof :: ===== end step 5: CUDA 10 ===== @@ -212,7 +207,7 @@ goto :eof :: ===== start step 7: xly agent ===== :xly-agent echo ">>>>>>>> step [7/7]: xly agent" -wget -O agent.jar "https://paddle-ci.gz.bcebos.com/window_requirement/agent.jar" +wget -O agent.jar "https://xly.bce.baidu.com/sa_server/agent/v1/download?version=1.2.8" :: ===== end step 8: xly agent ===== pause -- GitLab