fix Windows CI MP compile and environment install script and openblas CI (#32378)

* fix Windows CI MP compile and environment install script * clear Windows CI environment * clear Windows CI environment * clear Windows CI environment

fix Windows CI MP compile and environment install script and openblas CI (#32378)
* fix Windows CI MP compile and environment install script * clear Windows CI environment * clear Windows CI environment * clear Windows CI environment
7a681f0b · Zhou Wei · GitHub · 0e74eea2 · 7a681f0b · 7a681f0b
6 changed file
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -22,9 +22,6 @@ include(system)

 project(paddle CXX C)

-include(init)
-include(generic)            # simplify cmake module
-
 # enable language CUDA
 # TODO(Shibo Tao): remove find_package(CUDA) completely.
 find_package(CUDA QUIET)
@@ -38,6 +35,10 @@ option(WITH_ROCM        "Compile PaddlePaddle with ROCM platform"       OFF)
 # to develop some acl related functionality on x86
 option(WITH_ASCEND_CL         "Compile PaddlePaddle with ASCEND CL"        ${WITH_ASCEND})
 option(WITH_ASCEND_CXX11         "Compile PaddlePaddle with ASCEND and CXX11 ABI"        OFF)
+# Note(zhouwei): It use option above, so put here
+include(init)
+include(generic)            # simplify cmake module
+
 if (WITH_GPU  AND WITH_XPU)
    message(FATAL_ERROR "Error when compile GPU and XPU at the same time")
 endif()
@@ -104,8 +105,10 @@ if(WIN32)
        endforeach(flag_var)
    endif()

-    # NOTE(Avin0323): Less parallel count result in faster compilation.
-    math(EXPR PROCESS_MAX "${CPU_CORES} * 2 / 3")
+    # NOTE(zhouwei25): temporarily change MP to 1 for reducing CPU & memory utilization
+    set(PROCESS_MAX 1)
+    #math(EXPR PROCESS_MAX "${CPU_CORES} * 1 / 2")
+
    # windows build turn off warnings, use parallel compiling.
    foreach(flag_var
        CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE
@@ -133,6 +136,9 @@ if(WIN32)

    foreach(flag_var CMAKE_SHARED_LINKER_FLAGS CMAKE_STATIC_LINKER_FLAGS CMAKE_EXE_LINKER_FLAGS CMAKE_LINKER_FLAGS)
        set(${flag_var} "${${flag_var}} /ignore:4049 /ignore:4217 /ignore:4006 /ignore:4221")
+        if(MSVC_STATIC_CRT)
+            set(${flag_var} "${${flag_var}} /NODEFAULTLIB:MSVCRT.LIB")
+        endif()
    endforeach(flag_var)

    if (WITH_WIN_DUMP_DBG)

--- a/cmake/init.cmake
+++ b/cmake/init.cmake
@@ -18,10 +18,10 @@ if(NOT WIN32)
    set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g -DNDEBUG")
    set(CMAKE_CXX_FLAGS_MINSIZEREL "-Os -DNDEBUG")
 else()
-    # It has not been used now, it can specify CUDA compile flag manualy,
+    # It can specify CUDA compile flag manualy,
    # its use is to remvoe /Zi to reduce GPU static library size. But it's dangerous
    # because CUDA will update by nvidia, then error will occur.
-    # Now, it's used in CUDA:[10.0, 10.2]
+    # Now, it's only used in VS2015 + CUDA:[10.0, 10.2]
    set(WIN_PROPS ${CMAKE_SOURCE_DIR}/cmake/paddle_win.props)
 endif()


--- a/cmake/paddle_win.props
+++ b/cmake/paddle_win.props
@@ -88,4 +88,3 @@ set CUDAFE_FLAGS=--sdk_dir "$(WindowsSdkDir)"
        </ClCompile>
    </ItemDefinitionGroup>
 </Project>
-
--- a/paddle/scripts/paddle_build.bat
+++ b/paddle/scripts/paddle_build.bat
@@ -26,9 +26,20 @@ set cache_dir=%work_dir:Paddle=cache%
 if not exist %cache_dir%\tools (
    git clone https://github.com/zhouwei25/tools.git %cache_dir%\tools
 )
-taskkill /f /im op_function_generator.exe
-wmic process where name="op_function_generator.exe" call terminate
+taskkill /f /im op_function_generator.exe  2>NUL
+taskkill /f /im cmake.exe  2>NUL
+taskkill /f /im MSBuild.exe 2>NUL
+taskkill /f /im CL.exe 2>NUL
+taskkill /f /im Lib.exe 2>NUL
+taskkill /f /im link.exe 2>NUL
+taskkill /f /im vctip.exe 2>NUL
+taskkill /f /im cvtres.exe 2>NUL
+taskkill /f /im rc.exe 2>NUL
 taskkill /f /im python.exe  2>NUL
+wmic process where name="op_function_generator.exe" call terminate 2>NUL
+wmic process where name="python.exe" call terminate 2>NUL
+wmic process where name="cvtres.exe" call terminate 2>NUL
+wmic process where name="rc.exe" call terminate 2>NUL


 rem ------initialize common variable------
@@ -54,6 +65,7 @@ if not defined INFERENCE_DEMO_INSTALL_DIR set INFERENCE_DEMO_INSTALL_DIR=%cache_
 if not defined LOG_LEVEL set LOG_LEVEL=normal
 if not defined PRECISION_TEST set PRECISION_TEST=OFF
 if not defined NIGHTLY_MODE set PRECISION_TEST=OFF
+if not defined retry_times set retry_times=2

 rem -------set cache build directory-----------
 rmdir build\python /s/q
@@ -209,6 +221,7 @@ rem ------Build windows avx whl package------
 set WITH_AVX=ON
 set ON_INFER=OFF
 set CUDA_ARCH_NAME=All
+set retry_times=4

 call :cmake || goto cmake_error
 call :build || goto build_error
@@ -220,6 +233,7 @@ rem ------Build windows no-avx whl package------
 set WITH_AVX=OFF
 set ON_INFER=OFF
 set CUDA_ARCH_NAME=All
+set retry_times=4

 call :cmake || goto cmake_error
 call :build || goto build_error
@@ -249,6 +263,8 @@ echo    ========================================
 rem Configure the environment for 64-bit builds. 'DISTUTILS_USE_SDK' indicates that the user has selected the compiler.
 call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat"
 set DISTUTILS_USE_SDK=1
+rem Windows 10 Kit bin dir
+set PATH=C:\Program Files (x86)\Windows Kits\10\bin\10.0.17763.0\x64;%PATH%

 for /F %%# in ('wmic os get localdatetime^|findstr 20') do set start=%%#
 set start=%start:~4,10%
@@ -329,7 +345,7 @@ echo    ========================================
 echo    Step 2. Buile Paddle ...
 echo    ========================================

-for /F %%# in ('wmic cpu get NumberOfLogicalProcessors^|findstr [0-9]') do set /a PARALLEL_PROJECT_COUNT=%%#*4/5
+for /F %%# in ('wmic cpu get NumberOfLogicalProcessors^|findstr [0-9]') do set /a PARALLEL_PROJECT_COUNT=%%#*3/5
 echo "PARALLEL PROJECT COUNT is %PARALLEL_PROJECT_COUNT%"
 set build_times=1
 :build_tp
@@ -342,7 +358,7 @@ if %GENERATOR% == "Ninja" (
 )
 if %ERRORLEVEL% NEQ 0 (
    set /a build_times=%build_times%+1  
-    if %build_times% GTR 2 (
+    if %build_times% GTR %retry_times% (
        exit /b 7
    ) else (
        echo Build third_party failed, will retry!
@@ -356,6 +372,19 @@ set build_times=1
 :: reset clcache zero stats for collect PR's actual hit rate
 rem clcache.exe -z

+rem -------clean up environment again-----------
+taskkill /f /im MSBuild.exe 2>NUL
+taskkill /f /im cl.exe 2>NUL
+taskkill /f /im lib.exe 2>NUL
+taskkill /f /im link.exe 2>NUL
+taskkill /f /im vctip.exe 2>NUL
+taskkill /f /im cvtres.exe 2>NUL
+taskkill /f /im rc.exe 2>NUL
+taskkill /f /im test_api_impl.exe 2>NUL
+wmic process where name="op_function_generator.exe" call terminate 2>NUL
+wmic process where name="cvtres.exe" call terminate 2>NUL
+wmic process where name="rc.exe" call terminate 2>NUL
+
 echo Build Paddle the %build_times% time:
 if %GENERATOR% == "Ninja" (
    ninja -j %PARALLEL_PROJECT_COUNT%
@@ -369,7 +398,7 @@ if %GENERATOR% == "Ninja" (

 if %ERRORLEVEL% NEQ 0 (
    set /a build_times=%build_times%+1
-    if %build_times% GTR 1 (
+    if %build_times% GTR %retry_times% (
        exit /b 7
    ) else (
        echo Build Paddle failed, will retry!
@@ -706,9 +735,10 @@ taskkill /f /im git-remote-https.exe 2>NUL
 taskkill /f /im vctip.exe 2>NUL
 taskkill /f /im cvtres.exe 2>NUL
 taskkill /f /im rc.exe 2>NUL
+taskkill /f /im python.exe  2>NUL
 wmic process where name="op_function_generator.exe" call terminate 2>NUL
 wmic process where name="python.exe" call terminate 2>NUL
-taskkill /f /im python.exe  2>NUL
+wmic process where name="cvtres.exe" call terminate 2>NUL
 echo Windows CI run successfully!
 exit /b 0


--- a/tools/parallel_UT_rule.py
+++ b/tools/parallel_UT_rule.py
@@ -518,7 +518,6 @@ TWO_PARALLEL_JOB = [
    'test_softmax_with_cross_entropy_op',
    'test_imperative_gan',
    'test_simnet',
-    'test_instance_norm_op',
    'test_amp_check_finite_and_scale_op',
    'test_random_seed',
    'test_histogram_op',
@@ -561,7 +560,6 @@ TWO_PARALLEL_JOB = [
    'test_add_reader_dependency',
    'test_is_tensor',
    'test_variable',
-    'test_unsqueeze_op',
    'test_save_model_without_var',
    'test_unfold_op',
    'test_conv_bn_fuse_pass',
@@ -813,7 +811,6 @@ TWO_PARALLEL_JOB = [
    'test_sequence_expand_as',
    'test_sequence_reverse',
    'test_shape_op',
-    'test_lod_tensor',
    'test_diag',
    'test_strided_slice_op',
    'test_switch_case',
@@ -831,7 +828,6 @@ TWO_PARALLEL_JOB = [
    'test_arange',
    'test_lrn_mkldnn_op',
    'test_imperative_gnn',
-    'test_eager_deletion_while_op',
    'test_dequantize_abs_max_op',
    'test_elementwise_mul_op',
    'test_tensor_scalar_type_promotion_dynamic',

--- a/tools/windows/build_compile_environment.bat
+++ b/tools/windows/build_compile_environment.bat
@@ -22,9 +22,9 @@
 ::   Include:
 ::     1. CMake 3.17.0
 ::     2. Git 2.28.0
-::     3. Python 3.7.8
-::     4. Visual Studio 2015 with update 3
-::     5. CUDA 10
+::     3. Python 3.8.3
+::     4. Visual Studio 2017 Community
+::     5. CUDA 11.2
 ::     6. java jre
 ::     7. xly agent

@@ -73,7 +73,6 @@ if %errorlevel% == 0 (
 ) else (
  echo Error***** Install Cmake-3.17.0 failed, please re-install it manually.
 )
-del cmake-3.17.0-win64-x64.msi
 goto :eof
 :: ===== end step 1: cmake =====

@@ -99,91 +98,87 @@ if %errorlevel% == 0 (
 ) else (
  echo Error***** Install Git-2.28.0 failed, please re-install it manually.
 )
-del Git-2.28.0-64-bit.exe
 goto :eof
 :: ===== end step 2: Git =====

 :: ===== start step 3: Python =====
-:: Download Python-3.7.8 and add in PATH when it not installed.
-:: TODO: limit version >= 3.7.8
+:: Download Python-3.8.3 and add in PATH when it not installed.
+:: TODO: limit version >= 3.8.3
 :python
-echo ">>>>>>>> step [3/7]: Python 3.7.8"
-python -V 2>&1 | findstr /C:"Python 3.7.8" > nul 2> nul || call :install_python
-goto vs2015
+echo ">>>>>>>> step [3/7]: Python 3.8.3"
+python -V 2>&1 | findstr /C:"Python 3.8.3" > nul 2> nul || call :install_python
+goto vs

 :install_python
-echo There is not Python in this PC, will install Python-3.7.8.
-echo Download package from https://npm.taobao.org/mirrors/python/3.7.8/python-3.7.8-amd64.exe ...
-wget -O python-3.7.8-amd64.exe https://npm.taobao.org/mirrors/python/3.7.8/python-3.7.8-amd64.exe
-echo Install Python-3.7.8 ...
+echo There is not Python in this PC, will install Python-3.8.3
+echo Download package from https://paddle-ci.gz.bcebos.com/window_requirement/python-3.8.3-amd64.exe ...
+wget -O python-3.8.3-amd64.exe https://paddle-ci.gz.bcebos.com/window_requirement/python-3.8.3-amd64.exe
+echo Install Python-3.8.3 ...
 :: /passive [silent install]
 :: InstallAllUsers [add path for all users]
 :: PrependPath [add script/install into PATH]
 :: TargetDir [install directory]
-start /wait python-3.7.8-amd64.exe /passive InstallAllUsers=1 PrependPath=1 TargetDir=C:\Python37
+start /wait python-3.8.3-amd64.exe /passive InstallAllUsers=1 PrependPath=1 TargetDir=C:\Python38
 if %errorlevel% == 0 (
-  echo Install python-3.7.8 success!
+  echo Install python-3.8.3 success!
 ) else (
-  echo Error***** Install python-3.7.8 failed, please re-install it manually.
+  echo Error***** Install python-3.8.3 failed, please re-install it manually.
 )
-del python-3.7.8-amd64.exe
 goto :eof
 :: ===== end step 3: Python =====

-:: ===== start step 4: Visual Studio 2015 =====
-:: Download Visual Studio 2015 when it not installed.
-:vs2015
-echo ">>>>>>>> step [4/7]: Visual Studio 2015"
-cmd /C "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64 > nul 2> nul || call :install_visual_studio
+:: ===== start step 4: Visual Studio 2017 Community =====
+:: Download Visual Studio 2017 when it not installed.
+:vs
+echo ">>>>>>>> step [4/7]: Visual Studio 2017 "
+cmd /C "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat"  > nul 2> nul || call :install_visual_studio
 goto :cuda10

 :install_visual_studio
-echo There is not Visual Studio in this PC, will install VS2015.
-echo Download package from "https://paddle-ci.gz.bcebos.com/window_requirement/en_visual_studio_enterprise_2015_with_update_3_x86_x64_web_installer_8922986.exe"
-wget -O vs_installer.exe "https://paddle-ci.gz.bcebos.com/window_requirement/en_visual_studio_enterprise_2015_with_update_3_x86_x64_web_installer_8922986.exe"
-echo Install Visual Studio 2015 ...
+echo There is not Visual Studio in this PC, will install VS2017.
+echo Download package from "https://paddle-ci.gz.bcebos.com/window_requirement/VS2017/vs_Community.exe"
+wget -O vs_Community.exe "https://paddle-ci.gz.bcebos.com/window_requirement/VS2017/vs_Community.exe"
+echo Install Visual Studio 2017 ...
 :: /passive [silent install]
 :: /norestart [no restart]
 :: /NoRefresh [no refresh]
 :: /InstallSelectableItems NativeLanguageSupport_Group [select Visual C++ for installing]
-start /wait vs_installer.exe /passive /norestart /NoRefresh /InstallSelectableItems NativeLanguageSupport_Group
+start /wait vs_Community.exe --passive --add Microsoft.VisualStudio.Workload.NativeDesktop --add Microsoft.VisualStudio.Workload.Universal --includeRecommended
 if %errorlevel% == 0 (
-  echo Install Visual Studio 2015 success!
+  echo Install Visual Studio 2017 success!
 ) else (
-  echo Error***** Install Visual Studio 2015 failed, please re-install it manually.
+  echo Error***** Install Visual Studio 2017 failed, please re-install it manually.
 )
-del vs_installer.exe
 goto :eof
-:: ===== end step 4: Visual Studio 2015 =====
+:: ===== end step 4: Visual Studio 2017 =====

-:: ===== start step 5: CUDA 10 =====
+:: ===== start step 5: CUDA 11 =====
 :cuda10
-echo ">>>>>>>> step [5/7]: CUDA 10.2"
-cmd /C nvcc --version 2> nul | findstr /C:"10.2" > nul 2> nul || call :install_cuda
+echo ">>>>>>>> step [5/7]: CUDA 11.2"
+cmd /C nvcc --version 2> nul | findstr /C:"11.2" > nul 2> nul || call :install_cuda
 goto java-jre

 :install_cuda
-echo There is not CUDA in this PC, will install CUDA-10.2.
-echo Download package from "https://paddle-ci.gz.bcebos.com/window_requirement/cuda_10.2.89_441.22_win10.exe"
-wget -O cuda_installer.exe "https://paddle-ci.gz.bcebos.com/window_requirement/cuda_10.2.89_441.22_win10.exe"
-echo Install CUDA-10.2 ...
+echo There is not CUDA in this PC, will install CUDA-11.2.
+echo Download package from "https://paddle-ci.gz.bcebos.com/window_requirement/cuda_11.2.0_460.89_win10.exe"
+wget -O cuda_installer.exe "https://paddle-ci.gz.bcebos.com/window_requirement/cuda_11.2.0_460.89_win10.exe"
+echo Install CUDA-11.2 ...
 :: -s [silent install]
 start /wait cuda_installer.exe -s
 if %errorlevel% == 0 (
-  echo Install CUDA-10.2 success!
+  echo Install CUDA-11.2 success!
 ) else (
-  echo Error***** Install CUDA-10.2 failed, please re-install it manually.
+  echo Error***** Install CUDA-11.2 failed, please re-install it manually.
  goto :eof
 )
 del cuda_installer.exe
-echo Download cudnn from "https://paddle-ci.gz.bcebos.com/window_requirement/cudnn-10.2-windows10-x64-v7.6.5.32.zip"
-wget -O cudnn-10.2-windows10-x64-v7.6.5.32.zip "https://paddle-ci.gz.bcebos.com/window_requirement/cudnn-10.2-windows10-x64-v7.6.5.32.zip"
-tar xf cudnn-10.2-windows10-x64-v7.6.5.32.zip
-xcopy /E /Y /R "cuda\bin\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.2\bin"
-xcopy /E /Y /R "cuda\include\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.2\include"
-xcopy /E /Y /R "cuda\lib\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.2\lib"
+echo Download cudnn from "https://paddle-ci.gz.bcebos.com/window_requirement/cudnn-11.2-windows-x64-v8.1.0.77.zip"
+wget -O cudnn-11.2-windows-x64-v8.1.0.77.zip "https://paddle-ci.gz.bcebos.com/window_requirement/cudnn-11.2-windows-x64-v8.1.0.77.zip"
+tar xf cudnn-11.2-windows-x64-v8.1.0.77.zip
+xcopy /E /Y /R "cuda\bin\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2\bin"
+xcopy /E /Y /R "cuda\include\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2\include"
+xcopy /E /Y /R "cuda\lib\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2\lib"
 rd /s /q cuda
-del cudnn-10.2-windows10-x64-v7.6.5.32.zip
 goto :eof
 :: ===== end step 5: CUDA 10 =====

@@ -212,7 +207,7 @@ goto :eof
 :: ===== start step 7: xly agent =====
 :xly-agent
 echo ">>>>>>>> step [7/7]: xly agent"
-wget -O agent.jar "https://paddle-ci.gz.bcebos.com/window_requirement/agent.jar"
+wget -O agent.jar "https://xly.bce.baidu.com/sa_server/agent/v1/download?version=1.2.8"
 :: ===== end step 8: xly agent =====

 pause