Skip to content

P
Paddle

PaddlePaddle / Paddle
大约 1 年 前同步成功

通知 2298
Star 20931
Fork 5422
P
Paddle
未验证 提交 7a681f0b 编写于 作者: Z Zhou Wei 提交者： GitHub
浏览文件
操作

fix Windows CI MP compile and environment install script and openblas CI (#32378) 

* fix Windows CI MP compile and environment install script

* clear Windows CI environment

* clear Windows CI environment

* clear Windows CI environment
上级 0e74eea2
隐藏空白更改
内联 并排
Showing with 93 addition and 67 deletion
CMakeLists.txt
浏览文件 @ 7a681f0b
......@@ -22,9 +22,6 @@ include(system)
project(paddle CXX C)
include(init)
include(generic) # simplify cmake module
# enable language CUDA
# TODO(Shibo Tao): remove find_package(CUDA) completely.
find_package(CUDA QUIET)
......@@ -38,6 +35,10 @@ option(WITH_ROCM "Compile PaddlePaddle with ROCM platform" OFF)
# to develop some acl related functionality on x86
option(WITH_ASCEND_CL "Compile PaddlePaddle with ASCEND CL" ${WITH_ASCEND})
option(WITH_ASCEND_CXX11 "Compile PaddlePaddle with ASCEND and CXX11 ABI" OFF)
# Note(zhouwei): It use option above, so put here
include(init)
include(generic) # simplify cmake module
if (WITH_GPU AND WITH_XPU)
message(FATAL_ERROR "Error when compile GPU and XPU at the same time")
endif()
......@@ -104,8 +105,10 @@ if(WIN32)
endforeach(flag_var)
endif()
# NOTE(Avin0323): Less parallel count result in faster compilation.
math(EXPR PROCESS_MAX "${CPU_CORES} * 2 / 3")
# NOTE(zhouwei25): temporarily change MP to 1 for reducing CPU & memory utilization
set(PROCESS_MAX 1)
#math(EXPR PROCESS_MAX "${CPU_CORES} * 1 / 2")
# windows build turn off warnings, use parallel compiling.
foreach(flag_var
CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE
......@@ -133,6 +136,9 @@ if(WIN32)
foreach(flag_var CMAKE_SHARED_LINKER_FLAGS CMAKE_STATIC_LINKER_FLAGS CMAKE_EXE_LINKER_FLAGS CMAKE_LINKER_FLAGS)
set(${flag_var} "${${flag_var}} /ignore:4049 /ignore:4217 /ignore:4006 /ignore:4221")
if(MSVC_STATIC_CRT)
set(${flag_var} "${${flag_var}} /NODEFAULTLIB:MSVCRT.LIB")
endif()
endforeach(flag_var)
if (WITH_WIN_DUMP_DBG)
......
cmake/init.cmake
浏览文件 @ 7a681f0b
......@@ -18,10 +18,10 @@ if(NOT WIN32)
set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -g -DNDEBUG")
set(CMAKE_CXX_FLAGS_MINSIZEREL "-Os -DNDEBUG")
else()
# It has not been used now, it can specify CUDA compile flag manualy,
# It can specify CUDA compile flag manualy,
# its use is to remvoe /Zi to reduce GPU static library size. But it's dangerous
# because CUDA will update by nvidia, then error will occur.
# Now, it's used in CUDA:[10.0, 10.2]
# Now, it's only used in VS2015 + CUDA:[10.0, 10.2]
set(WIN_PROPS ${CMAKE_SOURCE_DIR}/cmake/paddle_win.props)
endif()
......
cmake/paddle_win.props
浏览文件 @ 7a681f0b
......@@ -88,4 +88,3 @@ set CUDAFE_FLAGS=--sdk_dir "$(WindowsSdkDir)"
</ClCompile>
</ItemDefinitionGroup>
</Project>
paddle/scripts/paddle_build.bat
浏览文件 @ 7a681f0b
......@@ -26,9 +26,20 @@ set cache_dir=%work_dir:Paddle=cache%
if not exist %cache_dir%\tools (
git clone https://github.com/zhouwei25/tools.git %cache_dir%\tools
)
taskkill /f /im op_function_generator.exe
wmic process where name="op_function_generator.exe" call terminate
taskkill /f /im op_function_generator.exe 2>NUL
taskkill /f /im cmake.exe 2>NUL
taskkill /f /im MSBuild.exe 2>NUL
taskkill /f /im CL.exe 2>NUL
taskkill /f /im Lib.exe 2>NUL
taskkill /f /im link.exe 2>NUL
taskkill /f /im vctip.exe 2>NUL
taskkill /f /im cvtres.exe 2>NUL
taskkill /f /im rc.exe 2>NUL
taskkill /f /im python.exe 2>NUL
wmic process where name="op_function_generator.exe" call terminate 2>NUL
wmic process where name="python.exe" call terminate 2>NUL
wmic process where name="cvtres.exe" call terminate 2>NUL
wmic process where name="rc.exe" call terminate 2>NUL
rem ------initialize common variable------
......@@ -54,6 +65,7 @@ if not defined INFERENCE_DEMO_INSTALL_DIR set INFERENCE_DEMO_INSTALL_DIR=%cache_
if not defined LOG_LEVEL set LOG_LEVEL=normal
if not defined PRECISION_TEST set PRECISION_TEST=OFF
if not defined NIGHTLY_MODE set PRECISION_TEST=OFF
if not defined retry_times set retry_times=2
rem -------set cache build directory-----------
rmdir build\python /s/q
......@@ -209,6 +221,7 @@ rem ------Build windows avx whl package------
set WITH_AVX=ON
set ON_INFER=OFF
set CUDA_ARCH_NAME=All
set retry_times=4
call :cmake || goto cmake_error
call :build || goto build_error
......@@ -220,6 +233,7 @@ rem ------Build windows no-avx whl package------
set WITH_AVX=OFF
set ON_INFER=OFF
set CUDA_ARCH_NAME=All
set retry_times=4
call :cmake || goto cmake_error
call :build || goto build_error
......@@ -249,6 +263,8 @@ echo ========================================
rem Configure the environment for 64-bit builds. 'DISTUTILS_USE_SDK' indicates that the user has selected the compiler.
call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat"
set DISTUTILS_USE_SDK=1
rem Windows 10 Kit bin dir
set PATH=C:\Program Files (x86)\Windows Kits\10\bin\10.0.17763.0\x64;%PATH%
for /F %%# in ('wmic os get localdatetime^|findstr 20') do set start=%%#
set start=%start:~4,10%
......@@ -329,7 +345,7 @@ echo ========================================
echo Step 2. Buile Paddle ...
echo ========================================
for /F %%# in ('wmic cpu get NumberOfLogicalProcessors^|findstr [0-9]') do set /a PARALLEL_PROJECT_COUNT=%%#*4/5
for /F %%# in ('wmic cpu get NumberOfLogicalProcessors^|findstr [0-9]') do set /a PARALLEL_PROJECT_COUNT=%%#*3/5
echo "PARALLEL PROJECT COUNT is %PARALLEL_PROJECT_COUNT%"
set build_times=1
:build_tp
......@@ -342,7 +358,7 @@ if %GENERATOR% == "Ninja" (
)
if %ERRORLEVEL% NEQ 0 (
set /a build_times=%build_times%+1
if %build_times% GTR 2 (
if %build_times% GTR %retry_times% (
exit /b 7
) else (
echo Build third_party failed, will retry!
......@@ -356,6 +372,19 @@ set build_times=1
:: reset clcache zero stats for collect PR's actual hit rate
rem clcache.exe -z
rem -------clean up environment again-----------
taskkill /f /im MSBuild.exe 2>NUL
taskkill /f /im cl.exe 2>NUL
taskkill /f /im lib.exe 2>NUL
taskkill /f /im link.exe 2>NUL
taskkill /f /im vctip.exe 2>NUL
taskkill /f /im cvtres.exe 2>NUL
taskkill /f /im rc.exe 2>NUL
taskkill /f /im test_api_impl.exe 2>NUL
wmic process where name="op_function_generator.exe" call terminate 2>NUL
wmic process where name="cvtres.exe" call terminate 2>NUL
wmic process where name="rc.exe" call terminate 2>NUL
echo Build Paddle the %build_times% time:
if %GENERATOR% == "Ninja" (
ninja -j %PARALLEL_PROJECT_COUNT%
......@@ -369,7 +398,7 @@ if %GENERATOR% == "Ninja" (
if %ERRORLEVEL% NEQ 0 (
set /a build_times=%build_times%+1
if %build_times% GTR 1 (
if %build_times% GTR %retry_times% (
exit /b 7
) else (
echo Build Paddle failed, will retry!
......@@ -706,9 +735,10 @@ taskkill /f /im git-remote-https.exe 2>NUL
taskkill /f /im vctip.exe 2>NUL
taskkill /f /im cvtres.exe 2>NUL
taskkill /f /im rc.exe 2>NUL
taskkill /f /im python.exe 2>NUL
wmic process where name="op_function_generator.exe" call terminate 2>NUL
wmic process where name="python.exe" call terminate 2>NUL
taskkill /f /im python.exe 2>NUL
wmic process where name="cvtres.exe" call terminate 2>NUL
echo Windows CI run successfully!
exit /b 0
......
tools/parallel_UT_rule.py
浏览文件 @ 7a681f0b
......@@ -518,7 +518,6 @@ TWO_PARALLEL_JOB = [
'test_softmax_with_cross_entropy_op',
'test_imperative_gan',
'test_simnet',
'test_instance_norm_op',
'test_amp_check_finite_and_scale_op',
'test_random_seed',
'test_histogram_op',
......@@ -561,7 +560,6 @@ TWO_PARALLEL_JOB = [
'test_add_reader_dependency',
'test_is_tensor',
'test_variable',
'test_unsqueeze_op',
'test_save_model_without_var',
'test_unfold_op',
'test_conv_bn_fuse_pass',
......@@ -813,7 +811,6 @@ TWO_PARALLEL_JOB = [
'test_sequence_expand_as',
'test_sequence_reverse',
'test_shape_op',
'test_lod_tensor',
'test_diag',
'test_strided_slice_op',
'test_switch_case',
......@@ -831,7 +828,6 @@ TWO_PARALLEL_JOB = [
'test_arange',
'test_lrn_mkldnn_op',
'test_imperative_gnn',
'test_eager_deletion_while_op',
'test_dequantize_abs_max_op',
'test_elementwise_mul_op',
'test_tensor_scalar_type_promotion_dynamic',
......
tools/windows/build_compile_environment.bat
浏览文件 @ 7a681f0b
......@@ -22,9 +22,9 @@
:: Include:
:: 1. CMake 3.17.0
:: 2. Git 2.28.0
:: 3. Python 3.7.8
:: 4. Visual Studio 2015 with update 3
:: 5. CUDA 10
:: 3. Python 3.8.3
:: 4. Visual Studio 2017 Community
:: 5. CUDA 11.2
:: 6. java jre
:: 7. xly agent
......@@ -73,7 +73,6 @@ if %errorlevel% == 0 (
) else (
echo Error***** Install Cmake-3.17.0 failed, please re-install it manually.
)
del cmake-3.17.0-win64-x64.msi
goto :eof
:: ===== end step 1: cmake =====
......@@ -99,91 +98,87 @@ if %errorlevel% == 0 (
) else (
echo Error***** Install Git-2.28.0 failed, please re-install it manually.
)
del Git-2.28.0-64-bit.exe
goto :eof
:: ===== end step 2: Git =====
:: ===== start step 3: Python =====
:: Download Python-3.7.8 and add in PATH when it not installed.
:: TODO: limit version >= 3.7.8
:: Download Python-3.8.3 and add in PATH when it not installed.
:: TODO: limit version >= 3.8.3
:python
echo ">>>>>>>> step [3/7]: Python 3.7.8"
python -V 2>&1 | findstr /C:"Python 3.7.8" > nul 2> nul || call :install_python
goto vs2015
echo ">>>>>>>> step [3/7]: Python 3.8.3"
python -V 2>&1 | findstr /C:"Python 3.8.3" > nul 2> nul || call :install_python
goto vs
:install_python
echo There is not Python in this PC, will install Python-3.7.8.
echo Download package from https://npm.taobao.org/mirrors/python/3.7.8/python-3.7.8-amd64.exe ...
wget -O python-3.7.8-amd64.exe https://npm.taobao.org/mirrors/python/3.7.8/python-3.7.8-amd64.exe
echo Install Python-3.7.8 ...
echo There is not Python in this PC, will install Python-3.8.3
echo Download package from https://paddle-ci.gz.bcebos.com/window_requirement/python-3.8.3-amd64.exe ...
wget -O python-3.8.3-amd64.exe https://paddle-ci.gz.bcebos.com/window_requirement/python-3.8.3-amd64.exe
echo Install Python-3.8.3 ...
:: /passive [silent install]
:: InstallAllUsers [add path for all users]
:: PrependPath [add script/install into PATH]
:: TargetDir [install directory]
start /wait python-3.7.8-amd64.exe /passive InstallAllUsers=1 PrependPath=1 TargetDir=C:\Python37
start /wait python-3.8.3-amd64.exe /passive InstallAllUsers=1 PrependPath=1 TargetDir=C:\Python38
if %errorlevel% == 0 (
echo Install python-3.7.8 success!
echo Install python-3.8.3 success!
) else (
echo Error***** Install python-3.7.8 failed, please re-install it manually.
echo Error***** Install python-3.8.3 failed, please re-install it manually.
)
del python-3.7.8-amd64.exe
goto :eof
:: ===== end step 3: Python =====
:: ===== start step 4: Visual Studio 2015 =====
:: Download Visual Studio 2015 when it not installed.
:vs2015
echo ">>>>>>>> step [4/7]: Visual Studio 2015"
cmd /C "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64 > nul 2> nul || call :install_visual_studio
:: ===== start step 4: Visual Studio 2017 Community =====
:: Download Visual Studio 2017 when it not installed.
:vs
echo ">>>>>>>> step [4/7]: Visual Studio 2017 "
cmd /C "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat" > nul 2> nul || call :install_visual_studio
goto :cuda10
:install_visual_studio
echo There is not Visual Studio in this PC, will install VS2015.
echo Download package from "https://paddle-ci.gz.bcebos.com/window_requirement/en_visual_studio_enterprise_2015_with_update_3_x86_x64_web_installer_8922986.exe"
wget -O vs_installer.exe "https://paddle-ci.gz.bcebos.com/window_requirement/en_visual_studio_enterprise_2015_with_update_3_x86_x64_web_installer_8922986.exe"
echo Install Visual Studio 2015 ...
echo There is not Visual Studio in this PC, will install VS2017.
echo Download package from "https://paddle-ci.gz.bcebos.com/window_requirement/VS2017/vs_Community.exe"
wget -O vs_Community.exe "https://paddle-ci.gz.bcebos.com/window_requirement/VS2017/vs_Community.exe"
echo Install Visual Studio 2017 ...
:: /passive [silent install]
:: /norestart [no restart]
:: /NoRefresh [no refresh]
:: /InstallSelectableItems NativeLanguageSupport_Group [select Visual C++ for installing]
start /wait vs_installer.exe /passive /norestart /NoRefresh /InstallSelectableItems NativeLanguageSupport_Group
start /wait vs_Community.exe --passive --add Microsoft.VisualStudio.Workload.NativeDesktop --add Microsoft.VisualStudio.Workload.Universal --includeRecommended
if %errorlevel% == 0 (
echo Install Visual Studio 2015 success!
echo Install Visual Studio 2017 success!
) else (
echo Error***** Install Visual Studio 2015 failed, please re-install it manually.
echo Error***** Install Visual Studio 2017 failed, please re-install it manually.
)
del vs_installer.exe
goto :eof
:: ===== end step 4: Visual Studio 2015 =====
:: ===== end step 4: Visual Studio 2017 =====
:: ===== start step 5: CUDA 10 =====
:: ===== start step 5: CUDA 11 =====
:cuda10
echo ">>>>>>>> step [5/7]: CUDA 10.2"
cmd /C nvcc --version 2> nul | findstr /C:"10.2" > nul 2> nul || call :install_cuda
echo ">>>>>>>> step [5/7]: CUDA 11.2"
cmd /C nvcc --version 2> nul | findstr /C:"11.2" > nul 2> nul || call :install_cuda
goto java-jre
:install_cuda
echo There is not CUDA in this PC, will install CUDA-10.2.
echo Download package from "https://paddle-ci.gz.bcebos.com/window_requirement/cuda_10.2.89_441.22_win10.exe"
wget -O cuda_installer.exe "https://paddle-ci.gz.bcebos.com/window_requirement/cuda_10.2.89_441.22_win10.exe"
echo Install CUDA-10.2 ...
echo There is not CUDA in this PC, will install CUDA-11.2.
echo Download package from "https://paddle-ci.gz.bcebos.com/window_requirement/cuda_11.2.0_460.89_win10.exe"
wget -O cuda_installer.exe "https://paddle-ci.gz.bcebos.com/window_requirement/cuda_11.2.0_460.89_win10.exe"
echo Install CUDA-11.2 ...
:: -s [silent install]
start /wait cuda_installer.exe -s
if %errorlevel% == 0 (
echo Install CUDA-10.2 success!
echo Install CUDA-11.2 success!
) else (
echo Error***** Install CUDA-10.2 failed, please re-install it manually.
echo Error***** Install CUDA-11.2 failed, please re-install it manually.
goto :eof
)
del cuda_installer.exe
echo Download cudnn from "https://paddle-ci.gz.bcebos.com/window_requirement/cudnn-10.2-windows10-x64-v7.6.5.32.zip"
wget -O cudnn-10.2-windows10-x64-v7.6.5.32.zip "https://paddle-ci.gz.bcebos.com/window_requirement/cudnn-10.2-windows10-x64-v7.6.5.32.zip"
tar xf cudnn-10.2-windows10-x64-v7.6.5.32.zip
xcopy /E /Y /R "cuda\bin\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.2\bin"
xcopy /E /Y /R "cuda\include\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.2\include"
xcopy /E /Y /R "cuda\lib\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.2\lib"
echo Download cudnn from "https://paddle-ci.gz.bcebos.com/window_requirement/cudnn-11.2-windows-x64-v8.1.0.77.zip"
wget -O cudnn-11.2-windows-x64-v8.1.0.77.zip "https://paddle-ci.gz.bcebos.com/window_requirement/cudnn-11.2-windows-x64-v8.1.0.77.zip"
tar xf cudnn-11.2-windows-x64-v8.1.0.77.zip
xcopy /E /Y /R "cuda\bin\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2\bin"
xcopy /E /Y /R "cuda\include\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2\include"
xcopy /E /Y /R "cuda\lib\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.2\lib"
rd /s /q cuda
del cudnn-10.2-windows10-x64-v7.6.5.32.zip
goto :eof
:: ===== end step 5: CUDA 10 =====
......@@ -212,7 +207,7 @@ goto :eof
:: ===== start step 7: xly agent =====
:xly-agent
echo ">>>>>>>> step [7/7]: xly agent"
wget -O agent.jar "https://paddle-ci.gz.bcebos.com/window_requirement/agent.jar"
wget -O agent.jar "https://xly.bce.baidu.com/sa_server/agent/v1/download?version=1.2.8"
:: ===== end step 8: xly agent =====
pause
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或