From ff8ca52f881d39a9e34267b28a6d86fb1bc7256f Mon Sep 17 00:00:00 2001 From: Zhou Wei <52485244+zhouwei25@users.noreply.github.com> Date: Wed, 10 Jun 2020 13:58:27 +0800 Subject: [PATCH] windows publish package scripts (#24851) * windows publish package scripts,test=develop * windows publish package scripts,test=develop * windows publish package scripts,test=develop --- paddle/scripts/windows_build/build.bat | 187 ++++++++++++++++++ paddle/scripts/windows_build/config.ini | 19 ++ paddle/scripts/windows_build/series_build.bat | 63 ++++++ 3 files changed, 269 insertions(+) create mode 100644 paddle/scripts/windows_build/build.bat create mode 100644 paddle/scripts/windows_build/config.ini create mode 100644 paddle/scripts/windows_build/series_build.bat diff --git a/paddle/scripts/windows_build/build.bat b/paddle/scripts/windows_build/build.bat new file mode 100644 index 00000000000..65d44877d12 --- /dev/null +++ b/paddle/scripts/windows_build/build.bat @@ -0,0 +1,187 @@ +@ECHO OFF +SETLOCAL +set source_path=%1 +set PYTHON_DIR=%2 +set WITH_GPU=%3 +set WITH_MKL=%4 +set ON_INFER=%5 +set PADDLE_VERSION=%6 +set BATDIR=%7 +set CUDA_DIR=%8 + +set RETRY_TIMES=3 + +set CUDA_DIR_WIN=%CUDA_DIR:/=\% +set PATH=%CUDA_DIR_WIN%\nvvm\bin\;%CUDA_DIR_WIN%\bin;%PATH% + + +for /f "tokens=1,2,* delims=\\" %%a in ("%PYTHON_DIR%") do ( + set c1=%%a + set c2=%%b +) +set PYTHONV=%c2% + +echo %CUDA_DIR% | findstr 10.0 > NULL +if %errorlevel% == 0 (set PADDLE_VERSION=%PADDLE_VERSION% +set CUDAV=v10.0) +echo %CUDA_DIR% | findstr 9.2 > NULL +if %errorlevel% == 0 (set PADDLE_VERSION=%PADDLE_VERSION%.post97 +set CUDAV=v9.2) +echo %CUDA_DIR% | findstr 9.0 > NULL +if %errorlevel% == 0 (set PADDLE_VERSION=%PADDLE_VERSION%.post97 +set CUDAV=v9.0) +echo %CUDA_DIR% | findstr 8.0 > NULL +if %errorlevel% == 0 (set PADDLE_VERSION=%PADDLE_VERSION%.post87 +set CUDAV=v8.0) +set PLAT=GPU +if "%WITH_GPU%"=="OFF" ( + set PLAT=CPU + set CUDAV=CPU +) + +if "%WITH_MKL%"=="ON" ( + set BLAS=MKL +) else ( + set BLAS=OPEN +) + +if "%ON_INFER%"=="ON" ( + goto :INFERENCE_LIBRARY +) + +echo "begin to do build noavx ..." + +set "dst_path=%source_path%\build_%PYTHONV%_%PLAT%_%BLAS%_%CUDAV%_noavx" + +if exist %dst_path% rmdir /q /s %dst_path% +mkdir %dst_path% + +cd /d %dst_path% +echo Current directory : %cd% + +call:rest_env + +echo cmake %dst_path%\..\Paddle -G "Visual Studio 14 2015 Win64" -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% -DWITH_AVX=OFF -DPYTHON_INCLUDE_DIR=%PYTHON_DIR%\include\ -DPYTHON_LIBRARY=%PYTHON_DIR%\libs\ -DPYTHON_EXECUTABLE=%PYTHON_DIR%\python.exe -DCMAKE_BUILD_TYPE=Release -DWITH_TESTING=OFF -DWITH_PYTHON=ON -DCUDA_TOOLKIT_ROOT_DIR=%CUDA_DIR% -DCUDA_ARCH_NAME=All +cmake %dst_path%\..\Paddle -G "Visual Studio 14 2015 Win64" -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% -DWITH_AVX=OFF -DPYTHON_INCLUDE_DIR=%PYTHON_DIR%\include\ -DPYTHON_LIBRARY=%PYTHON_DIR%\libs\ -DPYTHON_EXECUTABLE=%PYTHON_DIR%\python.exe -DCMAKE_BUILD_TYPE=Release -DWITH_TESTING=OFF -DWITH_PYTHON=ON -DCUDA_TOOLKIT_ROOT_DIR=%CUDA_DIR% -DCUDA_ARCH_NAME=All + +set MSBUILDDISABLENODEREUSE=1 + +set BUILD_TYPE=NO_AVX +call:Build + +REM ------------------------------------------------------------------------- + +echo "begin to do build avx ..." +set "dst_path=%source_path%\build_%PYTHONV%_%PLAT%_%BLAS%_%CUDAV%" + +if exist %dst_path% rmdir /q /s %dst_path% +mkdir %dst_path% + +cd /d %dst_path% +echo Current directory : %cd% + +call:rest_env + +echo cmake %dst_path%\..\Paddle -G "Visual Studio 14 2015 Win64" -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% -DWITH_AVX=ON -DPYTHON_INCLUDE_DIR=%PYTHON_DIR%\include\ -DPYTHON_LIBRARY=%PYTHON_DIR%\libs\ -DPYTHON_EXECUTABLE=%PYTHON_DIR%\python.exe -DCMAKE_BUILD_TYPE=Release -DWITH_TESTING=OFF -DWITH_PYTHON=ON -DCUDA_TOOLKIT_ROOT_DIR=%CUDA_DIR% -DCUDA_ARCH_NAME=All -DNOAVX_CORE_FILE=%dst_path%_noavx\python\paddle\fluid\core_noavx.pyd +cmake %dst_path%\..\Paddle -G "Visual Studio 14 2015 Win64" -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% -DWITH_AVX=ON -DPYTHON_INCLUDE_DIR=%PYTHON_DIR%\include\ -DPYTHON_LIBRARY=%PYTHON_DIR%\libs\ -DPYTHON_EXECUTABLE=%PYTHON_DIR%\python.exe -DCMAKE_BUILD_TYPE=Release -DWITH_TESTING=OFF -DWITH_PYTHON=ON -DCUDA_TOOLKIT_ROOT_DIR=%CUDA_DIR% -DCUDA_ARCH_NAME=All -DNOAVX_CORE_FILE=%dst_path%_noavx\python\paddle\fluid\core_noavx.pyd + +set MSBUILDDISABLENODEREUSE=1 + +set BUILD_TYPE=AVX +call:Build + +echo BUILD WHL PACKAGE COMPLETE +goto :END +REM ------------------------------------------------------------------------- + +:INFERENCE_LIBRARY + +echo "begin to do build inference library ..." +set "dst_path=%source_path%\build_INFERENCE_LIBRARY_%PLAT%_%BLAS%_%CUDAV%" + +if exist %dst_path% rmdir /q /s %dst_path% +mkdir %dst_path% + +cd /d %dst_path% +echo Current directory : %cd% + +call:rest_env + +echo cmake %dst_path%\..\Paddle -G "Visual Studio 14 2015 Win64" -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% -DCMAKE_BUILD_TYPE=Release -DWITH_PYTHON=OFF -DON_INFER=ON -DCUDA_TOOLKIT_ROOT_DIR=%CUDA_DIR% -DCUDA_ARCH_NAME=All +cmake %dst_path%\..\Paddle -G "Visual Studio 14 2015 Win64" -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% -DCMAKE_BUILD_TYPE=Release -DWITH_PYTHON=OFF -DON_INFER=ON -DCUDA_TOOLKIT_ROOT_DIR=%CUDA_DIR% -DCUDA_ARCH_NAME=All + +set MSBUILDDISABLENODEREUSE=1 + +set BUILD_TYPE=INFERENCE LIBRARY +call:Build + +echo PACKAGE INFERENCE LIBRARY + +mkdir inference_dist +%PYTHON_DIR%\python.exe -c "import shutil;shutil.make_archive('inference_dist/fluid_inference_install_dir', 'zip', root_dir='fluid_inference_install_dir')" +%PYTHON_DIR%\python.exe -c "import shutil;shutil.make_archive('inference_dist/fluid_install_dir', 'zip', root_dir='fluid_install_dir')" + +echo BUILD INFERENCE LIBRARY COMPLETE +goto :END + + +:Rest_env +echo "Reset Build Environment ..." +taskkill /f /im cmake.exe 2>NUL +taskkill /f /im msbuild.exe 2>NUL +taskkill /f /im git.exe 2>NUL +taskkill /f /im cl.exe 2>NUL +taskkill /f /im lib.exe 2>NUL +taskkill /f /im git-remote-https.exe 2>NUL +taskkill /f /im vctip.exe 2>NUL +goto:eof + +:Build +set build_times=1 +:build_thirdparty + +echo Build %BUILD_TYPE% Third Party Libraries, Round : %build_times% + +echo msbuild /m /p:Configuration=Release third_party.vcxproj ^>^> build_thirdparty_%build_times%.log +msbuild /m /p:Configuration=Release third_party.vcxproj >> build_thirdparty_%build_times%.log + +IF %ERRORLEVEL% NEQ 0 ( + echo Build %BUILD_TYPE% Third Party Libraries, Round : %build_times% Failed! + set /a build_times=%build_times%+1 + + if %build_times% GTR %RETRY_TIMES% ( + goto :FAILURE + ) else ( + goto :build_thirdparty + ) +) + +set build_times=1 +:build_paddle + +echo Build %BUILD_TYPE% Paddle Solutions, Round : %build_times% + +echo msbuild /m /p:Configuration=Release paddle.sln ^>^> build_%build_times%.log +msbuild /m /p:Configuration=Release paddle.sln >> build_%build_times%.log + +IF %ERRORLEVEL% NEQ 0 ( + echo Build %BUILD_TYPE% Paddle Solutions, Round : %build_times% Failed! + set /a build_times=%build_times%+1 + + if %build_times% GTR %RETRY_TIMES% ( + goto :FAILURE + ) else ( + goto :build_paddle + ) +) +goto:eof + + +:FAILURE +echo BUILD FAILED +exit /b 1 + +:END +echo BUILD SUCCESSFULLY + +ENDLOCAL \ No newline at end of file diff --git a/paddle/scripts/windows_build/config.ini b/paddle/scripts/windows_build/config.ini new file mode 100644 index 00000000000..32638d2873c --- /dev/null +++ b/paddle/scripts/windows_build/config.ini @@ -0,0 +1,19 @@ +############# the scripts for bulk publish package on windows platform ################ + +# Usage: please use # to comment, use space or ; to separate multiple variables +# Generally, you only need to change CUDA_PATH and PYTHON_PATH to publish package + +PADDLE_VERSION=2.0.0-alpha0 + +BRANCH=v2.0.0-alpha0 + +http_proxy=#please edit your proxy# +https_proxy=#please edit your proxy# + +# Just for example, please set by your windows environment +vcvarsall_dir="C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" +PYTHON3_PATH=C:\Python37 + +CUDA_PATH="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.0" "C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v9.0" + +PYTHON_PATH=C:\Python27 C:\Python37 \ No newline at end of file diff --git a/paddle/scripts/windows_build/series_build.bat b/paddle/scripts/windows_build/series_build.bat new file mode 100644 index 00000000000..e39ac761866 --- /dev/null +++ b/paddle/scripts/windows_build/series_build.bat @@ -0,0 +1,63 @@ +@ECHO OFF + +for /f "eol=# delims== tokens=1,2" %%i in (config.ini) do ( + set %%i=%%j +) + +set "source_path=%1" + +if "%source_path%"=="" ( + set /p source_path="Please input the dst path : =======>" +) + +if not exist %source_path% mkdir %source_path% +cd /d %source_path% +if %errorlevel% NEQ 0 GOTO END + + +echo "begin to download the source code from https://github.com/paddlepaddle/paddle" +git clone https://github.com/PaddlePaddle/Paddle +cd Paddle +git checkout %BRANCH% +rem sed -i "s/@PADDLE_VERSION@/%PADDLE_VERSION%/g" paddle\fluid\framework\commit.h.in +rem sed -i "s/add_definitions(-DPADDLE_VERSION=\${PADDLE_VERSION})//g" cmake\version.cmake +echo "download done!!!" + +cd .. +set start_path=%~dp0 +echo %start_path% + + +echo Init Visual Studio Env +call %vcvarsall_dir% amd64 + +rem source_path PYTHON_DIR WITH_GPU WITH_MKL ON_INFER PADDLE_VERSION BATDIR CUDA_DIR + + +rem ==============build GPU============ +for /D %%i in ( %CUDA_PATH% ) do ( + for /D %%j in ( %PYTHON_PATH% ) do ( + call %start_path%build.bat %source_path% %%j ON ON OFF %PADDLE_VERSION% %start_path% %%i + call %start_path%build.bat %source_path% %%j ON OFF OFF %PADDLE_VERSION% %start_path% %%i + ) + rem ===build inference library=== + call %start_path%build.bat %source_path% %PYTHON3_PATH% ON ON ON %PADDLE_VERSION% %start_path% %release_dir% %%i + call %start_path%build.bat %source_path% %PYTHON3_PATH% ON OFF ON %PADDLE_VERSION% %start_path% %release_dir% %%i +) + + +rem ==============build CPU============= + +for /D %%j in ( %PYTHON_PATH% ) do ( + call %start_path%build.bat %source_path% %%j OFF ON OFF %PADDLE_VERSION% %start_path% NEEDLESS + call %start_path%build.bat %source_path% %%j OFF OFF OFF %PADDLE_VERSION% %start_path% NEEDLESS +) +rem ===build inference library=== +call %start_path%build.bat %source_path% %PYTHON3_PATH% OFF ON ON %PADDLE_VERSION% %start_path% %release_dir% NEEDLESS +call %start_path%build.bat %source_path% %PYTHON3_PATH% OFF OFF ON %PADDLE_VERSION% %start_path% %release_dir% NEEDLESS + +:END +rem reset environment variable +for /f "eol=# delims== tokens=1,2" %%i in (config.ini) do ( + set %%i= +) \ No newline at end of file -- GitLab