paddle_build.bat

@ECHO OFF
rem Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
rem
rem Licensed under the Apache License, Version 2.0 (the "License");
rem you may not use this file except in compliance with the License.
rem You may obtain a copy of the License at
rem
rem     http://www.apache.org/licenses/LICENSE-2.0
rem
rem Unless required by applicable law or agreed to in writing, software
rem distributed under the License is distributed on an "AS IS" BASIS,
rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
rem See the License for the specific language governing permissions and
rem limitations under the License.

rem =================================================
rem       Paddle CI Task On Windows Platform
rem =================================================

@ECHO ON
setlocal enabledelayedexpansion

rem -------clean up environment-----------
set work_dir=%cd%
if not defined cache_dir set cache_dir=%work_dir:Paddle=cache%
if not exist %cache_dir%\tools (
    git clone https://github.com/zhouwei25/tools.git %cache_dir%\tools
)
taskkill /f /im cmake.exe /t 2>NUL
taskkill /f /im ninja.exe /t 2>NUL
taskkill /f /im MSBuild.exe /t 2>NUL
taskkill /f /im cl.exe /t 2>NUL
taskkill /f /im lib.exe /t 2>NUL
taskkill /f /im link.exe /t 2>NUL
taskkill /f /im vctip.exe /t 2>NUL
taskkill /f /im cvtres.exe /t 2>NUL
taskkill /f /im rc.exe /t 2>NUL
taskkill /f /im mspdbsrv.exe /t 2>NUL
taskkill /f /im csc.exe /t 2>NUL
taskkill /f /im python.exe /t 2>NUL
taskkill /f /im nvcc.exe /t 2>NUL
taskkill /f /im cicc.exe /t 2>NUL
taskkill /f /im ptxas.exe /t 2>NUL
taskkill /f /im op_function_generator.exe /t 2>NUL
taskkill /f /im eager_generator.exe /t 2>NUL
taskkill /f /im eager_op_function_generator.exe /t 2>NUL
wmic process where name="op_function_generator.exe" call terminate 2>NUL
wmic process where name="eager_generator.exe" call terminate 2>NUL
wmic process where name="eager_op_function_generator.exe" call terminate 2>NUL
wmic process where name="cvtres.exe" call terminate 2>NUL
wmic process where name="rc.exe" call terminate 2>NUL
wmic process where name="cl.exe" call terminate 2>NUL
wmic process where name="lib.exe" call terminate 2>NUL
wmic process where name="python.exe" call terminate 2>NUL

rem ------initialize common variable------
if not defined GENERATOR set GENERATOR="Visual Studio 15 2017 Win64"
if not defined WITH_TENSORRT set WITH_TENSORRT=ON
if not defined TENSORRT_ROOT set TENSORRT_ROOT=D:/TensorRT
if not defined CUDA_ARCH_NAME set CUDA_ARCH_NAME=Auto
if not defined WITH_GPU set WITH_GPU=ON
if not defined WITH_MKL set WITH_MKL=ON
if not defined WITH_AVX set WITH_AVX=ON
if not defined WITH_TESTING set WITH_TESTING=ON
if not defined MSVC_STATIC_CRT set MSVC_STATIC_CRT=ON
if not defined WITH_PYTHON set WITH_PYTHON=ON
if not defined ON_INFER set ON_INFER=ON
if not defined WITH_ONNXRUNTIME set WITH_ONNXRUNTIME=OFF
if not defined WITH_INFERENCE_API_TEST set WITH_INFERENCE_API_TEST=ON
if not defined WITH_STATIC_LIB set WITH_STATIC_LIB=ON
if not defined WITH_TPCACHE set WITH_TPCACHE=OFF
if not defined WITH_CACHE set WITH_CACHE=OFF
if not defined WITH_SCCACHE set WITH_SCCACHE=OFF
if not defined WITH_UNITY_BUILD set WITH_UNITY_BUILD=OFF
if not defined INFERENCE_DEMO_INSTALL_DIR set INFERENCE_DEMO_INSTALL_DIR=%cache_dir:\=/%/inference_demo
if not defined LOG_LEVEL set LOG_LEVEL=normal
if not defined PRECISION_TEST set PRECISION_TEST=OFF
if not defined NIGHTLY_MODE set PRECISION_TEST=OFF
if not defined retry_times set retry_times=1
if not defined PYTHON_ROOT set PYTHON_ROOT=C:\Python37
if not defined BUILD_DIR set BUILD_DIR=build
if not defined NEW_RELEASE_ALL set NEW_RELEASE_ALL=ON
if not defined NEW_RELEASE_PYPI set NEW_RELEASE_PYPI=OFF
if not defined NEW_RELEASE_JIT set NEW_RELEASE_JIT=OFF

set task_name=%1
set UPLOAD_TP_FILE=OFF

rem ------initialize the python environment------
set PYTHON_EXECUTABLE=%PYTHON_ROOT%\python.exe
set PATH=%PYTHON_ROOT%\Scripts;%PYTHON_ROOT%;%PATH%
if "%WITH_PYTHON%" == "ON" (
    where python
    where pip
    pip install wheel --user
    pip install pyyaml --user
    pip install wget --user
    pip install -r %work_dir%\python\requirements.txt --user
    if !ERRORLEVEL! NEQ 0 (
        echo pip install requirements.txt failed!
        exit /b 5
    )
)

rem -------Caching strategy 1: keep build directory for incremental compilation-----------
rmdir %BUILD_DIR%\python /s/q
rmdir %BUILD_DIR%\paddle\third_party\externalError /s/q
rem rmdir %BUILD_DIR%\paddle\fluid\pybind /s/q
rmdir %BUILD_DIR%\paddle_install_dir /s/q
rmdir %BUILD_DIR%\paddle_inference_install_dir /s/q
rmdir %BUILD_DIR%\paddle_inference_c_install_dir /s/q
del %BUILD_DIR%\CMakeCache.txt

if "%WITH_CACHE%"=="OFF" (
    rmdir %BUILD_DIR% /s/q
    goto :mkbuild
)

set error_code=0
type %cache_dir%\error_code.txt
: set /p error_code=< %cache_dir%\error_code.txt
if %error_code% NEQ 0 (
    rmdir %BUILD_DIR% /s/q
    goto :mkbuild
)

setlocal enabledelayedexpansion
git show-ref --verify --quiet refs/heads/last_pr
if %ERRORLEVEL% EQU 0 (
    git diff HEAD last_pr --stat --name-only
    git diff HEAD last_pr --stat --name-only | findstr "setup.py.in"
    if !ERRORLEVEL! EQU 0 (
        rmdir %BUILD_DIR% /s/q
    )
    git branch -D last_pr
    git branch last_pr
) else (
    rmdir %BUILD_DIR% /s/q
    git branch last_pr
)

for /F %%# in ('wmic os get localdatetime^|findstr 20') do set datetime=%%#
set day_now=%datetime:~6,2%
set day_before=-1
set /p day_before=< %cache_dir%\day.txt
if %day_now% NEQ %day_before% (
    echo %day_now% > %cache_dir%\day.txt
    type %cache_dir%\day.txt
    rmdir %BUILD_DIR% /s/q
    goto :mkbuild
)

:mkbuild
if not exist %BUILD_DIR% (
    echo Windows build cache FALSE
    set Windows_Build_Cache=FALSE
    mkdir %BUILD_DIR%
) else (
    echo Windows build cache TRUE
    set Windows_Build_Cache=TRUE
)
echo ipipe_log_param_Windows_Build_Cache: %Windows_Build_Cache%
cd /d %BUILD_DIR%
dir .
dir %cache_dir%
dir paddle\fluid\pybind\Release
rem -------Caching strategy 1: End --------------------------------


rem -------Caching strategy 2: sccache decorate compiler-----------
if not defined SCCACHE_ROOT set SCCACHE_ROOT=D:\sccache
if "%WITH_SCCACHE%"=="ON" (
    cmd /C sccache -V || call :install_sccache
    sccache --stop-server 2> NUL
    del %SCCACHE_ROOT%\sccache_log.txt

    :: Localy storage on windows
    if not exist %SCCACHE_ROOT% mkdir %SCCACHE_ROOT%
    set SCCACHE_DIR=%SCCACHE_ROOT%\.cache
    
    :: Sccache will shut down if a source file takes more than 10 mins to compile
    set SCCACHE_IDLE_TIMEOUT=0
    set SCCACHE_CACHE_SIZE=100G
    set SCCACHE_ERROR_LOG=%SCCACHE_ROOT%\sccache_log.txt
    set SCCACHE_LOG=quiet

    :: Distributed storage on windows
    set SCCACHE_ENDPOINT=s3.bj.bcebos.com
    set SCCACHE_BUCKET=paddle-windows
    set SCCACHE_S3_KEY_PREFIX=sccache/
    set SCCACHE_S3_USE_SSL=true

    sccache --start-server
    sccache -z
    goto :CASE_%1
) else (
    del %PYTHON_ROOT%\sccache.exe 2> NUL
    goto :CASE_%1
)

:install_sccache
echo There is not sccache in this PC, will install sccache.
echo Download package from https://paddle-ci.gz.bcebos.com/window_requirement/sccache.exe
%PYTHON_ROOT%\python.exe -c "import wget;wget.download('https://paddle-ci.gz.bcebos.com/window_requirement/sccache.exe')"
xcopy sccache.exe %PYTHON_ROOT%\ /Y
del sccache.exe
goto:eof
rem -------Caching strategy 2: End --------------------------------

echo "Usage: paddle_build.bat [OPTION]"
echo "OPTION:"
echo "wincheck_mkl: run Windows MKL/GPU PR CI tasks on Windows"
echo "wincheck_openbals: run Windows OPENBLAS/CPU PR CI tasks on Windows"
echo "build_avx_whl: build Windows avx whl package on Windows"
echo "build_no_avx_whl: build Windows no avx whl package on Windows"
echo "build_inference_lib: build Windows inference library on Windows"
exit /b 1

rem ------PR CI windows check for MKL/GPU----------
:CASE_wincheck_mkl
set WITH_MKL=ON
set WITH_GPU=ON
set WITH_AVX=ON
set MSVC_STATIC_CRT=OFF
set ON_INFER=OFF
set WITH_TENSORRT=ON
set WITH_INFERENCE_API_TEST=OFF

call :cmake || goto cmake_error
call :build || goto build_error
call :test_whl_pacakage || goto test_whl_pacakage_error
call :test_unit || goto test_unit_error
goto:success

rem ------PR CI windows check for OPENBLAS/CPU------
:CASE_wincheck_openblas
set WITH_MKL=OFF
set WITH_GPU=OFF
set WITH_AVX=OFF
set MSVC_STATIC_CRT=ON
set ON_INFER=OFF

call :cmake || goto cmake_error
call :build || goto build_error
call :test_whl_pacakage || goto test_whl_pacakage_error
call :test_unit || goto test_unit_error
goto:success

rem ------PR CI windows check for unittests and inference in CUDA11-MKL-AVX----------
:CASE_wincheck_inference
set WITH_MKL=ON
set WITH_GPU=ON
set WITH_AVX=ON
set MSVC_STATIC_CRT=ON
set ON_INFER=ON
set WITH_TENSORRT=ON
set WITH_INFERENCE_API_TEST=ON

call :cmake || goto cmake_error
call :build || goto build_error
call :test_whl_pacakage || goto test_whl_pacakage_error
call :test_unit || goto test_unit_error
::call :test_inference || goto test_inference_error
::call :test_inference_ut || goto test_inference_ut_error
::call :check_change_of_unittest || goto check_change_of_unittest_error
goto:success

rem ------Build windows avx whl package------
:CASE_build_avx_whl
set WITH_AVX=ON
set ON_INFER=OFF
set CUDA_ARCH_NAME=All

call :cmake || goto cmake_error
call :build || goto build_error
call :test_whl_pacakage || goto test_whl_pacakage_error
goto:success

rem ------Build windows no-avx whl package------
:CASE_build_no_avx_whl
set WITH_AVX=OFF
set ON_INFER=OFF
set CUDA_ARCH_NAME=All

call :cmake || goto cmake_error
call :build || goto build_error
call :test_whl_pacakage || goto test_whl_pacakage_error
goto:success

rem ------Build windows inference library------
:CASE_build_inference_lib
set ON_INFER=ON
set WITH_PYTHON=OFF
set CUDA_ARCH_NAME=All
python %work_dir%\tools\remove_grad_op_and_kernel.py
if %errorlevel% NEQ 0 exit /b 1

call :cmake || goto cmake_error
call :build || goto build_error
call :test_inference || goto test_inference_error
call :test_inference_ut || goto test_inference_ut_error
call :zip_cc_file || goto zip_cc_file_error
call :zip_c_file || goto zip_c_file_error
goto:success

rem "Other configurations are added here"
rem :CASE_wincheck_others
rem call ...


rem ---------------------------------------------------------------------------------------------
:cmake
@ECHO OFF
echo    ========================================
echo    Step 1. Cmake ...
echo    ========================================

rem set vs language to english to block showIncludes, this need vs has installed English language package.
set VSLANG=1033
rem Configure the environment for 64-bit builds. 'DISTUTILS_USE_SDK' indicates that the user has selected the compiler.
if not defined vcvars64_dir set vcvars64_dir="C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat"
call %vcvars64_dir%

set DISTUTILS_USE_SDK=1
rem Windows 10 Kit bin dir
set PATH=C:\Program Files (x86)\Windows Kits\10\bin\10.0.17763.0\x64;%PATH%
rem Use 64-bit ToolSet to compile
set PreferredToolArchitecture=x64

for /F %%# in ('wmic os get localdatetime^|findstr 20') do set start=%%#
set start=%start:~4,10%

if not defined CUDA_TOOLKIT_ROOT_DIR set CUDA_TOOLKIT_ROOT_DIR=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.2
set PATH=%TENSORRT_ROOT:/=\%\lib;%CUDA_TOOLKIT_ROOT_DIR:/=\%\bin;%CUDA_TOOLKIT_ROOT_DIR:/=\%\libnvvp;%PATH%

rem CUDA_TOOLKIT_ROOT_DIR in cmake must use / rather than \
set TENSORRT_ROOT=%TENSORRT_ROOT:\=/%
set CUDA_TOOLKIT_ROOT_DIR=%CUDA_TOOLKIT_ROOT_DIR:\=/%

rem install ninja if GENERATOR is Ninja
if %GENERATOR% == "Ninja" (
    pip install ninja
    if %errorlevel% NEQ 0 (
        echo pip install ninja failed!
        exit /b 5
    )
)

rem ------show summary of current GPU environment----------
cmake --version
if "%WITH_GPU%"=="ON" (
    nvcc --version
    nvidia-smi 2>NUL
)

rem ------set third_party cache dir------

if "%WITH_TPCACHE%"=="OFF" (
    set THIRD_PARTY_PATH=%work_dir:\=/%/%BUILD_DIR%/third_party
    goto :cmake_impl
)

rem clear third party cache every ten days
for /F %%# in ('wmic os get localdatetime^|findstr 20') do set datetime=%%#
set day_now=%datetime:~6,2%
set day_before=-1
set /p day_before=< %cache_dir%\day_third_party.txt
if %day_now% NEQ %day_before% (
    echo %day_now% > %cache_dir%\day_third_party.txt
    type %cache_dir%\day_third_party.txt
    if %day_now% EQU 21 (
        rmdir %cache_dir%\third_party /s/q
    )
    if %day_now% EQU 11 (
        rmdir %cache_dir%\third_party /s/q
    )
    if %day_now% EQU 01 (
        rmdir %cache_dir%\third_party /s/q
    )
)

echo set -ex > cache.sh
echo md5_content=$(cat %work_dir:\=/%/cmake/external/*.cmake  ^|md5sum ^| awk '{print $1}') >> cache.sh
echo echo ${md5_content}^>md5.txt >> cache.sh

%cache_dir%\tools\busybox64.exe cat cache.sh
%cache_dir%\tools\busybox64.exe bash cache.sh

set /p md5=< md5.txt
if "%WITH_GPU%"=="ON" (
    set cuda_version=%CUDA_TOOLKIT_ROOT_DIR:~-4%
    set sub_dir=cuda!cuda_version:.=!
) else (
    set sub_dir=cpu
)
set THIRD_PARTY_HOME=%cache_dir:\=/%/third_party/%sub_dir%
set THIRD_PARTY_PATH=%THIRD_PARTY_HOME%/%md5%

echo %task_name%|findstr build >nul && (
    echo %task_name% is a whl-build task, will only reuse local third_party cache.
    goto :cmake_impl
) || ( 
    echo %task_name% is a PR-CI-Windows task, will try to reuse bos and local third_party cache both. 
)

if not exist %THIRD_PARTY_PATH% (
    echo There is no usable third_party cache in %THIRD_PARTY_PATH%, will download from bos.
    pip install wget
    if not exist %THIRD_PARTY_HOME% mkdir "%THIRD_PARTY_HOME%"
    cd /d %THIRD_PARTY_HOME%
    echo Getting third party: downloading ...
    %PYTHON_ROOT%\python.exe -c "import wget;wget.download('https://paddle-windows.bj.bcebos.com/third_party/%sub_dir%/%md5%.tar.gz')" 2>nul
    if !ERRORLEVEL! EQU 0 (
        echo Getting third party: extracting ...
        tar -xf %md5%.tar.gz
        if !ERRORLEVEL! EQU 0 ( 
            echo Get third party from bos successfully.
        ) else (
            echo Get third party failed, reason: extract failed, will build locally.
        )
        del %md5%.tar.gz
    ) else (
        echo Get third party failed, reason: download failed, will build locally.
    )
    if not exist %THIRD_PARTY_PATH% set UPLOAD_TP_FILE=ON
    cd /d %work_dir%\%BUILD_DIR%
) else (
    echo Found reusable third_party cache in %THIRD_PARTY_PATH%, will reuse it.
)

:cmake_impl
echo cmake .. -G %GENERATOR% -DCMAKE_BUILD_TYPE=Release -DWITH_AVX=%WITH_AVX% -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% ^
-DWITH_TESTING=%WITH_TESTING% -DWITH_PYTHON=%WITH_PYTHON% -DPYTHON_EXECUTABLE=%PYTHON_EXECUTABLE% -DON_INFER=%ON_INFER% ^
-DWITH_INFERENCE_API_TEST=%WITH_INFERENCE_API_TEST% -DTHIRD_PARTY_PATH=%THIRD_PARTY_PATH% ^
-DINFERENCE_DEMO_INSTALL_DIR=%INFERENCE_DEMO_INSTALL_DIR% -DWITH_STATIC_LIB=%WITH_STATIC_LIB% ^
-DWITH_TENSORRT=%WITH_TENSORRT% -DTENSORRT_ROOT="%TENSORRT_ROOT%" -DMSVC_STATIC_CRT=%MSVC_STATIC_CRT% ^
-DWITH_UNITY_BUILD=%WITH_UNITY_BUILD% -DCUDA_ARCH_NAME=%CUDA_ARCH_NAME% -DCUB_PATH=%THIRD_PARTY_HOME%/cub ^
-DCUDA_TOOLKIT_ROOT_DIR="%CUDA_TOOLKIT_ROOT_DIR%" -DNEW_RELEASE_ALL=%NEW_RELEASE_ALL% -DNEW_RELEASE_PYPI=%NEW_RELEASE_PYPI% ^
-DNEW_RELEASE_JIT=%NEW_RELEASE_JIT%

cmake .. -G %GENERATOR% -DCMAKE_BUILD_TYPE=Release -DWITH_AVX=%WITH_AVX% -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% ^
-DWITH_TESTING=%WITH_TESTING% -DWITH_PYTHON=%WITH_PYTHON% -DPYTHON_EXECUTABLE=%PYTHON_EXECUTABLE% -DON_INFER=%ON_INFER% ^
-DWITH_INFERENCE_API_TEST=%WITH_INFERENCE_API_TEST% -DTHIRD_PARTY_PATH=%THIRD_PARTY_PATH% ^
-DINFERENCE_DEMO_INSTALL_DIR=%INFERENCE_DEMO_INSTALL_DIR% -DWITH_STATIC_LIB=%WITH_STATIC_LIB% ^
-DWITH_TENSORRT=%WITH_TENSORRT% -DTENSORRT_ROOT="%TENSORRT_ROOT%" -DMSVC_STATIC_CRT=%MSVC_STATIC_CRT% ^
-DWITH_UNITY_BUILD=%WITH_UNITY_BUILD% -DCUDA_ARCH_NAME=%CUDA_ARCH_NAME% -DCUB_PATH=%THIRD_PARTY_HOME%/cub ^
-DCUDA_TOOLKIT_ROOT_DIR="%CUDA_TOOLKIT_ROOT_DIR%" -DNEW_RELEASE_ALL=%NEW_RELEASE_ALL% -DNEW_RELEASE_PYPI=%NEW_RELEASE_PYPI% ^
-DNEW_RELEASE_JIT=%NEW_RELEASE_JIT%
goto:eof

:cmake_error
echo 7 > %cache_dir%\error_code.txt
type %cache_dir%\error_code.txt
echo Cmake failed, will exit!
exit /b 7

rem ---------------------------------------------------------------------------------------------
:build
@ECHO OFF
echo    ========================================
echo    Step 2. Build Paddle ...
echo    ========================================

for /F %%# in ('wmic cpu get NumberOfLogicalProcessors^|findstr [0-9]') do set /a PARALLEL_PROJECT_COUNT=%%#*4/5
echo "PARALLEL PROJECT COUNT is %PARALLEL_PROJECT_COUNT%"

set build_times=1
rem MSbuild will build third_party first to improve compiler stability.
if NOT %GENERATOR% == "Ninja" (
    goto :build_tp
) else (
    goto :build_paddle
)

:build_tp
echo Build third_party the %build_times% time:
if %GENERATOR% == "Ninja" (
    ninja third_party
) else (
    MSBuild /m /p:PreferredToolArchitecture=x64 /p:Configuration=Release /verbosity:%LOG_LEVEL% third_party.vcxproj
)

if %ERRORLEVEL% NEQ 0 (
    set /a build_times=%build_times%+1  
    if %build_times% GEQ %retry_times% (
        exit /b 7
    ) else (
        echo Build third_party failed, will retry!
        goto :build_tp
    )
)
echo Build third_party successfully!

set build_times=1

:build_paddle
:: reset clcache zero stats for collect PR's actual hit rate
rem clcache.exe -z

rem -------clean up environment again-----------
taskkill /f /im cmake.exe /t 2>NUL
taskkill /f /im MSBuild.exe /t 2>NUL
taskkill /f /im cl.exe /t 2>NUL
taskkill /f /im lib.exe /t 2>NUL
taskkill /f /im link.exe /t 2>NUL
taskkill /f /im vctip.exe /t 2>NUL
taskkill /f /im cvtres.exe /t 2>NUL
taskkill /f /im rc.exe /t 2>NUL
taskkill /f /im mspdbsrv.exe /t 2>NUL
taskkill /f /im csc.exe /t 2>NUL
taskkill /f /im nvcc.exe /t 2>NUL
taskkill /f /im cicc.exe /t 2>NUL
taskkill /f /im ptxas.exe /t 2>NUL
taskkill /f /im op_function_generator.exe /t 2>NUL
taskkill /f /im eager_generator.exe /t 2>NUL
taskkill /f /im eager_op_function_generator.exe /t 2>NUL
wmic process where name="op_function_generator.exe" call terminate 2>NUL
wmic process where name="eager_generator.exe" call terminate 2>NUL
wmic process where name="eager_op_function_generator.exe" call terminate 2>NUL
wmic process where name="cmake.exe" call terminate 2>NUL
wmic process where name="cvtres.exe" call terminate 2>NUL
wmic process where name="rc.exe" call terminate 2>NUL
wmic process where name="cl.exe" call terminate 2>NUL
wmic process where name="lib.exe" call terminate 2>NUL

if "%WITH_TESTING%"=="ON" (
    for /F "tokens=1 delims= " %%# in ('tasklist ^| findstr /i test') do taskkill /f /im %%# /t
)

echo Build Paddle the %build_times% time:
if %GENERATOR% == "Ninja" (
    ninja all
) else (
    MSBuild /m:%PARALLEL_PROJECT_COUNT% /p:PreferredToolArchitecture=x64 /p:TrackFileAccess=false /p:Configuration=Release /verbosity:%LOG_LEVEL% ALL_BUILD.vcxproj
)

if %ERRORLEVEL% NEQ 0 (
    set /a build_times=%build_times%+1
    if %build_times% GEQ %retry_times% (
        exit /b 7
    ) else (
        echo Build Paddle failed, will retry!
        goto :build_paddle
    )
)

if "%UPLOAD_TP_FILE%"=="ON" (
    set BCE_FILE=%cache_dir%\bce-python-sdk-0.8.33\BosClient.py
    echo Uploading third_party: checking bce ...
    if not exist %cache_dir%\bce-python-sdk-0.8.33 (
        echo There is no bce in this PC, will install bce.
        cd /d %cache_dir%
        echo Download package from https://paddle-windows.bj.bcebos.com/bce-python-sdk-0.8.33.tar.gz
        %PYTHON_ROOT%\python.exe -c "import wget;wget.download('https://paddle-windows.bj.bcebos.com/bce-python-sdk-0.8.33.tar.gz')"
        %PYTHON_ROOT%\python.exe -c "import shutil;shutil.unpack_archive('bce-python-sdk-0.8.33.tar.gz', extract_dir='./',format='gztar')"
        cd /d %cache_dir%\bce-python-sdk-0.8.33
        %PYTHON_ROOT%\python.exe setup.py install 1>nul
        del %cache_dir%\bce-python-sdk-0.8.33.tar.gz
    )
    if !errorlevel! EQU 0 (
        cd /d %THIRD_PARTY_HOME%
        echo Uploading third_party: compressing ...
        tar -zcf %md5%.tar.gz %md5%
        if !errorlevel! EQU 0 (
            echo Uploading third_party: uploading ...
            %PYTHON_ROOT%\python.exe !BCE_FILE! %md5%.tar.gz paddle-windows/third_party/%sub_dir% 1>nul
            if !errorlevel! EQU 0 (
                echo Upload third party %md5% to bos paddle-windows/third_party/%sub_dir% successfully.
            ) else (
                echo Failed upload third party to bos, reason: upload failed.
            )
        ) else (
            echo Failed upload third party to bos, reason: compress failed.
        )
        del %md5%.tar.gz
    ) else (
        echo Failed upload third party to bos, reason: install bce failed.
    )
    cd /d %work_dir%\%BUILD_DIR%
)

echo Build Paddle successfully!
echo 0 > %cache_dir%\error_code.txt
type %cache_dir%\error_code.txt

:: ci will collect sccache hit rate
if "%WITH_SCCACHE%"=="ON" (
    call :collect_sccache_hits
)

goto:eof

:build_error
echo 7 > %cache_dir%\error_code.txt
type %cache_dir%\error_code.txt
echo Build Paddle failed, will exit!
exit /b 7

rem ---------------------------------------------------------------------------------------------
:test_whl_pacakage
@ECHO OFF
echo    ========================================
echo    Step 3. Test pip install whl package ...
echo    ========================================

setlocal enabledelayedexpansion

for /F %%# in ('wmic os get localdatetime^|findstr 20') do set end=%%#
set end=%end:~4,10%
call :timestamp "%start%" "%end%" "Build"

%cache_dir%\tools\busybox64.exe du -h -d 0 %cd%\python\dist > whl_size.txt
set /p whlsize=< whl_size.txt
for /F %%i in ("%whlsize%") do echo "Windows PR whl Size: %%i"
for /F %%i in ("%whlsize%") do echo ipipe_log_param_Windows_PR_whl_Size: %%i

dir /s /b python\dist\*.whl > whl_file.txt
set /p PADDLE_WHL_FILE_WIN=< whl_file.txt

@ECHO ON
pip uninstall -y paddlepaddle
pip uninstall -y paddlepaddle-gpu
pip install %PADDLE_WHL_FILE_WIN% --user
if %ERRORLEVEL% NEQ 0 (
    call paddle_winci\Scripts\deactivate.bat 2>NUL
    echo pip install whl package failed!
    exit /b 1
)

set CUDA_VISIBLE_DEVICES=0
python %work_dir%\paddle\scripts\installation_validate.py
goto:eof

:test_whl_pacakage_error
::echo 1 > %cache_dir%\error_code.txt
::type %cache_dir%\error_code.txt
echo Test import paddle failed, will exit!
exit /b 1

rem ---------------------------------------------------------------------------------------------
:test_unit
@ECHO ON
echo    ========================================
echo    Step 4. Running unit tests ...
echo    ========================================

: set CI_SKIP_CPP_TEST if only *.py changed
git diff --name-only %BRANCH% | findstr /V "\.py" || set CI_SKIP_CPP_TEST=ON

pip install -r %work_dir%\python\unittest_py\requirements.txt --user
if %ERRORLEVEL% NEQ 0 (
    echo pip install unittest requirements.txt failed!
    exit /b 5
)

for /F %%# in ('wmic os get localdatetime^|findstr 20') do set start=%%#
set start=%start:~4,10%

set FLAGS_call_stack_level=2
dir %THIRD_PARTY_PATH:/=\%\install\openblas\lib
dir %THIRD_PARTY_PATH:/=\%\install\openblas\bin
dir %THIRD_PARTY_PATH:/=\%\install\zlib\bin
dir %THIRD_PARTY_PATH:/=\%\install\mklml\lib
dir %THIRD_PARTY_PATH:/=\%\install\mkldnn\bin
dir %THIRD_PARTY_PATH:/=\%\install\warpctc\bin

pip install requests

set PATH=%THIRD_PARTY_PATH:/=\%\install\openblas\lib;%THIRD_PARTY_PATH:/=\%\install\openblas\bin;^
%THIRD_PARTY_PATH:/=\%\install\zlib\bin;%THIRD_PARTY_PATH:/=\%\install\mklml\lib;^
%THIRD_PARTY_PATH:/=\%\install\mkldnn\bin;%THIRD_PARTY_PATH:/=\%\install\warpctc\bin;%PATH%

if "%WITH_GPU%"=="ON" (
    call:parallel_test_base_gpu
) else (
    call:parallel_test_base_cpu
)

set error_code=%ERRORLEVEL%

for /F %%# in ('wmic os get localdatetime^|findstr 20') do set end=%%#
set end=%end:~4,10%
call :timestamp "%start%" "%end%" "1 card TestCases Total"
call :timestamp "%start%" "%end%" "TestCases Total"

if %error_code% NEQ 0 (
    exit /b 8
) else ( 
    goto:eof 
)

:parallel_test_base_gpu
echo    ========================================
echo    Running GPU unit tests in parallel way ...
echo    ========================================

setlocal enabledelayedexpansion

:: set PATH=C:\Windows\System32;C:\Program Files\NVIDIA Corporation\NVSMI;%PATH%
:: cmd /C nvidia-smi -L
:: if %errorlevel% NEQ 0 exit /b 8
:: for /F %%# in ('cmd /C nvidia-smi -L ^|find "GPU" /C') do set CUDA_DEVICE_COUNT=%%#
set CUDA_DEVICE_COUNT=1

:: For hypothesis tests(mkldnn op and inference pass), we set use 'ci' profile
set HYPOTHESIS_TEST_PROFILE=ci
echo cmake .. -G %GENERATOR% -DCMAKE_BUILD_TYPE=Release -DWITH_AVX=%WITH_AVX% -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% ^
-DWITH_TESTING=%WITH_TESTING% -DWITH_PYTHON=%WITH_PYTHON% -DON_INFER=%ON_INFER% ^
-DWITH_INFERENCE_API_TEST=%WITH_INFERENCE_API_TEST% -DTHIRD_PARTY_PATH=%THIRD_PARTY_PATH% ^
-DINFERENCE_DEMO_INSTALL_DIR=%INFERENCE_DEMO_INSTALL_DIR% -DWITH_STATIC_LIB=%WITH_STATIC_LIB% ^
-DWITH_TENSORRT=%WITH_TENSORRT% -DTENSORRT_ROOT="%TENSORRT_ROOT%" -DMSVC_STATIC_CRT=%MSVC_STATIC_CRT% ^
-DWITH_UNITY_BUILD=%WITH_UNITY_BUILD% -DCUDA_ARCH_NAME=%CUDA_ARCH_NAME% -DCUB_PATH=%THIRD_PARTY_HOME%/cub ^
-DCUDA_TOOLKIT_ROOT_DIR="%CUDA_TOOLKIT_ROOT_DIR%" -DNEW_RELEASE_ALL=%NEW_RELEASE_ALL% -DNEW_RELEASE_PYPI=%NEW_RELEASE_PYPI% ^
-DNEW_RELEASE_JIT=%NEW_RELEASE_JIT% >> %work_dir%\win_cmake.sh

%cache_dir%\tools\busybox64.exe bash %work_dir%\tools\windows\run_unittests.sh %NIGHTLY_MODE% %PRECISION_TEST% %WITH_GPU%

goto:eof

:parallel_test_base_cpu
echo    ========================================
echo    Running CPU unit tests in parallel way ...
echo    ========================================

:: For hypothesis tests(mkldnn op and inference pass), we set use 'ci' profile
set HYPOTHESIS_TEST_PROFILE=ci
%cache_dir%\tools\busybox64.exe bash %work_dir%\tools\windows\run_unittests.sh %NIGHTLY_MODE% %PRECISION_TEST% %WITH_GPU%

goto:eof

:test_unit_error
:: echo 8 > %cache_dir%\error_code.txt
:: type %cache_dir%\error_code.txt
echo Running unit tests failed, will exit!
exit /b 8

rem ---------------------------------------------------------------------------------------------
:test_inference
@ECHO OFF
echo    ========================================
echo    Step 5. Testing fluid library for inference ...
echo    ========================================

tree /F %cd%\paddle_inference_install_dir\paddle
%cache_dir%\tools\busybox64.exe du -h -d 0 -k %cd%\paddle_inference_install_dir\paddle\lib > lib_size.txt
set /p libsize=< lib_size.txt
for /F %%i in ("%libsize%") do (
    set /a libsize_m=%%i/1024
    echo "Windows Paddle_Inference Size: !libsize_m!M"
    echo ipipe_log_param_Windows_Paddle_Inference_Size: !libsize_m!M
)

cd /d %work_dir%\paddle\fluid\inference\api\demo_ci
%cache_dir%\tools\busybox64.exe bash run.sh %work_dir:\=/% %WITH_MKL% %WITH_GPU% %cache_dir:\=/%/inference_demo %WITH_TENSORRT% %TENSORRT_ROOT% %WITH_ONNXRUNTIME% %MSVC_STATIC_CRT% "%CUDA_TOOLKIT_ROOT_DIR%"
goto:eof

:test_inference_error
::echo 1 > %cache_dir%\error_code.txt
::type %cache_dir%\error_code.txt
echo Testing fluid library for inference failed!
exit /b 1

rem ---------------------------------------------------------------------------------------------
:check_change_of_unittest
@ECHO OFF
echo    ========================================
echo    Step 6. Check whether deleting a unit test ...
echo    ========================================

%cache_dir%\tools\busybox64.exe bash %work_dir%\tools\windows\check_change_of_unittest.sh

goto:eof

:check_change_of_unittest_error
exit /b 1

rem ---------------------------------------------------------------------------------------------
:test_inference_ut
@ECHO OFF
echo    ========================================
echo    Step 7. Testing fluid library with infer_ut for inference ...
echo    ========================================

cd /d %work_dir%\paddle\fluid\inference\tests\infer_ut
%cache_dir%\tools\busybox64.exe bash run.sh %work_dir:\=/% %WITH_MKL% %WITH_GPU% %cache_dir:\=/%/inference_demo %TENSORRT_ROOT% %WITH_ONNXRUNTIME% %MSVC_STATIC_CRT% "%CUDA_TOOLKIT_ROOT_DIR%"
goto:eof

:test_inference_ut_error
::echo 1 > %cache_dir%\error_code.txt
::type %cache_dir%\error_code.txt
echo Testing fluid library with infer_ut for inference failed!
exit /b 1

rem ---------------------------------------------------------------------------------------------
:zip_cc_file
cd /d %work_dir%\%BUILD_DIR%
tree /F %cd%\paddle_inference_install_dir\paddle
if exist paddle_inference.zip del paddle_inference.zip
python -c "import shutil;shutil.make_archive('paddle_inference', 'zip', root_dir='paddle_inference_install_dir')"
%cache_dir%\tools\busybox64.exe du -h -k paddle_inference.zip > lib_size.txt
set /p libsize=< lib_size.txt
for /F %%i in ("%libsize%") do (
    set /a libsize_m=%%i/1024
    echo "Windows Paddle_Inference ZIP Size: !libsize_m!M"
)
goto:eof

:zip_cc_file_error
echo Tar inference library failed!
exit /b 1

rem ---------------------------------------------------------------------------------------------
:zip_c_file
cd /d %work_dir%\%BUILD_DIR%
tree /F %cd%\paddle_inference_c_install_dir\paddle
if exist paddle_inference_c.zip del paddle_inference_c.zip
python -c "import shutil;shutil.make_archive('paddle_inference_c', 'zip', root_dir='paddle_inference_c_install_dir')"
%cache_dir%\tools\busybox64.exe du -h -k paddle_inference_c.zip > lib_size.txt
set /p libsize=< lib_size.txt
for /F %%i in ("%libsize%") do (
    set /a libsize_m=%%i/1024
    echo "Windows Paddle_Inference CAPI ZIP Size: !libsize_m!M"
)
goto:eof

:zip_c_file_error
echo Tar inference capi library failed!
exit /b 1

:timestamp
setlocal enabledelayedexpansion
@ECHO OFF
set start=%~1
set dd=%start:~2,2%
set /a dd=100%dd%%%100
set hh=%start:~4,2%
set /a hh=100%hh%%%100
set nn=%start:~6,2%
set /a nn=100%nn%%%100
set ss=%start:~8,2%
set /a ss=100%ss%%%100
set /a start_sec=dd*86400+hh*3600+nn*60+ss
echo %start_sec%

set end=%~2
set dd=%end:~2,2%
set /a dd=100%dd%%%100
if %start:~0,2% NEQ %end:~0,2% (
    set month_day=0
    for %%i in (01 03 05 07 08 10 12) DO if %%i EQU %start:~0,2% set month_day=31
    for %%i in (04 06 09 11) DO if %%i EQU %start:~0,2% set month_day=30
    for %%i in (02) DO if %%i EQU %start:~0,2% set month_day=28
    set /a dd=%dd%+!month_day!
)
set hh=%end:~4,2%
set /a hh=100%hh%%%100
set nn=%end:~6,2%
set /a nn=100%nn%%%100
set ss=%end:~8,2%
set /a ss=100%ss%%%100
set /a end_secs=dd*86400+hh*3600+nn*60+ss
set /a cost_secs=end_secs-start_sec
echo "Windows %~3 Time: %cost_secs%s"
set tempTaskName=%~3
echo ipipe_log_param_Windows_%tempTaskName: =_%_Time: %cost_secs%s
goto:eof


:collect_sccache_hits
sccache -s > sccache_summary.txt
echo    ========================================
echo    sccache statistical summary ...
echo    ========================================
type sccache_summary.txt
for /f "tokens=2,3" %%i in ('type sccache_summary.txt ^| findstr "requests hits" ^| findstr /V "executed C/C++ CUDA"') do set %%i=%%j
if %requests% EQU 0 (
    echo "sccache hit rate: 0%"
    echo ipipe_log_param_sccache_Hit_Hate: 0%
) else (
    set /a rate=!hits!*10000/!requests!
    echo "sccache hit rate: !rate:~0,-2!.!rate:~-2!%%"
    echo ipipe_log_param_sccache_Hit_Hate: !rate:~0,-2!.!rate:~-2!%%
)

goto:eof


rem ---------------------------------------------------------------------------------------------
:success
echo    ========================================
echo    Clean up environment  at the end ...
echo    ========================================
taskkill /f /im cmake.exe /t 2>NUL
taskkill /f /im ninja.exe /t 2>NUL
taskkill /f /im MSBuild.exe /t 2>NUL
taskkill /f /im git.exe /t 2>NUL
taskkill /f /im cl.exe /t 2>NUL
taskkill /f /im lib.exe /t 2>NUL
taskkill /f /im link.exe /t 2>NUL
taskkill /f /im git-remote-https.exe /t 2>NUL
taskkill /f /im vctip.exe /t 2>NUL
taskkill /f /im cvtres.exe /t 2>NUL
taskkill /f /im rc.exe /t 2>NUL
taskkill /f /im mspdbsrv.exe /t 2>NUL
taskkill /f /im csc.exe /t 2>NUL
taskkill /f /im python.exe /t 2>NUL
taskkill /f /im nvcc.exe /t 2>NUL
taskkill /f /im cicc.exe /t 2>NUL
taskkill /f /im ptxas.exe /t 2>NUL
taskkill /f /im op_function_generator.exe /t 2>NUL
taskkill /f /im eager_generator.exe /t 2>NUL
taskkill /f /im eager_op_function_generator.exe /t 2>NUL
wmic process where name="op_function_generator.exe" call terminate 2>NUL
wmic process where name="eager_generator.exe" call terminate 2>NUL
wmic process where name="eager_op_function_generator.exe" call terminate 2>NUL
wmic process where name="cvtres.exe" call terminate 2>NUL
wmic process where name="rc.exe" call terminate 2>NUL
wmic process where name="cl.exe" call terminate 2>NUL
wmic process where name="lib.exe" call terminate 2>NUL
wmic process where name="python.exe" call terminate 2>NUL
if "%WITH_TESTING%"=="ON" (
    for /F "tokens=1 delims= " %%# in ('tasklist ^| findstr /i test') do taskkill /f /im %%# /t
)
echo Windows CI run successfully!
exit /b 0

ENDLOCAL