From e919620a9508584b7266f6afa0399cba5c9814bc Mon Sep 17 00:00:00 2001 From: Sing_chan <51314274+betterpig@users.noreply.github.com> Date: Tue, 14 Sep 2021 20:34:59 +0800 Subject: [PATCH] windows third party cache optimization: share third party cache among servers (#35368) * new function: share third party cache among servers to fasten build speed * modified code according to zhouwei25's comment * add wget install step, move cd build to the last of if condition * block note and error of third_party share; change bce upload method * change third_party sub_dir in bos, since third party in different cuda version cant share * set sub_dir by get nvcc version * change third_party local path to be same with bos path --- cmake/external/xbyak.cmake | 2 +- paddle/scripts/paddle_build.bat | 74 ++++++++++++++++++++++++++++++--- 2 files changed, 69 insertions(+), 7 deletions(-) diff --git a/cmake/external/xbyak.cmake b/cmake/external/xbyak.cmake index eabcabf7430..24cccb26b4b 100644 --- a/cmake/external/xbyak.cmake +++ b/cmake/external/xbyak.cmake @@ -44,7 +44,7 @@ ExternalProject_Add( DEPENDS "" PREFIX ${XBYAK_PREFIX_DIR} SOURCE_DIR ${XBYAK_SOURCE_DIR} - # UPDATE_COMMAND "" + UPDATE_COMMAND "" CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${XBYAK_INSTALL_ROOT} -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE} CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${XBYAK_INSTALL_ROOT} diff --git a/paddle/scripts/paddle_build.bat b/paddle/scripts/paddle_build.bat index 0ad2d799960..1cc4927e739 100644 --- a/paddle/scripts/paddle_build.bat +++ b/paddle/scripts/paddle_build.bat @@ -340,15 +340,12 @@ if %day_now% NEQ %day_before% ( echo %day_now% > %cache_dir%\day.txt type %cache_dir%\day.txt if %day_now% EQU 21 ( - rmdir %cache_dir%\third_party_GPU /s/q rmdir %cache_dir%\third_party /s/q ) if %day_now% EQU 11 ( - rmdir %cache_dir%\third_party_GPU /s/q rmdir %cache_dir%\third_party /s/q ) if %day_now% EQU 01 ( - rmdir %cache_dir%\third_party_GPU /s/q rmdir %cache_dir%\third_party /s/q ) ) @@ -367,11 +364,41 @@ echo echo ${md5_content}^>md5.txt >> cache.sh set /p md5=< md5.txt if "%WITH_GPU%"=="ON" ( - set THIRD_PARTY_HOME=%cache_dir:\=/%/third_party_GPU + for /F "delims=" %%# in ('nvcc --version^|findstr V1') do set cuda_version=%%# + set cuda_version=!cuda_version:~-7,4! + set sub_dir=cuda!cuda_version:.=! ) else ( - set THIRD_PARTY_HOME=%cache_dir:\=/%/third_party + set sub_dir=cpu ) + +set THIRD_PARTY_HOME=%cache_dir:\=/%/third_party/%sub_dir% set THIRD_PARTY_PATH=%THIRD_PARTY_HOME%/%md5% +set UPLOAD_TP_FILE=OFF + +if not exist %THIRD_PARTY_PATH% ( + echo There is no usable third_party cache in %THIRD_PARTY_PATH%, will download from bos. + pip install wget + if not exist %THIRD_PARTY_HOME% mkdir "%THIRD_PARTY_HOME%" + cd %THIRD_PARTY_HOME% + echo Getting third party: downloading ... + %PYTHON_ROOT%\python.exe -c "import wget;wget.download('https://paddle-windows.bj.bcebos.com/third_party/%sub_dir%/%md5%.tar.gz')" 2>nul + if !ERRORLEVEL! EQU 0 ( + echo Getting third party: extracting ... + tar -xf %md5%.tar.gz + if !ERRORLEVEL! EQU 0 ( + echo Get third party from bos successfully + ) else ( + echo Get third party failed, reason: extract failed, will build locally + ) + del %md5%.tar.gz + ) else ( + echo Get third party failed, reason: download failed, will build locally + ) + if not exist %THIRD_PARTY_PATH% ( set UPLOAD_TP_FILE=ON ) + cd %work_dir%\%BUILD_DIR% +) else ( + echo Found reusable third_party cache locally, will reuse it. +) :cmake_impl echo cmake .. -G %GENERATOR% -DCMAKE_BUILD_TYPE=Release -DWITH_AVX=%WITH_AVX% -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% ^ @@ -399,7 +426,7 @@ rem ---------------------------------------------------------------------------- :build @ECHO OFF echo ======================================== -echo Step 2. Buile Paddle ... +echo Step 2. Build Paddle ... echo ======================================== for /F %%# in ('wmic cpu get NumberOfLogicalProcessors^|findstr [0-9]') do set /a PARALLEL_PROJECT_COUNT=%%#*4/5 @@ -485,6 +512,41 @@ if %ERRORLEVEL% NEQ 0 ( ) ) +set BCE_FILE=%cache_dir%\bce-python-sdk-0.8.33\BosClient.py +if %UPLOAD_TP_FILE%==ON ( + echo Uploading third_party: checking bce ... + if not exist %cache_dir%\bce-python-sdk-0.8.33 ( + echo There is no bce in this PC, will install bce. + cd %cache_dir% + echo Download package from https://paddle-windows.bj.bcebos.com/bce-python-sdk-0.8.33.tar.gz + %PYTHON_ROOT%\python.exe -c "import wget;wget.download('https://paddle-windows.bj.bcebos.com/bce-python-sdk-0.8.33.tar.gz')" + %PYTHON_ROOT%\python.exe -c "import shutil;shutil.unpack_archive('bce-python-sdk-0.8.33.tar.gz', extract_dir='./',format='gztar')" + cd %cache_dir%\bce-python-sdk-0.8.33 + %PYTHON_ROOT%\python.exe setup.py install 1>nul + del %cache_dir%\bce-python-sdk-0.8.33.tar.gz + ) + if !errorlevel! EQU 0 ( + cd %THIRD_PARTY_HOME% + echo Uploading third_party: compressing ... + tar -zcf %md5%.tar.gz %md5% + if !errorlevel! EQU 0 ( + echo Uploading third_party: uploading ... + %PYTHON_ROOT%\python.exe %BCE_FILE% %md5%.tar.gz paddle-windows/third_party/%sub_dir% 1>nul + if !errorlevel! EQU 0 ( + echo Upload third party to bos paddle-windows/third_party/%sub_dir% successfully + ) else ( + echo Failed upload third party to bos, reason: upload failed + ) + ) else ( + echo Failed upload third party to bos, reason: compress failed + ) + del %md5%.tar.gz + ) else ( + echo Failed upload third party to bos, reason: install bce failed + ) + cd %work_dir%\%BUILD_DIR% +) + echo Build Paddle successfully! echo 0 > %cache_dir%\error_code.txt type %cache_dir%\error_code.txt -- GitLab