windows third party cache optimization: share third party cache among servers (#35368)

* new function: share third party cache among servers to fasten build speed

* modified code according to zhouwei25's comment

* add wget install step, move cd build to the last of if condition

* block note and error of third_party share; change bce upload method

* change third_party sub_dir in bos, since third party in different cuda version cant share

* set sub_dir by get nvcc version

* change third_party local path to be same with bos path

cmake/external/xbyak.cmake

@@ -44,7 +44,7 @@ ExternalProject_Add(
     DEPENDS             ""
     PREFIX              ${XBYAK_PREFIX_DIR}
     SOURCE_DIR          ${XBYAK_SOURCE_DIR}
-    # UPDATE_COMMAND      ""
+    UPDATE_COMMAND      ""
     CMAKE_ARGS          -DCMAKE_INSTALL_PREFIX=${XBYAK_INSTALL_ROOT}
                         -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
     CMAKE_CACHE_ARGS    -DCMAKE_INSTALL_PREFIX:PATH=${XBYAK_INSTALL_ROOT}

paddle/scripts/paddle_build.bat

@@ -340,15 +340,12 @@ if %day_now% NEQ %day_before% (
     echo %day_now% > %cache_dir%\day.txt
     type %cache_dir%\day.txt
     if %day_now% EQU 21 (
-        rmdir %cache_dir%\third_party_GPU /s/q
         rmdir %cache_dir%\third_party /s/q
     )
     if %day_now% EQU 11 (
-        rmdir %cache_dir%\third_party_GPU /s/q
         rmdir %cache_dir%\third_party /s/q
     )
     if %day_now% EQU 01 (
-        rmdir %cache_dir%\third_party_GPU /s/q
         rmdir %cache_dir%\third_party /s/q
     )
 )
@@ -367,11 +364,41 @@ echo echo ${md5_content}^>md5.txt >> cache.sh
 
 set /p md5=< md5.txt
 if "%WITH_GPU%"=="ON" (
-    set THIRD_PARTY_HOME=%cache_dir:\=/%/third_party_GPU
+    for /F "delims=" %%# in ('nvcc --version^|findstr V1') do set cuda_version=%%#
+    set cuda_version=!cuda_version:~-7,4!
+    set sub_dir=cuda!cuda_version:.=!
 ) else (
-    set THIRD_PARTY_HOME=%cache_dir:\=/%/third_party
+    set sub_dir=cpu
 )
+
+set THIRD_PARTY_HOME=%cache_dir:\=/%/third_party/%sub_dir%
 set THIRD_PARTY_PATH=%THIRD_PARTY_HOME%/%md5%
+set UPLOAD_TP_FILE=OFF
+
+if not exist %THIRD_PARTY_PATH% (
+    echo There is no usable third_party cache in %THIRD_PARTY_PATH%, will download from bos.
+    pip install wget
+    if not exist %THIRD_PARTY_HOME% mkdir "%THIRD_PARTY_HOME%"
+    cd %THIRD_PARTY_HOME%
+    echo Getting third party: downloading ...
+    %PYTHON_ROOT%\python.exe -c "import wget;wget.download('https://paddle-windows.bj.bcebos.com/third_party/%sub_dir%/%md5%.tar.gz')" 2>nul
+    if !ERRORLEVEL! EQU 0 (
+        echo Getting third party: extracting ...
+        tar -xf %md5%.tar.gz
+        if !ERRORLEVEL! EQU 0 ( 
+            echo Get third party from bos successfully
+        ) else (
+            echo Get third party failed, reason: extract failed, will build locally
+        )
+        del %md5%.tar.gz
+    ) else (
+        echo Get third party failed, reason: download failed, will build locally
+    )
+    if not exist %THIRD_PARTY_PATH% ( set UPLOAD_TP_FILE=ON ) 
+    cd %work_dir%\%BUILD_DIR%
+) else (
+    echo Found reusable third_party cache locally, will reuse it.
+)
 
 :cmake_impl
 echo cmake .. -G %GENERATOR% -DCMAKE_BUILD_TYPE=Release -DWITH_AVX=%WITH_AVX% -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% ^
@@ -399,7 +426,7 @@ rem ----------------------------------------------------------------------------
 :build
 @ECHO OFF
 echo    ========================================
-echo    Step 2. Buile Paddle ...
+echo    Step 2. Build Paddle ...
 echo    ========================================
 
 for /F %%# in ('wmic cpu get NumberOfLogicalProcessors^|findstr [0-9]') do set /a PARALLEL_PROJECT_COUNT=%%#*4/5
@@ -485,6 +512,41 @@ if %ERRORLEVEL% NEQ 0 (
     )
 )
 
+set BCE_FILE=%cache_dir%\bce-python-sdk-0.8.33\BosClient.py
+if %UPLOAD_TP_FILE%==ON (
+    echo Uploading third_party: checking bce ...
+    if not exist %cache_dir%\bce-python-sdk-0.8.33 (
+        echo There is no bce in this PC, will install bce.
+        cd %cache_dir%
+        echo Download package from https://paddle-windows.bj.bcebos.com/bce-python-sdk-0.8.33.tar.gz
+        %PYTHON_ROOT%\python.exe -c "import wget;wget.download('https://paddle-windows.bj.bcebos.com/bce-python-sdk-0.8.33.tar.gz')"
+        %PYTHON_ROOT%\python.exe -c "import shutil;shutil.unpack_archive('bce-python-sdk-0.8.33.tar.gz', extract_dir='./',format='gztar')"
+        cd %cache_dir%\bce-python-sdk-0.8.33
+        %PYTHON_ROOT%\python.exe setup.py install 1>nul
+        del %cache_dir%\bce-python-sdk-0.8.33.tar.gz
+    )
+    if !errorlevel! EQU 0 (
+        cd %THIRD_PARTY_HOME%
+        echo Uploading third_party: compressing ...
+        tar -zcf %md5%.tar.gz %md5%
+        if !errorlevel! EQU 0 (
+            echo Uploading third_party: uploading ...
+            %PYTHON_ROOT%\python.exe %BCE_FILE% %md5%.tar.gz paddle-windows/third_party/%sub_dir% 1>nul
+            if !errorlevel! EQU 0 (
+                echo Upload third party to bos paddle-windows/third_party/%sub_dir% successfully 
+            ) else (
+                echo Failed upload third party to bos, reason: upload failed
+            )
+        ) else (
+            echo Failed upload third party to bos, reason: compress failed
+        )
+        del %md5%.tar.gz
+    ) else (
+        echo Failed upload third party to bos, reason: install bce failed
+    )
+    cd %work_dir%\%BUILD_DIR%
+)
+
 echo Build Paddle successfully!
 echo 0 > %cache_dir%\error_code.txt
 type %cache_dir%\error_code.txt