From 8094ac686ec66b524ff28520653543fdf6d1a64c Mon Sep 17 00:00:00 2001
From: LoveAn <mr.avin0323@gmail.com>
Date: Wed, 9 Dec 2020 12:36:08 +0800
Subject: [PATCH] Print ccache/clcache hit rate (#29341)

* test ccache hit statistics, test=develop

* test ccache hit statistics, test=develop

* add cache hit statistics, test=develop

* fix no percent symbol erro on windows, test=develop

* remove switch, test=develop
---
 paddle/scripts/paddle_build.bat | 20 +++++++++++++++++++-
 paddle/scripts/paddle_build.sh  | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/paddle/scripts/paddle_build.bat b/paddle/scripts/paddle_build.bat
index 375a691e933..03c08d8a56d 100644
--- a/paddle/scripts/paddle_build.bat
+++ b/paddle/scripts/paddle_build.bat
@@ -268,14 +268,21 @@ echo Build third_party successfully!
 
 set build_times=1
 :build_paddle
+:: reset clcache zero stats for collect PR's actual hit rate
+clcache.exe -z
+
 echo Build Paddle the %build_times% time:
 if "%WITH_CLCACHE%"=="OFF" (
     msbuild /m:%PARALLEL_PROJECT_COUNT% /p:Configuration=Release /verbosity:minimal paddle.sln
 ) else (
     msbuild /m:%PARALLEL_PROJECT_COUNT% /p:TrackFileAccess=false /p:CLToolExe=clcache.exe /p:CLToolPath=%PYTHON_ROOT%\Scripts /p:Configuration=Release /verbosity:minimal paddle.sln
 )
+set build_error=%ERRORLEVEL%
 
-if %ERRORLEVEL% NEQ 0 (
+:: ci will collect clcache hit rate
+goto :collect_clcache_hits
+
+if %build_error% NEQ 0 (
     set /a build_times=%build_times%+1
     if %build_times% GTR 1 (
         exit /b 7
@@ -613,6 +620,17 @@ echo "Windows %~3 Time: %cost_secs%s"
 goto:eof
 
 
+:collect_clcache_hits
+for /f "tokens=2,4" %%i in ('clcache.exe -s ^| findstr "entries hits"') do set %%i=%%j
+if %hits% EQU 0 (
+    echo "clcache hit rate: 0%%"
+) else (
+    set /a rate=%hits%*10000/%entries%
+    echo "clcache hit rate: %rate:~0,-2%.%rate:~-2%%%"
+)
+goto:eof
+
+
 rem ---------------------------------------------------------------------------------------------
 :success
 echo    ========================================
diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh
index 1ca4465fb28..0bbdd388c7f 100755
--- a/paddle/scripts/paddle_build.sh
+++ b/paddle/scripts/paddle_build.sh
@@ -358,7 +358,14 @@ function build_base() {
         make clean
     fi
 
+    # reset ccache zero stats for collect PR's actual hit rate
+    ccache -z
+
     make install -j ${parallel_number};build_error=$?
+
+    # ci will collect ccache hit rate
+    collect_ccache_hits
+
     if [ "$build_error" != 0 ];then
         exit 7;
     fi
@@ -425,10 +432,19 @@ EOF
     if [[ "$ENABLE_MAKE_CLEAN" != "OFF" ]]; then
         make clean
     fi
+
+    # reset ccache zero stats for collect PR's actual hit rate
+    ccache -z
+
     make install -j 8;build_error=$?
+
+    # ci will collect ccache hit rate
+    collect_ccache_hits
+
     if [ "$build_error" != 0 ];then
         exit 7;
     fi
+
     set -e
     build_size
 }
@@ -1547,13 +1563,22 @@ EOF
     startTime_s=`date +%s`
     set +e
     cmake .. -DWITH_DISTRIBUTE=OFF -DON_INFER=ON -DCUDA_ARCH_NAME=${CUDA_ARCH_NAME:-Auto};build_error=$?
+
+    # reset ccache zero stats for collect PR's actual hit rate
+    ccache -z
+
     make -j ${parallel_number} fluid_lib_dist;build_error=$?
     make -j ${parallel_number} inference_lib_dist;build_error=$?
+
+    # ci will collect ccache hit rate
+    collect_ccache_hits
+
     if [ "$build_error" != 0 ];then
         exit 7;
     fi
     endTime_s=`date +%s`
     echo "Build Time: $[ $endTime_s - $startTime_s ]s"
+
     build_size "paddle_inference"
 }
 
@@ -1625,6 +1650,13 @@ function example() {
     fi
 }
 
+
+function collect_ccache_hits() {
+    rate=$(ccache -s | grep 'cache hit rate' | awk '{print $4}')
+    echo "ccache hit rate: ${rate}%"
+}
+
+
 function test_op_benchmark() {
     bash ${PADDLE_ROOT}/tools/test_op_benchmark.sh
 }
-- 
GitLab