From 95df0e144799a15888eed06f36c4a3ec582d8857 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=20Wei=20=28=E4=BB=BB=E5=8D=AB=29?= Date: Fri, 25 Dec 2020 17:04:35 +0800 Subject: [PATCH] Add the ipipe log param prefix (#29545) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add the ipipe log param prefix 1. add the prefix; 2. using Colon before the metric values; * 增加效率云日志指标收集前缀 暂未验证windows bat的这个字符串替换是否正常 * Preserve The Old Format Metrics During The Transition Period Please DELETE the old format metrics log finally. The period man last for a week. * ipipe_log_param + ccache and clcache .. --- paddle/scripts/paddle_build.bat | 9 +++++++++ paddle/scripts/paddle_build.sh | 17 ++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/paddle/scripts/paddle_build.bat b/paddle/scripts/paddle_build.bat index f59bfe7755..8c103f0d7c 100644 --- a/paddle/scripts/paddle_build.bat +++ b/paddle/scripts/paddle_build.bat @@ -98,9 +98,11 @@ git diff --name-only %BRANCH% | findstr /V "\.py" || set CI_SKIP_CPP_TEST=ON :mkbuild if not exist build ( echo Windows build cache FALSE + echo "ipipe_log_param_Windows_Build_Cache: FALSE" mkdir build ) else ( echo Windows build cache TRUE + echo "ipipe_log_param_Windows_Build_Cache: TRUE" ) cd /d build dir . @@ -329,10 +331,12 @@ set /p libsize=< lib_size.txt for /F %%i in ("%libsize%") do ( set /a libsize_m=%%i/1024 echo "Windows Paddle_Inference Size: !libsize_m!M" + echo "ipipe_log_param_Windows_Paddle_Inference_Size: !libsize_m!M" ) %cache_dir%\tools\busybox64.exe du -h -d 0 %cd%\python\dist > whl_size.txt set /p whlsize=< whl_size.txt for /F %%i in ("%whlsize%") do echo "Windows PR whl Size: %%i" +for /F %%i in ("%whlsize%") do echo "ipipe_log_param_Windows_PR_whl_Size: %%i" dir /s /b python\dist\*.whl > whl_file.txt set /p PADDLE_WHL_FILE_WIN=< whl_file.txt @@ -485,6 +489,7 @@ echo spec_path=$(pwd)/UNITTEST_PR.spec>> check_change_of_unittest.sh echo ctest -N ^| awk -F ':' '{print $2}' ^| sed '/^^$/d' ^| sed '$d' ^> ${spec_path}>> check_change_of_unittest.sh echo num=$(awk 'END{print NR}' ${spec_path})>> check_change_of_unittest.sh echo echo "Windows 1 card TestCases count is $num">> check_change_of_unittest.sh +echo echo "ipipe_log_param_Windows_1_Card_TestCases_Count: $num">> check_change_of_unittest.sh echo UPSTREAM_URL='https://github.com/PaddlePaddle/Paddle'>> check_change_of_unittest.sh echo origin_upstream_url=`git remote -v ^| awk '{print $1, $2}' ^| uniq ^| grep upstream ^| awk '{print $2}'`>> check_change_of_unittest.sh echo if [ "$origin_upstream_url" == "" ]; then>> check_change_of_unittest.sh @@ -575,6 +580,8 @@ set /a ss=100%ss%%%100 set /a end_secs=dd*86400+hh*3600+nn*60+ss set /a cost_secs=end_secs-start_sec echo "Windows %~3 Time: %cost_secs%s" +set tempTaskName=%~3 +echo "ipipe_log_param_Windows_%tempTaskName: =_%_Time: %cost_secs%s" goto:eof @@ -582,9 +589,11 @@ goto:eof for /f "tokens=2,4" %%i in ('clcache.exe -s ^| findstr "entries hits"') do set %%i=%%j if %hits% EQU 0 ( echo "clcache hit rate: 0%%" + echo "ipipe_log_param_Clcache_Hit_Rate: 0%%" ) else ( set /a rate=%hits%*10000/%entries% echo "clcache hit rate: %rate:~0,-2%.%rate:~-2%%%" + echo "ipipe_log_param_Clcache_Hit_Hate: %rate:~0,-2%.%rate:~-2%%%" ) goto:eof diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index fde8cdc6b7..1232f213e9 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -391,6 +391,7 @@ EOF tar -czf paddle_inference.tgz paddle_inference buildSize=$(du -h --max-depth=0 ${PADDLE_ROOT}/build/paddle_inference.tgz |awk '{print $1}') echo "Paddle_Inference Size: $buildSize" + echo "ipipe_log_param_Paddle_Inference_Size: $buildSize" else SYSTEM=`uname -s` if [ "$SYSTEM" == "Darwin" ]; then @@ -400,8 +401,10 @@ EOF fi buildSize=$($com ${PADDLE_ROOT}/build |awk '{print $1}') echo "Build Size: $buildSize" + echo "ipipe_log_param_Build_Size: $buildSize" PR_whlSize=$($com ${PADDLE_ROOT}/build/python/dist |awk '{print $1}') echo "PR whl Size: $PR_whlSize" + echo "ipipe_log_param_PR_whl_Size: $PR_whlSize" fi } @@ -426,6 +429,7 @@ function cmake_gen_and_build() { build $2 endTime_s=`date +%s` echo "Build Time: $[ $endTime_s - $startTime_s ]s" + echo "ipipe_log_param_Build_Time: $[ $endTime_s - $startTime_s ]s" } function build_mac() { @@ -463,6 +467,7 @@ function cmake_gen_and_build_mac() { build_mac endTime_s=`date +%s` echo "Build Time: $[ $endTime_s - $startTime_s ]s" + echo "ipipe_log_param_Build_Time: $[ $endTime_s - $startTime_s ]s" } function run_test() { @@ -650,6 +655,7 @@ EOF #mactest_error=$? ut_endTime_s=`date +%s` echo "Mac testCase Time: $[ $ut_endTime_s - $ut_startTime_s ]s" + echo "ipipe_log_param_Mac_TestCases_Time: $[ $ut_endTime_s - $ut_startTime_s ]s" paddle version # Recovery proxy to avoid failure in later steps set +x @@ -932,8 +938,10 @@ EOF num=$(echo $testcases|grep -o '\^'|wc -l) if [ "$2" == "" ]; then echo "exclusive TestCases count is $num" + echo "ipipe_log_param_Exclusive_TestCases_Count: $num" else echo "$2 card TestCases count is $num" + echo "ipipe_log_param_${2}_Cards_TestCases_Count $num" fi } @@ -1025,8 +1033,10 @@ function card_test() { ut_endTime_s=`date +%s` if [ "$2" == "" ]; then echo "exclusive TestCases Total Time: $[ $ut_endTime_s - $ut_startTime_s ]s" + echo "ipipe_log_param_Exclusive_TestCases_Total_Time: $[ $ut_endTime_s - $ut_startTime_s ]s" else echo "$2 card TestCases Total Time: $[ $ut_endTime_s - $ut_startTime_s ]s" + echo "ipipe_log_param_${2}_Cards_TestCases_Total_Time: $[ $ut_endTime_s - $ut_startTime_s ]s" fi set +m } @@ -1325,6 +1335,7 @@ function parallel_test() { fi ut_total_endTime_s=`date +%s` echo "TestCases Total Time: $[ $ut_total_endTime_s - $ut_total_startTime_s ]s" + echo "ipipe_log_param_TestCases_Total_Time: $[ $ut_total_endTime_s - $ut_total_startTime_s ]s" } function enable_unused_var_check() { @@ -1604,6 +1615,7 @@ EOF fi endTime_s=`date +%s` echo "Build Time: $[ $endTime_s - $startTime_s ]s" + echo "ipipe_log_param_Build_Time: $[ $endTime_s - $startTime_s ]s" build_size "paddle_inference" } @@ -1634,7 +1646,8 @@ EOF ${TENSORRT_LIB_DIR:-/usr/local/TensorRT/lib} EXIT_CODE=$? fluid_endTime_s=`date +%s` - echo "test_fluid_lib Total Time: $[ $fluid_endTime_s - $fluid_startTime_s ]s" + echo "test_fluid_lib Total Time: $[ $fluid_endTime_s - $fluid_startTime_s ]s" + echo "ipipe_log_param_Test_Fluid_Lib_Total_Time: $[ $fluid_endTime_s - $fluid_startTime_s ]s" ./clean.sh if [[ "$EXIT_CODE" != "0" ]]; then exit 8; @@ -1653,6 +1666,7 @@ EOF EXIT_CODE=$? fluid_train_endTime_s=`date +%s` echo "test_fluid_lib_train Total Time: $[ $fluid_train_endTime_s - $fluid_train_startTime_s ]s" + echo "ipipe_log_param_Test_Fluid_Lib_Train_Total_Time: $[ $fluid_train_endTime_s - $fluid_train_startTime_s ]s" ./clean.sh if [[ "$EXIT_CODE" != "0" ]]; then exit 8; @@ -1680,6 +1694,7 @@ function example() { function collect_ccache_hits() { rate=$(ccache -s | grep 'cache hit rate' | awk '{print $4}') echo "ccache hit rate: ${rate}%" + echo "ipipe_log_param_Ccache_Hit_Rate: ${rate}%" } -- GitLab