From 576d68083e1ec3dc4251275e3ae03750bea1763b Mon Sep 17 00:00:00 2001
From: zhangchunle <clzhang_cauc@163.com>
Date: Thu, 4 Jun 2020 10:56:06 +0800
Subject: [PATCH] generate ci index (#24792)

---
 paddle/scripts/paddle_build.sh | 107 ++++++++++++++++++++++++++++-----
 1 file changed, 92 insertions(+), 15 deletions(-)

diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh
index cc5ce29b8a7..6c716cb1110 100755
--- a/paddle/scripts/paddle_build.sh
+++ b/paddle/scripts/paddle_build.sh
@@ -315,6 +315,32 @@ function build_base() {
     make install -j ${parallel_number}
 }
 
+function build_size() {
+    cat <<EOF
+    ============================================
+    Calculate /paddle/build size and PR whl size
+    ============================================
+EOF
+    if [ "$1" == "fluid_inference" ]; then
+        cd ${PADDLE_ROOT}/build
+        cp -r fluid_inference_install_dir fluid_inference
+        tar -czf fluid_inference.tgz fluid_inference
+        buildSize=$(du -h --max-depth=0 ${PADDLE_ROOT}/build/fluid_inference.tgz |awk '{print $1}')
+        echo "FLuid_Inference Size: $buildSize"
+    else
+        SYSTEM=`uname -s`
+        if [ "$SYSTEM" == "Darwin" ]; then
+            com='du -h -d 0'
+        else
+            com='du -h --max-depth=0'
+        fi
+        buildSize=$($com ${PADDLE_ROOT}/build |awk '{print $1}')
+        echo "Build Size: $buildSize"
+        PR_whlSize=$($com ${PADDLE_ROOT}/build/python/dist |awk '{print $1}')
+        echo "PR whl Size: $PR_whlSize"
+    fi
+}
+
 function build() {
     mkdir -p ${PADDLE_ROOT}/build
     cd ${PADDLE_ROOT}/build
@@ -324,6 +350,18 @@ function build() {
     ============================================
 EOF
     build_base $@
+    current_branch=`git branch | grep \* | cut -d ' ' -f2`
+    if [ "$current_branch" != "develop_base_pr" ];then
+        build_size
+    fi
+}
+
+function cmake_gen_and_build() {
+    startTime_s=`date +%s`
+    cmake_gen $1
+    build $2
+    endTime_s=`date +%s`
+    echo "Build Time: $[ $endTime_s - $startTime_s ]s"
 }
 
 function build_mac() {
@@ -338,6 +376,15 @@ EOF
         make clean
     fi
     make install -j 8
+    build_size
+}
+
+function cmake_gen_and_build_mac() {
+    startTime_s=`date +%s`
+    cmake_gen $1
+    build_mac
+    endTime_s=`date +%s`
+    echo "Build Time: $[ $endTime_s - $startTime_s ]s"
 }
 
 function run_test() {
@@ -454,8 +501,10 @@ EOF
         elif [ "$1" == "cp37-cp37m" ]; then
             pip3.7 install --user ${INSTALL_PREFIX:-/paddle/build}/opt/paddle/share/wheels/*.whl
         fi
-
+        ut_startTime_s=`date +%s`
         ctest --output-on-failure -j $2
+        ut_endTime_s=`date +%s`
+        echo "Mac testCase Time: $[ $ut_endTime_s - $ut_startTime_s ]s"
         paddle version
         # Recovery proxy to avoid failure in later steps
         export http_proxy=$my_proxy
@@ -680,9 +729,25 @@ function caught_error() {
     done
 }
 
+function case_count(){
+    cat <<EOF
+    ============================================
+    Generating TestCases Count ... 
+    ============================================
+EOF
+    testcases=$1
+    num=$(echo $testcases|grep -o '\^'|wc -l)
+    if [ "$2" == "" ]; then
+        echo "exclusive TestCases count is $num"
+    else
+        echo "$2 card TestCases count is $num"
+    fi
+}
+
 function card_test() {
     set -m
-
+    case_count $1 $2
+    ut_startTime_s=`date +%s` 
     # get the CUDA device count
     CUDA_DEVICE_COUNT=$(nvidia-smi -L | wc -l)
 
@@ -730,6 +795,12 @@ function card_test() {
     done
 
     wait; # wait for all subshells to finish
+    ut_endTime_s=`date +%s`
+    if [ "$2" == "" ]; then
+        echo "exclusive TestCases Total Time: $[ $ut_endTime_s - $ut_startTime_s ]s"
+    else
+        echo "$2 card TestCases Total Time: $[ $ut_endTime_s - $ut_startTime_s ]s"
+    fi
     set +m
 }
 
@@ -815,9 +886,12 @@ set -ex
 }
 
 function parallel_test() {
+    ut_total_startTime_s=`date +%s`
     mkdir -p ${PADDLE_ROOT}/build
     cd ${PADDLE_ROOT}/build
     parallel_test_base
+    ut_total_endTime_s=`date +%s`
+    echo "TestCases Total Time: $[ $ut_total_endTime_s - $ut_total_startTime_s ]s"
 }
 
 function enable_unused_var_check() {
@@ -1075,10 +1149,14 @@ EOF
     if [[ "$1" != "" ]]; then
       parallel_number=$1
     fi
+    startTime_s=`date +%s` 
     cmake .. -DWITH_DISTRIBUTE=OFF -DON_INFER=ON -DCUDA_ARCH_NAME=${CUDA_ARCH_NAME:-Auto}
 
     make -j ${parallel_number} fluid_lib_dist
     make -j ${parallel_number} inference_lib_dist
+    endTime_s=`date +%s`
+    echo "Build Time: $[ $endTime_s - $startTime_s ]s"
+    build_size "fluid_inference"
 }
 
 function tar_fluid_lib() {
@@ -1100,10 +1178,13 @@ function test_fluid_lib() {
     Testing fluid library for inference ...
     ========================================
 EOF
+    fluid_startTime_s=`date +%s`
     cd ${PADDLE_ROOT}/paddle/fluid/inference/api/demo_ci
     ./run.sh ${PADDLE_ROOT} ${WITH_MKL:-ON} ${WITH_GPU:-OFF} ${INFERENCE_DEMO_INSTALL_DIR} \
              ${TENSORRT_INCLUDE_DIR:-/usr/local/TensorRT/include} \
              ${TENSORRT_LIB_DIR:-/usr/local/TensorRT/lib}
+    fluid_endTime_s=`date +%s`
+    echo "test_fluid_lib Total Time: $[ $fluid_endTime_s - $fluid_startTime_s ]s"          
     ./clean.sh
 }
 
@@ -1113,8 +1194,11 @@ function test_fluid_lib_train() {
     Testing fluid library for training ...
     ========================================
 EOF
+    fluid_train_startTime_s=`date +%s`
     cd ${PADDLE_ROOT}/paddle/fluid/train/demo
     ./run.sh ${PADDLE_ROOT} ${WITH_MKL:-ON}
+    fluid_train_endTime_s=`date +%s`
+    echo "test_fluid_lib_train Total Time: $[ $fluid_train_endTime_s - $fluid_train_startTime_s ]s"
     ./clean.sh
 }
 
@@ -1134,21 +1218,18 @@ function example() {
     fi
 }
 
-
 function main() {
     local CMD=$1 
     local parallel_number=$2
     init
     case $CMD in
       build_only)
-        cmake_gen ${PYTHON_ABI:-""}
-        build ${parallel_number}
+        cmake_gen_and_build ${PYTHON_ABI:-""} ${parallel_number}
         ;;
       build_and_check)
         check_style
         generate_upstream_develop_api_spec ${PYTHON_ABI:-""} ${parallel_number}
-        cmake_gen ${PYTHON_ABI:-""}
-        build ${parallel_number} 
+        cmake_gen_and_build ${PYTHON_ABI:-""} ${parallel_number}
         check_sequence_op_unittest
         generate_api_spec ${PYTHON_ABI:-""} "PR"
         example
@@ -1204,8 +1285,7 @@ function main() {
         ;;
       cicheck_coverage)
         check_approvals_of_unittest 1
-        cmake_gen ${PYTHON_ABI:-""}
-        build ${parallel_number}
+        cmake_gen_and_build ${PYTHON_ABI:-""} ${parallel_number}
         enable_unused_var_check
         parallel_test
         check_coverage
@@ -1233,13 +1313,11 @@ function main() {
         assert_api_spec_approvals
         ;;
       maccheck)
-        cmake_gen ${PYTHON_ABI:-""}
-        build_mac
+        cmake_gen_and_build_mac ${PYTHON_ABI:-""}
         run_mac_test ${PYTHON_ABI:-""} ${PROC_RUN:-1}
         ;;
       maccheck_py35)
-        cmake_gen ${PYTHON_ABI:-""}
-        build_mac
+        cmake_gen_and_build_mac ${PYTHON_ABI:-""}
         run_mac_test ${PYTHON_ABI:-""} ${PROC_RUN:-1}
         check_change_of_unittest ${PYTHON_ABI:-""}
         ;;
@@ -1248,8 +1326,7 @@ function main() {
         build_mac
         ;;
       cicheck_py35)
-        cmake_gen ${PYTHON_ABI:-""}
-        build ${parallel_number}
+        cmake_gen_and_build ${PYTHON_ABI:-""} ${parallel_number}
         parallel_test
         ;;
       cmake_gen)
-- 
GitLab