From a7e1b9d2e29a54debb1d581bf5057f3e4530ff70 Mon Sep 17 00:00:00 2001
From: risemeup1 <62429225+risemeup1@users.noreply.github.com>
Date: Mon, 10 Oct 2022 10:07:23 +0800
Subject: [PATCH] Delete gpu_memory information of every unnitest to less time
 consuming (#46472)

* it is a test

* it is a test

* it is a test,test=coverage

* optimizing the proceess of generating ut_file_map.json

* optimizing the proceess of generating ut_file_map.json

* optimizing the proceess of generating ut_file_map.json

* optimizing the proceess of generating ut_file_map.json
---
 paddle/scripts/paddle_build.sh | 52 +++++++++++++++++++++++++---------
 1 file changed, 39 insertions(+), 13 deletions(-)

diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh
index 9f8e36b67c..0b20a000ca 100755
--- a/paddle/scripts/paddle_build.sh
+++ b/paddle/scripts/paddle_build.sh
@@ -1810,14 +1810,10 @@ EOF
 }
 
 function insert_pile_to_h_cu_diff {
-    # TODO get develop h/cu md5
+    # get all .cu files of develop branch
     cd ${PADDLE_ROOT}
     find ${PADDLE_ROOT} -name '*.cu'| grep -v ${PADDLE_ROOT}/build >> ${PADDLE_ROOT}/tools/h_cu_files.log
-    python ${PADDLE_ROOT}/tools/handle_h_cu_file.py 'get_h_file_md5' ${PADDLE_ROOT}
-    
-    # TODO insert pile to diff h/cu file 
-
-    #insert pile to full h/cu file 
+    #insert pile to all .cu files
     python ${PADDLE_ROOT}/tools/handle_h_cu_file.py 'insert_pile_to_h_file' ${PADDLE_ROOT}
 }
 
@@ -1859,12 +1855,11 @@ function parallel_card_test_single {
     num=$2
     for case in $(echo $testcases | tr "$|^" "\n")
     do
-        cd ${PADDLE_ROOT}/build
-        precise_card_test "^${case}$" $num
+        cd ${PADDLE_ROOT}/build 
+        parallel_card_test "^${case}$" $num 
     done
 }
-
-function precise_card_test() {
+function parallel_card_test() {
     set -m
     testcases=$1
     if (( $# > 1 )); then
@@ -1900,13 +1895,42 @@ function precise_card_test() {
     set +m
 }
 
+function precise_card_test() {
+    set -m
+    testcases=$1
+    if (( $# > 1 )); then
+        cardnumber=$2
+        cuda_list="0"
+        if [ $cardnumber -eq 2 ]; then
+            cuda_list=${CUDA_VISIBLE_DEVICES}
+        else
+            cuda_list="0"
+        fi
+    else
+        cardnumber=2
+        cuda_list=${CUDA_VISIBLE_DEVICES}
+    fi
+
+    if [[ "$testcases" == "" ]]; then
+        return 0
+    fi
+
+    echo "****************************************************************"
+    echo "***Running ut: $testcases***"
+    echo "****************************************************************"
+    
+    tmpfile=$tmp_dir/$testcases".log"
+    env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I 0,,1 -R "($testcases)" --timeout 500 --output-on-failure -V -j 1 > $tmpfile 
+    set +m
+}
+
 function get_precise_tests_map_file {
     cd ${PADDLE_ROOT}/build
     pip install ${PADDLE_ROOT}/build/python/dist/*whl
     ut_total_startTime_s=`date +%s`
     EXIT_CODE=0;
     test_cases=$(ctest -N -V) # get all test cases
-    single_card_tests='' # all cases list which would take one graph card
+    single_card_tests=''      # all cases list which would take one graph card
     exclusive_tests=''        # cases list which would be run exclusively
     multiple_card_tests=''    # cases list which would take multiple GPUs, most cases would be two GPUs
     is_exclusive=''           # indicate whether the case is exclusive type
@@ -1973,18 +1997,20 @@ set +x
 set -x
     mkdir -p ${PADDLE_ROOT}/build/ut_map
     mkdir -p ${PADDLE_ROOT}/build/pytest
-
+    #run all unittest to get the coverage information of .c and .h files
     precise_card_test_single "$single_card_tests" 1
     precise_card_test_single "$single_card_tests_1" 1
     precise_card_test_single "$multiple_card_tests" 2
     precise_card_test_single "$exclusive_tests"
     wait;
+    #get notSuccessut including the failed uniitests and not executed unittests
     python ${PADDLE_ROOT}/tools/get_ut_file_map.py 'get_not_success_ut' ${PADDLE_ROOT}
     
-    #analy h/cu to Map file
+    #analyze the mapping between unit tests and .cu files
     python ${PADDLE_ROOT}/tools/handle_h_cu_file.py 'analy_h_cu_file' $tmp_dir ${PADDLE_ROOT}
 
     wait;
+    #rerun the notSuccessut and get the mapping between notSuccessut and .cu files
     get_failedUts_precise_map_file
 
     #generate python coverage and generate python file to tests_map_file
-- 
GitLab