fix ut retry (#37301)

* fix ut retry

paddle/scripts/paddle_build.sh

@@ -1051,9 +1051,7 @@ failed_test_lists=''
 tmp_dir=`mktemp -d`
 
 function collect_failed_tests() {
-    echo "collect_failed_tests begineee!!!"
     for file in `ls $tmp_dir`; do
-        echo "fileis $file"
         exit_code=0
         grep -q 'The following tests FAILED:' $tmp_dir/$file||exit_code=$?
         if [ $exit_code -ne 0 ]; then
@@ -1064,7 +1062,6 @@ function collect_failed_tests() {
             ${failuretest}"
         fi
     done
-    echo "collect_failed_tests finished!!!"
 }
 
 # getting qucik disable ut list 
@@ -1277,7 +1274,6 @@ set +x
                   # trick: treat all test case with prefix "test_dist" as dist case, and would run on 2 GPUs
                   read is_multicard <<< $(echo "$testcase"|grep -oEi "test_dist_")
                 fi
-
                 if [[ "$is_exclusive" != "" ]]; then
                     if [[ $(echo $high_parallel_job | grep -o "\^$testcase\\$") != "" ]]; then
                         exclusive_tests_high_parallel="$exclusive_tests_high_parallel|^$testcase$"
@@ -1385,7 +1381,7 @@ set +x
                         retry_unittests_record="$retry_unittests_record$failed_test_lists"
                         failed_test_lists_ult=`echo "${failed_test_lists}" |grep -Po '[^ ].*$'`
                         set -e
-                        if [[ "${exec_times}" == "1" ]] || [[ "${exec_times}" == "3" ]];then
+                        if [[ "${exec_times}" == "1" ]] || [[ "${exec_times}" == "2" ]];then
                             if [[ "${failed_test_lists}" == "" ]];then
                                 break
                             else
@@ -1424,45 +1420,34 @@ set +x
                                 fi
 
                             done
-                        echo "rerun one_card_retry beginee!!!"
                         if [[ "$one_card_retry" != "" ]]; then
                             card_test "$one_card_retry" 1 4
                         fi
-                        echo "rerun one_card_retry finished!!!"
-                        
-                        echo "rerun multiple_card_retry beginee!!!"
                         if [[ "$multiple_card_retry" != "" ]]; then
                             card_test "$multiple_card_retry" 2
                         fi
-                        echo "rerun multiple_card_retry finished!!!"
-                        
-                        echo "rerun exclusive_retry beginee!!!"
                         if [[ "$exclusive_retry" != "" ]]; then
                             card_test "$exclusive_retry" -1
                         fi
-                        echo "rerun exclusive_retry finished!!!"
-                        
-                        exec_times=$[$exec_times+1]
                         echo "exec_times: $exec_times"
+                        exec_times=$[$exec_times+1]
                         failed_test_lists=''
                         collect_failed_tests
-                        echo "failed_test_listsssssss: $failed_test_lists"
+                        echo "failed_test_lists: $failed_test_lists"
                         rm -f $tmp_dir/*
                         one_card_retry=''
                         multiple_card_retry=''
                         exclusive_retry='' 
-                    
                     else 
                         break
                     fi 
                 done
+            retry_unittests_record="$retry_unittests_record$failed_test_lists"
         fi
 
         rerun_ut_endTime_s=`date +%s`
-        echo "ipipe_log_param_Rerunaaaa_TestCases_Total_Time: $[ $rerun_ut_endTime_s - $rerun_ut_startTime_s ]s" 
         echo "ipipe_log_param_Rerun_TestCases_Total_Time: $[ $rerun_ut_endTime_s - $rerun_ut_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt
         ut_actual_total_endTime_s=`date +%s`
-        echo "ipipe_log_param_actualaaaaa_TestCases_Total_Time: $[ $ut_actual_total_endTime_s - $ut_actual_total_startTime_s ]s"
         echo "ipipe_log_param_actual_TestCases_Total_Time: $[ $ut_actual_total_endTime_s - $ut_actual_total_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt
         if [[ "$EXIT_CODE" != "0" ]]; then
             show_ut_retry_result
@@ -1495,7 +1480,7 @@ function show_ut_retry_result() {
         exit 8;
     else
         retry_unittests_ut_name=$(echo "$retry_unittests_record" | grep -oEi "\-.+\(" | sed 's/(//' | sed 's/- //' )
-        retry_unittests_record_judge=$(echo ${retry_unittests_ut_name}| tr ' ' '\n' | sort | uniq -c | awk '{if ($1 >=3) {print $2}}')
+        retry_unittests_record_judge=$(echo ${retry_unittests_ut_name}| tr ' ' '\n' | sort | uniq -c | awk '{if ($1 >=4) {print $2}}')
         if [ -z "${retry_unittests_record_judge}" ];then
             echo "========================================"
             echo "There are failed tests, which have been successful after re-run:"

tools/parallel_UT_rule.py

@@ -2265,7 +2265,7 @@ def main():
         print("{};{};{};{}".format(high_parallel_job, fourth_high_parallel_job,
                                    fifth_high_parallel_job, non_parallel_job))
     else:
-        print("{};{};{};{};{};{};{}".format(
+        print("{};{};{};{};{};{};{};{}".format(
             high_parallel_job, secondary_high_parallel_job,
             third_high_parallel_job, fourth_high_parallel_job,
             fifth_high_parallel_job, sixth_high_parallel_job,