test=document_fix (#35606)

paddle/scripts/paddle_build.sh

@@ -1037,7 +1037,9 @@ failed_test_lists=''
 tmp_dir=`mktemp -d`
 
 function collect_failed_tests() {
+    echo "collect_failed_tests begineee!!!"
     for file in `ls $tmp_dir`; do
+        echo "fileis $file"
         exit_code=0
         grep -q 'The following tests FAILED:' $tmp_dir/$file||exit_code=$?
         if [ $exit_code -ne 0 ]; then
@@ -1048,6 +1050,7 @@ function collect_failed_tests() {
             ${failuretest}"
         fi
     done
+    echo "collect_failed_tests finished!!!"
 }
 
 # getting qucik disable ut list 
@@ -1065,7 +1068,7 @@ function get_quickly_disable_ut() {
 
 function card_test() {
     set -m
-
+    echo "$2 bengingggggg!!!!!"
     case_count $1 $2
     ut_startTime_s=`date +%s` 
 
@@ -1137,6 +1140,7 @@ function card_test() {
     else
         echo "$2 card TestCases Total Time: $[ $ut_endTime_s - $ut_startTime_s ]s"
     fi
+    echo "$2 card TestCases finished!!!! "
     set +m
 }
 
@@ -1292,22 +1296,41 @@ set +x
 
         single_ut_startTime_s=`date +%s`
         card_test "$single_card_tests_high_parallel" 1 24               # run cases 24 job each time with single GPU
+        echo "single_card_tests_high_parallel finished!!!"
         card_test "$single_card_tests_secondary_high_parallel" 1 15     # run cases 15 job each time with single GPU
+        echo "single_card_tests_secondary_high_parallel finished!!!"
         card_test "$single_card_tests_third_high_parallel" 1 12         # run cases 12 job each time with single GPU
+        echo "single_card_tests_third_high_parallel finished!!!"
         card_test "$single_card_tests_medium_parallel" 1 7              # run cases 7 job each time with single GPU
+        echo "single_card_tests_medium_parallel finished!!!"
         card_test "$single_card_tests_non_parallel" 1 2                 # run cases 2 job each time with single GPU
+        echo "single_card_tests_non_parallel finished!!!"
         single_ut_endTime_s=`date +%s`
+        echo "single_card_tests finished!!!"
         
         multi_ut_startTime_s=`date +%s`
+        echo "multiple_card_tests begined!!!!!"
         card_test "$multiple_card_tests_medium_parallel" 2 4            # run cases 2 job each time with two GPUs
+        echo "multiple_card_tests_medium_parallel finished!!!"
         card_test "$multiple_card_tests_non_parallel" 2 2               # run cases 1 job each time with two GPUs
+        echo "multiple_card_tests_non_parallel finished!!!"
         multi_ut_endTime_s=`date +%s`
+        echo "multiple_card_tests finished!!!"
         
         exclu_ut_startTime_s=`date +%s`
+        echo "exclu_card_tests begined!!!!!"
         card_test "$exclusive_tests_high_parallel" -1 5                 # run cases exclusively, in this cases would be run with 2/4/8 GPUs
+        echo "exclusive_tests_high_parallel finished!!!"
         card_test "$exclusive_tests_medium_parallel" -1 3                  # run cases exclusively, in this cases would be run with 2/4/8 GPUs
+        echo "exclusive_tests_medium_parallel finished!!!"
         card_test "$exclusive_tests_non_parallel" -1 2                  # run cases exclusively, in this cases would be run with 2/4/8 GPUs
+        echo "exclusive_tests_non_parallel finished!!!"
         exclu_ut_endTime_s=`date +%s`
+        echo "exclusive_tests finished!!!"
+
+        echo "ipipe_log_param_1aaa_TestCases_Total_Time: $[ $single_ut_endTime_s - $single_ut_startTime_s ]s" 
+        echo "ipipe_log_param_2aaa_TestCases_Total_Time: $[ $multi_ut_endTime_s - $multi_ut_startTime_s ]s"
+        echo "ipipe_log_param_Exclusiveaaaa_TestCases_Total_Time: $[ $exclu_ut_endTime_s - $exclu_ut_startTime_s ]s"
 
         echo "ipipe_log_param_1_TestCases_Total_Time: $[ $single_ut_endTime_s - $single_ut_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt
         echo "ipipe_log_param_2_TestCases_Total_Time: $[ $multi_ut_endTime_s - $multi_ut_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt
@@ -1393,21 +1416,29 @@ set +x
                                 fi
 
                             done
-
+                        echo "rerun one_card_retry beginee!!!"
                         if [[ "$one_card_retry" != "" ]]; then
                             card_test "$one_card_retry" 1 4
                         fi
+                        echo "rerun one_card_retry finished!!!"
                         
+                        echo "rerun multiple_card_retry beginee!!!"
                         if [[ "$multiple_card_retry" != "" ]]; then
                             card_test "$multiple_card_retry" 2
                         fi
+                        echo "rerun multiple_card_retry finished!!!"
                         
+                        echo "rerun exclusive_retry beginee!!!"
                         if [[ "$exclusive_retry" != "" ]]; then
                             card_test "$exclusive_retry" -1
                         fi
+                        echo "rerun exclusive_retry finished!!!"
+                        
                         exec_times=$[$exec_times+1]
+                        echo "exec_times: $exec_times"
                         failed_test_lists=''
                         collect_failed_tests
+                        echo "failed_test_listsssssss: $failed_test_lists"
                         rm -f $tmp_dir/*
                         one_card_retry=''
                         multiple_card_retry=''
@@ -1417,9 +1448,10 @@ set +x
         fi
 
         rerun_ut_endTime_s=`date +%s`
-        
+        echo "ipipe_log_param_Rerunaaaa_TestCases_Total_Time: $[ $rerun_ut_endTime_s - $rerun_ut_startTime_s ]s" 
         echo "ipipe_log_param_Rerun_TestCases_Total_Time: $[ $rerun_ut_endTime_s - $rerun_ut_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt
         ut_actual_total_endTime_s=`date +%s`
+        echo "ipipe_log_param_actualaaaaa_TestCases_Total_Time: $[ $ut_actual_total_endTime_s - $ut_actual_total_startTime_s ]s"
         echo "ipipe_log_param_actual_TestCases_Total_Time: $[ $ut_actual_total_endTime_s - $ut_actual_total_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt
         if [[ "$EXIT_CODE" != "0" ]]; then
             show_ut_retry_result