diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index a524b4291f627fbebbf08ea764b01f040794cb55..feafca26cd32badf09e0cb52e7207ddb97ec5ccf 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -1037,7 +1037,9 @@ failed_test_lists='' tmp_dir=`mktemp -d` function collect_failed_tests() { + echo "collect_failed_tests begineee!!!" for file in `ls $tmp_dir`; do + echo "fileis $file" exit_code=0 grep -q 'The following tests FAILED:' $tmp_dir/$file||exit_code=$? if [ $exit_code -ne 0 ]; then @@ -1048,6 +1050,7 @@ function collect_failed_tests() { ${failuretest}" fi done + echo "collect_failed_tests finished!!!" } # getting qucik disable ut list @@ -1065,7 +1068,7 @@ function get_quickly_disable_ut() { function card_test() { set -m - + echo "$2 bengingggggg!!!!!" case_count $1 $2 ut_startTime_s=`date +%s` @@ -1137,6 +1140,7 @@ function card_test() { else echo "$2 card TestCases Total Time: $[ $ut_endTime_s - $ut_startTime_s ]s" fi + echo "$2 card TestCases finished!!!! " set +m } @@ -1292,22 +1296,41 @@ set +x single_ut_startTime_s=`date +%s` card_test "$single_card_tests_high_parallel" 1 24 # run cases 24 job each time with single GPU + echo "single_card_tests_high_parallel finished!!!" card_test "$single_card_tests_secondary_high_parallel" 1 15 # run cases 15 job each time with single GPU + echo "single_card_tests_secondary_high_parallel finished!!!" card_test "$single_card_tests_third_high_parallel" 1 12 # run cases 12 job each time with single GPU + echo "single_card_tests_third_high_parallel finished!!!" card_test "$single_card_tests_medium_parallel" 1 7 # run cases 7 job each time with single GPU + echo "single_card_tests_medium_parallel finished!!!" card_test "$single_card_tests_non_parallel" 1 2 # run cases 2 job each time with single GPU + echo "single_card_tests_non_parallel finished!!!" single_ut_endTime_s=`date +%s` - + echo "single_card_tests finished!!!" + multi_ut_startTime_s=`date +%s` + echo "multiple_card_tests begined!!!!!" card_test "$multiple_card_tests_medium_parallel" 2 4 # run cases 2 job each time with two GPUs + echo "multiple_card_tests_medium_parallel finished!!!" card_test "$multiple_card_tests_non_parallel" 2 2 # run cases 1 job each time with two GPUs + echo "multiple_card_tests_non_parallel finished!!!" multi_ut_endTime_s=`date +%s` - + echo "multiple_card_tests finished!!!" + exclu_ut_startTime_s=`date +%s` + echo "exclu_card_tests begined!!!!!" card_test "$exclusive_tests_high_parallel" -1 5 # run cases exclusively, in this cases would be run with 2/4/8 GPUs + echo "exclusive_tests_high_parallel finished!!!" card_test "$exclusive_tests_medium_parallel" -1 3 # run cases exclusively, in this cases would be run with 2/4/8 GPUs + echo "exclusive_tests_medium_parallel finished!!!" card_test "$exclusive_tests_non_parallel" -1 2 # run cases exclusively, in this cases would be run with 2/4/8 GPUs + echo "exclusive_tests_non_parallel finished!!!" exclu_ut_endTime_s=`date +%s` + echo "exclusive_tests finished!!!" + + echo "ipipe_log_param_1aaa_TestCases_Total_Time: $[ $single_ut_endTime_s - $single_ut_startTime_s ]s" + echo "ipipe_log_param_2aaa_TestCases_Total_Time: $[ $multi_ut_endTime_s - $multi_ut_startTime_s ]s" + echo "ipipe_log_param_Exclusiveaaaa_TestCases_Total_Time: $[ $exclu_ut_endTime_s - $exclu_ut_startTime_s ]s" echo "ipipe_log_param_1_TestCases_Total_Time: $[ $single_ut_endTime_s - $single_ut_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt echo "ipipe_log_param_2_TestCases_Total_Time: $[ $multi_ut_endTime_s - $multi_ut_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt @@ -1393,21 +1416,29 @@ set +x fi done - + echo "rerun one_card_retry beginee!!!" if [[ "$one_card_retry" != "" ]]; then card_test "$one_card_retry" 1 4 fi - + echo "rerun one_card_retry finished!!!" + + echo "rerun multiple_card_retry beginee!!!" if [[ "$multiple_card_retry" != "" ]]; then card_test "$multiple_card_retry" 2 fi - + echo "rerun multiple_card_retry finished!!!" + + echo "rerun exclusive_retry beginee!!!" if [[ "$exclusive_retry" != "" ]]; then card_test "$exclusive_retry" -1 fi + echo "rerun exclusive_retry finished!!!" + exec_times=$[$exec_times+1] + echo "exec_times: $exec_times" failed_test_lists='' collect_failed_tests + echo "failed_test_listsssssss: $failed_test_lists" rm -f $tmp_dir/* one_card_retry='' multiple_card_retry='' @@ -1417,9 +1448,10 @@ set +x fi rerun_ut_endTime_s=`date +%s` - + echo "ipipe_log_param_Rerunaaaa_TestCases_Total_Time: $[ $rerun_ut_endTime_s - $rerun_ut_startTime_s ]s" echo "ipipe_log_param_Rerun_TestCases_Total_Time: $[ $rerun_ut_endTime_s - $rerun_ut_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt ut_actual_total_endTime_s=`date +%s` + echo "ipipe_log_param_actualaaaaa_TestCases_Total_Time: $[ $ut_actual_total_endTime_s - $ut_actual_total_startTime_s ]s" echo "ipipe_log_param_actual_TestCases_Total_Time: $[ $ut_actual_total_endTime_s - $ut_actual_total_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt if [[ "$EXIT_CODE" != "0" ]]; then show_ut_retry_result