未验证 提交 c0443835 编写于 作者: Z zhangchunle 提交者: GitHub

fix ut retry (#37301)

* fix ut retry
上级 bc150edc
...@@ -1051,9 +1051,7 @@ failed_test_lists='' ...@@ -1051,9 +1051,7 @@ failed_test_lists=''
tmp_dir=`mktemp -d` tmp_dir=`mktemp -d`
function collect_failed_tests() { function collect_failed_tests() {
echo "collect_failed_tests begineee!!!"
for file in `ls $tmp_dir`; do for file in `ls $tmp_dir`; do
echo "fileis $file"
exit_code=0 exit_code=0
grep -q 'The following tests FAILED:' $tmp_dir/$file||exit_code=$? grep -q 'The following tests FAILED:' $tmp_dir/$file||exit_code=$?
if [ $exit_code -ne 0 ]; then if [ $exit_code -ne 0 ]; then
...@@ -1064,7 +1062,6 @@ function collect_failed_tests() { ...@@ -1064,7 +1062,6 @@ function collect_failed_tests() {
${failuretest}" ${failuretest}"
fi fi
done done
echo "collect_failed_tests finished!!!"
} }
# getting qucik disable ut list # getting qucik disable ut list
...@@ -1277,7 +1274,6 @@ set +x ...@@ -1277,7 +1274,6 @@ set +x
# trick: treat all test case with prefix "test_dist" as dist case, and would run on 2 GPUs # trick: treat all test case with prefix "test_dist" as dist case, and would run on 2 GPUs
read is_multicard <<< $(echo "$testcase"|grep -oEi "test_dist_") read is_multicard <<< $(echo "$testcase"|grep -oEi "test_dist_")
fi fi
if [[ "$is_exclusive" != "" ]]; then if [[ "$is_exclusive" != "" ]]; then
if [[ $(echo $high_parallel_job | grep -o "\^$testcase\\$") != "" ]]; then if [[ $(echo $high_parallel_job | grep -o "\^$testcase\\$") != "" ]]; then
exclusive_tests_high_parallel="$exclusive_tests_high_parallel|^$testcase$" exclusive_tests_high_parallel="$exclusive_tests_high_parallel|^$testcase$"
...@@ -1385,7 +1381,7 @@ set +x ...@@ -1385,7 +1381,7 @@ set +x
retry_unittests_record="$retry_unittests_record$failed_test_lists" retry_unittests_record="$retry_unittests_record$failed_test_lists"
failed_test_lists_ult=`echo "${failed_test_lists}" |grep -Po '[^ ].*$'` failed_test_lists_ult=`echo "${failed_test_lists}" |grep -Po '[^ ].*$'`
set -e set -e
if [[ "${exec_times}" == "1" ]] || [[ "${exec_times}" == "3" ]];then if [[ "${exec_times}" == "1" ]] || [[ "${exec_times}" == "2" ]];then
if [[ "${failed_test_lists}" == "" ]];then if [[ "${failed_test_lists}" == "" ]];then
break break
else else
...@@ -1424,45 +1420,34 @@ set +x ...@@ -1424,45 +1420,34 @@ set +x
fi fi
done done
echo "rerun one_card_retry beginee!!!"
if [[ "$one_card_retry" != "" ]]; then if [[ "$one_card_retry" != "" ]]; then
card_test "$one_card_retry" 1 4 card_test "$one_card_retry" 1 4
fi fi
echo "rerun one_card_retry finished!!!"
echo "rerun multiple_card_retry beginee!!!"
if [[ "$multiple_card_retry" != "" ]]; then if [[ "$multiple_card_retry" != "" ]]; then
card_test "$multiple_card_retry" 2 card_test "$multiple_card_retry" 2
fi fi
echo "rerun multiple_card_retry finished!!!"
echo "rerun exclusive_retry beginee!!!"
if [[ "$exclusive_retry" != "" ]]; then if [[ "$exclusive_retry" != "" ]]; then
card_test "$exclusive_retry" -1 card_test "$exclusive_retry" -1
fi fi
echo "rerun exclusive_retry finished!!!"
exec_times=$[$exec_times+1]
echo "exec_times: $exec_times" echo "exec_times: $exec_times"
exec_times=$[$exec_times+1]
failed_test_lists='' failed_test_lists=''
collect_failed_tests collect_failed_tests
echo "failed_test_listsssssss: $failed_test_lists" echo "failed_test_lists: $failed_test_lists"
rm -f $tmp_dir/* rm -f $tmp_dir/*
one_card_retry='' one_card_retry=''
multiple_card_retry='' multiple_card_retry=''
exclusive_retry='' exclusive_retry=''
else else
break break
fi fi
done done
retry_unittests_record="$retry_unittests_record$failed_test_lists"
fi fi
rerun_ut_endTime_s=`date +%s` rerun_ut_endTime_s=`date +%s`
echo "ipipe_log_param_Rerunaaaa_TestCases_Total_Time: $[ $rerun_ut_endTime_s - $rerun_ut_startTime_s ]s"
echo "ipipe_log_param_Rerun_TestCases_Total_Time: $[ $rerun_ut_endTime_s - $rerun_ut_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt echo "ipipe_log_param_Rerun_TestCases_Total_Time: $[ $rerun_ut_endTime_s - $rerun_ut_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt
ut_actual_total_endTime_s=`date +%s` ut_actual_total_endTime_s=`date +%s`
echo "ipipe_log_param_actualaaaaa_TestCases_Total_Time: $[ $ut_actual_total_endTime_s - $ut_actual_total_startTime_s ]s"
echo "ipipe_log_param_actual_TestCases_Total_Time: $[ $ut_actual_total_endTime_s - $ut_actual_total_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt echo "ipipe_log_param_actual_TestCases_Total_Time: $[ $ut_actual_total_endTime_s - $ut_actual_total_startTime_s ]s" >> ${PADDLE_ROOT}/build/build_summary.txt
if [[ "$EXIT_CODE" != "0" ]]; then if [[ "$EXIT_CODE" != "0" ]]; then
show_ut_retry_result show_ut_retry_result
...@@ -1495,7 +1480,7 @@ function show_ut_retry_result() { ...@@ -1495,7 +1480,7 @@ function show_ut_retry_result() {
exit 8; exit 8;
else else
retry_unittests_ut_name=$(echo "$retry_unittests_record" | grep -oEi "\-.+\(" | sed 's/(//' | sed 's/- //' ) retry_unittests_ut_name=$(echo "$retry_unittests_record" | grep -oEi "\-.+\(" | sed 's/(//' | sed 's/- //' )
retry_unittests_record_judge=$(echo ${retry_unittests_ut_name}| tr ' ' '\n' | sort | uniq -c | awk '{if ($1 >=3) {print $2}}') retry_unittests_record_judge=$(echo ${retry_unittests_ut_name}| tr ' ' '\n' | sort | uniq -c | awk '{if ($1 >=4) {print $2}}')
if [ -z "${retry_unittests_record_judge}" ];then if [ -z "${retry_unittests_record_judge}" ];then
echo "========================================" echo "========================================"
echo "There are failed tests, which have been successful after re-run:" echo "There are failed tests, which have been successful after re-run:"
......
...@@ -2265,7 +2265,7 @@ def main(): ...@@ -2265,7 +2265,7 @@ def main():
print("{};{};{};{}".format(high_parallel_job, fourth_high_parallel_job, print("{};{};{};{}".format(high_parallel_job, fourth_high_parallel_job,
fifth_high_parallel_job, non_parallel_job)) fifth_high_parallel_job, non_parallel_job))
else: else:
print("{};{};{};{};{};{};{}".format( print("{};{};{};{};{};{};{};{}".format(
high_parallel_job, secondary_high_parallel_job, high_parallel_job, secondary_high_parallel_job,
third_high_parallel_job, fourth_high_parallel_job, third_high_parallel_job, fourth_high_parallel_job,
fifth_high_parallel_job, sixth_high_parallel_job, fifth_high_parallel_job, sixth_high_parallel_job,
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册