From 2111d912d4e7203d4cf3a5d4de2d69bef23f9840 Mon Sep 17 00:00:00 2001 From: YUNSHEN XIE <1084314248@qq.com> Date: Tue, 2 Mar 2021 17:07:55 +0800 Subject: [PATCH] Decrease threshold for failed ut retry (#30903) * Decrease threshold for failed ut retry * retry Method upgrade * second method upgrade * fix error * Remove the comment lines * test for modified_retry_times * fix error * fix some error * fix error * fix error * remove test content * fix error * Reduce duplicate code * fix more than 10 ut failed bug * fix more than 10 ut failed bug on mac --- paddle/scripts/paddle_build.sh | 91 +++++++++++++++++----------------- 1 file changed, 46 insertions(+), 45 deletions(-) diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index 9ca426ae029..ad9ee67f2e5 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -615,18 +615,19 @@ EOF retry_time=3 exec_times=0 exec_time_array=('first' 'second' 'third') - exec_retry_threshold=20 + exec_retry_threshold=10 + is_retry_execuate=0 if [ -n "$failed_test_lists" ];then mactest_error=1 read need_retry_ut_str <<< $(echo "$failed_test_lists" | grep -oEi "\-.+\(" | sed 's/(//' | sed 's/- //' ) need_retry_ut_arr=(${need_retry_ut_str}) need_retry_ut_count=${#need_retry_ut_arr[@]} + read retry_unittests <<< $(echo "$failed_test_lists" | grep -oEi "\-.+\(" | sed 's/(//' | sed 's/- //' ) if [ $need_retry_ut_count -lt $exec_retry_threshold ];then - while ( [ $exec_times -lt $retry_time ] && [ -n "${failed_test_lists}" ] ) + while ( [ $exec_times -lt $retry_time ] ) do retry_unittests_record="$retry_unittests_record$failed_test_lists" failed_test_lists_ult=`echo "${failed_test_lists}"` - read retry_unittests <<< $(echo "$failed_test_lists" | grep -oEi "\-.+\(" | sed 's/(//' | sed 's/- //' ) echo "=========================================" echo "This is the ${exec_time_array[$exec_times]} time to re-run" echo "=========================================" @@ -650,9 +651,8 @@ EOF exec_times=$[$exec_times+1] done else - echo "=========================================" - echo "There are more than 20 failed unit tests, so no unit test retry!!!" - echo "=========================================" + # There are more than 10 failed unit tests, so no unit test retry + is_retry_execuate=1 fi fi @@ -665,24 +665,10 @@ EOF set +x export http_proxy=$my_proxy export https_proxy=$my_proxy - set -x if [ "$mactest_error" != 0 ];then - if [[ "$failed_test_lists" == "" ]]; then - echo "========================================" - echo "There are failed tests, which have been successful after re-run:" - echo "========================================" - echo "The following tests have been re-ran:" - echo "${retry_unittests_record}" - else - failed_test_lists_ult=`echo "${failed_test_lists}"` - echo "========================================" - echo "Summary Failed Tests... " - echo "========================================" - echo "The following tests FAILED: " - echo "${failed_test_lists_ult}" - exit 8; - fi + show_ut_retry_result fi + set -x fi } @@ -1204,18 +1190,18 @@ set +x retry_unittests_record='' retry_time=3 exec_time_array=('first' 'second' 'third') - exec_retry_threshold=20 + exec_retry_threshold=10 + is_retry_execuate=0 if [ -n "$failed_test_lists" ];then read need_retry_ut_str <<< $(echo "$failed_test_lists" | grep -oEi "\-.+\(.+\)" | sed 's/(.\+)//' | sed 's/- //' ) need_retry_ut_arr=(${need_retry_ut_str}) need_retry_ut_count=${#need_retry_ut_arr[@]} + read retry_unittests <<< $(echo "$failed_test_lists" | grep -oEi "\-.+\(.+\)" | sed 's/(.\+)//' | sed 's/- //' ) if [ $need_retry_ut_count -lt $exec_retry_threshold ];then - while ( [ $exec_times -lt $retry_time ] && [ -n "${failed_test_lists}" ] ) + while ( [ $exec_times -lt $retry_time ] ) do - retry_unittests_record="$retry_unittests_record$failed_test_lists" failed_test_lists_ult=`echo "${failed_test_lists}" |grep -Po '[^ ].*$'` - read retry_unittests <<< $(echo "$failed_test_lists" | grep -oEi "\-.+\(.+\)" | sed 's/(.\+)//' | sed 's/- //' ) echo "=========================================" echo "This is the ${exec_time_array[$exec_times]} time to re-run" echo "=========================================" @@ -1270,36 +1256,51 @@ set +x one_card_retry='' multiple_card_retry='' exclusive_retry='' - retry_unittests='' done else - echo "=========================================" - echo "There are more than 20 failed unit tests, so no unit test retry!!!" - echo "=========================================" + # There are more than 10 failed unit tests, so no unit test retry + is_retry_execuate=1 fi fi if [[ "$EXIT_CODE" != "0" ]]; then - if [[ "$failed_test_lists" == "" ]]; then - echo "========================================" - echo "There are failed tests, which have been successful after re-run:" - echo "========================================" - echo "The following tests have been re-ran:" - echo "${retry_unittests_record}" - else - failed_test_lists_ult=`echo "${failed_test_lists}" |grep -Po '[^ ].*$'` - echo "========================================" - echo "Summary Failed Tests... " - echo "========================================" - echo "The following tests FAILED: " - echo "${failed_test_lists_ult}" - exit 8; - fi + show_ut_retry_result fi set -ex fi } +function show_ut_retry_result() { + if [[ "$is_retry_execuate" != "0" ]];then + failed_test_lists_ult=`echo "${failed_test_lists}" | grep -Po '[^ ].*$'` + echo "=========================================" + echo "There are more than 10 failed unit tests, so no unit test retry!!!" + echo "=========================================" + echo "The following tests FAILED: " + echo "${failed_test_lists_ult}" + exit 8; + else + read retry_unittests_ut_name <<< $(echo "$retry_unittests_record" | grep -oEi "\-.+\(" | sed 's/(//' | sed 's/- //' ) + retry_unittests_record_judge=$(echo ${retry_unittests_ut_name}| tr ' ' '\n' | sort | uniq -c | awk '{if ($1 >=3) {print $2}}') + if [ -z "${retry_unittests_record_judge}" ];then + echo "========================================" + echo "There are failed tests, which have been successful after re-run:" + echo "========================================" + echo "The following tests have been re-ran:" + echo "${retry_unittests_record}" + else + failed_ut_re=$(echo "${retry_unittests_record_judge}" | awk BEGIN{RS=EOF}'{gsub(/\n/,"|");print}') + echo "========================================" + echo "There are failed tests, which have been executed re-run,but success rate is less than 50%:" + echo "Summary Failed Tests... " + echo "========================================" + echo "The following tests FAILED: " + echo "${retry_unittests_record}" | grep -E "$failed_ut_re" + exit 8; + fi + fi +} + function parallel_test_base_cpu() { mkdir -p ${PADDLE_ROOT}/build cd ${PADDLE_ROOT}/build -- GitLab