Retry when download failed for precision test

paddle/scripts/paddle_build.sh

@@ -1044,12 +1044,17 @@ function parallel_test_base_gpu() {
     ========================================
 EOF
 
-set +x
+set -x
         precison_cases=""
+        bash $PADDLE_ROOT/tools/check_added_ut.sh
         if [ ${PRECISION_TEST:-OFF} == "ON" ]; then
-            precision_cases=`python $PADDLE_ROOT/tools/get_pr_ut.py`
+            python3.7 $PADDLE_ROOT/tools/get_pr_ut.py
+            if [[ -f "ut_list" ]]; then
+                set +x
+                precision_cases=`cat ut_list`
+                set -x
+            fi
         fi
-        bash $PADDLE_ROOT/tools/check_added_ut.sh
         if [ -a "$PADDLE_ROOT/added_ut" ];then
             added_uts=^$(awk BEGIN{RS=EOF}'{gsub(/\n/,"$|^");print}' $PADDLE_ROOT/added_ut)$
             ctest -R "(${added_uts})" --output-on-failure --repeat-until-fail 3 --timeout 15;added_ut_error=$?
@@ -1060,6 +1065,7 @@ set +x
                 exit 8;
             fi
         fi
+set +x
         EXIT_CODE=0;
         test_cases=$(ctest -N -V) # get all test cases
         exclusive_tests=''        # cases list which would be run exclusively

tools/check_added_ut.sh

@@ -14,7 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -e
+set +e
+set -x
 if [ -z ${BRANCH} ]; then
     BRANCH="develop"
 fi
@@ -25,6 +26,7 @@ CURDIR=`pwd`
 cd $PADDLE_ROOT
 cp $PADDLE_ROOT/paddle/scripts/paddle_build.sh $PADDLE_ROOT/paddle/scripts/paddle_build_pre.sh
 CURBRANCH=`git rev-parse --abbrev-ref HEAD`
+echo $CURBRANCH
 git checkout -b prec_added_ut upstream/${BRANCH}
 mkdir prec_build
 cd prec_build
@@ -32,13 +34,14 @@ bash $PADDLE_ROOT/paddle/scripts/paddle_build_pre.sh cmake_gen_in_current_dir >p
 ctest -N | awk -F ':' '{print $2}' | sed '/^$/d' | sed '$d' | sed 's/ //g' > /$PADDLE_ROOT/br-ut
 cd $PADDLE_ROOT/build
 ctest -N | awk -F ':' '{print $2}' | sed '/^$/d' | sed '$d' | sed 's/ //g' > /$PADDLE_ROOT/pr-ut
-cd /$PADDLE_ROOT
-grep -F -x -v -f br-ut pr-ut > /$PADDLE_ROOT/added_ut
+cd $PADDLE_ROOT
+grep -F -x -v -f br-ut pr-ut > $PADDLE_ROOT/added_ut
 echo "New-UT:"
-cat /$PADDLE_ROOT/added_ut
+cat $PADDLE_ROOT/added_ut
 rm -rf prec_build
-rm /$PADDLE_ROOT/br-ut /$PADDLE_ROOT/pr-ut $PADDLE_ROOT/paddle/scripts/paddle_build_pre.sh
+rm $PADDLE_ROOT/br-ut $PADDLE_ROOT/pr-ut $PADDLE_ROOT/paddle/scripts/paddle_build_pre.sh
 git checkout $CURBRANCH
+echo $CURBRANCH
 git branch -D prec_added_ut
 cd $CURDIR
 export CI_SKIP_CPP_TEST=

tools/coverage/paddle_coverage.sh

@@ -28,7 +28,7 @@ make install
 
 cd /paddle/build
 
-python3 ${PADDLE_ROOT}/tools/coverage/gcda_clean.py ${GIT_PR_ID}
+python3.7 ${PADDLE_ROOT}/tools/coverage/gcda_clean.py ${GIT_PR_ID}
 
 lcov --capture -d ./ -o coverage.info --rc lcov_branch_coverage=0
 
@@ -67,9 +67,9 @@ gen_full_html_report || true
 function gen_diff_html_report() {
     if [ "${GIT_PR_ID}" != "" ]; then
 
-        COVERAGE_DIFF_PATTERN="`python3 ${PADDLE_ROOT}/tools/coverage/pull_request.py files ${GIT_PR_ID}`"
+        COVERAGE_DIFF_PATTERN="`python3.7 ${PADDLE_ROOT}/tools/coverage/pull_request.py files ${GIT_PR_ID}`"
 
-        python3 ${PADDLE_ROOT}/tools/coverage/pull_request.py diff ${GIT_PR_ID} > git-diff.out
+        python3.7 ${PADDLE_ROOT}/tools/coverage/pull_request.py diff ${GIT_PR_ID} > git-diff.out
     fi
 
     lcov --extract coverage-full.info \
@@ -77,7 +77,7 @@ function gen_diff_html_report() {
         -o coverage-diff.info \
         --rc lcov_branch_coverage=0
 
-    python3 ${PADDLE_ROOT}/tools/coverage/coverage_diff.py coverage-diff.info git-diff.out > coverage-diff.tmp
+    python3.7 ${PADDLE_ROOT}/tools/coverage/coverage_diff.py coverage-diff.info git-diff.out > coverage-diff.tmp
 
     mv -f coverage-diff.tmp coverage-diff.info
 
@@ -96,7 +96,7 @@ set -x
 
 coverage xml -i -o python-coverage.xml
 
-python3 ${PADDLE_ROOT}/tools/coverage/python_coverage.py > python-coverage.info
+python3.7 ${PADDLE_ROOT}/tools/coverage/python_coverage.py > python-coverage.info
 
 # python full html report
 #
@@ -122,9 +122,9 @@ gen_python_full_html_report || true
 
 function gen_python_diff_html_report() {
     if [ "${GIT_PR_ID}" != "" ]; then
-        COVERAGE_DIFF_PATTERN="`python ${PADDLE_ROOT}/tools/coverage/pull_request.py files ${GIT_PR_ID}`"
+        COVERAGE_DIFF_PATTERN="`python3.7 ${PADDLE_ROOT}/tools/coverage/pull_request.py files ${GIT_PR_ID}`"
 
-        python ${PADDLE_ROOT}/tools/coverage/pull_request.py diff ${GIT_PR_ID} > python-git-diff.out
+        python3.7 ${PADDLE_ROOT}/tools/coverage/pull_request.py diff ${GIT_PR_ID} > python-git-diff.out
     fi
 
     lcov --extract python-coverage-full.info \
@@ -132,7 +132,7 @@ function gen_python_diff_html_report() {
         -o python-coverage-diff.info \
         --rc lcov_branch_coverage=0
 
-    python ${PADDLE_ROOT}/tools/coverage/coverage_diff.py python-coverage-diff.info python-git-diff.out > python-coverage-diff.tmp
+    python3.7 ${PADDLE_ROOT}/tools/coverage/coverage_diff.py python-coverage-diff.info python-git-diff.out > python-coverage-diff.tmp
 
     mv -f python-coverage-diff.tmp python-coverage-diff.info
 
@@ -150,11 +150,11 @@ gen_python_diff_html_report || true
 
 echo "Assert Diff Coverage"
 
-python ${PADDLE_ROOT}/tools/coverage/coverage_lines.py coverage-diff.info 0.9 || COVERAGE_LINES_ASSERT=1
+python3.7 ${PADDLE_ROOT}/tools/coverage/coverage_lines.py coverage-diff.info 0.9 || COVERAGE_LINES_ASSERT=1
 
 echo "Assert Python Diff Coverage"
 
-python ${PADDLE_ROOT}/tools/coverage/coverage_lines.py python-coverage-diff.info 0.9 || PYTHON_COVERAGE_LINES_ASSERT=1
+python3.7 ${PADDLE_ROOT}/tools/coverage/coverage_lines.py python-coverage-diff.info 0.9 || PYTHON_COVERAGE_LINES_ASSERT=1
 
 if [ "$COVERAGE_LINES_ASSERT" = "1" ] || [ "$PYTHON_COVERAGE_LINES_ASSERT" = "1" ]; then
     echo "exit 9" > /tmp/paddle_coverage.result

tools/get_pr_ut.py

@@ -17,6 +17,8 @@ import os
 import json
 import re
 import sys
+import time
+import subprocess
 import requests
 from github import Github
 
@@ -45,7 +47,7 @@ class PRChecker(object):
         """ Get pull request. """
         pr_id = os.getenv('GIT_PR_ID')
         if not pr_id:
-            print('No PR ID')
+            print('PREC No PR ID')
             exit(0)
         suffix = os.getenv('PREC_SUFFIX')
         if suffix:
@@ -60,9 +62,31 @@ class PRChecker(object):
             else:
                 break
             ix = ix + 1
-        if last_commit.message.find('test=full_case') != -1:
+        if last_commit.message.find('test=allcase') != -1:
+            print('PREC test=allcase is set')
             self.full_case = True
 
+    #todo: exception
+    def __wget_with_retry(self, url):
+        ix = 1
+        proxy = '--no-proxy'
+        while ix < 6:
+            if ix // 2 == 0:
+                proxy = ''
+            else:
+                proxy = '--no-proxy'
+            code = subprocess.call(
+                'wget -q {} --no-check-certificate {}'.format(proxy, url),
+                shell=True)
+            if code == 0:
+                return True
+            print(
+                'PREC download {} error, retry {} time(s) after {} secs.[proxy_option={}]'.
+                format(url, ix, ix * 10, proxy))
+            time.sleep(ix * 10)
+            ix += 1
+        return False
+
     def get_pr_files(self):
         """ Get files in pull request. """
         page = 0
@@ -101,6 +125,7 @@ class PRChecker(object):
 
     def get_comment_of_file(self, f):
         #content = self.repo.get_contents(f.replace(PADDLE_ROOT, ''), 'pull/').decoded_content
+        #todo: get file from github
         with open(f) as fd:
             lines = fd.readlines()
         lineno = 1
@@ -166,6 +191,7 @@ class PRChecker(object):
         for l in diff_lines:
             if l not in comment_lines:
                 return False
+        print('PREC {} is only comment'.format(f))
         return True
 
     def get_pr_ut(self):
@@ -175,8 +201,12 @@ class PRChecker(object):
         check_added_ut = False
         ut_list = []
         file_ut_map = None
-        cmd = 'wget -q --no-proxy --no-check-certificate https://sys-p0.bj.bcebos.com/prec/file_ut.json' + self.suffix
-        os.system(cmd)
+        ret = self.__wget_with_retry(
+            'https://sys-p0.bj.bcebos.com/prec/file_ut.json{}'.format(
+                self.suffix))
+        if not ret:
+            print('PREC download file_ut.json failed')
+            exit(1)
         with open('file_ut.json' + self.suffix) as jsonfile:
             file_ut_map = json.load(jsonfile)
         for f in self.get_pr_files():
@@ -187,16 +217,24 @@ class PRChecker(object):
                     if self.is_only_comment(f):
                         ut_list.append('h_cu_comment_placeholder')
                     else:
+                        print(
+                            'PREC dismatch: {} not in file ut map and not md or comment'.
+                            format(f))
                         return ''
                 elif f.endswith('.cc') or f.endswith('.py') or f.endswith(
                         '.cu'):
                     if f.find('test_') != -1 or f.find('_test') != -1:
+                        print('PREC {} need check new ut'.format(f))
                         check_added_ut = True
                     elif self.is_only_comment(f):
                         ut_list.append('nomap_comment_placeholder')
                     else:
+                        print(
+                            'PREC dismatch: {} not in file ut map and not new ut or comment'.
+                            format(f))
                         return ''
                 else:
+                    print('PREC dismatch: {} not in file ut map'.format(f))
                     return ''
             else:
                 if self.is_only_comment(f):
@@ -204,24 +242,28 @@ class PRChecker(object):
                 else:
                     ut_list.extend(file_ut_map.get(f))
         ut_list = list(set(ut_list))
-        cmd = 'wget -q --no-proxy --no-check-certificate https://sys-p0.bj.bcebos.com/prec/prec_delta' + self.suffix
-        os.system(cmd)
+        ret = self.__wget_with_retry(
+            'https://sys-p0.bj.bcebos.com/prec/prec_delta{}'.format(
+                self.suffix))
+        if ret:
             with open('prec_delta' + self.suffix) as delta:
                 for ut in delta:
                     ut_list.append(ut.rstrip('\r\n'))
+        else:
+            print('PREC download prec_delta failed')
 
         if check_added_ut:
-            cmd = 'bash {}/tools/check_added_ut.sh >/tmp/pre_ut 2>&1'.format(
-                PADDLE_ROOT)
-            os.system(cmd)
             with open('{}/added_ut'.format(PADDLE_ROOT)) as utfile:
                 for ut in utfile:
+                    print('PREC NEW UT: {}'.format(ut.rstrip('\r\n')))
                     ut_list.append(ut.rstrip('\r\n'))
 
-        return ' '.join(ut_list)
+        return '\n'.join(ut_list)
 
 
 if __name__ == '__main__':
     pr_checker = PRChecker()
     pr_checker.init()
-    print(pr_checker.get_pr_ut())
+    #print(pr_checker.get_pr_ut())
+    with open('ut_list', 'w') as f:
+        f.write(pr_checker.get_pr_ut())