update get_pr_ut.py (#33037)

paddle/scripts/paddle_build.sh

@@ -1578,7 +1578,6 @@ set -x
 
     #analy h/cu to Map file
     python ${PADDLE_ROOT}/tools/handle_h_cu_file.py 'analy_h_cu_file' $tmp_dir ${PADDLE_ROOT}
-
     #generate ut map
     python ${PADDLE_ROOT}/tools/get_ut_file_map.py 'get_ut_map' ${PADDLE_ROOT}
     wait;

tools/get_pr_ut.py

@@ -124,18 +124,34 @@ class PRChecker(object):
     def get_pr_files(self):
         """ Get files in pull request. """
         page = 0
-        file_list = []
+        file_dict = {}
         while True:
             files = self.pr.get_files().get_page(page)
             if not files:
                 break
             for f in files:
-                if f.status == 'removed':
-                    file_list.append('removed')
-                else:
-                    file_list.append(PADDLE_ROOT + f.filename)
+                file_dict[PADDLE_ROOT + f.filename] = f.status
             page += 1
-        return file_list
+        print("pr modify files: %s" % file_dict)
+        return file_dict
+
+    def get_is_white_file(self, filename):
+        """ judge is white file in pr's files. """
+        isWhiteFile = False
+        white_files = (PADDLE_ROOT + 'cmake/', PADDLE_ROOT + 'patches/',
+                       PADDLE_ROOT + 'tools/dockerfile/',
+                       PADDLE_ROOT + 'tools/windows/',
+                       PADDLE_ROOT + 'tools/test_runner.py',
+                       PADDLE_ROOT + 'tools/parallel_UT_rule.py',
+                       PADDLE_ROOT + 'paddle/scripts/paddle_build.sh',
+                       PADDLE_ROOT + 'paddle/scripts/paddle_build.bat')
+        if 'cmakelist' in filename.lower():
+            isWhiteFile = False
+        elif filename.startswith((white_files)):
+            isWhiteFile = False
+        else:
+            isWhiteFile = True
+        return isWhiteFile
 
     def __get_comment_by_filetype(self, content, filetype):
         result = []
@@ -247,24 +263,41 @@ class PRChecker(object):
         check_added_ut = False
         ut_list = []
         file_ut_map = None
+
         ret = self.__urlretrieve(
             'https://paddle-docker-tar.bj.bcebos.com/pre_test/ut_file_map.json',
             'ut_file_map.json')
         if not ret:
             print('PREC download file_ut.json failed')
             exit(1)
+
         with open('ut_file_map.json') as jsonfile:
             file_ut_map = json.load(jsonfile)
 
         current_system = platform.system()
         notHitMapFiles = []
-        hitMapFiles = []
+        hitMapFiles = {}
         onlyCommentsFilesOrXpu = []
-        file_list = self.get_pr_files()
-        if 'removed' in file_list:
-            print("ipipe_log_param_PRECISION_TEST: false")
-            print("notHitMapFiles: [rm file]")
-            return ''
+        filterFiles = []
+        file_list = []
+        file_dict = self.get_pr_files()
+        for filename in file_dict:
+            if filename.startswith(
+                (PADDLE_ROOT + 'python/', PADDLE_ROOT + 'paddle/fluid/')):
+                file_list.append(filename)
+            else:
+                isWhiteFile = self.get_is_white_file(filename)
+                if isWhiteFile == False:
+                    file_list.append(filename)
+                else:
+                    filterFiles.append(filename)
+        if len(file_list) == 0:
+            ut_list.append('filterfiles_placeholder')
+            print("filterFiles: %s" % filterFiles)
+            print("ipipe_log_param_PRECISION_TEST: true")
+            print("ipipe_log_param_PRECISION_TEST_Cases_count: 0")
+            print("ipipe_log_param_PRECISION_TEST_Cases_ratio: 0")
+            return '\n'.join(ut_list)
         else:
             for f in file_list:
                 if current_system == "Darwin" or current_system == "Windows" or self.suffix == ".py3":
@@ -283,24 +316,36 @@ class PRChecker(object):
                         if f_judge.find('test_') != -1 or f_judge.find(
                                 '_test') != -1:
                             check_added_ut = True
+                        if file_dict[f] not in ['removed']:
                             if self.is_only_comment(f):
                                 ut_list.append('comment_placeholder')
                                 onlyCommentsFilesOrXpu.append(f_judge)
                             else:
                                 notHitMapFiles.append(f_judge)
                         else:
-                        notHitMapFiles.append(f_judge)
+                            print("remove file not hit mapFiles: %s" % f_judge)
                     else:
+                        notHitMapFiles.append(f_judge) if file_dict[
+                            f] != 'removed' else print(
+                                "remove file not hit mapFiles: %s" % f_judge)
+                else:
+                    if file_dict[f] not in ['removed']:
                         if self.is_only_comment(f):
                             ut_list.append('comment_placeholder')
                             onlyCommentsFilesOrXpu.append(f_judge)
                         else:
-                        hitMapFiles.append(f_judge)
+                            hitMapFiles[f_judge] = len(file_ut_map[f_judge])
+                            ut_list.extend(file_ut_map.get(f_judge))
+                    else:
+                        hitMapFiles[f_judge] = len(file_ut_map[f_judge])
                         ut_list.extend(file_ut_map.get(f_judge))
+
             ut_list = list(set(ut_list))
             if len(notHitMapFiles) != 0:
                 print("ipipe_log_param_PRECISION_TEST: false")
                 print("notHitMapFiles: %s" % notHitMapFiles)
+                if len(filterFiles) != 0:
+                    print("filterFiles: %s" % filterFiles)
                 return ''
             else:
                 if check_added_ut:
@@ -318,6 +363,7 @@ class PRChecker(object):
                     else:
                         print('PREC download prec_delta failed')
                         exit(1)
+                    print("hitMapFiles: %s" % hitMapFiles)
                     print("ipipe_log_param_PRECISION_TEST: true")
                     print("ipipe_log_param_PRECISION_TEST_Cases_count: %s" %
                           len(ut_list))
@@ -326,6 +372,8 @@ class PRChecker(object):
                         '.2f')
                     print("ipipe_log_param_PRECISION_TEST_Cases_ratio: %s" %
                           PRECISION_TEST_Cases_ratio)
+                    if len(filterFiles) != 0:
+                        print("filterFiles: %s" % filterFiles)
                 return '\n'.join(ut_list)