From 9066b578152c85fea543a7d5d58deb244a3ddee0 Mon Sep 17 00:00:00 2001 From: zhangchunle Date: Mon, 31 May 2021 14:20:16 +0800 Subject: [PATCH] update get_pr_ut.py (#33037) --- paddle/scripts/paddle_build.sh | 1 - tools/get_pr_ut.py | 88 ++++++++++++++++++++++++++-------- 2 files changed, 68 insertions(+), 21 deletions(-) diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index d88f80bafea..6d5c7fc6700 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -1578,7 +1578,6 @@ set -x #analy h/cu to Map file python ${PADDLE_ROOT}/tools/handle_h_cu_file.py 'analy_h_cu_file' $tmp_dir ${PADDLE_ROOT} - #generate ut map python ${PADDLE_ROOT}/tools/get_ut_file_map.py 'get_ut_map' ${PADDLE_ROOT} wait; diff --git a/tools/get_pr_ut.py b/tools/get_pr_ut.py index 0df3b4914f5..4180a37eef2 100644 --- a/tools/get_pr_ut.py +++ b/tools/get_pr_ut.py @@ -124,18 +124,34 @@ class PRChecker(object): def get_pr_files(self): """ Get files in pull request. """ page = 0 - file_list = [] + file_dict = {} while True: files = self.pr.get_files().get_page(page) if not files: break for f in files: - if f.status == 'removed': - file_list.append('removed') - else: - file_list.append(PADDLE_ROOT + f.filename) + file_dict[PADDLE_ROOT + f.filename] = f.status page += 1 - return file_list + print("pr modify files: %s" % file_dict) + return file_dict + + def get_is_white_file(self, filename): + """ judge is white file in pr's files. """ + isWhiteFile = False + white_files = (PADDLE_ROOT + 'cmake/', PADDLE_ROOT + 'patches/', + PADDLE_ROOT + 'tools/dockerfile/', + PADDLE_ROOT + 'tools/windows/', + PADDLE_ROOT + 'tools/test_runner.py', + PADDLE_ROOT + 'tools/parallel_UT_rule.py', + PADDLE_ROOT + 'paddle/scripts/paddle_build.sh', + PADDLE_ROOT + 'paddle/scripts/paddle_build.bat') + if 'cmakelist' in filename.lower(): + isWhiteFile = False + elif filename.startswith((white_files)): + isWhiteFile = False + else: + isWhiteFile = True + return isWhiteFile def __get_comment_by_filetype(self, content, filetype): result = [] @@ -247,24 +263,41 @@ class PRChecker(object): check_added_ut = False ut_list = [] file_ut_map = None + ret = self.__urlretrieve( 'https://paddle-docker-tar.bj.bcebos.com/pre_test/ut_file_map.json', 'ut_file_map.json') if not ret: print('PREC download file_ut.json failed') exit(1) + with open('ut_file_map.json') as jsonfile: file_ut_map = json.load(jsonfile) current_system = platform.system() notHitMapFiles = [] - hitMapFiles = [] + hitMapFiles = {} onlyCommentsFilesOrXpu = [] - file_list = self.get_pr_files() - if 'removed' in file_list: - print("ipipe_log_param_PRECISION_TEST: false") - print("notHitMapFiles: [rm file]") - return '' + filterFiles = [] + file_list = [] + file_dict = self.get_pr_files() + for filename in file_dict: + if filename.startswith( + (PADDLE_ROOT + 'python/', PADDLE_ROOT + 'paddle/fluid/')): + file_list.append(filename) + else: + isWhiteFile = self.get_is_white_file(filename) + if isWhiteFile == False: + file_list.append(filename) + else: + filterFiles.append(filename) + if len(file_list) == 0: + ut_list.append('filterfiles_placeholder') + print("filterFiles: %s" % filterFiles) + print("ipipe_log_param_PRECISION_TEST: true") + print("ipipe_log_param_PRECISION_TEST_Cases_count: 0") + print("ipipe_log_param_PRECISION_TEST_Cases_ratio: 0") + return '\n'.join(ut_list) else: for f in file_list: if current_system == "Darwin" or current_system == "Windows" or self.suffix == ".py3": @@ -283,24 +316,36 @@ class PRChecker(object): if f_judge.find('test_') != -1 or f_judge.find( '_test') != -1: check_added_ut = True + if file_dict[f] not in ['removed']: + if self.is_only_comment(f): + ut_list.append('comment_placeholder') + onlyCommentsFilesOrXpu.append(f_judge) + else: + notHitMapFiles.append(f_judge) + else: + print("remove file not hit mapFiles: %s" % f_judge) + else: + notHitMapFiles.append(f_judge) if file_dict[ + f] != 'removed' else print( + "remove file not hit mapFiles: %s" % f_judge) + else: + if file_dict[f] not in ['removed']: if self.is_only_comment(f): ut_list.append('comment_placeholder') onlyCommentsFilesOrXpu.append(f_judge) else: - notHitMapFiles.append(f_judge) + hitMapFiles[f_judge] = len(file_ut_map[f_judge]) + ut_list.extend(file_ut_map.get(f_judge)) else: - notHitMapFiles.append(f_judge) - else: - if self.is_only_comment(f): - ut_list.append('comment_placeholder') - onlyCommentsFilesOrXpu.append(f_judge) - else: - hitMapFiles.append(f_judge) + hitMapFiles[f_judge] = len(file_ut_map[f_judge]) ut_list.extend(file_ut_map.get(f_judge)) + ut_list = list(set(ut_list)) if len(notHitMapFiles) != 0: print("ipipe_log_param_PRECISION_TEST: false") print("notHitMapFiles: %s" % notHitMapFiles) + if len(filterFiles) != 0: + print("filterFiles: %s" % filterFiles) return '' else: if check_added_ut: @@ -318,6 +363,7 @@ class PRChecker(object): else: print('PREC download prec_delta failed') exit(1) + print("hitMapFiles: %s" % hitMapFiles) print("ipipe_log_param_PRECISION_TEST: true") print("ipipe_log_param_PRECISION_TEST_Cases_count: %s" % len(ut_list)) @@ -326,6 +372,8 @@ class PRChecker(object): '.2f') print("ipipe_log_param_PRECISION_TEST_Cases_ratio: %s" % PRECISION_TEST_Cases_ratio) + if len(filterFiles) != 0: + print("filterFiles: %s" % filterFiles) return '\n'.join(ut_list) -- GitLab