diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index 6d5c7fc6700ed84b8623a87f5b788edf1c196107..d88f80bafea4287512f6c54cbf97348407107019 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -1578,6 +1578,7 @@ set -x #analy h/cu to Map file python ${PADDLE_ROOT}/tools/handle_h_cu_file.py 'analy_h_cu_file' $tmp_dir ${PADDLE_ROOT} + #generate ut map python ${PADDLE_ROOT}/tools/get_ut_file_map.py 'get_ut_map' ${PADDLE_ROOT} wait; diff --git a/tools/get_pr_ut.py b/tools/get_pr_ut.py index 4180a37eef22f59e76b43e76d69a48c6e2be5bbf..0df3b4914f5df972aab26768bc4d18b1ffc00163 100644 --- a/tools/get_pr_ut.py +++ b/tools/get_pr_ut.py @@ -124,34 +124,18 @@ class PRChecker(object): def get_pr_files(self): """ Get files in pull request. """ page = 0 - file_dict = {} + file_list = [] while True: files = self.pr.get_files().get_page(page) if not files: break for f in files: - file_dict[PADDLE_ROOT + f.filename] = f.status + if f.status == 'removed': + file_list.append('removed') + else: + file_list.append(PADDLE_ROOT + f.filename) page += 1 - print("pr modify files: %s" % file_dict) - return file_dict - - def get_is_white_file(self, filename): - """ judge is white file in pr's files. """ - isWhiteFile = False - white_files = (PADDLE_ROOT + 'cmake/', PADDLE_ROOT + 'patches/', - PADDLE_ROOT + 'tools/dockerfile/', - PADDLE_ROOT + 'tools/windows/', - PADDLE_ROOT + 'tools/test_runner.py', - PADDLE_ROOT + 'tools/parallel_UT_rule.py', - PADDLE_ROOT + 'paddle/scripts/paddle_build.sh', - PADDLE_ROOT + 'paddle/scripts/paddle_build.bat') - if 'cmakelist' in filename.lower(): - isWhiteFile = False - elif filename.startswith((white_files)): - isWhiteFile = False - else: - isWhiteFile = True - return isWhiteFile + return file_list def __get_comment_by_filetype(self, content, filetype): result = [] @@ -263,41 +247,24 @@ class PRChecker(object): check_added_ut = False ut_list = [] file_ut_map = None - ret = self.__urlretrieve( 'https://paddle-docker-tar.bj.bcebos.com/pre_test/ut_file_map.json', 'ut_file_map.json') if not ret: print('PREC download file_ut.json failed') exit(1) - with open('ut_file_map.json') as jsonfile: file_ut_map = json.load(jsonfile) current_system = platform.system() notHitMapFiles = [] - hitMapFiles = {} + hitMapFiles = [] onlyCommentsFilesOrXpu = [] - filterFiles = [] - file_list = [] - file_dict = self.get_pr_files() - for filename in file_dict: - if filename.startswith( - (PADDLE_ROOT + 'python/', PADDLE_ROOT + 'paddle/fluid/')): - file_list.append(filename) - else: - isWhiteFile = self.get_is_white_file(filename) - if isWhiteFile == False: - file_list.append(filename) - else: - filterFiles.append(filename) - if len(file_list) == 0: - ut_list.append('filterfiles_placeholder') - print("filterFiles: %s" % filterFiles) - print("ipipe_log_param_PRECISION_TEST: true") - print("ipipe_log_param_PRECISION_TEST_Cases_count: 0") - print("ipipe_log_param_PRECISION_TEST_Cases_ratio: 0") - return '\n'.join(ut_list) + file_list = self.get_pr_files() + if 'removed' in file_list: + print("ipipe_log_param_PRECISION_TEST: false") + print("notHitMapFiles: [rm file]") + return '' else: for f in file_list: if current_system == "Darwin" or current_system == "Windows" or self.suffix == ".py3": @@ -316,36 +283,24 @@ class PRChecker(object): if f_judge.find('test_') != -1 or f_judge.find( '_test') != -1: check_added_ut = True - if file_dict[f] not in ['removed']: - if self.is_only_comment(f): - ut_list.append('comment_placeholder') - onlyCommentsFilesOrXpu.append(f_judge) - else: - notHitMapFiles.append(f_judge) - else: - print("remove file not hit mapFiles: %s" % f_judge) - else: - notHitMapFiles.append(f_judge) if file_dict[ - f] != 'removed' else print( - "remove file not hit mapFiles: %s" % f_judge) - else: - if file_dict[f] not in ['removed']: if self.is_only_comment(f): ut_list.append('comment_placeholder') onlyCommentsFilesOrXpu.append(f_judge) else: - hitMapFiles[f_judge] = len(file_ut_map[f_judge]) - ut_list.extend(file_ut_map.get(f_judge)) + notHitMapFiles.append(f_judge) else: - hitMapFiles[f_judge] = len(file_ut_map[f_judge]) + notHitMapFiles.append(f_judge) + else: + if self.is_only_comment(f): + ut_list.append('comment_placeholder') + onlyCommentsFilesOrXpu.append(f_judge) + else: + hitMapFiles.append(f_judge) ut_list.extend(file_ut_map.get(f_judge)) - ut_list = list(set(ut_list)) if len(notHitMapFiles) != 0: print("ipipe_log_param_PRECISION_TEST: false") print("notHitMapFiles: %s" % notHitMapFiles) - if len(filterFiles) != 0: - print("filterFiles: %s" % filterFiles) return '' else: if check_added_ut: @@ -363,7 +318,6 @@ class PRChecker(object): else: print('PREC download prec_delta failed') exit(1) - print("hitMapFiles: %s" % hitMapFiles) print("ipipe_log_param_PRECISION_TEST: true") print("ipipe_log_param_PRECISION_TEST_Cases_count: %s" % len(ut_list)) @@ -372,8 +326,6 @@ class PRChecker(object): '.2f') print("ipipe_log_param_PRECISION_TEST_Cases_ratio: %s" % PRECISION_TEST_Cases_ratio) - if len(filterFiles) != 0: - print("filterFiles: %s" % filterFiles) return '\n'.join(ut_list)