未验证 提交 9066b578 编写于 作者: Z zhangchunle 提交者: GitHub

update get_pr_ut.py (#33037)

上级 2a771c06
...@@ -1578,7 +1578,6 @@ set -x ...@@ -1578,7 +1578,6 @@ set -x
#analy h/cu to Map file #analy h/cu to Map file
python ${PADDLE_ROOT}/tools/handle_h_cu_file.py 'analy_h_cu_file' $tmp_dir ${PADDLE_ROOT} python ${PADDLE_ROOT}/tools/handle_h_cu_file.py 'analy_h_cu_file' $tmp_dir ${PADDLE_ROOT}
#generate ut map #generate ut map
python ${PADDLE_ROOT}/tools/get_ut_file_map.py 'get_ut_map' ${PADDLE_ROOT} python ${PADDLE_ROOT}/tools/get_ut_file_map.py 'get_ut_map' ${PADDLE_ROOT}
wait; wait;
......
...@@ -124,18 +124,34 @@ class PRChecker(object): ...@@ -124,18 +124,34 @@ class PRChecker(object):
def get_pr_files(self): def get_pr_files(self):
""" Get files in pull request. """ """ Get files in pull request. """
page = 0 page = 0
file_list = [] file_dict = {}
while True: while True:
files = self.pr.get_files().get_page(page) files = self.pr.get_files().get_page(page)
if not files: if not files:
break break
for f in files: for f in files:
if f.status == 'removed': file_dict[PADDLE_ROOT + f.filename] = f.status
file_list.append('removed')
else:
file_list.append(PADDLE_ROOT + f.filename)
page += 1 page += 1
return file_list print("pr modify files: %s" % file_dict)
return file_dict
def get_is_white_file(self, filename):
""" judge is white file in pr's files. """
isWhiteFile = False
white_files = (PADDLE_ROOT + 'cmake/', PADDLE_ROOT + 'patches/',
PADDLE_ROOT + 'tools/dockerfile/',
PADDLE_ROOT + 'tools/windows/',
PADDLE_ROOT + 'tools/test_runner.py',
PADDLE_ROOT + 'tools/parallel_UT_rule.py',
PADDLE_ROOT + 'paddle/scripts/paddle_build.sh',
PADDLE_ROOT + 'paddle/scripts/paddle_build.bat')
if 'cmakelist' in filename.lower():
isWhiteFile = False
elif filename.startswith((white_files)):
isWhiteFile = False
else:
isWhiteFile = True
return isWhiteFile
def __get_comment_by_filetype(self, content, filetype): def __get_comment_by_filetype(self, content, filetype):
result = [] result = []
...@@ -247,24 +263,41 @@ class PRChecker(object): ...@@ -247,24 +263,41 @@ class PRChecker(object):
check_added_ut = False check_added_ut = False
ut_list = [] ut_list = []
file_ut_map = None file_ut_map = None
ret = self.__urlretrieve( ret = self.__urlretrieve(
'https://paddle-docker-tar.bj.bcebos.com/pre_test/ut_file_map.json', 'https://paddle-docker-tar.bj.bcebos.com/pre_test/ut_file_map.json',
'ut_file_map.json') 'ut_file_map.json')
if not ret: if not ret:
print('PREC download file_ut.json failed') print('PREC download file_ut.json failed')
exit(1) exit(1)
with open('ut_file_map.json') as jsonfile: with open('ut_file_map.json') as jsonfile:
file_ut_map = json.load(jsonfile) file_ut_map = json.load(jsonfile)
current_system = platform.system() current_system = platform.system()
notHitMapFiles = [] notHitMapFiles = []
hitMapFiles = [] hitMapFiles = {}
onlyCommentsFilesOrXpu = [] onlyCommentsFilesOrXpu = []
file_list = self.get_pr_files() filterFiles = []
if 'removed' in file_list: file_list = []
print("ipipe_log_param_PRECISION_TEST: false") file_dict = self.get_pr_files()
print("notHitMapFiles: [rm file]") for filename in file_dict:
return '' if filename.startswith(
(PADDLE_ROOT + 'python/', PADDLE_ROOT + 'paddle/fluid/')):
file_list.append(filename)
else:
isWhiteFile = self.get_is_white_file(filename)
if isWhiteFile == False:
file_list.append(filename)
else:
filterFiles.append(filename)
if len(file_list) == 0:
ut_list.append('filterfiles_placeholder')
print("filterFiles: %s" % filterFiles)
print("ipipe_log_param_PRECISION_TEST: true")
print("ipipe_log_param_PRECISION_TEST_Cases_count: 0")
print("ipipe_log_param_PRECISION_TEST_Cases_ratio: 0")
return '\n'.join(ut_list)
else: else:
for f in file_list: for f in file_list:
if current_system == "Darwin" or current_system == "Windows" or self.suffix == ".py3": if current_system == "Darwin" or current_system == "Windows" or self.suffix == ".py3":
...@@ -283,24 +316,36 @@ class PRChecker(object): ...@@ -283,24 +316,36 @@ class PRChecker(object):
if f_judge.find('test_') != -1 or f_judge.find( if f_judge.find('test_') != -1 or f_judge.find(
'_test') != -1: '_test') != -1:
check_added_ut = True check_added_ut = True
if file_dict[f] not in ['removed']:
if self.is_only_comment(f):
ut_list.append('comment_placeholder')
onlyCommentsFilesOrXpu.append(f_judge)
else:
notHitMapFiles.append(f_judge)
else:
print("remove file not hit mapFiles: %s" % f_judge)
else:
notHitMapFiles.append(f_judge) if file_dict[
f] != 'removed' else print(
"remove file not hit mapFiles: %s" % f_judge)
else:
if file_dict[f] not in ['removed']:
if self.is_only_comment(f): if self.is_only_comment(f):
ut_list.append('comment_placeholder') ut_list.append('comment_placeholder')
onlyCommentsFilesOrXpu.append(f_judge) onlyCommentsFilesOrXpu.append(f_judge)
else: else:
notHitMapFiles.append(f_judge) hitMapFiles[f_judge] = len(file_ut_map[f_judge])
ut_list.extend(file_ut_map.get(f_judge))
else: else:
notHitMapFiles.append(f_judge) hitMapFiles[f_judge] = len(file_ut_map[f_judge])
else:
if self.is_only_comment(f):
ut_list.append('comment_placeholder')
onlyCommentsFilesOrXpu.append(f_judge)
else:
hitMapFiles.append(f_judge)
ut_list.extend(file_ut_map.get(f_judge)) ut_list.extend(file_ut_map.get(f_judge))
ut_list = list(set(ut_list)) ut_list = list(set(ut_list))
if len(notHitMapFiles) != 0: if len(notHitMapFiles) != 0:
print("ipipe_log_param_PRECISION_TEST: false") print("ipipe_log_param_PRECISION_TEST: false")
print("notHitMapFiles: %s" % notHitMapFiles) print("notHitMapFiles: %s" % notHitMapFiles)
if len(filterFiles) != 0:
print("filterFiles: %s" % filterFiles)
return '' return ''
else: else:
if check_added_ut: if check_added_ut:
...@@ -318,6 +363,7 @@ class PRChecker(object): ...@@ -318,6 +363,7 @@ class PRChecker(object):
else: else:
print('PREC download prec_delta failed') print('PREC download prec_delta failed')
exit(1) exit(1)
print("hitMapFiles: %s" % hitMapFiles)
print("ipipe_log_param_PRECISION_TEST: true") print("ipipe_log_param_PRECISION_TEST: true")
print("ipipe_log_param_PRECISION_TEST_Cases_count: %s" % print("ipipe_log_param_PRECISION_TEST_Cases_count: %s" %
len(ut_list)) len(ut_list))
...@@ -326,6 +372,8 @@ class PRChecker(object): ...@@ -326,6 +372,8 @@ class PRChecker(object):
'.2f') '.2f')
print("ipipe_log_param_PRECISION_TEST_Cases_ratio: %s" % print("ipipe_log_param_PRECISION_TEST_Cases_ratio: %s" %
PRECISION_TEST_Cases_ratio) PRECISION_TEST_Cases_ratio)
if len(filterFiles) != 0:
print("filterFiles: %s" % filterFiles)
return '\n'.join(ut_list) return '\n'.join(ut_list)
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册