未验证 提交 1542c60a 编写于 作者: Z zhangchunle 提交者: GitHub

test=document_fix (#33799)

上级 0f31ed71
......@@ -1445,7 +1445,6 @@ function precise_card_test_single {
mkdir ${PADDLE_ROOT}/build/ut_map/$case
fi
set -x
mkdir ${PADDLE_ROOT}/build/ut_map/$case
find paddle/fluid -name '*.gcda'|xargs -I {} cp --path {} ut_map/$case
find paddle/fluid -name '*.gcno'|xargs -I {} cp --path {} ut_map/$case
python ${PADDLE_ROOT}/tools/get_single_test_cov.py ${PADDLE_ROOT} $case &
......
......@@ -25,7 +25,10 @@ import sys
def analysisPyXml(rootPath, ut):
xml_path = '%s/build/pytest/%s/python-coverage.xml' % (rootPath, ut)
ut_map_file = '%s/build/ut_map/%s/%s.txt' % (rootPath, ut, ut)
related_ut_map_file = '%s/build/ut_map/%s/related_%s.txt' % (rootPath, ut,
ut)
notrelated_ut_map_file = '%s/build/ut_map/%s/notrelated_%s.txt' % (rootPath,
ut, ut)
tree = ElementTree.parse(xml_path)
root = tree.getroot()
error_files = []
......@@ -46,16 +49,27 @@ def analysisPyXml(rootPath, ut):
'@', '\'\'\'', 'logger', '_logger', 'logging', 'r"""',
'pass', 'try', 'except', 'if __name__ == "__main__"'
)) == False:
#print(line_hits, line_number)
pattern = "(.*) = ('*')|(.*) = (\"*\")|(.*) = (\d)|(.*) = (-\d)|(.*) = (None)|(.*) = (True)|(.*) = (False)|(.*) = (URL_PREFIX*)|(.*) = (\[)|(.*) = (\{)|(.*) = (\()" #a='b'/a="b"/a=0
if re.match(pattern, output.strip()) == None:
pyCov_file.append(clazz_filename)
os.system('echo %s >> %s' %
(clazz_filename, ut_map_file))
coverageMessage = 'RELATED'
break
else:
coverageMessage = 'FILTER' #hit filter logic
else:
coverageMessage = 'FILTER'
else:
coverageMessage = 'ERROR'
error_files.append(clazz_filename)
break
else:
coverageMessage = 'NOT_RELATED'
if coverageMessage in ['NOT_RELATED', 'ERROR', 'FILTER']:
os.system('echo %s >> %s' %
(clazz_filename, notrelated_ut_map_file))
elif coverageMessage == 'RELATED':
os.system('echo %s >> %s' % (clazz_filename, related_ut_map_file))
print("============len(pyCov_file)")
print(len(pyCov_file))
print("============error")
......
......@@ -37,24 +37,47 @@ def getFNDAFile(rootPath, test):
def analysisFNDAFile(rootPath, test):
ut_map_file = '%s/build/ut_map/%s/%s.txt' % (rootPath, test, test)
os.system('touch %s' % ut_map_file)
related_ut_map_file = '%s/build/ut_map/%s/related_%s.txt' % (rootPath, test,
test)
notrelated_ut_map_file = '%s/build/ut_map/%s/notrelated_%s.txt' % (
rootPath, test, test)
os.system('touch %s' % related_ut_map_file)
os.system('touch %s' % notrelated_ut_map_file)
fn_filename = '%s/build/ut_map/%s/fnda.tmp' % (rootPath, test)
f = open(fn_filename)
data = f.read().split('SF:')
related_file_list = []
for message in data:
if 'FNDA:' in message:
message_list = message.split('\n')
clazz_filename = message_list[0]
#if not clazz_filename.endswith('.h'): #filter .h's Analysis
if '/build/' in clazz_filename:
clazz_filename = clazz_filename.replace('/build', '')
if '.pb.h' in clazz_filename:
clazz_filename = clazz_filename.replace('.pb.h', '.proto')
if '.pb.cc' in clazz_filename:
clazz_filename = clazz_filename.replace('.pb.cc', '.proto')
if 'FNDA:' in message:
OP_REGIST = True
for i in range(1, len(message_list) - 1):
fn = message_list[i]
matchObj = re.match(
r'(.*)Maker(.*)|(.*)Touch(.*)Regist(.*)|(.*)Touch(.*)JitKernel(.*)|(.*)converterC2Ev(.*)',
fn, re.I)
if matchObj == None:
os.system('echo %s >> %s' % (clazz_filename, ut_map_file))
OP_REGIST = False
break
if OP_REGIST == False:
related_file_list.append(clazz_filename)
os.system('echo %s >> %s' %
(clazz_filename, related_ut_map_file))
else:
os.system('echo %s >> %s' %
(clazz_filename, notrelated_ut_map_file))
else:
if clazz_filename != '':
if clazz_filename not in related_file_list: # xx.pb.cc in RELATED xx.pb.h not in RELATED
os.system('echo %s >> %s' %
(clazz_filename, notrelated_ut_map_file))
f.close()
......@@ -64,7 +87,7 @@ def getCovinfo(rootPath, test):
'cd %s && lcov --capture -d . -o coverage.info --rc lcov_branch_coverage=0 > /dev/null 2>&1'
% ut_map_path)
os.system(
"cd %s && lcov --extract coverage.info '/paddle/paddle/fluid/framework/*' '/paddle/paddle/fluid/imperative/*' '/paddle/paddle/fluid/inference/*' '/paddle/paddle/fluid/memory/*' '/paddle/paddle/fluid/operators/*' '/paddle/paddle/fluid/string/*' '/paddle/paddle/fluid/distributed/*' '/paddle/paddle/fluid/extension/*' '/paddle/paddle/fluid/platform/*' '/paddle/paddle/fluid/pybind/*' -o coverage.info.tmp --rc lcov_branch_coverage=0 > /dev/null 2>&1"
"cd %s && lcov --extract coverage.info '/paddle/paddle/fluid/framework/*' '/paddle/paddle/fluid/imperative/*' '/paddle/paddle/fluid/inference/*' '/paddle/paddle/fluid/memory/*' '/paddle/paddle/fluid/operators/*' '/paddle/paddle/fluid/string/*' '/paddle/paddle/fluid/distributed/*' '/paddle/paddle/fluid/extension/*' '/paddle/paddle/fluid/platform/*' '/paddle/paddle/fluid/pybind/*' '/paddle/build/*' -o coverage.info.tmp --rc lcov_branch_coverage=0 > /dev/null 2>&1"
% ut_map_path)
os.system('rm -rf %s/paddle' % ut_map_path)
os.system('rm -rf %s/coverage.info' % ut_map_path)
......
......@@ -20,7 +20,7 @@ import json
def get_all_paddle_file(rootPath):
"""get all file in Paddle repo: paddle/fluild, python"""
traverse_files = ['%s/paddle/fluid' % rootPath, '%s/python' % rootPath]
traverse_files = ['%s' % rootPath]
all_file_paddle = '%s/build/all_file_paddle' % rootPath
all_file_paddle_list = []
with open(all_file_paddle, 'w') as f:
......@@ -56,7 +56,7 @@ def remove_useless_file(rootPath):
def handle_ut_file_map(rootPath):
utNotSuccess = ''
utNotSuccess_list = []
ut_map_path = "%s/build/ut_map" % rootPath
files = os.listdir(ut_map_path)
ut_file_map = {}
......@@ -67,7 +67,7 @@ def handle_ut_file_map(rootPath):
print("ut %s: %s" % (count, ut))
coverage_info = '%s/%s/coverage.info.tmp' % (ut_map_path, ut)
if os.path.exists(coverage_info):
filename = '%s/%s/%s.txt' % (ut_map_path, ut, ut)
filename = '%s/%s/related_%s.txt' % (ut_map_path, ut, ut)
f = open(filename)
lines = f.readlines()
for line in lines:
......@@ -86,19 +86,33 @@ def handle_ut_file_map(rootPath):
ut_file_map[source_file] = []
if ut not in ut_file_map[source_file]:
ut_file_map[source_file].append(ut)
else:
not_success_file.write('%s\n' % ut)
utNotSuccess = utNotSuccess + '^%s$|' % ut
utNotSuccess_list.append(ut)
not_success_file.close()
print("utNotSuccess:")
print(utNotSuccess_list)
for ut in files:
if ut not in utNotSuccess_list:
filename = '%s/%s/notrelated_%s.txt' % (ut_map_path, ut, ut)
f = open(filename)
lines = f.readlines()
for line in lines:
line = line.replace('\n', '').strip()
if line == '':
continue
elif line.startswith('/paddle/build'):
source_file = line.replace('/build', '')
else:
source_file = line
if source_file not in ut_file_map:
ut_file_map[source_file] = []
with open("%s/build/ut_file_map.json" % rootPath, "w") as f:
json.dump(ut_file_map, f, indent=4)
print("utNotSuccess:")
print(utNotSuccess)
def notsuccessfuc(rootPath):
utNotSuccess = ''
......@@ -153,9 +167,6 @@ def ut_file_map_supplement(rootPath):
for filename in load_dict_old:
if filename not in load_dict_new:
if filename.endswith(('.h')):
load_dict_new[filename] = []
else:
load_dict_new[filename] = load_dict_old[filename]
with open("/pre_test/ut_file_map.json", "w") as f:
......@@ -182,6 +193,8 @@ def ut_file_map_supplement(rootPath):
if ut in all_uts_paddle_list:
if not os.path.exists(filename) and ut not in prec_delta_new_list:
prec_delta_new_list.append(ut)
prec_delta_new_list.append(
'test_py_reader_error_msg') #add a python case for pycoverage
prec_delta_file = open("/pre_test/prec_delta", 'w')
for ut in prec_delta_new_list:
prec_delta_file.write(ut + '\n')
......@@ -189,6 +202,15 @@ def ut_file_map_supplement(rootPath):
prec_delta_file.close()
def utmap_analysis(rootPath):
ut_file_map_new = "%s/build/ut_file_map.json" % rootPath
with open(ut_file_map_new, 'r') as load_f:
load_dict_new = json.load(load_f)
print(len(load_dict_new))
for filename in load_dict_new:
print(filename, len(load_dict_new[filename]))
if __name__ == "__main__":
func = sys.argv[1]
if func == 'get_not_success_ut':
......
......@@ -85,7 +85,7 @@ def get_h_cu_file(file_path):
filename = file_path[2]
ut = filename.replace('^', '').replace('$', '').replace('.log', '')
os.system(
"cat %s/%s | grep 'precise test map fileeee:'| uniq >> %s/build/ut_map/%s/%s.txt"
"cat %s/%s | grep 'precise test map fileeee:'| uniq >> %s/build/ut_map/%s/related_%s.txt"
% (dir_path, filename, rootPath, ut, ut))
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册