Skip to content

P
Paddle

BaiXuePrincess / Paddle
与 Fork 源项目一致

Fork自 PaddlePaddle / Paddle

通知 1
Star 1
Fork 0
P
Paddle
未验证 提交 32804fe8 编写于 作者: R risemeup1 提交者： GitHub
浏览文件
操作

Add eager jit dir (#49369) 

* add_eager_and_jit,test=ljd_test

* test

* test,test=ljd_test

* test,test=ljd_test

* test,test=ljd_test

* test,test=ljd_test

* test,test=ljd_test

* test,test=ljd_test

* add_eager_jit_dir,test=ljd_test

* fix conflict,test=ljd_test

* test,test=ljd_test

* get new precise_map,test=ljd_test

* test
上级 aec1e4ce
隐藏空白更改
内联 并排
Showing with 65 addition and 40 deletion
paddle/scripts/paddle_build.sh
浏览文件 @ 32804fe8
...@@ -1861,12 +1861,12 @@ function precise_card_test_single { ...@@ -1861,12 +1861,12 @@ function precise_card_test_single {
mkdir ${PADDLE_ROOT}/build/ut_map/$case mkdir ${PADDLE_ROOT}/build/ut_map/$case
fi fi
set -x set -x
find paddle/fluid -name '*.gcda'|xargs -I {} cp --path {} ut_map/$case find paddle/fluid -name '*.gcda'|xargs -I {} cp --parents {} ut_map/$case
find paddle/phi -name '*.gcda'|xargs -I {} cp --path {} ut_map/$case find paddle/phi -name '*.gcda'|xargs -I {} cp --parents {} ut_map/$case
find paddle/utils -name '*.gcda'|xargs -I {} cp --path {} ut_map/$case find paddle/utils -name '*.gcda'|xargs -I {} cp --parents {} ut_map/$case
find paddle/phi -name '*.gcno'|xargs -I {} cp --path {} ut_map/$case find paddle/phi -name '*.gcno'|xargs -I {} cp --parents {} ut_map/$case
find paddle/utils -name '*.gcno'|xargs -I {} cp --path {} ut_map/$case find paddle/utils -name '*.gcno'|xargs -I {} cp --parents {} ut_map/$case
find paddle/fluid -name '*.gcno'|xargs -I {} cp --path {} ut_map/$case find paddle/fluid -name '*.gcno'|xargs -I {} cp --parents {} ut_map/$case
python ${PADDLE_ROOT}/tools/get_single_test_cov.py ${PADDLE_ROOT} $case & python ${PADDLE_ROOT}/tools/get_single_test_cov.py ${PADDLE_ROOT} $case &
# python # python
...@@ -1953,7 +1953,7 @@ function precise_card_test() { ...@@ -1953,7 +1953,7 @@ function precise_card_test() {
echo "****************************************************************" echo "****************************************************************"
tmpfile=$tmp_dir/$testcases".log" tmpfile=$tmp_dir/$testcases".log"
env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I 0,,1 -R "($testcases)" --timeout 500 --output-on-failure -V -j 1 > $tmpfile env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I 0,,1 -R "($testcases)" -E "($disable_ut_quickly)" --timeout 500 --output-on-failure -V -j 1 > $tmpfile
set +m set +m
} }
...@@ -2010,7 +2010,7 @@ set +x ...@@ -2010,7 +2010,7 @@ set +x
fi fi
else else
single_card_test_num=$(($single_card_test_num+1)) single_card_test_num=$(($single_card_test_num+1))
if [[ $single_card_test_num -gt 3000 ]];then if [[ $single_card_test_num -gt 1000 ]];then
if [[ "$single_card_tests_1" == "" ]]; then if [[ "$single_card_tests_1" == "" ]]; then
single_card_tests_1="^$testcase$" single_card_tests_1="^$testcase$"
else else
...@@ -2034,14 +2034,19 @@ set +x ...@@ -2034,14 +2034,19 @@ set +x
set -x set -x
mkdir -p ${PADDLE_ROOT}/build/ut_map mkdir -p ${PADDLE_ROOT}/build/ut_map
mkdir -p ${PADDLE_ROOT}/build/pytest mkdir -p ${PADDLE_ROOT}/build/pytest
#run all unittest to get the coverage information of .c and .h files #get disabled tests,not run these disabled ut
get_quickly_disable_ut||disable_ut_quickly='disable_ut'
# run all unittest to get the coverage information of .c and .h files
precise_card_test_single "^simple_precision_test$" 1 precise_card_test_single "^simple_precision_test$" 1
wait; wait;
precise_card_test_single "$single_card_tests" 1 precise_card_test_single "$single_card_tests" 1
wait;
precise_card_test_single "$single_card_tests_1" 1 precise_card_test_single "$single_card_tests_1" 1
precise_card_test_single "$multiple_card_tests" 2 precise_card_test_single "$multiple_card_tests" 2
precise_card_test_single "$exclusive_tests" precise_card_test_single "$exclusive_tests"
wait; wait;
#get notSuccessut including the failed uniitests and not executed unittests #get notSuccessut including the failed uniitests and not executed unittests
python ${PADDLE_ROOT}/tools/get_ut_file_map.py 'get_not_success_ut' ${PADDLE_ROOT} python ${PADDLE_ROOT}/tools/get_ut_file_map.py 'get_not_success_ut' ${PADDLE_ROOT}
......
tools/get_single_test_cov.py
浏览文件 @ 32804fe8
...@@ -17,6 +17,7 @@ import os ...@@ -17,6 +17,7 @@ import os
import re import re
import subprocess import subprocess
import sys import sys
import time
def getFNDAFile(rootPath, test): def getFNDAFile(rootPath, test):
...@@ -168,25 +169,28 @@ def getCovinfo(rootPath, test): ...@@ -168,25 +169,28 @@ def getCovinfo(rootPath, test):
ut_map_path = '%s/build/ut_map/%s' % (rootPath, test) ut_map_path = '%s/build/ut_map/%s' % (rootPath, test)
print("start get fluid ===>") print("start get fluid ===>")
cmd_fluid = ( cmd_fluid = (
'cd %s && lcov --capture -d ./paddle/fluid/ -o ./paddle/fluid/coverage_fluid.info --rc lcov_branch_coverage=0' 'cd %s && lcov --capture -d paddle/fluid/ -o paddle/fluid/coverage_fluid.info --rc lcov_branch_coverage=0'
% ut_map_path % ut_map_path
) )
p_fluid = subprocess.Popen(cmd_fluid, shell=True, stdout=subprocess.DEVNULL) p_fluid = subprocess.Popen(cmd_fluid, shell=True, stdout=subprocess.DEVNULL)
print("start get phi ===>") print("start get phi ===>")
cmd_phi = ( cmd_phi = (
'cd %s && lcov --capture -d ./paddle/phi -o ./paddle/phi/coverage_phi.info --rc lcov_branch_coverage=0' 'cd %s && lcov --capture -d paddle/phi -o paddle/phi/coverage_phi.info --rc lcov_branch_coverage=0'
% ut_map_path % ut_map_path
) )
p_phi = subprocess.Popen(cmd_phi, shell=True, stdout=subprocess.DEVNULL) if os.path.exists("%s/paddle/phi" % ut_map_path):
p_phi = subprocess.Popen(cmd_phi, shell=True, stdout=subprocess.DEVNULL)
print("start get utils ===>") print("start get utils ===>")
cmd_utils = ( cmd_utils = (
'cd %s && lcov --capture -d ./paddle/utils -o ./paddle/utils/coverage_utils.info --rc lcov_branch_coverage=0' 'cd %s && lcov --capture -d paddle/utils -o paddle/utils/coverage_utils.info --rc lcov_branch_coverage=0'
% ut_map_path % ut_map_path
) )
p_utils = subprocess.Popen(cmd_utils, shell=True, stdout=subprocess.DEVNULL) if os.path.exists("%s/paddle/utils" % ut_map_path):
p_utils = subprocess.Popen(
cmd_utils, shell=True, stdout=subprocess.DEVNULL
)
print("start wait fluid ===>") print("start wait fluid ===>")
p_fluid.wait() p_fluid.wait()
print("start wait phi ===>") print("start wait phi ===>")
...@@ -194,10 +198,22 @@ def getCovinfo(rootPath, test): ...@@ -194,10 +198,22 @@ def getCovinfo(rootPath, test):
print("start wait utils ===>") print("start wait utils ===>")
p_utils.wait() p_utils.wait()
print("end wait...") print("end wait...")
os.system( coverage_utils_info_path = (
'cd %s && lcov -a paddle/fluid/coverage_fluid.info -a paddle/phi/coverage_phi.info -a paddle/utils/coverage_utils.info -o coverage.info --rc lcov_branch_coverage=0 > /dev/null 2>&1' "%s/paddle/utils/coverage_utils.info" % ut_map_path
% ut_map_path
) )
if (
os.path.exists(coverage_utils_info_path)
and os.path.getsize(coverage_utils_info_path) != 0
):
os.system(
'cd %s && lcov -a paddle/fluid/coverage_fluid.info -a paddle/phi/coverage_phi.info -a paddle/utils/coverage_utils.info -o coverage.info --rc lcov_branch_coverage=0 > /dev/null 2>&1'
% ut_map_path
)
else:
os.system(
'cd %s && lcov -a paddle/fluid/coverage_fluid.info -a paddle/phi/coverage_phi.info -o coverage.info --rc lcov_branch_coverage=0 > /dev/null 2>&1'
% ut_map_path
)
coverage_info_path = ut_map_path + '/coverage.info' coverage_info_path = ut_map_path + '/coverage.info'
file_size = os.path.getsize(coverage_info_path) file_size = os.path.getsize(coverage_info_path)
if file_size == 0: if file_size == 0:
...@@ -206,7 +222,7 @@ def getCovinfo(rootPath, test): ...@@ -206,7 +222,7 @@ def getCovinfo(rootPath, test):
else: else:
print("get coverage.info succesfully") print("get coverage.info succesfully")
os.system( os.system(
"cd %s && lcov --extract coverage.info '/paddle/paddle/phi/*' '/paddle/paddle/utils/*' '/paddle/paddle/fluid/framework/*' '/paddle/paddle/fluid/imperative/*' '/paddle/paddle/fluid/inference/*' '/paddle/paddle/fluid/memory/*' '/paddle/paddle/fluid/operators/*' '/paddle/paddle/fluid/string/*' '/paddle/paddle/fluid/distributed/*' '/paddle/paddle/fluid/platform/*' '/paddle/paddle/fluid/pybind/*' '/paddle/build/*' -o coverage.info.tmp --rc lcov_branch_coverage=0 > /dev/null 2>&1" "cd %s && lcov --extract coverage.info '/paddle/paddle/phi/*' '/paddle/paddle/utils/*' '/paddle/paddle/fluid/*' '/paddle/build/*' -o coverage.info.tmp --rc lcov_branch_coverage=0 > /dev/null 2>&1"
% ut_map_path % ut_map_path
) )
coverage_info_tmp = ut_map_path + '/coverage.info.tmp' coverage_info_tmp = ut_map_path + '/coverage.info.tmp'
...@@ -222,12 +238,24 @@ def getCovinfo(rootPath, test): ...@@ -222,12 +238,24 @@ def getCovinfo(rootPath, test):
if test == "simple_precision_test": if test == "simple_precision_test":
getBaseFnda(rootPath, test) getBaseFnda(rootPath, test)
else: else:
start_getFNDAFile = time.time()
getFNDAFile(rootPath, test) getFNDAFile(rootPath, test)
end_getFNDAFile = time.time()
print("getFNDAFile time:", end_getFNDAFile - start_getFNDAFile)
start_analysisFNDAFile = time.time()
analysisFNDAFile(rootPath, test) analysisFNDAFile(rootPath, test)
end_analysisFNDAFile = time.time()
print(
"analysisFNDAFile time :",
end_analysisFNDAFile - start_analysisFNDAFile,
)
os.system('rm -rf %s/coverage.info.tmp' % ut_map_path) os.system('rm -rf %s/coverage.info.tmp' % ut_map_path)
if __name__ == "__main__": if __name__ == "__main__":
rootPath = sys.argv[1] rootPath = sys.argv[1]
case = sys.argv[2] case = sys.argv[2]
start_getCovinfo = time.time()
getCovinfo(rootPath, case) getCovinfo(rootPath, case)
end_getCovinfo = time.time()
print("getConvinfo time :", end_getCovinfo - start_getCovinfo)
tools/get_ut_file_map.py
浏览文件 @ 32804fe8
...@@ -145,8 +145,11 @@ def notsuccessfuc(rootPath): ...@@ -145,8 +145,11 @@ def notsuccessfuc(rootPath):
ut_map_path = "%s/build/ut_map" % rootPath ut_map_path = "%s/build/ut_map" % rootPath
files = os.listdir(ut_map_path) files = os.listdir(ut_map_path)
count = 0 count = 0
# ut failed!! # ut failed!!
for ut in files: for ut in files:
if ut == 'simple_precise_test':
continue
coverage_info = '%s/%s/fnda.tmp' % (ut_map_path, ut) coverage_info = '%s/%s/fnda.tmp' % (ut_map_path, ut)
if os.path.exists(coverage_info): if os.path.exists(coverage_info):
pass pass
...@@ -175,15 +178,15 @@ def notsuccessfuc(rootPath): ...@@ -175,15 +178,15 @@ def notsuccessfuc(rootPath):
def ut_file_map_supplement(rootPath): def ut_file_map_supplement(rootPath):
ut_file_map_new = "%s/build/ut_file_map.json" % rootPath ut_file_map_new = "%s/build/ut_file_map.json" % rootPath
os.system('mkdir /pre_test_tmp') precision_test_map_store_dir = "/precision_test_map_store"
os.system('mkdir %s' % precision_test_map_store_dir)
os.system( os.system(
'cd /pre_test_tmp && wget --no-proxy https://paddle-docker-tar.bj.bcebos.com/pre_test/ut_file_map.json --no-check-certificate' 'cd %s && wget --no-proxy https://paddle-docker-tar.bj.bcebos.com/tmp_test/ut_file_map.json --no-check-certificate'
% precision_test_map_store_dir
) )
ut_file_map_old = "/pre_test_tmp/ut_file_map.json" ut_file_map_old = "%s/ut_file_map.json" % precision_test_map_store_dir
with open(ut_file_map_new, 'r') as load_f: with open(ut_file_map_new, 'r') as load_f:
load_dict_new = json.load(load_f) load_dict_new = json.load(load_f)
with open(ut_file_map_old, 'r') as f:
load_dict_old = json.load(f)
all_uts_paddle = '%s/build/all_uts_paddle' % rootPath all_uts_paddle = '%s/build/all_uts_paddle' % rootPath
...@@ -193,35 +196,24 @@ def ut_file_map_supplement(rootPath): ...@@ -193,35 +196,24 @@ def ut_file_map_supplement(rootPath):
all_uts_paddle_list.append(ut.strip()) all_uts_paddle_list.append(ut.strip())
f.close() f.close()
with open("/pre_test_tmp/ut_file_map.json", "w") as f: with open("%s/ut_file_map.json" % precision_test_map_store_dir, "w") as f:
json.dump(load_dict_new, f, indent=4) json.dump(load_dict_new, f, indent=4)
print("load_dict_new success!!") print("load_dict_new success!!")
os.system( os.system(
'cd /pre_test_tmp && wget --no-proxy https://paddle-docker-tar.bj.bcebos.com/pre_test/prec_delta --no-check-certificate' 'cd %s && wget --no-proxy https://paddle-docker-tar.bj.bcebos.com/tmp_test/prec_delta --no-check-certificate'
% precision_test_map_store_dir
) )
prec_delta_old = '/pre_test_tmp/prec_delta'
prec_delta_new = "%s/build/prec_delta" % rootPath prec_delta_new = "%s/build/prec_delta" % rootPath
with open(prec_delta_old, 'r') as f:
prec_delta_old_list = []
for ut in f.readlines():
prec_delta_old_list.append(ut.strip())
f.close()
with open(prec_delta_new, 'r') as f: with open(prec_delta_new, 'r') as f:
prec_delta_new_list = [] prec_delta_new_list = []
for ut in f.readlines(): for ut in f.readlines():
prec_delta_new_list.append(ut.strip()) prec_delta_new_list.append(ut.strip())
f.close() f.close()
for ut in prec_delta_old_list:
filename = '%s/build/ut_map/%s/fnda.tmp' % (rootPath, ut)
if ut in all_uts_paddle_list:
if not os.path.exists(filename) and ut not in prec_delta_new_list:
prec_delta_new_list.append(ut)
prec_delta_new_list.append( prec_delta_new_list.append(
'test_py_reader_error_msg' 'test_py_reader_error_msg'
) # add a python case for pycoverage ) # add a python case for pycoverage
prec_delta_file = open("/pre_test_tmp/prec_delta", 'w') prec_delta_file = open("%s/prec_delta" % precision_test_map_store_dir, 'w')
for ut in prec_delta_new_list: for ut in prec_delta_new_list:
prec_delta_file.write(ut + '\n') prec_delta_file.write(ut + '\n')
print("prec_delta_file success!!") print("prec_delta_file success!!")
......
tools/handle_h_cu_file.py
浏览文件 @ 32804fe8
...@@ -66,7 +66,7 @@ def insert_pile_to_h_file(rootPath): ...@@ -66,7 +66,7 @@ def insert_pile_to_h_file(rootPath):
% (func, line) % (func, line)
) )
os.system( os.system(
'echo \' printf("precise test map fileeee: %%s\\\\n", __FILE__);\n}\' >> %s' 'echo \' fprintf(stderr,"precise test map fileeee: %%s\\\\n", __FILE__);\n}\' >> %s'
% line % line
) )
os.system('echo "\n#endif" >> %s' % line) os.system('echo "\n#endif" >> %s' % line)
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或