diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index 1945803b2db764b845c1629157e6ddfd2eab522b..a0e630818d853ff5d09a08d01c6cac9e01ce3de6 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -1427,7 +1427,6 @@ EOF function insert_pile_to_h_cu_diff { # TODO get develop h/cu md5 cd ${PADDLE_ROOT} - find ${PADDLE_ROOT} -name '*.h'| grep -v ${PADDLE_ROOT}/build >> ${PADDLE_ROOT}/tools/h_cu_files.log find ${PADDLE_ROOT} -name '*.cu'| grep -v ${PADDLE_ROOT}/build >> ${PADDLE_ROOT}/tools/h_cu_files.log python ${PADDLE_ROOT}/tools/handle_h_cu_file.py 'get_h_file_md5' ${PADDLE_ROOT} @@ -1447,8 +1446,8 @@ function precise_card_test_single { cd ${PADDLE_ROOT}/build precise_card_test "^${case}$" $num # c++ - if [ -d "${PADDLE_ROOT}/build/ut_map/$case" ];then - rm -rf ${PADDLE_ROOT}/build/ut_map/$case + if [ ! -d "${PADDLE_ROOT}/build/ut_map/$case" ];then + mkdir ${PADDLE_ROOT}/build/ut_map/$case fi set -x mkdir ${PADDLE_ROOT}/build/ut_map/$case @@ -1460,7 +1459,9 @@ function precise_card_test_single { ls python-coverage.data.* if [[ $? == 0 ]] then - mkdir -p ${PADDLE_ROOT}/build/pytest/$case + if [ ! -d "${PADDLE_ROOT}/build/pytest/$case" ];then + mkdir -p ${PADDLE_ROOT}/build/pytest/$case + fi mv python-coverage.data.* ${PADDLE_ROOT}/build/pytest/$case fi find paddle/fluid -name *.gcda | xargs rm -f #delete gcda @@ -1571,26 +1572,38 @@ set -x precise_card_test_single "$single_card_tests_1" 1 precise_card_test_single "$multiple_card_tests" 2 precise_card_test_single "$exclusive_tests" - + wait; python ${PADDLE_ROOT}/tools/get_ut_file_map.py 'get_not_success_ut' ${PADDLE_ROOT} - if [[ -f "${PADDLE_ROOT}/build/utNotSuccess" ]]; then - rerun_tests=`cat ${PADDLE_ROOT}/build/utNotSuccess` - precise_card_test_single "$rerun_tests" - fi + #analy h/cu to Map file + python ${PADDLE_ROOT}/tools/handle_h_cu_file.py 'analy_h_cu_file' $tmp_dir ${PADDLE_ROOT} + wait; + get_failedUts_precise_map_file #generate python coverage and generate python file to tests_map_file python ${PADDLE_ROOT}/tools/pyCov_multithreading.py ${PADDLE_ROOT} + wait; - #analy h/cu to Map file - python ${PADDLE_ROOT}/tools/handle_h_cu_file.py 'analy_h_cu_file' $tmp_dir ${PADDLE_ROOT} #generate ut map python ${PADDLE_ROOT}/tools/get_ut_file_map.py 'get_ut_map' ${PADDLE_ROOT} - wait; } - +function get_failedUts_precise_map_file { + if [[ -f "${PADDLE_ROOT}/build/utNotSuccess" ]]; then + rerun_tests=`cat ${PADDLE_ROOT}/build/utNotSuccess` + #remove pile to full h/cu file + python ${PADDLE_ROOT}/tools/handle_h_cu_file.py 'remove_pile_from_h_file' ${PADDLE_ROOT} + cd ${PADDLE_ROOT}/build + cmake_base ${PYTHON_ABI:-""} + build ${parallel_number} + pip uninstall -y paddlepaddle-gpu + pip install ${PADDLE_ROOT}/build/python/dist/*whl + precise_card_test_single "$rerun_tests" + wait; + + fi +} function parallel_test_base_xpu() { mkdir -p ${PADDLE_ROOT}/build diff --git a/tools/get_single_test_cov.py b/tools/get_single_test_cov.py index 42940386ca077398b4d5c7fe310f33fd9068f2d3..088471364f21d040b7759cfe0561aaf649263986 100644 --- a/tools/get_single_test_cov.py +++ b/tools/get_single_test_cov.py @@ -46,16 +46,15 @@ def analysisFNDAFile(rootPath, test): if 'FNDA:' in message: message_list = message.split('\n') clazz_filename = message_list[0] - if not clazz_filename.endswith('.h'): #filter .h's Analysis - for i in range(1, len(message_list) - 1): - fn = message_list[i] - matchObj = re.match( - r'(.*)Maker(.*)|(.*)Touch(.*)Regist(.*)|(.*)Touch(.*)JitKernel(.*)|(.*)converterC2Ev(.*)', - fn, re.I) - if matchObj == None: - os.system('echo %s >> %s' % - (clazz_filename, ut_map_file)) - break + #if not clazz_filename.endswith('.h'): #filter .h's Analysis + for i in range(1, len(message_list) - 1): + fn = message_list[i] + matchObj = re.match( + r'(.*)Maker(.*)|(.*)Touch(.*)Regist(.*)|(.*)Touch(.*)JitKernel(.*)|(.*)converterC2Ev(.*)', + fn, re.I) + if matchObj == None: + os.system('echo %s >> %s' % (clazz_filename, ut_map_file)) + break f.close() diff --git a/tools/get_ut_file_map.py b/tools/get_ut_file_map.py index d952a299d490e5adb574c05b357aeede587ddec1..59325b91d8e0ea0aaffbc183db8503d96b48ad1a 100644 --- a/tools/get_ut_file_map.py +++ b/tools/get_ut_file_map.py @@ -139,48 +139,53 @@ def ut_file_map_supplement(rootPath): 'cd /pre_test && wget --no-proxy https://paddle-docker-tar.bj.bcebos.com/pre_test/ut_file_map.json --no-check-certificate' ) ut_file_map_old = "/pre_test/ut_file_map.json" - ut_file_map_full = {} with open(ut_file_map_new, 'r') as load_f: load_dict_new = json.load(load_f) with open(ut_file_map_old, 'r') as f: load_dict_old = json.load(f) - for filename in load_dict_new: - ut_file_map_full[filename] = load_dict_new[filename] - if filename in load_dict_old: - for ut in load_dict_old[filename]: - if ut not in ut_file_map_full[filename]: - ut_file_map_full[filename].append(ut) + all_uts_paddle = '%s/build/all_uts_paddle' % rootPath + with open(all_uts_paddle, 'r') as f: + all_uts_paddle_list = [] + for ut in f.readlines(): + all_uts_paddle_list.append(ut.strip()) + f.close() for filename in load_dict_old: if filename not in load_dict_new: - ut_file_map_full[filename] = load_dict_old[filename] + if filename.endswith(('.h')): + load_dict_new[filename] = [] + else: + load_dict_new[filename] = load_dict_old[filename] with open("/pre_test/ut_file_map.json", "w") as f: - json.dump(ut_file_map_full, f, indent=4) - print("ut_file_map_full success!!") + json.dump(load_dict_new, f, indent=4) + print("load_dict_new success!!") - all_uts_paddle = '%s/build/all_uts_paddle' % rootPath - with open(all_uts_paddle, 'r') as f: - all_uts_paddle_list = f.readlines() - f.close() os.system( 'cd /pre_test && wget --no-proxy https://paddle-docker-tar.bj.bcebos.com/pre_test/prec_delta --no-check-certificate' ) prec_delta_old = '/pre_test/prec_delta' prec_delta_new = "%s/build/prec_delta" % rootPath with open(prec_delta_old, 'r') as f: - prec_delta_old_list = f.readlines() + prec_delta_old_list = [] + for ut in f.readlines(): + prec_delta_old_list.append(ut.strip()) f.close() with open(prec_delta_new, 'r') as f: - prec_delta_new_list = f.readlines() + prec_delta_new_list = [] + for ut in f.readlines(): + prec_delta_new_list.append(ut.strip()) f.close() for ut in prec_delta_old_list: - if ut not in prec_delta_new_list and ut not in all_uts_paddle_list: - prec_delta_new_list.append(ut) + filename = '%s/build/ut_map/%s/coverage.info.tmp' % (rootPath, ut) + if ut in all_uts_paddle_list: + if not os.path.exists(filename) and ut not in prec_delta_new_list: + prec_delta_new_list.append(ut) prec_delta_file = open("/pre_test/prec_delta", 'w') for ut in prec_delta_new_list: - prec_delta_file.write(ut) + prec_delta_file.write(ut + '\n') + print("prec_delta_file success!!") prec_delta_file.close() diff --git a/tools/handle_h_cu_file.py b/tools/handle_h_cu_file.py index 7c300d96c84618ec6874f65b838542a837ccc4f9..eb66a3d1dc48d6c15432dff5f963f99235b95d31 100644 --- a/tools/handle_h_cu_file.py +++ b/tools/handle_h_cu_file.py @@ -66,6 +66,19 @@ def insert_pile_to_h_file(rootPath): os.system('echo "\n#endif" >> %s' % line) +def remove_pile_from_h_file(rootPath): + h_cu_files = '%s/tools/h_cu_files.log' % rootPath + f = open(h_cu_files) + lines = f.readlines() + count = 12 + for line in lines: + line = line.strip() + while count > 0: + os.system("sed -i '$d' %s" % line) + count = count - 1 + count = 12 + + def get_h_cu_file(file_path): rootPath = file_path[0] dir_path = file_path[1] @@ -110,3 +123,6 @@ if __name__ == "__main__": dir_path = sys.argv[2] rootPath = sys.argv[3] main(rootPath, dir_path) + elif func == 'remove_pile_from_h_file': + rootPath = sys.argv[2] + remove_pile_from_h_file(rootPath) diff --git a/tools/pyCov_multithreading.py b/tools/pyCov_multithreading.py index 2df4ac2ef6b3fb918322d65c2c3b9f60964bb5d6..20181fb6f93cb16df1749c10868c67c9769e00fd 100644 --- a/tools/pyCov_multithreading.py +++ b/tools/pyCov_multithreading.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import commands from xml.etree import ElementTree import re import time @@ -51,7 +50,7 @@ def getPyCovResult(params): os.system('cd %s && coverage combine `ls python-coverage.data.*`' % path) os.system('cd %s && pwd && coverage xml -i -o python-coverage.xml' % path) xml_path = '%s/python-coverage.xml' % path - os.system("python %s/tools/analysisPyXml.py %s %s" % + os.system("python2.7 %s/tools/analysisPyXml.py %s %s" % (rootPath, rootPath, ut)) endTime = int(time.time()) print('pyCov Time: %s' % (endTime - startTime))