diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index dd0af79c94d5d51de7fed2c4016945e59eb917bd..578b23dd31f8e95bbdb39ef41af5a2995655e19f 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -1871,6 +1871,18 @@ function precise_card_test_single { done } +function parallel_card_test_single { + set +e + set +x + testcases=$1 + num=$2 + for case in $(echo $testcases | tr "$|^" "\n") + do + cd ${PADDLE_ROOT}/build + precise_card_test "^${case}$" $num + done +} + function precise_card_test() { set -m testcases=$1 @@ -1918,6 +1930,8 @@ function get_precise_tests_map_file { multiple_card_tests='' # cases list which would take multiple GPUs, most cases would be two GPUs is_exclusive='' # indicate whether the case is exclusive type is_multicard='' # indicate whether the case is multiple GPUs type + + single_card_test_num=0 set +x while read -r line; do @@ -1953,7 +1967,8 @@ set +x multiple_card_tests="$multiple_card_tests|^$testcase$" fi else - if [[ "${single_card_tests}" -gt 3000 ]];then + single_card_test_num=$(($single_card_test_num+1)) + if [[ $single_card_test_num -gt 3000 ]];then if [[ "$single_card_tests_1" == "" ]]; then single_card_tests_1="^$testcase$" else @@ -1997,7 +2012,86 @@ set -x #generate ut file map python ${PADDLE_ROOT}/tools/get_ut_file_map.py 'get_ut_map' ${PADDLE_ROOT} + +} + +function get_parallel_tests_map_file { + cd ${PADDLE_ROOT}/build + pip install ${PADDLE_ROOT}/build/python/dist/*whl + ut_total_startTime_s=`date +%s` + EXIT_CODE=0; + test_cases=$(ctest -N -V) # get all test cases + single_card_tests='' # all cases list which would take one graph card + exclusive_tests='' # cases list which would be run exclusively + multiple_card_tests='' # cases list which would take multiple GPUs, most cases would be two GPUs + is_exclusive='' # indicate whether the case is exclusive type + is_multicard='' # indicate whether the case is multiple GPUs type + single_card_test_num=0 +set +x + + while read -r line; do + if [[ "$line" == "" ]]; then + continue + fi + read matchstr <<< $(echo "$line"|grep -oEi 'Test[ \t]+#') + if [[ "$matchstr" == "" ]]; then + # Any test case with LABELS property would be parse here + # RUN_TYPE=EXCLUSIVE mean the case would run exclusively + # RUN_TYPE=DIST mean the case would take two graph GPUs during runtime + read is_exclusive <<< $(echo "$line"|grep -oEi "RUN_TYPE=EXCLUSIVE") + read is_multicard <<< $(echo "$line"|grep -oEi "RUN_TYPE=DIST") + continue + fi + read testcase <<< $(echo "$line"|grep -oEi "\w+$") + + if [[ "$is_multicard" == "" ]]; then + # trick: treat all test case with prefix "test_dist" as dist case, and would run on 2 GPUs + read is_multicard <<< $(echo "$testcase"|grep -oEi "test_dist_") + fi + if [[ "$is_exclusive" != "" ]]; then + if [[ "$exclusive_tests" == "" ]]; then + exclusive_tests="^$testcase$" + else + exclusive_tests="$exclusive_tests|^$testcase$" + fi + elif [[ "$is_multicard" != "" ]]; then + if [[ "$multiple_card_tests" == "" ]]; then + multiple_card_tests="^$testcase$" + else + multiple_card_tests="$multiple_card_tests|^$testcase$" + fi + else + single_card_test_num=$(($single_card_test_num+1)) + if [[ $single_card_test_num -gt 3000 ]];then + if [[ "$single_card_tests_1" == "" ]]; then + single_card_tests_1="^$testcase$" + else + single_card_tests_1="$single_card_tests_1|^$testcase$" + fi + continue + fi + if [[ "$single_card_tests" == "" ]]; then + single_card_tests="^$testcase$" + else + single_card_tests="$single_card_tests|^$testcase$" + fi + fi + is_exclusive='' + is_multicard='' + is_nightly='' + matchstr='' + testcase='' + done <<< "$test_cases"; + +set -x + mkdir -p ${PADDLE_ROOT}/build/ut_map + mkdir -p ${PADDLE_ROOT}/build/pytest + + parallel_card_test_single "$single_card_tests" 1 + parallel_card_test_single "$single_card_tests_1" 1 + parallel_card_test_single "$multiple_card_tests" 2 + parallel_card_test_single "$exclusive_tests" wait; #classify_case_by_cardNum @@ -2383,7 +2477,6 @@ set +x else break fi - done fi @@ -3465,9 +3558,11 @@ function main() { ci_preciseTest) insert_pile_to_h_cu_diff cmake_gen_and_build ${PYTHON_ABI:-""} ${parallel_number} - enable_unused_var_check get_precise_tests_map_file ;; + ci_parallelTest) + get_parallel_tests_map_file + ;; cicheck_brpc) cmake_gen ${PYTHON_ABI:-""} build ${parallel_number} diff --git a/tools/final_ut_parallel_rule.py b/tools/final_ut_parallel_rule.py index 7a25eee71b227fc0238a65059e6c3f37d941e1fa..2c696e3139ddb8ba634374452d7896e88c126fd4 100644 --- a/tools/final_ut_parallel_rule.py +++ b/tools/final_ut_parallel_rule.py @@ -85,6 +85,9 @@ def classify_cases_by_mem(rootPath): case = case.replace('^', '').replace('$', '').strip() all_tests_by_card['exclusive_card_tests'].append(case) + if not os.path.exists("/pre_test"): + os.mkdir("/pre_test") + with open("/pre_test/classify_case_by_cardNum.json", "w") as f: json.dump(all_tests_by_card, f) @@ -120,7 +123,7 @@ def classify_cases_by_mem(rootPath): mem_1_sum = 0 with open('/pre_test/%s' % cardType, 'w') as f_not_0: for index in case_mem_1_sort: - if mem_1_sum < 16 * 1024 * 2: + if mem_1_sum < 14 * 1024 * 2: mem_1_sum += index[1] case_mem_1_line = case_mem_1_line + '|^' + index[0] + '$' else: diff --git a/tools/get_ut_mem_map.py b/tools/get_ut_mem_map.py index a8fd94c6f762bf4587c1e09687d54fa18c5d4718..dd1511a7cf293d92cf2a35ec458717e1b31c3af5 100644 --- a/tools/get_ut_mem_map.py +++ b/tools/get_ut_mem_map.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + # Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,12 +28,13 @@ def get_ut_mem(rootPath): ut = f.replace('^', '').replace('$.log', '') case_dic[ut] = {} filename = '%s/%s' % (parent, f) - fi = open(filename) + fi = open(filename, mode='rb') lines = fi.readlines() mem_reserved1 = -1 mem_nvidia1 = -1 caseTime = -1 for line in lines: + line = line.decode('utf-8', errors='ignore') if '[Memory Usage (Byte)] gpu' in line: mem_reserved = round( float( @@ -56,7 +59,10 @@ def get_ut_mem(rootPath): case_dic[ut]['mem_nvidia'] = mem_nvidia1 if caseTime != -1: case_dic[ut]['time'] = caseTime + fi.close() + if not os.path.exists("/pre_test"): + os.mkdir("/pre_test") ut_mem_map_file = "/pre_test/ut_mem_map.json" with open(ut_mem_map_file, "w") as f: json.dump(case_dic, f) diff --git a/tools/group_case_for_parallel.py b/tools/group_case_for_parallel.py index e4aea8f39f565d937b671504f943013ed7528bf4..e2c8eb9ff04e9bc3f14f1b782fd46aa68a418925 100644 --- a/tools/group_case_for_parallel.py +++ b/tools/group_case_for_parallel.py @@ -21,16 +21,16 @@ def group_case_for_parallel(rootPath): #wget file for filename in [ - 'nightly_tests', 'single_card_tests', 'single_card_tests_mem0', + 'nightly_case', 'single_card_tests', 'single_card_tests_mem0', 'multiple_card_tests', 'multiple_card_tests_mem0', 'exclusive_card_tests', 'exclusive_card_tests_mem0' ]: os.system( - 'cd %s/tools && wget --no-proxy https://paddle-docker-tar.bj.bcebos.com/pre_test/%s --no-check-certificate' + 'cd %s/tools && wget --no-proxy https://paddle-docker-tar.bj.bcebos.com/pre_test_bak/%s --no-check-certificate' % (rootPath, filename)) #get nightly tests - nightly_tests_file = open('%s/tools/nightly_tests' % rootPath, 'r') + nightly_tests_file = open('%s/tools/nightly_case' % rootPath, 'r') nightly_tests = nightly_tests_file.read().strip().split('\n') nightly_tests_file.close()