未验证 提交 ff216f18 编写于 作者: Z zhangbo9674 提交者: GitHub

Improve CI unittest parallel execution strategy (#44334)

* paralle_test_daily

* test=paralle_test_daily

* test=paralle_test_daily

* test=parallel_test_daily

* test=paralle_test_daily

* test=paralle_test_daily

* test=paralle_test_daily

* test=paralle_test_daily

* test=paralle_test_daily

* test=paralle_test_daily

* test=paralle_test_daily

* test=paralle_test_daily

* test pre_test_bak

* test cfs

* test_cfs,test=paralle_test_daily

* test_cfs,test=paralle_test_daily

* fix nightly test name,test=paralle_test_daily

* fix nightly test name,test=paralle_test_daily

* test ci parallel speed

* refine parallel rule,test=paralle_test_daily
上级 d0f4465d
...@@ -1871,6 +1871,18 @@ function precise_card_test_single { ...@@ -1871,6 +1871,18 @@ function precise_card_test_single {
done done
} }
function parallel_card_test_single {
set +e
set +x
testcases=$1
num=$2
for case in $(echo $testcases | tr "$|^" "\n")
do
cd ${PADDLE_ROOT}/build
precise_card_test "^${case}$" $num
done
}
function precise_card_test() { function precise_card_test() {
set -m set -m
testcases=$1 testcases=$1
...@@ -1918,6 +1930,8 @@ function get_precise_tests_map_file { ...@@ -1918,6 +1930,8 @@ function get_precise_tests_map_file {
multiple_card_tests='' # cases list which would take multiple GPUs, most cases would be two GPUs multiple_card_tests='' # cases list which would take multiple GPUs, most cases would be two GPUs
is_exclusive='' # indicate whether the case is exclusive type is_exclusive='' # indicate whether the case is exclusive type
is_multicard='' # indicate whether the case is multiple GPUs type is_multicard='' # indicate whether the case is multiple GPUs type
single_card_test_num=0
set +x set +x
while read -r line; do while read -r line; do
...@@ -1953,7 +1967,8 @@ set +x ...@@ -1953,7 +1967,8 @@ set +x
multiple_card_tests="$multiple_card_tests|^$testcase$" multiple_card_tests="$multiple_card_tests|^$testcase$"
fi fi
else else
if [[ "${single_card_tests}" -gt 3000 ]];then single_card_test_num=$(($single_card_test_num+1))
if [[ $single_card_test_num -gt 3000 ]];then
if [[ "$single_card_tests_1" == "" ]]; then if [[ "$single_card_tests_1" == "" ]]; then
single_card_tests_1="^$testcase$" single_card_tests_1="^$testcase$"
else else
...@@ -1997,7 +2012,86 @@ set -x ...@@ -1997,7 +2012,86 @@ set -x
#generate ut file map #generate ut file map
python ${PADDLE_ROOT}/tools/get_ut_file_map.py 'get_ut_map' ${PADDLE_ROOT} python ${PADDLE_ROOT}/tools/get_ut_file_map.py 'get_ut_map' ${PADDLE_ROOT}
}
function get_parallel_tests_map_file {
cd ${PADDLE_ROOT}/build
pip install ${PADDLE_ROOT}/build/python/dist/*whl
ut_total_startTime_s=`date +%s`
EXIT_CODE=0;
test_cases=$(ctest -N -V) # get all test cases
single_card_tests='' # all cases list which would take one graph card
exclusive_tests='' # cases list which would be run exclusively
multiple_card_tests='' # cases list which would take multiple GPUs, most cases would be two GPUs
is_exclusive='' # indicate whether the case is exclusive type
is_multicard='' # indicate whether the case is multiple GPUs type
single_card_test_num=0
set +x
while read -r line; do
if [[ "$line" == "" ]]; then
continue
fi
read matchstr <<< $(echo "$line"|grep -oEi 'Test[ \t]+#')
if [[ "$matchstr" == "" ]]; then
# Any test case with LABELS property would be parse here
# RUN_TYPE=EXCLUSIVE mean the case would run exclusively
# RUN_TYPE=DIST mean the case would take two graph GPUs during runtime
read is_exclusive <<< $(echo "$line"|grep -oEi "RUN_TYPE=EXCLUSIVE")
read is_multicard <<< $(echo "$line"|grep -oEi "RUN_TYPE=DIST")
continue
fi
read testcase <<< $(echo "$line"|grep -oEi "\w+$")
if [[ "$is_multicard" == "" ]]; then
# trick: treat all test case with prefix "test_dist" as dist case, and would run on 2 GPUs
read is_multicard <<< $(echo "$testcase"|grep -oEi "test_dist_")
fi
if [[ "$is_exclusive" != "" ]]; then
if [[ "$exclusive_tests" == "" ]]; then
exclusive_tests="^$testcase$"
else
exclusive_tests="$exclusive_tests|^$testcase$"
fi
elif [[ "$is_multicard" != "" ]]; then
if [[ "$multiple_card_tests" == "" ]]; then
multiple_card_tests="^$testcase$"
else
multiple_card_tests="$multiple_card_tests|^$testcase$"
fi
else
single_card_test_num=$(($single_card_test_num+1))
if [[ $single_card_test_num -gt 3000 ]];then
if [[ "$single_card_tests_1" == "" ]]; then
single_card_tests_1="^$testcase$"
else
single_card_tests_1="$single_card_tests_1|^$testcase$"
fi
continue
fi
if [[ "$single_card_tests" == "" ]]; then
single_card_tests="^$testcase$"
else
single_card_tests="$single_card_tests|^$testcase$"
fi
fi
is_exclusive=''
is_multicard=''
is_nightly=''
matchstr=''
testcase=''
done <<< "$test_cases";
set -x
mkdir -p ${PADDLE_ROOT}/build/ut_map
mkdir -p ${PADDLE_ROOT}/build/pytest
parallel_card_test_single "$single_card_tests" 1
parallel_card_test_single "$single_card_tests_1" 1
parallel_card_test_single "$multiple_card_tests" 2
parallel_card_test_single "$exclusive_tests"
wait; wait;
#classify_case_by_cardNum #classify_case_by_cardNum
...@@ -2383,7 +2477,6 @@ set +x ...@@ -2383,7 +2477,6 @@ set +x
else else
break break
fi fi
done done
fi fi
...@@ -3465,9 +3558,11 @@ function main() { ...@@ -3465,9 +3558,11 @@ function main() {
ci_preciseTest) ci_preciseTest)
insert_pile_to_h_cu_diff insert_pile_to_h_cu_diff
cmake_gen_and_build ${PYTHON_ABI:-""} ${parallel_number} cmake_gen_and_build ${PYTHON_ABI:-""} ${parallel_number}
enable_unused_var_check
get_precise_tests_map_file get_precise_tests_map_file
;; ;;
ci_parallelTest)
get_parallel_tests_map_file
;;
cicheck_brpc) cicheck_brpc)
cmake_gen ${PYTHON_ABI:-""} cmake_gen ${PYTHON_ABI:-""}
build ${parallel_number} build ${parallel_number}
......
...@@ -85,6 +85,9 @@ def classify_cases_by_mem(rootPath): ...@@ -85,6 +85,9 @@ def classify_cases_by_mem(rootPath):
case = case.replace('^', '').replace('$', '').strip() case = case.replace('^', '').replace('$', '').strip()
all_tests_by_card['exclusive_card_tests'].append(case) all_tests_by_card['exclusive_card_tests'].append(case)
if not os.path.exists("/pre_test"):
os.mkdir("/pre_test")
with open("/pre_test/classify_case_by_cardNum.json", "w") as f: with open("/pre_test/classify_case_by_cardNum.json", "w") as f:
json.dump(all_tests_by_card, f) json.dump(all_tests_by_card, f)
...@@ -120,7 +123,7 @@ def classify_cases_by_mem(rootPath): ...@@ -120,7 +123,7 @@ def classify_cases_by_mem(rootPath):
mem_1_sum = 0 mem_1_sum = 0
with open('/pre_test/%s' % cardType, 'w') as f_not_0: with open('/pre_test/%s' % cardType, 'w') as f_not_0:
for index in case_mem_1_sort: for index in case_mem_1_sort:
if mem_1_sum < 16 * 1024 * 2: if mem_1_sum < 14 * 1024 * 2:
mem_1_sum += index[1] mem_1_sum += index[1]
case_mem_1_line = case_mem_1_line + '|^' + index[0] + '$' case_mem_1_line = case_mem_1_line + '|^' + index[0] + '$'
else: else:
......
# -*- coding: utf-8 -*-
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. # Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
# #
# Licensed under the Apache License, Version 2.0 (the "License"); # Licensed under the Apache License, Version 2.0 (the "License");
...@@ -26,12 +28,13 @@ def get_ut_mem(rootPath): ...@@ -26,12 +28,13 @@ def get_ut_mem(rootPath):
ut = f.replace('^', '').replace('$.log', '') ut = f.replace('^', '').replace('$.log', '')
case_dic[ut] = {} case_dic[ut] = {}
filename = '%s/%s' % (parent, f) filename = '%s/%s' % (parent, f)
fi = open(filename) fi = open(filename, mode='rb')
lines = fi.readlines() lines = fi.readlines()
mem_reserved1 = -1 mem_reserved1 = -1
mem_nvidia1 = -1 mem_nvidia1 = -1
caseTime = -1 caseTime = -1
for line in lines: for line in lines:
line = line.decode('utf-8', errors='ignore')
if '[Memory Usage (Byte)] gpu' in line: if '[Memory Usage (Byte)] gpu' in line:
mem_reserved = round( mem_reserved = round(
float( float(
...@@ -56,7 +59,10 @@ def get_ut_mem(rootPath): ...@@ -56,7 +59,10 @@ def get_ut_mem(rootPath):
case_dic[ut]['mem_nvidia'] = mem_nvidia1 case_dic[ut]['mem_nvidia'] = mem_nvidia1
if caseTime != -1: if caseTime != -1:
case_dic[ut]['time'] = caseTime case_dic[ut]['time'] = caseTime
fi.close()
if not os.path.exists("/pre_test"):
os.mkdir("/pre_test")
ut_mem_map_file = "/pre_test/ut_mem_map.json" ut_mem_map_file = "/pre_test/ut_mem_map.json"
with open(ut_mem_map_file, "w") as f: with open(ut_mem_map_file, "w") as f:
json.dump(case_dic, f) json.dump(case_dic, f)
......
...@@ -21,16 +21,16 @@ def group_case_for_parallel(rootPath): ...@@ -21,16 +21,16 @@ def group_case_for_parallel(rootPath):
#wget file #wget file
for filename in [ for filename in [
'nightly_tests', 'single_card_tests', 'single_card_tests_mem0', 'nightly_case', 'single_card_tests', 'single_card_tests_mem0',
'multiple_card_tests', 'multiple_card_tests_mem0', 'multiple_card_tests', 'multiple_card_tests_mem0',
'exclusive_card_tests', 'exclusive_card_tests_mem0' 'exclusive_card_tests', 'exclusive_card_tests_mem0'
]: ]:
os.system( os.system(
'cd %s/tools && wget --no-proxy https://paddle-docker-tar.bj.bcebos.com/pre_test/%s --no-check-certificate' 'cd %s/tools && wget --no-proxy https://paddle-docker-tar.bj.bcebos.com/pre_test_bak/%s --no-check-certificate'
% (rootPath, filename)) % (rootPath, filename))
#get nightly tests #get nightly tests
nightly_tests_file = open('%s/tools/nightly_tests' % rootPath, 'r') nightly_tests_file = open('%s/tools/nightly_case' % rootPath, 'r')
nightly_tests = nightly_tests_file.read().strip().split('\n') nightly_tests = nightly_tests_file.read().strip().split('\n')
nightly_tests_file.close() nightly_tests_file.close()
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册