diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index 5f0a70dc0e69f29869be5b3f082dca7028415fe3..3a2c51fe72b205675e564f93b96e2f77ad184d2a 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -1817,7 +1817,14 @@ function precise_card_test() { echo "****************************************************************" tmpfile=$tmp_dir/$testcases".log" + tmpfile1=$tmp_dir/$testcases"-gpu.log" + nvidia-smi --id=0 --query-compute-apps=used_memory --format=csv -lms 10 > $tmpfile1 2>&1 & + gpu_memory_pid=$! env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I 0,,1 -R "($testcases)" --timeout 500 --output-on-failure -V -j 1 > $tmpfile + kill ${gpu_memory_pid} + cat $tmpfile1 | tr -d ' MiB' | awk 'BEGIN {max = 0} {if(NR>1){if ($1 > max) max=$1}} END {print "MAX_GPU_MEMORY_USE=", max}' >> $tmpfile + cat $tmpfile1 | tr -d ' MiB' | awk 'BEGIN {sum = 0} {if(NR>1){sum = sum + $1 }} END {print "AVG_GPU_MEMORY_USE=", sum / (NR-2)}' >> $tmpfile + rm -rf $tmpfile1 set +m } @@ -1909,8 +1916,11 @@ set -x python ${PADDLE_ROOT}/tools/pyCov_multithreading.py ${PADDLE_ROOT} wait; - #generate ut map + #generate ut file map python ${PADDLE_ROOT}/tools/get_ut_file_map.py 'get_ut_map' ${PADDLE_ROOT} + + #generate ut mem map + python ${PADDLE_ROOT}/tools/get_ut_mem_map.py $tmp_dir } function get_failedUts_precise_map_file { diff --git a/tools/get_ut_mem_map.py b/tools/get_ut_mem_map.py new file mode 100644 index 0000000000000000000000000000000000000000..daf80597d3ad0057bcad0194fc51b24f3ba6949a --- /dev/null +++ b/tools/get_ut_mem_map.py @@ -0,0 +1,66 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import json + + +def get_ut_mem(rootPath): + case_dic = {} + for parent, dirs, files in os.walk(rootPath): + for f in files: + if f.endswith('$-gpu.log'): + continue + ut = f.replace('^', '').replace('$.log', '') + case_dic[ut] = {} + filename = '%s%s' % (parent, f) + fi = open(filename) + lines = fi.readlines() + mem_reserved1 = -1 + mem_nvidia1 = -1 + caseTime = -1 + for line in lines: + if '[Memory Usage (Byte)] gpu' in line: + mem_reserved = round( + float( + line.split('[max memory reserved] gpu')[1].split( + ':')[1].split('\\n')[0].strip()), 2) + if mem_reserved > mem_reserved1: + mem_reserved1 = mem_reserved + if 'MAX_GPU_MEMORY_USE=' in line: + mem_nvidia = round( + float( + line.split('MAX_GPU_MEMORY_USE=')[1].split('\\n')[0] + .strip()), 2) + if mem_nvidia > mem_nvidia1: + mem_nvidia1 = mem_nvidia + if 'Total Test time (real)' in line: + caseTime = float( + line.split('Total Test time (real) =')[1].split('sec')[ + 0].strip()) + if mem_reserved1 != -1: + case_dic[ut]['mem_reserved'] = mem_reserved1 + if mem_nvidia1 != -1: + case_dic[ut]['mem_nvidia'] = mem_nvidia1 + if caseTime != -1: + case_dic[ut]['time'] = caseTime + + ut_mem_map_file = "/pre_test/ut_mem_map.json" % rootPath + with open(ut_mem_map_file, "w") as f: + json.dump(case_dic, f) + + +if __name__ == "__main__": + rootPath = sys.argv[1] + get_ut_mem(rootPath)