diff --git a/tools/statistics_UT_resource.sh b/tools/statistics_UT_resource.sh new file mode 100644 index 0000000000000000000000000000000000000000..a6f1f264c4cd2b883a649d8b5a93ff9700fb6bb8 --- /dev/null +++ b/tools/statistics_UT_resource.sh @@ -0,0 +1,58 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +unset GREP_OPTIONS +rm ./run_detail.log +rm ./UT_resource.log +rm ./UT_resource_sort.log +rm ./while_list.log + +export LD_LIBRARY_PATH="$PWD/python/paddle/libs;$LD_LIBRARY_PATH" +export CUDA_VISIBLE_DEVICES=0,1 + +test_cases=$(ctest -N | awk -F ': ' '{print $2}' | sed '/^$/d' | sed '$d') +use_memory_base=$(nvidia-smi -q -i 0 | grep "Used" | head -1 | grep -o "[0-9]*") +for unittest in $test_cases +do + use_memory=0 + gpu_utilization=0 + memory_utilization=0 + ctest -R "^${unittest}$" --repeat-until-fail 5 -j 1 & + PID=$! + echo -e "******************************************************" + echo -e "[$unittest]: PID:$PID \n" + while [[ $(ps aux | awk '{print $2}' | grep "^$PID$" | grep -v "grep" | wc -l) -ge 1 ]] + do + use_memory_current=$(nvidia-smi -q -i 0 | grep "Used" | head -1 | grep -o "[0-9]*") + if [[ $use_memory_current -gt $use_memory ]];then + use_memory=$use_memory_current + fi + memory_utilization_current=$(nvidia-smi -q -i 0 | grep "Memory" | sed -n '3p' | grep -o "[0-9]*") + if [[ $memory_utilization_current -gt $memory_utilization ]];then + memory_utilization=$memory_utilization_current + fi + + gpu_utilization_current=$(nvidia-smi -q -i 0 | grep "Gpu" | grep -o "[0-9]*") + if [[ $gpu_utilization_current -gt $gpu_utilization ]];then + gpu_utilization=$gpu_utilization_current + fi + done + use_memory=`expr $use_memory - $use_memory_base` + echo -e " use_memory:$use_memory \n memory_utilization:$memory_utilization \n gpu_utilization:$gpu_utilization\n" + echo -e "[$unittest]: \n use_memory:$use_memory \n memory_utilization:$memory_utilization \n gpu_utilization:$gpu_utilization\n" >> run_detail.log + echo -e "$unittest:$use_memory:$memory_utilization:$gpu_utilization" >> UT_resource.log +done + +sort -r -n -k 2 -t : UT_resource.log > UT_resource_sort.log +cat UT_resource_sort.log | awk -F ':' '{print $1}' > while_list.log