add statistics_UT_resource.sh for imporving UT parallel level (#32220)

tools/statistics_UT_resource.sh

+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+unset GREP_OPTIONS
+rm ./run_detail.log
+rm ./UT_resource.log
+rm ./UT_resource_sort.log
+rm ./while_list.log
+
+export LD_LIBRARY_PATH="$PWD/python/paddle/libs;$LD_LIBRARY_PATH"
+export CUDA_VISIBLE_DEVICES=0,1
+
+test_cases=$(ctest -N | awk -F ': ' '{print $2}' | sed '/^$/d' | sed '$d')
+use_memory_base=$(nvidia-smi -q -i 0  | grep "Used"  | head -1 | grep -o "[0-9]*")
+for unittest in $test_cases
+do
+    use_memory=0
+    gpu_utilization=0
+    memory_utilization=0
+    ctest -R "^${unittest}$" --repeat-until-fail 5 -j 1 &
+    PID=$!
+    echo -e "******************************************************"
+    echo -e "[$unittest]:    PID:$PID \n"
+    while [[ $(ps aux | awk '{print $2}' | grep "^$PID$" | grep -v "grep" | wc -l) -ge 1 ]]
+    do
+        use_memory_current=$(nvidia-smi -q -i 0  | grep "Used"  | head -1 | grep -o "[0-9]*")
+        if [[ $use_memory_current -gt $use_memory ]];then
+            use_memory=$use_memory_current
+        fi
+        memory_utilization_current=$(nvidia-smi -q -i 0 |  grep "Memory" | sed -n '3p' | grep -o "[0-9]*")
+        if [[ $memory_utilization_current -gt $memory_utilization ]];then
+            memory_utilization=$memory_utilization_current
+        fi
+
+        gpu_utilization_current=$(nvidia-smi -q -i 0  | grep "Gpu"  | grep -o "[0-9]*")
+        if [[ $gpu_utilization_current -gt $gpu_utilization ]];then
+            gpu_utilization=$gpu_utilization_current
+        fi
+    done
+    use_memory=`expr $use_memory - $use_memory_base`
+    echo -e "     use_memory:$use_memory \n     memory_utilization:$memory_utilization \n     gpu_utilization:$gpu_utilization\n"
+    echo -e "[$unittest]: \n     use_memory:$use_memory \n     memory_utilization:$memory_utilization \n     gpu_utilization:$gpu_utilization\n" >> run_detail.log
+    echo -e "$unittest:$use_memory:$memory_utilization:$gpu_utilization" >> UT_resource.log
+done
+
+sort -r -n -k 2 -t : UT_resource.log > UT_resource_sort.log
+cat UT_resource_sort.log | awk -F ':' '{print $1}' > while_list.log