get ut mem and save in /pre_test/ut_mem_map.json (#42497)

paddle/scripts/paddle_build.sh

@@ -1817,7 +1817,14 @@ function precise_card_test() {
     echo "****************************************************************"
     
     tmpfile=$tmp_dir/$testcases".log"
+    tmpfile1=$tmp_dir/$testcases"-gpu.log"
+    nvidia-smi --id=0 --query-compute-apps=used_memory --format=csv -lms 10 > $tmpfile1 2>&1 &
+    gpu_memory_pid=$!
     env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I 0,,1 -R "($testcases)" --timeout 500 --output-on-failure -V -j 1 > $tmpfile 
+    kill ${gpu_memory_pid}
+    cat $tmpfile1 | tr -d ' MiB' | awk 'BEGIN {max = 0} {if(NR>1){if ($1 > max) max=$1}} END {print "MAX_GPU_MEMORY_USE=", max}' >> $tmpfile 
+    cat $tmpfile1 | tr -d ' MiB' | awk 'BEGIN {sum = 0} {if(NR>1){sum = sum + $1 }} END {print "AVG_GPU_MEMORY_USE=", sum / (NR-2)}' >> $tmpfile 
+    rm -rf $tmpfile1
     set +m
 }
 
@@ -1909,8 +1916,11 @@ set -x
     python ${PADDLE_ROOT}/tools/pyCov_multithreading.py ${PADDLE_ROOT}
     wait;
 
-    #generate ut map
+    #generate ut file map
     python ${PADDLE_ROOT}/tools/get_ut_file_map.py 'get_ut_map' ${PADDLE_ROOT}
+
+    #generate ut mem map
+    python ${PADDLE_ROOT}/tools/get_ut_mem_map.py $tmp_dir 
 }
 
 function get_failedUts_precise_map_file {

tools/get_ut_mem_map.py

+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import json
+
+
+def get_ut_mem(rootPath):
+    case_dic = {}
+    for parent, dirs, files in os.walk(rootPath):
+        for f in files:
+            if f.endswith('$-gpu.log'):
+                continue
+            ut = f.replace('^', '').replace('$.log', '')
+            case_dic[ut] = {}
+            filename = '%s%s' % (parent, f)
+            fi = open(filename)
+            lines = fi.readlines()
+            mem_reserved1 = -1
+            mem_nvidia1 = -1
+            caseTime = -1
+            for line in lines:
+                if '[Memory Usage (Byte)] gpu' in line:
+                    mem_reserved = round(
+                        float(
+                            line.split('[max memory reserved] gpu')[1].split(
+                                ':')[1].split('\\n')[0].strip()), 2)
+                    if mem_reserved > mem_reserved1:
+                        mem_reserved1 = mem_reserved
+                if 'MAX_GPU_MEMORY_USE=' in line:
+                    mem_nvidia = round(
+                        float(
+                            line.split('MAX_GPU_MEMORY_USE=')[1].split('\\n')[0]
+                            .strip()), 2)
+                    if mem_nvidia > mem_nvidia1:
+                        mem_nvidia1 = mem_nvidia
+                if 'Total Test time (real)' in line:
+                    caseTime = float(
+                        line.split('Total Test time (real) =')[1].split('sec')[
+                            0].strip())
+            if mem_reserved1 != -1:
+                case_dic[ut]['mem_reserved'] = mem_reserved1
+            if mem_nvidia1 != -1:
+                case_dic[ut]['mem_nvidia'] = mem_nvidia1
+            if caseTime != -1:
+                case_dic[ut]['time'] = caseTime
+
+    ut_mem_map_file = "/pre_test/ut_mem_map.json" % rootPath
+    with open(ut_mem_map_file, "w") as f:
+        json.dump(case_dic, f)
+
+
+if __name__ == "__main__":
+    rootPath = sys.argv[1]
+    get_ut_mem(rootPath)