From 430fbd5df9e58ca9342403195f4cf1cb07cc1bc2 Mon Sep 17 00:00:00 2001
From: root <root@bjyz-sys-gpu-kongming9.bjyz.baidu.com>
Date: Tue, 2 Aug 2022 03:07:14 +0000
Subject: [PATCH] tipc: support fp32/fp16 training benchmark of PP-Shitu

---
 test_tipc/benchmark_train.sh                  | 27 ++++++++++++-----
 ...nition_PPLCNet_x2_5_train_infer_python.txt |  6 ++++
 test_tipc/prepare.sh                          | 29 ++++++++++++++-----
 3 files changed, 48 insertions(+), 14 deletions(-)

diff --git a/test_tipc/benchmark_train.sh b/test_tipc/benchmark_train.sh
index 7a3087d4..2a73388a 100644
--- a/test_tipc/benchmark_train.sh
+++ b/test_tipc/benchmark_train.sh
@@ -184,17 +184,30 @@ for batch_size in ${batch_size_list[*]}; do
             # At least 25 log number would be good to calculate ips for benchmark system.
             # So the copy number for train_list is as follows:
             total_batch_size=`echo $[$batch_size*${device_num:1:1}*${device_num:3:3}]`
-            copy_num=`echo $[$total_batch_size/200]`
-            if [[ $copy_num -gt 1 ]];then
+            if [[ $model_name == *GeneralRecognition* ]]; then
+                cd dataset/
+                train_list_length=`cat train_reg_all_data.txt | wc -l`
+                copy_num=`echo $[25*10*$total_batch_size/$train_list_length]`
+                if [[ $copy_num -gt 1 ]];then
+                    rm -rf train_reg_all_data.txt
+                    for ((i=1; i <=$copy_num; i++));do
+                        cat tipc_shitu_demo_data/demo_train.txt >> train_reg_all_data.txt
+                    done
+                fi
+                cd ..
+            else
                 cd dataset/ILSVRC2012
-                rm -rf train_list.txt
-                for ((i=1; i <=$copy_num; i++));do
-                    cat val_list.txt >> train_list.txt
-                done
+                train_list_length=`cat train_list.txt | wc -l`
+                copy_num=`echo $[25*10*$total_batch_size/$train_list_length]`
+                if [[ $copy_num -gt 1 ]];then
+                    rm -rf train_list.txt
+                    for ((i=1; i <=$copy_num; i++));do
+                        cat val_list.txt >> train_list.txt
+                    done
+                fi
                 cd ../../
             fi
 
-
             if [[ ${#gpu_id} -le 1 ]];then
                 log_path="$SAVE_LOG/profiling_log"
                 mkdir -p $log_path
diff --git a/test_tipc/configs/GeneralRecognition/GeneralRecognition_PPLCNet_x2_5_train_infer_python.txt b/test_tipc/configs/GeneralRecognition/GeneralRecognition_PPLCNet_x2_5_train_infer_python.txt
index b76d5d1b..ff3bd088 100644
--- a/test_tipc/configs/GeneralRecognition/GeneralRecognition_PPLCNet_x2_5_train_infer_python.txt
+++ b/test_tipc/configs/GeneralRecognition/GeneralRecognition_PPLCNet_x2_5_train_infer_python.txt
@@ -50,5 +50,11 @@ inference:python/predict_rec.py -c configs/inference_rec.yaml
 -o Global.benchmark:False
 null:null
 null:null
+===========================train_benchmark_params==========================
+batch_size:256
+fp_items:fp32|fp16
+epoch:1
+--profiler_options:batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile
+flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98;FLAGS_conv_workspace_size_limit=4096
 ===========================infer_benchmark_params==========================
 random_infer_input:[{float32,[3,224,224]}]
\ No newline at end of file
diff --git a/test_tipc/prepare.sh b/test_tipc/prepare.sh
index 15adf957..0ccb8a95 100644
--- a/test_tipc/prepare.sh
+++ b/test_tipc/prepare.sh
@@ -346,12 +346,27 @@ fi
 if [[ ${MODE} = "benchmark_train" ]]; then
     pip install -r requirements.txt
     cd dataset
-    rm -rf ILSVRC2012
-    wget -nc https://paddle-imagenet-models-name.bj.bcebos.com/data/ImageNet1k/ILSVRC2012_val.tar
-    tar xf ILSVRC2012_val.tar
-    ln -s ILSVRC2012_val ILSVRC2012
-    cd ILSVRC2012
-    rm -rf train_list.txt
-    ln -s val_list.txt train_list.txt
+
+    if [[ $model_name == *GeneralRecognition* ]]; then
+        rm -rf Aliproduct
+        rm -rf train_reg_all_data.txt
+        rm -rf demo_train
+        wget -nc https://paddle-imagenet-models-name.bj.bcebos.com/data/whole_chain/tipc_shitu_demo_data.tar --no-check-certificate
+        tar -xf tipc_shitu_demo_data.tar
+        ln -s tipc_shitu_demo_data Aliproduct
+        ln -s tipc_shitu_demo_data/demo_train.txt train_reg_all_data.txt
+        ln -s tipc_shitu_demo_data/demo_train demo_train
+        cd tipc_shitu_demo_data
+        ln -s demo_test.txt val_list.txt
+    else
+        rm -rf ILSVRC2012
+        wget -nc https://paddle-imagenet-models-name.bj.bcebos.com/data/ImageNet1k/ILSVRC2012_val.tar
+        tar xf ILSVRC2012_val.tar
+        ln -s ILSVRC2012_val ILSVRC2012
+        cd ILSVRC2012
+        rm -rf train_list.txt
+        ln -s val_list.txt train_list.txt
+    fi
     cd ../../
+
 fi
-- 
GitLab