update tipc to_static (#9369)

* add d2s train for slanet and v3

* fix bug

* udpate tipc to_static

* update db

* remove_print

* update benchmark_train.sh

* update maybe_download_params

benchmark/PaddleOCR_DBNet/test_tipc/benchmark_train.sh

@@ -102,7 +102,7 @@ export frame_version=${str_tmp%%.post*}
 export frame_commit=$(echo `${python} -c "import paddle;print(paddle.version.commit)"`)
 
 # 获取benchmark_params所在的行数
-line_num=`grep -n "train_benchmark_params" $FILENAME  | cut -d ":" -f 1`
+line_num=`grep -n -w "train_benchmark_params" $FILENAME  | cut -d ":" -f 1`
 # for train log parser
 batch_size=$(func_parser_value "${lines[line_num]}")
 line_num=`expr $line_num + 1`

benchmark/PaddleOCR_DBNet/test_tipc/configs/det_res50_db/train_infer_python.txt

@@ -17,7 +17,7 @@ norm_train:tools/train.py --config_file config/icdar2015_resnet50_FPN_DBhead_pol
 quant_export:null
 fpgm_export:null
 distill_train:null
-to_static_train:trainer.to_static=true
+null:null
 null:null
 ##
 ===========================eval_params=========================== 
@@ -56,4 +56,6 @@ batch_size:8
 fp_items:fp32|fp16
 epoch:2
 --profiler_options:batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile
-flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98;FLAGS_conv_workspace_size_limit=4096
+flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98;FLAGS_conv_workspace_size_limit=4096
\ No newline at end of file
+===========================to_static_train_benchmark_params===========================
+to_static_train:trainer.to_static=true
\ No newline at end of file

benchmark/PaddleOCR_DBNet/test_tipc/test_train_inference_python.sh

@@ -5,7 +5,7 @@ FILENAME=$1
 # MODE be one of ['lite_train_lite_infer' 'lite_train_whole_infer' 'whole_train_whole_infer', 'whole_infer']
 MODE=$2
 
-dataline=$(awk 'NR==1, NR==51{print}'  $FILENAME)
+dataline=$(awk 'NR>=1{print}'  $FILENAME)
 
 # parser params
 IFS=$'\n'
@@ -40,8 +40,8 @@ fpgm_key=$(func_parser_key "${lines[17]}")
 fpgm_trainer=$(func_parser_value "${lines[17]}")
 distill_key=$(func_parser_key "${lines[18]}")
 distill_trainer=$(func_parser_value "${lines[18]}")
-to_static_key=$(func_parser_key "${lines[19]}")
-to_static_value=$(func_parser_value "${lines[19]}")
+trainer_key1=$(func_parser_key "${lines[19]}")
+trainer_value1=$(func_parser_value "${lines[19]}")
 trainer_key2=$(func_parser_key "${lines[20]}")
 trainer_value2=$(func_parser_value "${lines[20]}")
 
@@ -92,6 +92,9 @@ LOG_PATH="./test_tipc/output/${model_name}/${MODE}"
 mkdir -p ${LOG_PATH}
 status_log="${LOG_PATH}/results_python.log"
 
+line_num=`grep -n -w "to_static_train_benchmark_params" $FILENAME  | cut -d ":" -f 1`
+to_static_key=$(func_parser_key "${lines[line_num]}")
+to_static_trainer=$(func_parser_value "${lines[line_num]}")
 
 function func_inference(){
     IFS='|'
@@ -254,7 +257,7 @@ else
                     run_train=${distill_trainer}
                     run_export=${distill_export}
                 elif [ ${trainer} = "${to_static_key}" ]; then
-                    run_train="${norm_trainer}  ${to_static_value}"
+                    run_train="${norm_trainer}  ${to_static_trainer}"
                     run_export=${norm_export}
                 elif [[ ${trainer} = ${trainer_key2} ]]; then
                     run_train=${trainer_value2}

ppocr/utils/network.py

@@ -70,10 +70,8 @@ def maybe_download(model_storage_directory, url):
 
 
 def maybe_download_params(model_path):
-    if os.path.exists(model_path):
+    if os.path.exists(model_path) or not is_link(model_path):
         return model_path
-    elif not is_link(model_path):
-        url = 'https://paddleocr.bj.bcebos.com/' + model_path
     else:
         url = model_path
     tmp_path = os.path.join(MODELS_DIR, url.split('/')[-1])

test_tipc/benchmark_train.sh

@@ -102,7 +102,7 @@ export frame_version=${str_tmp%%.post*}
 export frame_commit=$(echo `${python} -c "import paddle;print(paddle.version.commit)"`)
 
 # 获取benchmark_params所在的行数
-line_num=`grep -n "train_benchmark_params" $FILENAME  | cut -d ":" -f 1`
+line_num=`grep -n -w "train_benchmark_params" $FILENAME  | cut -d ":" -f 1`
 # for train log parser
 batch_size=$(func_parser_value "${lines[line_num]}")
 line_num=`expr $line_num + 1`

test_tipc/configs/ch_PP-OCRv3_det/train_infer_python.txt

@@ -57,3 +57,5 @@ fp_items:fp32|fp16
 epoch:2
 --profiler_options:batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile
 flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98;FLAGS_conv_workspace_size_limit=4096
+===========================to_static_train_benchmark_params===========================
+to_static_train:Global.to_static=true
\ No newline at end of file

test_tipc/configs/ch_PP-OCRv3_rec/train_infer_python.txt

@@ -57,4 +57,5 @@ fp_items:fp32|fp16
 epoch:1
 --profiler_options:batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile
 flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98;FLAGS_conv_workspace_size_limit=4096
-
+===========================to_static_train_benchmark_params===========================
+to_static_train:Global.to_static=true

test_tipc/configs/slanet/train_infer_python.txt

@@ -57,3 +57,5 @@ fp_items:fp32|fp16
 epoch:2
 --profiler_options:batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile
 flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98;FLAGS_conv_workspace_size_limit=4096
+===========================to_static_train_benchmark_params===========================
+to_static_train:Global.to_static=true
\ No newline at end of file

test_tipc/configs/table_master/train_infer_python.txt

@@ -17,7 +17,7 @@ norm_train:tools/train.py -c test_tipc/configs/table_master/table_master.yml -o
 pact_train:null
 fpgm_train:null
 distill_train:null
-to_static_train:Global.to_static=true
+null:null
 null:null
 ##
 ===========================eval_params=========================== 
@@ -56,4 +56,6 @@ batch_size:10
 fp_items:fp32|fp16
 epoch:2
 --profiler_options:batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile
-flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98;FLAGS_conv_workspace_size_limit=4096
+flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98;FLAGS_conv_workspace_size_limit=4096
\ No newline at end of file
+===========================to_static_train_benchmark_params===========================
+to_static_train:Global.to_static=true
\ No newline at end of file

test_tipc/configs/vi_layoutxlm_ser/train_infer_python.txt

@@ -57,3 +57,5 @@ fp_items:fp32|fp16
 epoch:3
 --profiler_options:batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile
 flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98
+===========================to_static_train_benchmark_params===========================
+to_static_train:Global.to_static=true

test_tipc/test_train_inference_python.sh

@@ -5,7 +5,7 @@ FILENAME=$1
 # MODE be one of ['lite_train_lite_infer' 'lite_train_whole_infer' 'whole_train_whole_infer', 'whole_infer']
 MODE=$2
 
-dataline=$(awk 'NR==1, NR==51{print}'  $FILENAME)
+dataline=$(awk 'NR>=1{print}'  $FILENAME)
 
 # parser params
 IFS=$'\n'
@@ -40,8 +40,8 @@ fpgm_key=$(func_parser_key "${lines[17]}")
 fpgm_trainer=$(func_parser_value "${lines[17]}")
 distill_key=$(func_parser_key "${lines[18]}")
 distill_trainer=$(func_parser_value "${lines[18]}")
-to_static_key=$(func_parser_key "${lines[19]}")
-to_static_value=$(func_parser_value "${lines[19]}")
+trainer_key1=$(func_parser_key "${lines[19]}")
+trainer_value1=$(func_parser_value "${lines[19]}")
 trainer_key2=$(func_parser_key "${lines[20]}")
 trainer_value2=$(func_parser_value "${lines[20]}")
 
@@ -88,11 +88,14 @@ benchmark_value=$(func_parser_value "${lines[49]}")
 infer_key1=$(func_parser_key "${lines[50]}")
 infer_value1=$(func_parser_value "${lines[50]}")
 
+line_num=`grep -n -w "to_static_train_benchmark_params" $FILENAME  | cut -d ":" -f 1`
+to_static_key=$(func_parser_key "${lines[line_num]}")
+to_static_trainer=$(func_parser_value "${lines[line_num]}")
+
 LOG_PATH="./test_tipc/output/${model_name}/${MODE}"
 mkdir -p ${LOG_PATH}
 status_log="${LOG_PATH}/results_python.log"
 
-
 function func_inference(){
     IFS='|'
     _python=$1
@@ -254,7 +257,7 @@ else
                     run_train=${distill_trainer}
                     run_export=${distill_export}
                 elif [ ${trainer} = "${to_static_key}" ]; then
-                    run_train="${norm_trainer}  ${to_static_value}"
+                    run_train="${norm_trainer}  ${to_static_trainer}"
                     run_export=${norm_export}
                 elif [[ ${trainer} = ${trainer_key2} ]]; then
                     run_train=${trainer_value2}