[TIPC] upgrade benchmark in TIPC (#5194)

test_tipc/benchmark_train.sh

+#!/bin/bash
+source test_tipc/utils_func.sh
+
+# set env
+python=python
+export model_branch=`git symbolic-ref HEAD 2>/dev/null | cut -d"/" -f 3`
+export model_commit=$(git log|head -n1|awk '{print $2}')
+export str_tmp=$(echo `pip list|grep paddlepaddle-gpu|awk -F ' ' '{print $2}'`)
+export frame_version=${str_tmp%%.post*}
+export frame_commit=$(echo `${python} -c "import paddle;print(paddle.version.commit)"`)
+
+# run benchmark sh
+# Usage:
+# bash run_benchmark_train.sh config.txt params
+# or
+# bash run_benchmark_train.sh config.txt
+
+function func_parser_params(){
+    strs=$1
+    IFS="="
+    array=(${strs})
+    tmp=${array[1]}
+    echo ${tmp}
+}
+
+function func_sed_params(){
+    filename=$1
+    line=$2
+    param_value=$3
+    params=`sed -n "${line}p" $filename`
+    IFS=":"
+    array=(${params})
+    key=${array[0]}
+    new_params="${key}:${param_value}"
+    IFS=";"
+    cmd="sed -i '${line}s/.*/${new_params}/' '${filename}'"
+    eval $cmd
+}
+
+function set_gpu_id(){
+    string=$1
+    _str=${string:1:6}
+    IFS="C"
+    arr=(${_str})
+    M=${arr[0]}
+    P=${arr[1]}
+    gn=`expr $P - 1`
+    gpu_num=`expr $gn / $M`
+    seq=`seq -s "," 0 $gpu_num`
+    echo $seq
+}
+
+function get_repo_name(){
+    IFS=";"
+    cur_dir=$(pwd)
+    IFS="/"
+    arr=(${cur_dir})
+    echo ${arr[-1]}
+}
+
+FILENAME=$1
+# copy FILENAME as new
+new_filename="./test_tipc/benchmark_train.txt"
+cmd=`yes|cp $FILENAME $new_filename`
+FILENAME=$new_filename
+# MODE must be one of ['benchmark_train']
+MODE=$2
+PARAMS=$3
+# bash test_tipc/benchmark_train.sh test_tipc/configs/det_mv3_db_v2_0/train_benchmark.txt  benchmark_train dynamic_bs8_null_DP_N1C1
+IFS=$'\n'
+# parser params from train_benchmark.txt
+dataline=`cat $FILENAME`
+# parser params
+IFS=$'\n'
+lines=(${dataline})
+model_name=$(func_parser_value "${lines[1]}")
+
+# 获取benchmark_params所在的行数
+line_num=`grep -n "train_benchmark_params" $FILENAME  | cut -d ":" -f 1`
+# for train log parser
+batch_size=$(func_parser_value "${lines[line_num]}")
+line_num=`expr $line_num + 1`
+fp_items=$(func_parser_value "${lines[line_num]}")
+line_num=`expr $line_num + 1`
+epoch=$(func_parser_value "${lines[line_num]}")
+
+line_num=`expr $line_num + 1`
+profile_option_key=$(func_parser_key "${lines[line_num]}")
+profile_option_params=$(func_parser_value "${lines[line_num]}")
+profile_option="${profile_option_key}:${profile_option_params}"
+
+line_num=`expr $line_num + 1`
+flags_value=$(func_parser_value "${lines[line_num]}")
+# set flags
+IFS=";"
+flags_list=(${flags_value})
+for _flag in ${flags_list[*]}; do
+    cmd="export ${_flag}"
+    eval $cmd
+done
+
+# set log_name
+repo_name=$(get_repo_name )
+SAVE_LOG=${BENCHMARK_LOG_DIR:-$(pwd)}   # */benchmark_log
+mkdir -p "${SAVE_LOG}/benchmark_log/"
+status_log="${SAVE_LOG}/benchmark_log/results.log"
+
+# The number of lines in which train params can be replaced.
+line_python=3
+line_gpuid=4
+line_precision=6
+line_epoch=7
+line_batchsize=9
+line_profile=13
+line_eval_py=24
+line_export_py=30
+
+func_sed_params "$FILENAME" "${line_eval_py}" "null"
+func_sed_params "$FILENAME" "${line_export_py}" "null"
+func_sed_params "$FILENAME" "${line_python}"  "${python}"
+
+# if params
+if  [ ! -n "$PARAMS" ] ;then
+    # PARAMS input is not a word.
+    IFS="|"
+    batch_size_list=(${batch_size})
+    fp_items_list=(${fp_items})
+    device_num_list=(N1C4)
+    run_mode="DP"
+else
+    # parser params from input: modeltype_bs${bs_item}_${fp_item}_${run_mode}_${device_num}
+    IFS="_"
+    params_list=(${PARAMS})
+    model_type=${params_list[0]}
+    batch_size=${params_list[1]}
+    batch_size=`echo  ${batch_size} | tr -cd "[0-9]" `
+    precision=${params_list[2]}
+    # run_process_type=${params_list[3]}
+    run_mode=${params_list[3]}
+    device_num=${params_list[4]}
+    IFS=";"
+
+    if [ ${precision} = "null" ];then
+        precision="fp32"
+    fi
+
+    fp_items_list=($precision)
+    batch_size_list=($batch_size)
+    device_num_list=($device_num)
+fi
+
+IFS="|"
+for batch_size in ${batch_size_list[*]}; do
+    for precision in ${fp_items_list[*]}; do
+        for device_num in ${device_num_list[*]}; do
+            # sed batchsize and precision
+            func_sed_params "$FILENAME" "${line_precision}" "$precision"
+            func_sed_params "$FILENAME" "${line_batchsize}" "$MODE=$batch_size"
+            func_sed_params "$FILENAME" "${line_epoch}" "$MODE=$epoch"
+            gpu_id=$(set_gpu_id $device_num)
+
+            if [ ${#gpu_id} -le 1 ];then
+                run_process_type="SingleP"
+                log_path="$SAVE_LOG/profiling_log"
+                mkdir -p $log_path
+                log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}_${device_num}_profiling"
+                func_sed_params "$FILENAME" "${line_gpuid}" "0"  # sed used gpu_id
+                # set profile_option params
+                tmp=`sed -i "${line_profile}s/.*/${profile_option}/" "${FILENAME}"`
+
+                # run test_train_inference_python.sh
+                cmd="bash test_tipc/test_train_inference_python.sh ${FILENAME} benchmark_train > ${log_path}/${log_name} 2>&1 "
+                echo $cmd
+                eval $cmd
+                eval "cat ${log_path}/${log_name}"
+
+                # without profile
+                log_path="$SAVE_LOG/train_log"
+                speed_log_path="$SAVE_LOG/index"
+                mkdir -p $log_path
+                mkdir -p $speed_log_path
+                log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}_${device_num}_log"
+                speed_log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}_${device_num}_speed"
+                func_sed_params "$FILENAME" "${line_profile}" "null"  # sed profile_id as null
+                cmd="bash test_tipc/test_train_inference_python.sh ${FILENAME} benchmark_train > ${log_path}/${log_name} 2>&1 "
+                echo $cmd
+                job_bt=`date '+%Y%m%d%H%M%S'`
+                eval $cmd
+                job_et=`date '+%Y%m%d%H%M%S'`
+                export model_run_time=$((${job_et}-${job_bt}))
+                eval "cat ${log_path}/${log_name}"
+
+                # parser log
+                _model_name="${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}"
+                cmd="${python} ${BENCHMARK_ROOT}/scripts/analysis.py --filename ${log_path}/${log_name} \
+                        --speed_log_file '${speed_log_path}/${speed_log_name}' \
+                        --model_name ${_model_name} \
+                        --base_batch_size ${batch_size} \
+                        --run_mode ${run_mode} \
+                        --run_process_type ${run_process_type} \
+                        --fp_item ${precision} \
+                        --keyword ips: \
+                        --skip_steps 2 \
+                        --device_num ${device_num} \
+                        --speed_unit images/s \
+                        --convergence_key loss: "
+                echo $cmd
+                eval $cmd
+                last_status=${PIPESTATUS[0]}
+                status_check $last_status "${cmd}" "${status_log}"
+            else
+                IFS=";"
+                unset_env=`unset CUDA_VISIBLE_DEVICES`
+                run_process_type="MultiP"
+                log_path="$SAVE_LOG/train_log"
+                speed_log_path="$SAVE_LOG/index"
+                mkdir -p $log_path
+                mkdir -p $speed_log_path
+                log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}_${device_num}_log"
+                speed_log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}_${device_num}_speed"
+                func_sed_params "$FILENAME" "${line_gpuid}" "$gpu_id"  # sed used gpu_id
+                func_sed_params "$FILENAME" "${line_profile}" "null"  # sed --profile_option as null
+                cmd="bash test_tipc/test_train_inference_python.sh ${FILENAME} benchmark_train > ${log_path}/${log_name} 2>&1 "
+                echo $cmd
+                job_bt=`date '+%Y%m%d%H%M%S'`
+                eval $cmd
+                job_et=`date '+%Y%m%d%H%M%S'`
+                export model_run_time=$((${job_et}-${job_bt}))
+                eval "cat ${log_path}/${log_name}"
+                # parser log
+                _model_name="${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}"
+
+                cmd="${python} ${BENCHMARK_ROOT}/scripts/analysis.py --filename ${log_path}/${log_name} \
+                        --speed_log_file '${speed_log_path}/${speed_log_name}' \
+                        --model_name ${_model_name} \
+                        --base_batch_size ${batch_size} \
+                        --run_mode ${run_mode} \
+                        --run_process_type ${run_process_type} \
+                        --fp_item ${precision} \
+                        --keyword ips: \
+                        --skip_steps 2 \
+                        --device_num ${device_num} \
+                        --speed_unit images/s \
+                        --convergence_key loss: "
+                echo $cmd
+                eval $cmd
+                last_status=${PIPESTATUS[0]}
+                status_check $last_status "${cmd}" "${status_log}"
+            fi
+        done
+    done
+done

test_tipc/configs/deformable_detr/deformable_detr_r50_1x_coco_train_infer_python.txt

+===========================train_params===========================
+model_name:deformable_detr_r50_1x_coco
+python:python3.7
+gpu_list:0|0,1
+use_gpu:True
+auto_cast:null
+epoch:lite_train_lite_infer=1|lite_train_whole_infer=1|whole_train_whole_infer=50
+save_dir:null
+TrainReader.batch_size:lite_train_lite_infer=2|lite_train_whole_infer=2|whole_train_whole_infer=2
+pretrain_weights:https://paddledet.bj.bcebos.com/models/deformable_detr_r50_1x_coco.pdparams
+trained_model_name:model_final.pdparams
+train_infer_img_dir:./dataset/coco/test2017/
+filename:null
+##
+trainer:norm_train
+norm_train:tools/train.py -c configs/deformable_detr/deformable_detr_r50_1x_coco.yml -o
+pact_train:tools/train.py -c configs/deformable_detr/deformable_detr_r50_1x_coco.yml --slim_config _template_pact -o
+fpgm_train:tools/train.py -c configs/deformable_detr/deformable_detr_r50_1x_coco.yml --slim_config _template_fpgm -o
+distill_train:null
+null:null
+null:null
+##
+===========================eval_params===========================
+eval:tools/eval.py -c configs/deformable_detr/deformable_detr_r50_1x_coco.yml -o
+null:null
+##
+===========================infer_params===========================
+--output_dir:./output_inference
+weights:https://paddledet.bj.bcebos.com/models/deformable_detr_r50_1x_coco.pdparams
+norm_export:tools/export_model.py -c configs/deformable_detr/deformable_detr_r50_1x_coco.yml -o
+pact_export:tools/export_model.py -c configs/deformable_detr/deformable_detr_r50_1x_coco.yml --slim_config _template_pact -o
+fpgm_export:tools/export_model.py -c configs/deformable_detr/deformable_detr_r50_1x_coco.yml --slim_config _template_fpgm -o
+distill_export:null
+export1:null
+export2:null
+kl_quant_export:tools/post_quant.py -c configs/deformable_detr/deformable_detr_r50_1x_coco.yml --slim_config _template_kl_quant -o
+##
+infer_mode:norm
+infer_quant:False
+inference:./deploy/python/infer.py
+--device:gpu|cpu
+--enable_mkldnn:True|False
+--cpu_threads:1|6
+--batch_size:1|2
+--use_tensorrt:null
+--run_mode:paddle|trt_fp32|trt_fp16
+--model_dir:
+--image_dir:./dataset/coco/test2017/
+--save_log_path:null
+--run_benchmark:False
+--trt_max_shape:1600
+===========================train_benchmark_params==========================
+batch_size:2
+fp_items:fp32|fp16
+epoch:1
+--profiler_options:batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile
+flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98;FLAGS_conv_workspace_size_limit=4096
\ No newline at end of file

test_tipc/configs/faster_rcnn/faster_rcnn_r50_fpn_1x_coco_train_infer_python.txt

@@ -49,3 +49,9 @@ inference:./deploy/python/infer.py
 --save_log_path:null
 --run_benchmark:True
 --trt_max_shape:1600
+===========================train_benchmark_params==========================
+batch_size:2|8
+fp_items:fp32|fp16
+epoch:1
+--profiler_options:batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile
+flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98;FLAGS_conv_workspace_size_limit=4096
\ No newline at end of file

test_tipc/configs/fcos/fcos_r50_fpn_1x_coco_train_infer_python.txt

@@ -49,3 +49,9 @@ inference:./deploy/python/infer.py
 --save_log_path:null
 --run_benchmark:True
 --trt_max_shape:1600
+===========================train_benchmark_params==========================
+batch_size:2|8
+fp_items:fp32|fp16
+epoch:1
+--profiler_options:batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile
+flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98;FLAGS_conv_workspace_size_limit=4096
\ No newline at end of file

test_tipc/configs/gfl/gfl_r50_fpn_1x_coco_train_infer_python.txt

+===========================train_params===========================
+model_name:gfl_r50_fpn_1x_coco
+python:python3.7
+gpu_list:0|0,1
+use_gpu:True
+auto_cast:null
+epoch:lite_train_lite_infer=1|lite_train_whole_infer=1|whole_train_whole_infer=12
+save_dir:null
+TrainReader.batch_size:lite_train_lite_infer=2|lite_train_whole_infer=2|whole_train_whole_infer=2
+pretrain_weights:https://paddledet.bj.bcebos.com/models/gfl_r50_fpn_1x_coco.pdparams
+trained_model_name:model_final.pdparams
+train_infer_img_dir:./dataset/coco/test2017/
+filename:null
+##
+trainer:norm_train
+norm_train:tools/train.py -c configs/gfl/gfl_r50_fpn_1x_coco.yml -o
+pact_train:tools/train.py -c configs/gfl/gfl_r50_fpn_1x_coco.yml --slim_config _template_pact -o
+fpgm_train:tools/train.py -c configs/gfl/gfl_r50_fpn_1x_coco.yml --slim_config _template_fpgm -o
+distill_train:null
+null:null
+null:null
+##
+===========================eval_params===========================
+eval:tools/eval.py -c configs/gfl/gfl_r50_fpn_1x_coco.yml -o
+null:null
+##
+===========================infer_params===========================
+--output_dir:./output_inference
+weights:https://paddledet.bj.bcebos.com/models/gfl_r50_fpn_1x_coco.pdparams
+norm_export:tools/export_model.py -c configs/gfl/gfl_r50_fpn_1x_coco.yml -o
+pact_export:tools/export_model.py -c configs/gfl/gfl_r50_fpn_1x_coco.yml --slim_config _template_pact -o
+fpgm_export:tools/export_model.py -c configs/gfl/gfl_r50_fpn_1x_coco.yml --slim_config _template_fpgm -o
+distill_export:null
+export1:null
+export2:null
+kl_quant_export:tools/post_quant.py -c configs/gfl/gfl_r50_fpn_1x_coco.yml --slim_config _template_kl_quant -o
+##
+infer_mode:norm
+infer_quant:False
+inference:./deploy/python/infer.py
+--device:gpu|cpu
+--enable_mkldnn:True|False
+--cpu_threads:1|6
+--batch_size:1|2
+--use_tensorrt:null
+--run_mode:paddle|trt_fp32|trt_fp16
+--model_dir:
+--image_dir:./dataset/coco/test2017/
+--save_log_path:null
+--run_benchmark:False
+--trt_max_shape:1600
+===========================train_benchmark_params==========================
+batch_size:2|8
+fp_items:fp32|fp16
+epoch:1
+--profiler_options:batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile
+flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98;FLAGS_conv_workspace_size_limit=4096
\ No newline at end of file

test_tipc/configs/keypoint/higherhrnet_hrnet_w32_512_train_infer_python.txt

@@ -49,3 +49,9 @@ inference:./deploy/python/keypoint_infer.py
 --save_log_path:null
 --run_benchmark:True
 --trt_max_shape:1600
+===========================train_benchmark_params==========================
+batch_size:20|24
+fp_items:fp32|fp16
+epoch:1
+--profiler_options:batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile
+flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98;FLAGS_conv_workspace_size_limit=4096
\ No newline at end of file

test_tipc/configs/keypoint/hrnet_w32_256x192_train_infer_python.txt

@@ -49,3 +49,9 @@ inference:./deploy/python/keypoint_infer.py
 --save_log_path:null
 --run_benchmark:True
 null:null
+===========================train_benchmark_params==========================
+batch_size:64|160
+fp_items:fp32|fp16
+epoch:1
+--profiler_options:batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile
+flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98;FLAGS_conv_workspace_size_limit=4096
\ No newline at end of file

test_tipc/configs/mot/fairmot_dla34_30e_1088x608_train_infer_python.txt

+===========================train_params===========================
+model_name:fairmot_dla34_30e_1088x608
+python:python3.7
+gpu_list:0|0,1
+use_gpu:True
+auto_cast:null
+epoch:lite_train_lite_infer=1|lite_train_whole_infer=1|whole_train_whole_infer=30
+save_dir:null
+TrainReader.batch_size:lite_train_lite_infer=2|lite_train_whole_infer=2|whole_train_whole_infer=6
+pretrain_weights:https://paddledet.bj.bcebos.com/models/mot/fairmot_dla34_30e_1088x608.pdparams
+trained_model_name:model_final.pdparams
+train_infer_img_dir:./dataset/coco/test2017/
+filename:null
+##
+trainer:norm_train
+norm_train:tools/train.py -c configs/mot/fairmot/fairmot_dla34_30e_1088x608.yml -o
+pact_train:tools/train.py -c configs/mot/fairmot/fairmot_dla34_30e_1088x608.yml --slim_config _template_pact -o
+fpgm_train:tools/train.py -c configs/mot/fairmot/fairmot_dla34_30e_1088x608.yml --slim_config _template_fpgm -o
+distill_train:null
+null:null
+null:null
+##
+===========================eval_params===========================
+eval:tools/eval.py -c configs/mot/fairmot/fairmot_dla34_30e_1088x608.yml -o
+null:null
+##
+===========================infer_params===========================
+--output_dir:./output_inference
+weights:https://paddledet.bj.bcebos.com/models/mot/fairmot_dla34_30e_1088x608.pdparams
+norm_export:tools/export_model.py -c configs/mot/fairmot/fairmot_dla34_30e_1088x608.yml -o
+pact_export:tools/export_model.py -c configs/mot/fairmot/fairmot_dla34_30e_1088x608.yml --slim_config _template_pact -o
+fpgm_export:tools/export_model.py -c configs/mot/fairmot/fairmot_dla34_30e_1088x608.yml --slim_config _template_fpgm -o
+distill_export:null
+export1:null
+export2:null
+kl_quant_export:tools/post_quant.py -c configs/mot/fairmot/fairmot_dla34_30e_1088x608.yml --slim_config _template_kl_quant -o
+##
+infer_mode:norm
+infer_quant:False
+inference:./deploy/python/infer.py
+--device:gpu|cpu
+--enable_mkldnn:True|False
+--cpu_threads:1|6
+--batch_size:1|2
+--use_tensorrt:null
+--run_mode:paddle|trt_fp32|trt_fp16
+--model_dir:
+--image_dir:./dataset/coco/test2017/
+--save_log_path:null
+--run_benchmark:False
+--trt_max_shape:1600
+===========================train_benchmark_params==========================
+batch_size:6|22
+fp_items:fp32|fp16
+epoch:1
+--profiler_options:batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile
+flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98;FLAGS_conv_workspace_size_limit=4096
\ No newline at end of file

test_tipc/configs/mot/jde_darknet53_30e_1088x608_train_infer_python.txt

+===========================train_params===========================
+model_name:jde_darknet53_30e_1088x608
+python:python3.7
+gpu_list:0|0,1
+use_gpu:True
+auto_cast:null
+epoch:lite_train_lite_infer=1|lite_train_whole_infer=1|whole_train_whole_infer=30
+save_dir:null
+TrainReader.batch_size:lite_train_lite_infer=2|lite_train_whole_infer=2|whole_train_whole_infer=4
+pretrain_weights:https://paddledet.bj.bcebos.com/models/mot/jde_darknet53_30e_1088x608.pdparams
+trained_model_name:model_final.pdparams
+train_infer_img_dir:./dataset/coco/test2017/
+filename:null
+##
+trainer:norm_train
+norm_train:tools/train.py -c configs/mot/jde/jde_darknet53_30e_1088x608.yml -o
+pact_train:tools/train.py -c configs/mot/jde/jde_darknet53_30e_1088x608.yml --slim_config _template_pact -o
+fpgm_train:tools/train.py -c configs/mot/jde/jde_darknet53_30e_1088x608.yml --slim_config _template_fpgm -o
+distill_train:null
+null:null
+null:null
+##
+===========================eval_params===========================
+eval:tools/eval.py -c configs/mot/jde/jde_darknet53_30e_1088x608.yml -o
+null:null
+##
+===========================infer_params===========================
+--output_dir:./output_inference
+weights:https://paddledet.bj.bcebos.com/models/mot/jde_darknet53_30e_1088x608.pdparams
+norm_export:tools/export_model.py -c configs/mot/jde/jde_darknet53_30e_1088x608.yml -o
+pact_export:tools/export_model.py -c configs/mot/jde/jde_darknet53_30e_1088x608.yml --slim_config _template_pact -o
+fpgm_export:tools/export_model.py -c configs/mot/jde/jde_darknet53_30e_1088x608.yml --slim_config _template_fpgm -o
+distill_export:null
+export1:null
+export2:null
+kl_quant_export:tools/post_quant.py -c configs/mot/jde/jde_darknet53_30e_1088x608.yml --slim_config _template_kl_quant -o
+##
+infer_mode:norm
+infer_quant:False
+inference:./deploy/python/infer.py
+--device:gpu|cpu
+--enable_mkldnn:True|False
+--cpu_threads:1|6
+--batch_size:1|2
+--use_tensorrt:null
+--run_mode:paddle|trt_fp32|trt_fp16
+--model_dir:
+--image_dir:./dataset/coco/test2017/
+--save_log_path:null
+--run_benchmark:False
+--trt_max_shape:1600
+===========================train_benchmark_params==========================
+batch_size:4|14
+fp_items:fp32|fp16
+epoch:1
+--profiler_options:batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile
+flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98;FLAGS_conv_workspace_size_limit=4096
\ No newline at end of file

test_tipc/configs/solov2/solov2_r50_fpn_1x_coco_train_infer_python.txt

@@ -49,3 +49,9 @@ inference:./deploy/python/infer.py
 --save_log_path:null
 --run_benchmark:True
 --trt_max_shape:1600
+===========================train_benchmark_params==========================
+batch_size:2|4
+fp_items:fp32|fp16
+epoch:1
+--profiler_options:batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile
+flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98;FLAGS_conv_workspace_size_limit=4096
\ No newline at end of file

test_tipc/docs/benchmark_train.md

+# TIPC Linux端Benchmark测试文档
+
+该文档为Benchmark测试说明，Benchmark预测功能测试的主程序为`benchmark_train.sh`，用于验证监控模型训练的性能。
+
+# 1. 测试流程
+## 1.1 准备数据和环境安装
+运行`test_tipc/prepare.sh`，完成训练数据准备和安装环境流程。
+
+```shell
+# 运行格式：bash test_tipc/prepare.sh  train_benchmark.txt  mode
+bash test_tipc/prepare.sh test_tipc/configs/faster_rcnn/faster_rcnn_r50_fpn_1x_coco_train_infer_python.txt benchmark_train
+```
+
+## 1.2 功能测试
+执行`test_tipc/benchmark_train.sh`，完成模型训练和日志解析
+
+```shell
+# 运行格式：bash test_tipc/benchmark_train.sh train_benchmark.txt mode
+bash test_tipc/benchmark_train.sh test_tipc/configs/faster_rcnn/faster_rcnn_r50_fpn_1x_coco_train_infer_python.txt benchmark_train
+```
+
+`test_tipc/benchmark_train.sh`支持根据传入的第三个参数实现只运行某一个训练配置，如下：
+```shell
+# 运行格式：bash test_tipc/benchmark_train.sh train_benchmark.txt mode
+bash test_tipc/benchmark_train.sh test_tipc/configs/faster_rcnn/faster_rcnn_r50_fpn_1x_coco_train_infer_python.txt benchmark_train  dynamic_bs2_fp32_DP_N1C1
+```
+dynamic_bs2_fp32_DP_N1C1为test_tipc/benchmark_train.sh传入的参数，格式如下：
+`${modeltype}_${batch_size}_${fp_item}_${run_mode}_${device_num}`
+包含的信息有：模型类型、batchsize大小、训练精度如fp32,fp16等、分布式运行模式以及分布式训练使用的机器信息如单机单卡（N1C1）。
+
+
+## 2. 日志输出
+
+运行后将保存模型的训练日志和解析日志，使用 `test_tipc/configs/faster_rcnn/faster_rcnn_r50_fpn_1x_coco_train_infer_python.txt` 参数文件的训练日志解析结果是：
+
+```
+{"model_branch": "tipc_fuse_benchmark", "model_commit": "4cce901d231f7954468045cf96302505bd6be495", "model_name": "faster_rcnn_r50_fpn_1x_coco_bs2_fp32_SingleP_DP", "batch_size": 2, "fp_item": "fp32", "run_process_type": "SingleP", "run_mode": "DP", "convergence_value": "0.556966", "convergence_key": "loss:", "ips": 4.857, "speed_unit": "images/s", "device_num": "N1C1", "model_run_time": "590", "frame_commit": "6b0c57cf65945e97d87a8fba89c0a2fc18dd8544", "frame_version": "0.0.0"}
+```
+
+训练日志和日志解析结果保存在benchmark_log目录下，文件组织格式如下：
+```
+train_log/
+├── index
+│   └── PaddleDetection_faster_rcnn_r50_fpn_1x_coco_bs2_fp32_SingleP_DP_N1C1_speed
+├── profiling_log
+│   └── PaddleDetection_faster_rcnn_r50_fpn_1x_coco_bs2_fp32_SingleP_DP_N1C1_profiling
+└── train_log
+    ├── PaddleDetection_faster_rcnn_r50_fpn_1x_coco_bs2_fp32_SingleP_DP_N1C1_log
+    └── PaddleDetection_faster_rcnn_r50_fpn_1x_coco_bs2_fp32_MultiP_DP_N1C4_log
+```

test_tipc/prepare.sh

@@ -56,6 +56,17 @@ elif [ ${MODE} = "cpp_infer" ];then
         fi
     fi
     cd ../../
+elif [ ${MODE} = "benchmark_train" ];then
+    pip install -U pip Cython
+    pip install -r requirements.txt
+    # prepare lite benchmark coco data
+    wget -nc -P ./dataset/coco/ https://paddledet.bj.bcebos.com/data/coco_benchmark.tar
+    cd ./dataset/coco/ && tar -xvf coco_benchmark.tar && mv -u coco_benchmark/* .
+    rm -rf coco_benchmark/ && cd ../../
+    # prepare lite benchmark mot data
+    wget -nc -P ./dataset/mot/ https://paddledet.bj.bcebos.com/data/mot_benchmark.tar
+    cd ./dataset/mot/ && tar -xvf mot_benchmark.tar && mv -u mot_benchmark/* .
+    rm -rf mot_benchmark/ && cd ../../
 else
     # download coco lite data
     wget -nc -P ./dataset/coco/ https://paddledet.bj.bcebos.com/data/tipc/coco_tipc.tar

test_tipc/test_train_inference_python.sh

@@ -29,7 +29,8 @@ pretrain_model_key=$(func_parser_key "${lines[9]}")
 pretrain_model_value=$(func_parser_value "${lines[9]}")
 train_model_name=$(func_parser_value "${lines[10]}")
 train_infer_img_dir=$(func_parser_value "${lines[11]}")
-filename_key=$(func_parser_key "${lines[12]}")
+train_param_key1=$(func_parser_key "${lines[12]}")
+train_param_value1=$(func_parser_value "${lines[12]}")
 
 trainer_list=$(func_parser_value "${lines[14]}")
 norm_key=$(func_parser_key "${lines[15]}")
@@ -173,6 +174,9 @@ if [ ${MODE} = "whole_infer" ] || [ ${MODE} = "klquant_whole_infer" ]; then
     IFS="|"
     infer_quant_flag=(${infer_is_quant_list})
     for infer_mode in ${infer_mode_list[*]}; do
+        if [ ${infer_mode} = "null" ]; then
+            continue
+        fi
         if [ ${MODE} = "klquant_whole_infer" ] && [ ${infer_mode} != "kl_quant" ]; then
             continue
         fi
@@ -188,12 +192,9 @@ if [ ${MODE} = "whole_infer" ] || [ ${MODE} = "klquant_whole_infer" ]; then
             kl_quant) run_export=${kl_quant_export} ;;
             *) echo "Undefined infer_mode!"; exit 1;
         esac
-        if [ ${run_export} = "null" ]; then
-            continue
-        fi
         set_export_weight=$(func_set_params "${export_weight_key}" "${export_weight_value}")
         set_save_export_dir=$(func_set_params "${save_export_key}" "${save_export_value}")
-        set_filename=$(func_set_params "${filename_key}" "${model_name}")
+        set_filename=$(func_set_params "filename" "${model_name}")
         export_cmd="${python} ${run_export} ${set_export_weight} ${set_filename} ${set_save_export_dir} "
         echo  $export_cmd
         eval $export_cmd
@@ -269,17 +270,18 @@ else
                 set_epoch=$(func_set_params "${epoch_key}" "${epoch_num}")
                 set_pretrain=$(func_set_params "${pretrain_model_key}" "${pretrain_model_value}")
                 set_batchsize=$(func_set_params "${train_batch_key}" "${train_batch_value}")
-                set_filename=$(func_set_params "${filename_key}" "${model_name}")
+                set_filename=$(func_set_params "filename" "${model_name}")
                 set_use_gpu=$(func_set_params "${train_use_gpu_key}" "${use_gpu}")
+                set_train_params1=$(func_set_params "${train_param_key1}" "${train_param_value1}")
                 save_log="${LOG_PATH}/${trainer}_gpus_${gpu}_autocast_${autocast}"
 
                 set_save_model=$(func_set_params "${save_model_key}" "${save_log}")
                 if [ ${#gpu} -le 2 ];then  # train with cpu or single gpu
-                    cmd="${python} ${run_train} ${set_use_gpu} ${set_save_model} ${set_epoch} ${set_pretrain} ${set_batchsize} ${set_filename} ${set_autocast}"
+                    cmd="${python} ${run_train} LearningRate.base_lr=0.0001 log_iter=1 ${set_use_gpu} ${set_save_model} ${set_epoch} ${set_pretrain} ${set_batchsize} ${set_filename} ${set_train_params1} ${set_autocast}"
                 elif [ ${#ips} -le 26 ];then  # train with multi-gpu
-                    cmd="${python} -m paddle.distributed.launch --gpus=${gpu} ${run_train} ${set_use_gpu} ${set_save_model} ${set_epoch} ${set_pretrain} ${set_batchsize} ${set_filename} ${set_autocast}"
+                    cmd="${python} -m paddle.distributed.launch --gpus=${gpu} ${run_train} log_iter=1 ${set_use_gpu} ${set_save_model} ${set_epoch} ${set_pretrain} ${set_batchsize} ${set_filename} ${set_train_params1} ${set_autocast}"
                 else     # train with multi-machine
-                    cmd="${python} -m paddle.distributed.launch --ips=${ips} --gpus=${gpu} ${set_use_gpu} ${run_train} ${set_save_model} ${set_pretrain} ${set_epoch} ${set_batchsize} ${set_filename} ${set_autocast}"
+                    cmd="${python} -m paddle.distributed.launch --ips=${ips} --gpus=${gpu} ${set_use_gpu} ${run_train} log_iter=1 ${set_save_model} ${set_epoch} ${set_pretrain} ${set_batchsize} ${set_filename} ${set_train_params1} ${set_autocast}"
                 fi
                 # run train
                 eval $cmd

tests/README.md

-# 全链条CI/CE脚本详细说明
-
-```
-├── tests
-│   ├── ppdet_params
-│   │   ├── ppyolo_mbv3_large_coco_params.txt
-│   │   ├── ppyolo_r50vd_dcn_1x_coco_params.txt
-│   │   ├── ppyolov2_r50vd_dcn_365e_coco_params.txt
-│   │   ├── yolov3_darknet53_270e_coco_params.txt
-│   │   ├── ...
-│   ├── prepare.sh
-│   ├── README.md
-│   ├── requirements.txt
-│   ├── test.sh
-```
-
-## 脚本说明
-
-### test.sh
-主要运行脚本，以 `ppdet_params/xxx_params.txt` 作为参数配置，可完成相关测试方案
-### prepare.sh
-相关数据准备脚本，以 `ppdet_params/xxx_params.txt` 作为参数配置，完成数据、预训练模型、预测模型的自动下载
-
-## test.sh使用方法
-以 `yolov3_darknet53_270e_coco_params.txt` 为例，进入到PaddleDetection目录下
-
-### 模式1： 少量数据训练，少量数据预测（lite_train_infer）
-```bash
-bash ./tests/prepare.sh ./tests/ppdet_params/yolov3_darknet53_270e_coco_params.txt lite_train_infer
-bash ./tests/test.sh ./tests/ppdet_params/yolov3_darknet53_270e_coco_params.txt lite_train_infer
-```
-
-### 模式2： 少量数据训练，全量数据预测（whole_infer）
-```bash
-bash ./tests/prepare.sh ./tests/ppdet_params/yolov3_darknet53_270e_coco_params.txt whole_infer
-bash ./tests/test.sh ./tests/ppdet_params/yolov3_darknet53_270e_coco_params.txt whole_infer
-```
-
-### 模式3： 全量数据训练，全量数据预测（whole_train_infer）
-```bash
-bash ./tests/prepare.sh ./tests/ppdet_params/yolov3_darknet53_270e_coco_params.txt whole_train_infer
-bash ./tests/test.sh ./tests/ppdet_params/yolov3_darknet53_270e_coco_params.txt whole_train_infer
-```
-
-### 模式4： 不训练，全量数据预测（infer）
-```bash
-bash ./tests/prepare.sh ./tests/ppdet_params/yolov3_darknet53_270e_coco_params.txt infer
-bash ./tests/test.sh ./tests/ppdet_params/yolov3_darknet53_270e_coco_params.txt infer
-```
-**注：**
-运行`prepare.sh`时，会清空`dataset/coco/`下所有文件

tests/ppdet_params/ppyolo_mbv3_large_coco_params.txt

-===========================train_params===========================
-model_name:ppyolo_mbv3_large_coco
-python:python3.7
-gpu_list:0
-use_gpu:True
-auto_cast:null|amp
-epoch:lite_train_infer=1|whole_train_infer=405
-save_dir:./output/
-TrainReader.batch_size:lite_train_infer=2|whole_train_infer=24
-pretrain_weights:null
-train_model_name:ppyolo_mbv3_large_coco/model_final.pdparams
-train_infer_img_dir:./tests/demo/
-filename:ppyolo_mbv3_large_coco
-##
-trainer:norm_train|pact_train|fpgm_train
-norm_train:tools/train.py -c configs/ppyolo/ppyolo_mbv3_large_coco.yml -o
-pact_train:tools/train.py -c configs/ppyolo/ppyolo_mbv3_large_coco.yml --slim_config configs/slim/quant/ppyolo_mbv3_large_qat.yml -o
-fpgm_train:tools/train.py -c configs/ppyolo/ppyolo_mbv3_large_coco.yml --slim_config configs/slim/prune/ppyolo_mbv3_large_prune_fpgm.yml -o
-distill_train:null
-null:null
-null:null
-##
-===========================eval_params===========================
-eval:tools/eval.py -c configs/ppyolo/ppyolo_mbv3_large_coco.yml -o
-null:null
-##
-===========================infer_params===========================
---output_dir:null
-weights:
-norm_export:tools/export_model.py -c configs/ppyolo/ppyolo_mbv3_large_coco.yml -o
-quant_export:tools/export_model.py -c configs/ppyolo/ppyolo_mbv3_large_coco.yml --slim_config configs/slim/quant/ppyolo_mbv3_large_qat.yml -o
-fpgm_export:tools/export_model.py -c configs/ppyolo/ppyolo_mbv3_large_coco.yml --slim_config configs/slim/prune/ppyolo_mbv3_large_prune_fpgm.yml -o
-distill_export:null
-kl_quant:tools/post_quant.py -c configs/ppyolo/ppyolo_mbv3_large_coco.yml --slim_config configs/slim/post_quant/ppyolo_mbv3_large_ptq.yml -o
-export2:null
-##
-infer_model:ppyolo_mbv3_large_coco.pdparams|ppyolo_mbv3_large_qat.pdparams|ppyolo_mbv3_large_prune_fpgm.pdparams
-infer_model_dir:./tests/weights
-infer_quant:False|True|False
-inference:deploy/python/infer.py
---device:gpu|cpu
---enable_mkldnn:False|True
---cpu_threads:1|4
---batch_size:1|2
---use_tensorrt:null
---run_mode:fluid|trt_fp32|trt_fp16|trt_int8
---model_dir:
---image_dir:./dataset/coco/val2017/
---save_log_path:null
---run_benchmark:True
-null:null

tests/ppdet_params/ppyolo_r50vd_dcn_1x_coco_params.txt

-===========================train_params===========================
-model_name:ppyolo_r50vd_dcn_1x_coco
-python:python3.7
-gpu_list:0
-use_gpu:True
-auto_cast:null|amp
-epoch:lite_train_infer=1|whole_train_infer=405
-save_dir:./output/
-TrainReader.batch_size:lite_train_infer=2|whole_train_infer=24
-pretrain_weights:null
-train_model_name:ppyolo_r50vd_dcn_1x_coco/model_final.pdparams
-train_infer_img_dir:./tests/demo/
-filename:ppyolo_r50vd_dcn_1x_coco
-##
-trainer:norm_train|pact_train|fpgm_train
-norm_train:tools/train.py -c configs/ppyolo/ppyolo_r50vd_dcn_1x_coco.yml -o
-pact_train:tools/train.py -c configs/ppyolo/ppyolo_r50vd_dcn_1x_coco.yml --slim_config configs/slim/quant/ppyolo_r50vd_qat_pact.yml -o
-fpgm_train:tools/train.py -c configs/ppyolo/ppyolo_r50vd_dcn_1x_coco.yml --slim_config configs/slim/prune/ppyolo_r50vd_prune_fpgm.yml -o
-distill_train:null
-null:null
-null:null
-##
-===========================eval_params===========================
-eval:tools/eval.py -c configs/ppyolo/ppyolo_r50vd_dcn_1x_coco.yml -o
-null:null
-##
-===========================infer_params===========================
---output_dir:null
-weights:
-norm_export:tools/export_model.py -c configs/ppyolo/ppyolo_r50vd_dcn_1x_coco.yml -o
-quant_export:tools/export_model.py -c configs/ppyolo/ppyolo_r50vd_dcn_1x_coco.yml --slim_config configs/slim/quant/ppyolo_r50vd_qat_pact.yml -o
-fpgm_export:tools/export_model.py -c configs/ppyolo/ppyolo_r50vd_dcn_1x_coco.yml --slim_config configs/slim/prune/ppyolo_r50vd_prune_fpgm.yml -o
-distill_export:null
-kl_quant:tools/post_quant.py -c configs/ppyolo/ppyolo_r50vd_dcn_1x_coco.yml --slim_config configs/slim/post_quant/ppyolo_r50vd_dcn_ptq.yml -o
-export2:null
-##
-infer_model:ppyolo_r50vd_dcn_1x_coco.pdparams|ppyolo_r50vd_qat_pact.pdparams|ppyolo_r50vd_prune_fpgm.pdparams
-infer_model_dir:./tests/weights
-infer_quant:False|True|False
-inference:deploy/python/infer.py
---device:gpu|cpu
---enable_mkldnn:False|True
---cpu_threads:1|4
---batch_size:1|2
---use_tensorrt:null
---run_mode:fluid|trt_fp32|trt_fp16|trt_int8
---model_dir:
---image_dir:./dataset/coco/val2017/
---save_log_path:null
---run_benchmark:True
-null:null

tests/ppdet_params/ppyolov2_r50vd_dcn_365e_coco_params.txt

-===========================train_params===========================
-model_name:ppyolov2_r50vd_dcn_365e_coco
-python:python3.7
-gpu_list:0
-use_gpu:True
-auto_cast:null|amp
-epoch:lite_train_infer=1|whole_train_infer=365
-save_dir:./output/
-TrainReader.batch_size:lite_train_infer=2|whole_train_infer=12
-pretrain_weights:./tests/weights/ppyolov2_r50vd_dcn_365e_coco.pdparams
-train_model_name:ppyolov2_r50vd_dcn_365e_coco/model_final.pdparams
-train_infer_img_dir:./tests/demo/
-filename:ppyolov2_r50vd_dcn_365e_coco
-##
-trainer:norm_train
-norm_train:tools/train.py -c configs/ppyolo/ppyolov2_r50vd_dcn_365e_coco.yml -o
-pact_train:tools/train.py -c configs/ppyolo/ppyolov2_r50vd_dcn_365e_coco.yml --slim_config configs/slim/quant/ppyolov2_r50vd_dcn_qat.yml -o
-fpgm_train:tools/train.py -c configs/ppyolo/ppyolov2_r50vd_dcn_365e_coco.yml --slim_config configs/slim/prune/ppyolov2_r50vd_dcn_prune_fpgm.yml -o
-distill_train:null
-null:null
-null:null
-##
-===========================eval_params===========================
-eval:tools/eval.py -c configs/ppyolo/ppyolov2_r50vd_dcn_365e_coco.yml -o
-null:null
-##
-===========================infer_params===========================
---output_dir:null
-weights:
-norm_export:tools/export_model.py -c configs/ppyolo/ppyolov2_r50vd_dcn_365e_coco.yml -o
-quant_export:tools/export_model.py -c configs/ppyolo/ppyolov2_r50vd_dcn_365e_coco.yml --slim_config configs/slim/quant/ppyolov2_r50vd_dcn_qat.yml -o
-fpgm_export:tools/export_model.py -c configs/ppyolo/ppyolov2_r50vd_dcn_365e_coco.yml --slim_config configs/slim/prune/ppyolov2_r50vd_dcn_prune_fpgm.yml -o
-distill_export:null
-export1:null
-export2:null
-##
-infer_model:./tests/weights/ppyolov2_r50vd_dcn_365e_coco.pdparams
-infer_export:True
-infer_quant:False
-inference:deploy/python/infer.py
---device:gpu|cpu
---enable_mkldnn:False|True
---cpu_threads:1|4
---batch_size:1|2
---use_tensorrt:null
---run_mode:fluid|trt_fp32|trt_fp16|trt_int8
---model_dir:
---image_dir:./dataset/coco/val2017/
---save_log_path:null
---run_benchmark:True
-null:null

tests/ppdet_params/yolov3_darknet53_270e_coco_params.txt

-===========================train_params===========================
-model_name:yolov3_darknet53_270e_coco
-python:python3.7
-gpu_list:0
-use_gpu:True
-auto_cast:null|amp
-epoch:lite_train_infer=1|whole_train_infer=270
-save_dir:./output/
-TrainReader.batch_size:lite_train_infer=2|whole_train_infer=8
-pretrain_weights:null
-train_model_name:yolov3_darknet53_270e_coco/model_final.pdparams
-train_infer_img_dir:./tests/demo/
-filename:yolov3_darknet53_270e_coco
-##
-trainer:norm_train|pact_train|fpgm_train
-norm_train:tools/train.py -c configs/yolov3/yolov3_darknet53_270e_coco.yml -o
-pact_train:tools/train.py -c configs/yolov3/yolov3_darknet53_270e_coco.yml --slim_config configs/slim/quant/yolov3_darknet_qat.yml -o
-fpgm_train:tools/train.py -c configs/yolov3/yolov3_darknet53_270e_coco.yml --slim_config configs/slim/prune/yolov3_darknet_prune_fpgm.yml -o
-distill_train:null
-null:null
-null:null
-##
-===========================eval_params===========================
-eval:tools/eval.py -c configs/yolov3/yolov3_darknet53_270e_coco.yml -o
-null:null
-##
-===========================infer_params===========================
---output_dir:null
-weights:
-norm_export:tools/export_model.py -c configs/yolov3/yolov3_darknet53_270e_coco.yml -o
-quant_export:tools/export_model.py -c configs/yolov3/yolov3_darknet53_270e_coco.yml --slim_config configs/slim/quant/yolov3_darknet_qat.yml -o
-fpgm_export:tools/export_model.py -c configs/yolov3/yolov3_darknet53_270e_coco.yml --slim_config configs/slim/prune/yolov3_darknet_prune_fpgm.yml -o
-distill_export:null
-kl_quant:tools/post_quant.py -c configs/yolov3/yolov3_darknet53_270e_coco.yml --slim_config configs/slim/post_quant/yolov3_darknet53_ptq.yml -o
-export2:null
-##
-infer_model:yolov3_darknet53_270e_coco.pdparams|yolov3_darknet_coco_qat.pdparams|yolov3_darknet_prune_fpgm.pdparams
-infer_model_dir:./tests/weights
-infer_quant:False|True|False
-inference:deploy/python/infer.py
---device:gpu|cpu
---enable_mkldnn:False|True
---cpu_threads:1|4
---batch_size:1|2
---use_tensorrt:null
---run_mode:fluid|trt_fp32|trt_fp16|trt_int8
---model_dir:
---image_dir:./dataset/coco/val2017/
---save_log_path:null
---run_benchmark:True
-null:null

tests/prepare.sh

-#!/bin/bash
-FILENAME=$1
-# MODE be one of ['lite_train_infer' 'whole_infer' 'whole_train_infer', 'infer']
-MODE=$2
-
-dataline=$(cat ${FILENAME})
-
-# parser params
-IFS=$'\n'
-lines=(${dataline})
-function func_parser_key(){
-    strs=$1
-    IFS=":"
-    array=(${strs})
-    tmp=${array[0]}
-    echo ${tmp}
-}
-function func_parser_value(){
-    strs=$1
-    IFS=":"
-    array=(${strs})
-    tmp=${array[1]}
-    echo ${tmp}
-}
-IFS=$'\n'
-# The training params
-model_name=$(func_parser_value "${lines[1]}")
-python=$(func_parser_value "${lines[2]}")
-trainer_list=$(func_parser_value "${lines[14]}")
-
-# MODE be one of ['lite_train_infer' 'whole_infer' 'whole_train_infer']
-MODE=$2
-
-if [ ${MODE} = "lite_train_infer" ];then
-    mv ./dataset/coco/download_coco.py . && rm -rf ./dataset/coco/* && mv ./download_coco.py ./dataset/coco/
-    # prepare lite train data
-    wget -nc -P ./dataset/coco/ https://paddledet.bj.bcebos.com/data/coco_ce.tar
-    cd ./dataset/coco/ && tar -xvf coco_ce.tar && mv -u coco_ce/* .
-    rm -rf coco_ce/
-    # prepare lite train infer_img_dir
-    cd ../../ && mkdir -p ./tests/demo/
-    cp -u ./demo/road554.png ./demo/orange_71.jpg ./tests/demo/
-    if [[ ${model_name} = "ppyolov2_r50vd_dcn_365e_coco" ]]; then
-        wget -nc -P ./tests/weights/ https://paddledet.bj.bcebos.com/models/ppyolov2_r50vd_dcn_365e_coco.pdparams
-    fi
-elif [ ${MODE} = "whole_train_infer" ];then
-    mv ./dataset/coco/download_coco.py . && rm -rf ./dataset/coco/* && mv ./download_coco.py ./dataset/coco/
-    # prepare whole train data
-    eval "${python} ./dataset/coco/download_coco.py"
-    # prepare whole train infer_img_dir
-    mkdir -p ./tests/demo/
-    cp -u dataset/coco/val2017/* ./tests/demo/
-elif [ ${MODE} = "whole_infer" ];then
-    mv ./dataset/coco/download_coco.py . && rm -rf ./dataset/coco/* && mv ./download_coco.py ./dataset/coco/
-    # prepare whole infer data
-    wget -nc -P ./dataset/coco/ https://paddledet.bj.bcebos.com/data/coco_ce.tar
-    cd ./dataset/coco/ && tar -xvf coco_ce.tar && mv -u coco_ce/* .
-    rm -rf coco_ce/
-    # prepare whole infer infer_img_dir
-    cd ../../ && mkdir -p ./tests/demo/
-    cp -u dataset/coco/val2017/* ./tests/demo/
-else
-    mv ./dataset/coco/download_coco.py . && rm -rf ./dataset/coco/* && mv ./download_coco.py ./dataset/coco/
-    # prepare infer data
-    wget -nc -P ./dataset/coco/ https://paddledet.bj.bcebos.com/data/coco_ce.tar
-    cd ./dataset/coco/ && tar -xvf coco_ce.tar && mv -u coco_ce/* .
-    rm -rf coco_ce/ && cd ../../
-    if [[ ${model_name} = "yolov3_darknet53_270e_coco" ]]; then
-        wget -nc -P ./tests/weights/ https://paddledet.bj.bcebos.com/models/yolov3_darknet53_270e_coco.pdparams
-        wget -nc -P ./tests/weights/ https://paddledet.bj.bcebos.com/models/slim/yolov3_darknet_coco_qat.pdparams
-        wget -nc -P ./tests/weights/ https://paddledet.bj.bcebos.com/models/slim/yolov3_darknet_prune_fpgm.pdparams
-    elif [[ ${model_name} = "ppyolo_r50vd_dcn_1x_coco" ]]; then
-        wget -nc -P ./tests/weights/ https://paddledet.bj.bcebos.com/models/ppyolo_r50vd_dcn_1x_coco.pdparams
-        wget -nc -P ./tests/weights/ https://paddledet.bj.bcebos.com/models/slim/ppyolo_r50vd_qat_pact.pdparams
-        wget -nc -P ./tests/weights/ https://paddledet.bj.bcebos.com/models/slim/ppyolo_r50vd_prune_fpgm.pdparams
-    elif [[ ${model_name} = "ppyolo_mbv3_large_coco" ]]; then
-        wget -nc -P ./tests/weights/ https://paddledet.bj.bcebos.com/models/ppyolo_mbv3_large_coco.pdparams
-        wget -nc -P ./tests/weights/ https://paddledet.bj.bcebos.com/models/slim/ppyolo_mbv3_large_qat.pdparams
-        wget -nc -P ./tests/weights/ https://paddledet.bj.bcebos.com/models/slim/ppyolo_mbv3_large_prune_fpgm.pdparams
-    elif [[ ${model_name} = "ppyolov2_r50vd_dcn_365e_coco" ]]; then
-        wget -nc -P ./tests/weights/ https://paddledet.bj.bcebos.com/models/ppyolov2_r50vd_dcn_365e_coco.pdparams
-    elif [[ ${model_name} = "mask_rcnn_r50_fpn_1x_coco" ]]; then
-        wget -nc -P ./tests/weights/ https://paddledet.bj.bcebos.com/models/mask_rcnn_r50_fpn_1x_coco.pdparams
-    elif [[ ${model_name} = "solov2_r50_fpn_1x_coco" ]]; then
-        wget -nc -P ./tests/weights/ https://paddledet.bj.bcebos.com/models/solov2_r50_fpn_1x_coco.pdparams
-    else
-        sleep 2
-    fi
-fi

tests/requirements.txt

-pynvml
-psutil
-GPUtil
-paddleslim

tests/test.sh

-#!/bin/bash
-FILENAME=$1
-# MODE be one of ['lite_train_infer' 'whole_infer' 'whole_train_infer', 'infer']
-MODE=$2
-
-dataline=$(cat ${FILENAME})
-
-# parser params
-IFS=$'\n'
-lines=(${dataline})
-
-function func_parser_key(){
-    strs=$1
-    IFS=":"
-    array=(${strs})
-    tmp=${array[0]}
-    echo ${tmp}
-}
-function func_parser_value(){
-    strs=$1
-    IFS=":"
-    array=(${strs})
-    tmp=${array[1]}
-    echo ${tmp}
-}
-function func_set_params(){
-    key=$1
-    value=$2
-    if [ ${key} = "null" ];then
-        echo " "
-    elif [[ ${value} = "null" ]] || [[ ${value} = " " ]] || [ ${#value} -le 0 ];then
-        echo " "
-    else
-        echo "${key}=${value}"
-    fi
-}
-function func_parser_params(){
-    strs=$1
-    IFS=":"
-    array=(${strs})
-    key=${array[0]}
-    tmp=${array[1]}
-    IFS="|"
-    res=""
-    for _params in ${tmp[*]}; do
-        IFS="="
-        array=(${_params})
-        mode=${array[0]}
-        value=${array[1]}
-        if [[ ${mode} = ${MODE} ]]; then
-            IFS="|"
-            #echo $(func_set_params "${mode}" "${value}")
-            echo $value
-            break
-        fi
-        IFS="|"
-    done
-    echo ${res}
-}
-function status_check(){
-    last_status=$1   # the exit code
-    run_command=$2
-    run_log=$3
-    if [ $last_status -eq 0 ]; then
-        echo -e "\033[33m Run successfully with command - ${run_command}!  \033[0m" | tee -a ${run_log}
-    else
-        echo -e "\033[33m Run failed with command - ${run_command}!  \033[0m" | tee -a ${run_log}
-    fi
-}
-
-IFS=$'\n'
-# The training params
-model_name=$(func_parser_value "${lines[1]}")
-python=$(func_parser_value "${lines[2]}")
-gpu_list=$(func_parser_value "${lines[3]}")
-train_use_gpu_key=$(func_parser_key "${lines[4]}")
-train_use_gpu_value=$(func_parser_value "${lines[4]}")
-autocast_list=$(func_parser_value "${lines[5]}")
-autocast_key=$(func_parser_key "${lines[5]}")
-epoch_key=$(func_parser_key "${lines[6]}")
-epoch_num=$(func_parser_params "${lines[6]}")
-save_model_key=$(func_parser_key "${lines[7]}")
-train_batch_key=$(func_parser_key "${lines[8]}")
-train_batch_value=$(func_parser_params "${lines[8]}")
-pretrain_model_key=$(func_parser_key "${lines[9]}")
-pretrain_model_value=$(func_parser_value "${lines[9]}")
-train_model_name=$(func_parser_value "${lines[10]}")
-train_infer_img_dir=$(func_parser_value "${lines[11]}")
-train_param_key1=$(func_parser_key "${lines[12]}")
-train_param_value1=$(func_parser_value "${lines[12]}")
-
-trainer_list=$(func_parser_value "${lines[14]}")
-trainer_norm=$(func_parser_key "${lines[15]}")
-norm_trainer=$(func_parser_value "${lines[15]}")
-pact_key=$(func_parser_key "${lines[16]}")
-pact_trainer=$(func_parser_value "${lines[16]}")
-fpgm_key=$(func_parser_key "${lines[17]}")
-fpgm_trainer=$(func_parser_value "${lines[17]}")
-distill_key=$(func_parser_key "${lines[18]}")
-distill_trainer=$(func_parser_value "${lines[18]}")
-trainer_key1=$(func_parser_key "${lines[19]}")
-trainer_value1=$(func_parser_value "${lines[19]}")
-trainer_key2=$(func_parser_key "${lines[20]}")
-trainer_value2=$(func_parser_value "${lines[20]}")
-
-eval_py=$(func_parser_value "${lines[23]}")
-eval_key1=$(func_parser_key "${lines[24]}")
-eval_value1=$(func_parser_value "${lines[24]}")
-
-save_infer_key=$(func_parser_key "${lines[27]}")
-export_weight=$(func_parser_key "${lines[28]}")
-norm_export=$(func_parser_value "${lines[29]}")
-pact_export=$(func_parser_value "${lines[30]}")
-fpgm_export=$(func_parser_value "${lines[31]}")
-distill_export=$(func_parser_value "${lines[32]}")
-export_key1=$(func_parser_key "${lines[33]}")
-export_value1=$(func_parser_value "${lines[33]}")
-export_key2=$(func_parser_key "${lines[34]}")
-export_value2=$(func_parser_value "${lines[34]}")
-
-# parser inference model
-infer_model_name_list=$(func_parser_value "${lines[36]}")
-infer_model_dir=$(func_parser_value "${lines[37]}")
-infer_is_quant=$(func_parser_value "${lines[38]}")
-# parser inference
-inference_py=$(func_parser_value "${lines[39]}")
-device_key=$(func_parser_key "${lines[40]}")
-device_list=$(func_parser_value "${lines[40]}")
-use_mkldnn_key=$(func_parser_key "${lines[41]}")
-use_mkldnn_list=$(func_parser_value "${lines[41]}")
-cpu_threads_key=$(func_parser_key "${lines[42]}")
-cpu_threads_list=$(func_parser_value "${lines[42]}")
-batch_size_key=$(func_parser_key "${lines[43]}")
-batch_size_list=$(func_parser_value "${lines[43]}")
-use_trt_key=$(func_parser_key "${lines[44]}")
-use_trt_list=$(func_parser_value "${lines[44]}")
-precision_key=$(func_parser_key "${lines[45]}")
-precision_list=$(func_parser_value "${lines[45]}")
-infer_model_key=$(func_parser_key "${lines[46]}")
-image_dir_key=$(func_parser_key "${lines[47]}")
-infer_img_dir=$(func_parser_value "${lines[47]}")
-save_log_key=$(func_parser_key "${lines[48]}")
-benchmark_key=$(func_parser_key "${lines[49]}")
-benchmark_value=$(func_parser_value "${lines[49]}")
-infer_key1=$(func_parser_key "${lines[50]}")
-infer_value1=$(func_parser_value "${lines[50]}")
-
-LOG_PATH="./tests/output"
-mkdir -p ${LOG_PATH}
-status_log="${LOG_PATH}/results.log"
-
-
-function func_inference(){
-    IFS='|'
-    _python=$1
-    _script=$2
-    _model_dir=$3
-    _log_path=$4
-    _img_dir=$5
-    _flag_quant=$6
-    # inference
-    for device in ${device_list[*]}; do
-        if [ ${device} = "False" ] || [ ${device} = "cpu" ]; then
-            for use_mkldnn in ${use_mkldnn_list[*]}; do
-                if [ ${use_mkldnn} = "False" ] && [ ${_flag_quant} = "True" ]; then
-                    continue
-                fi
-                for threads in ${cpu_threads_list[*]}; do
-                    for batch_size in ${batch_size_list[*]}; do
-                        _save_log_path="${_log_path}/infer_cpu_usemkldnn_${use_mkldnn}_threads_${threads}_batchsize_${batch_size}.log"
-                        set_infer_data=$(func_set_params "${image_dir_key}" "${_img_dir}")
-                        set_benchmark=$(func_set_params "${benchmark_key}" "${benchmark_value}")
-                        set_batchsize=$(func_set_params "${batch_size_key}" "${batch_size}")
-                        set_cpu_threads=$(func_set_params "${cpu_threads_key}" "${threads}")
-                        set_model_dir=$(func_set_params "${infer_model_key}" "${_model_dir}")
-                        set_infer_params1=$(func_set_params "${infer_key1}" "${infer_value1}")
-                        command="${_python} ${_script} ${device_key}=${device} ${use_mkldnn_key}=${use_mkldnn} ${set_cpu_threads} ${set_model_dir} ${set_batchsize} ${set_infer_data} ${set_benchmark} ${set_infer_params1} > ${_save_log_path} 2>&1 "
-                        eval $command
-                        last_status=${PIPESTATUS[0]}
-                        eval "cat ${_save_log_path}"
-                        status_check $last_status "${command}" "${status_log}"
-                    done
-                done
-            done
-        elif [ ${device} = "True" ] || [ ${device} = "gpu" ]; then
-            for use_trt in ${use_trt_list[*]}; do
-                for precision in ${precision_list[*]}; do
-                    if [[ ${precision} != "fluid" ]]; then
-                        if [[ ${_flag_quant} = "False" ]] && [[ ${precision} = "trt_int8" ]]; then
-                            continue
-                        fi
-                        if [[ ${_flag_quant} = "True" ]] && [[ ${precision} != "trt_int8" ]]; then
-                            continue
-                        fi
-                    fi
-                    for batch_size in ${batch_size_list[*]}; do
-                        _save_log_path="${_log_path}/infer_gpu_usetrt_${use_trt}_precision_${precision}_batchsize_${batch_size}.log"
-                        set_infer_data=$(func_set_params "${image_dir_key}" "${_img_dir}")
-                        set_benchmark=$(func_set_params "${benchmark_key}" "${benchmark_value}")
-                        set_batchsize=$(func_set_params "${batch_size_key}" "${batch_size}")
-                        set_tensorrt=$(func_set_params "${use_trt_key}" "${use_trt}")
-                        set_precision=$(func_set_params "${precision_key}" "${precision}")
-                        set_model_dir=$(func_set_params "${infer_model_key}" "${_model_dir}")
-                        set_infer_params1=$(func_set_params "${infer_key1}" "${infer_value1}")
-                        command="${_python} ${_script} ${device_key}=${device} ${set_tensorrt} ${set_precision} ${set_model_dir} ${set_batchsize} ${set_infer_data} ${set_benchmark} ${set_infer_params1} > ${_save_log_path} 2>&1 "
-                        eval $command
-                        last_status=${PIPESTATUS[0]}
-                        eval "cat ${_save_log_path}"
-                        status_check $last_status "${command}" "${status_log}"
-
-                    done
-                done
-            done
-        else
-            echo "Does not support hardware other than CPU and GPU Currently!"
-        fi
-    done
-}
-
-if [ ${MODE} = "infer" ]; then
-    GPUID=$3
-    if [ ${#GPUID} -le 0 ];then
-        env=" "
-    else
-        env="export CUDA_VISIBLE_DEVICES=${GPUID}"
-    fi
-    # set CUDA_VISIBLE_DEVICES
-    eval $env
-    export Count=0
-    IFS="|"
-    infer_quant_flag=(${infer_is_quant})
-    set_train_params1=$(func_set_params "${train_param_key1}" "${train_param_value1}")
-    set_save_infer_key=$(func_set_params "${save_infer_key}" "${infer_model_dir}")
-    infer_model="${infer_model_dir}/${train_param_value1}"
-    for infer_model_name in ${infer_model_name_list[*]}; do
-        # run export
-        case ${Count} in
-            0) run_export=${norm_export} ;;
-            1) run_export=${pact_export} ;;
-            2) run_export=${fpgm_export} ;;
-            *) echo "Undefined run_export"; exit 1;
-        esac
-        set_export_weight=$(func_set_params "${export_weight}" "${infer_model_dir}/${infer_model_name}")
-        export_cmd="${python} ${run_export} ${set_export_weight} ${set_train_params1} ${set_save_infer_key}"
-        eval $export_cmd
-        status_export=$?
-        if [ ${status_export} = 0 ];then
-            status_check $status_export "${export_cmd}" "${status_log}"
-        fi
-        #run inference
-        is_quant=${infer_quant_flag[Count]}
-        func_inference "${python}" "${inference_py}" "${infer_model}" "${LOG_PATH}" "${infer_img_dir}" ${is_quant}
-        Count=$(($Count + 1))
-    done
-
-    # kl quant
-    if [ ${export_key1} = "kl_quant" ]; then
-        # run kl quant
-        kl_cmd="${python} ${export_value1} ${set_train_params1} ${set_save_infer_key}"
-        eval $kl_cmd
-        status_export=$?
-        if [ ${status_export} = 0 ];then
-            status_check $status_export "${kl_cmd}" "${status_log}"
-        fi
-        # run inference
-        is_quant=True
-        func_inference "${python}" "${inference_py}" "${infer_model}" "${LOG_PATH}" "${infer_img_dir}" ${is_quant}
-    fi
-
-else
-    IFS="|"
-    export Count=0
-    USE_GPU_KEY=(${train_use_gpu_value})
-    for gpu in ${gpu_list[*]}; do
-        use_gpu=${USE_GPU_KEY[Count]}
-        Count=$(($Count + 1))
-        if [ ${gpu} = "-1" ];then
-            env=""
-        elif [ ${#gpu} -le 1 ];then
-            env="export CUDA_VISIBLE_DEVICES=${gpu}"
-            eval ${env}
-        elif [ ${#gpu} -le 15 ];then
-            IFS=","
-            array=(${gpu})
-            env="export CUDA_VISIBLE_DEVICES=${array[0]}"
-            IFS="|"
-        else
-            IFS=";"
-            array=(${gpu})
-            ips=${array[0]}
-            gpu=${array[1]}
-            IFS="|"
-            env=" "
-        fi
-        for autocast in ${autocast_list[*]}; do
-            for trainer in ${trainer_list[*]}; do
-                flag_quant=False
-                if [ ${trainer} = ${pact_key} ]; then
-                    run_train=${pact_trainer}
-                    run_export=${pact_export}
-                    flag_quant=True
-                    if [ ${autocast} = "amp" ]; then
-                        continue
-                    fi
-                elif [ ${trainer} = "${fpgm_key}" ]; then
-                    run_train=${fpgm_trainer}
-                    run_export=${fpgm_export}
-                elif [ ${trainer} = "${distill_key}" ]; then
-                    run_train=${distill_trainer}
-                    run_export=${distill_export}
-                elif [ ${trainer} = ${trainer_key1} ]; then
-                    run_train=${trainer_value1}
-                    run_export=${export_value1}
-                elif [[ ${trainer} = ${trainer_key2} ]]; then
-                    run_train=${trainer_value2}
-                    run_export=${export_value2}
-                else
-                    run_train=${norm_trainer}
-                    run_export=${norm_export}
-                fi
-
-                if [ ${run_train} = "null" ]; then
-                    continue
-                fi
-
-                if [ ${autocast} = "amp" ]; then
-                    set_autocast="--fp16"
-                else
-                    set_autocast=$(func_set_params "${autocast_key}" "${autocast}")
-                fi
-                set_epoch=$(func_set_params "${epoch_key}" "${epoch_num}")
-                set_pretrain=$(func_set_params "${pretrain_model_key}" "${pretrain_model_value}")
-                set_batchsize=$(func_set_params "${train_batch_key}" "${train_batch_value}")
-                set_train_params1=$(func_set_params "${train_param_key1}" "${train_param_value1}")
-                set_use_gpu=$(func_set_params "${train_use_gpu_key}" "${use_gpu}")
-                save_log="${LOG_PATH}/${trainer}_gpus_${gpu}_autocast_${autocast}"
-
-                # load pretrain from norm training if current trainer is pact or fpgm trainer
-                if [ ${trainer} = ${pact_key} ] || [ ${trainer} = ${fpgm_key} ]; then
-                    set_pretrain=$(func_set_params "${pretrain_model_key}" "${save_log}/${train_model_name}")
-                fi
-
-                set_save_model=$(func_set_params "${save_model_key}" "${save_log}")
-                if [ ${#gpu} -le 2 ];then  # train with cpu or single gpu
-                    cmd="${python} ${run_train} ${set_use_gpu}  ${set_save_model} ${set_epoch} ${set_pretrain} ${set_batchsize} ${set_train_params1} ${set_autocast} "
-                elif [ ${#gpu} -le 15 ];then  # train with multi-gpu
-                    cmd="${python} -m paddle.distributed.launch --gpus=${gpu} ${run_train} ${set_save_model} ${set_epoch} ${set_pretrain} ${set_batchsize} ${set_train_params1} ${set_autocast}"
-                else     # train with multi-machine
-                    cmd="${python} -m paddle.distributed.launch --ips=${ips} --gpus=${gpu} ${run_train} ${set_save_model} ${set_pretrain} ${set_epoch} ${set_batchsize} ${set_train_params1} ${set_autocast}"
-                fi
-                # run train
-                # eval "unset CUDA_VISIBLE_DEVICES"
-                eval $cmd
-                status_check $? "${cmd}" "${status_log}"
-
-                set_eval_trained_weight=$(func_set_params "weights" "${save_log}/${train_model_name}")
-                # run eval
-                if [ ${eval_py} != "null" ]; then
-                    set_eval_params1=$(func_set_params "${eval_key1}" "${eval_value1}")
-                    eval_cmd="${python} ${eval_py} ${set_eval_trained_weight} ${set_use_gpu} ${set_eval_params1}"
-                    eval $eval_cmd
-                    status_check $? "${eval_cmd}" "${status_log}"
-                fi
-                # run export model
-                if [ ${run_export} != "null" ]; then
-                    # run export model
-                    save_infer_path="${save_log}"
-                    set_export_weight=$(func_set_params "${export_weight}" "${save_log}/${train_model_name}")
-                    set_save_infer_key=$(func_set_params "${save_infer_key}" "${save_infer_path}")
-                    export_cmd="${python} ${run_export} ${set_export_weight} ${set_train_params1} ${set_save_infer_key}"
-                    eval $export_cmd
-                    status_check $? "${export_cmd}" "${status_log}"
-
-                    #run inference
-                    eval $env
-                    save_infer_path="${save_log}/${train_param_value1}"
-                    func_inference "${python}" "${inference_py}" "${save_infer_path}" "${LOG_PATH}" "${train_infer_img_dir}" "${flag_quant}"
-                    # eval "unset CUDA_VISIBLE_DEVICES"
-                fi
-            done  # done with:    for trainer in ${trainer_list[*]}; do
-        done      # done with:    for autocast in ${autocast_list[*]}; do
-    done          # done with:    for gpu in ${gpu_list[*]}; do
-fi  # end if [ ${MODE} = "infer" ]; then