From 462e273f87563cf29b5c7dae5669c5adf8641b2e Mon Sep 17 00:00:00 2001 From: shangliang Xu Date: Thu, 10 Feb 2022 19:25:18 +0800 Subject: [PATCH] [TIPC] upgrade benchmark in TIPC (#5194) --- test_tipc/benchmark_train.sh | 252 ++++++++++++ ...le_detr_r50_1x_coco_train_infer_python.txt | 57 +++ ...cnn_r50_fpn_1x_coco_train_infer_python.txt | 8 +- ...cos_r50_fpn_1x_coco_train_infer_python.txt | 8 +- ...gfl_r50_fpn_1x_coco_train_infer_python.txt | 57 +++ ...hrnet_hrnet_w32_512_train_infer_python.txt | 8 +- .../hrnet_w32_256x192_train_infer_python.txt | 8 +- ..._dla34_30e_1088x608_train_infer_python.txt | 57 +++ ...knet53_30e_1088x608_train_infer_python.txt | 57 +++ ...ov2_r50_fpn_1x_coco_train_infer_python.txt | 8 +- test_tipc/docs/benchmark_train.md | 50 +++ test_tipc/prepare.sh | 11 + test_tipc/test_train_inference_python.sh | 20 +- tests/README.md | 51 --- .../ppyolo_mbv3_large_coco_params.txt | 51 --- .../ppyolo_r50vd_dcn_1x_coco_params.txt | 51 --- .../ppyolov2_r50vd_dcn_365e_coco_params.txt | 51 --- .../yolov3_darknet53_270e_coco_params.txt | 51 --- tests/prepare.sh | 89 ---- tests/requirements.txt | 4 - tests/test.sh | 383 ------------------ 21 files changed, 587 insertions(+), 745 deletions(-) create mode 100644 test_tipc/benchmark_train.sh create mode 100644 test_tipc/configs/deformable_detr/deformable_detr_r50_1x_coco_train_infer_python.txt create mode 100644 test_tipc/configs/gfl/gfl_r50_fpn_1x_coco_train_infer_python.txt create mode 100644 test_tipc/configs/mot/fairmot_dla34_30e_1088x608_train_infer_python.txt create mode 100644 test_tipc/configs/mot/jde_darknet53_30e_1088x608_train_infer_python.txt create mode 100644 test_tipc/docs/benchmark_train.md delete mode 100644 tests/README.md delete mode 100644 tests/ppdet_params/ppyolo_mbv3_large_coco_params.txt delete mode 100644 tests/ppdet_params/ppyolo_r50vd_dcn_1x_coco_params.txt delete mode 100644 tests/ppdet_params/ppyolov2_r50vd_dcn_365e_coco_params.txt delete mode 100644 tests/ppdet_params/yolov3_darknet53_270e_coco_params.txt delete mode 100644 tests/prepare.sh delete mode 100644 tests/requirements.txt delete mode 100644 tests/test.sh diff --git a/test_tipc/benchmark_train.sh b/test_tipc/benchmark_train.sh new file mode 100644 index 000000000..6741e51bf --- /dev/null +++ b/test_tipc/benchmark_train.sh @@ -0,0 +1,252 @@ +#!/bin/bash +source test_tipc/utils_func.sh + +# set env +python=python +export model_branch=`git symbolic-ref HEAD 2>/dev/null | cut -d"/" -f 3` +export model_commit=$(git log|head -n1|awk '{print $2}') +export str_tmp=$(echo `pip list|grep paddlepaddle-gpu|awk -F ' ' '{print $2}'`) +export frame_version=${str_tmp%%.post*} +export frame_commit=$(echo `${python} -c "import paddle;print(paddle.version.commit)"`) + +# run benchmark sh +# Usage: +# bash run_benchmark_train.sh config.txt params +# or +# bash run_benchmark_train.sh config.txt + +function func_parser_params(){ + strs=$1 + IFS="=" + array=(${strs}) + tmp=${array[1]} + echo ${tmp} +} + +function func_sed_params(){ + filename=$1 + line=$2 + param_value=$3 + params=`sed -n "${line}p" $filename` + IFS=":" + array=(${params}) + key=${array[0]} + new_params="${key}:${param_value}" + IFS=";" + cmd="sed -i '${line}s/.*/${new_params}/' '${filename}'" + eval $cmd +} + +function set_gpu_id(){ + string=$1 + _str=${string:1:6} + IFS="C" + arr=(${_str}) + M=${arr[0]} + P=${arr[1]} + gn=`expr $P - 1` + gpu_num=`expr $gn / $M` + seq=`seq -s "," 0 $gpu_num` + echo $seq +} + +function get_repo_name(){ + IFS=";" + cur_dir=$(pwd) + IFS="/" + arr=(${cur_dir}) + echo ${arr[-1]} +} + +FILENAME=$1 +# copy FILENAME as new +new_filename="./test_tipc/benchmark_train.txt" +cmd=`yes|cp $FILENAME $new_filename` +FILENAME=$new_filename +# MODE must be one of ['benchmark_train'] +MODE=$2 +PARAMS=$3 +# bash test_tipc/benchmark_train.sh test_tipc/configs/det_mv3_db_v2_0/train_benchmark.txt benchmark_train dynamic_bs8_null_DP_N1C1 +IFS=$'\n' +# parser params from train_benchmark.txt +dataline=`cat $FILENAME` +# parser params +IFS=$'\n' +lines=(${dataline}) +model_name=$(func_parser_value "${lines[1]}") + +# 获取benchmark_params所在的行数 +line_num=`grep -n "train_benchmark_params" $FILENAME | cut -d ":" -f 1` +# for train log parser +batch_size=$(func_parser_value "${lines[line_num]}") +line_num=`expr $line_num + 1` +fp_items=$(func_parser_value "${lines[line_num]}") +line_num=`expr $line_num + 1` +epoch=$(func_parser_value "${lines[line_num]}") + +line_num=`expr $line_num + 1` +profile_option_key=$(func_parser_key "${lines[line_num]}") +profile_option_params=$(func_parser_value "${lines[line_num]}") +profile_option="${profile_option_key}:${profile_option_params}" + +line_num=`expr $line_num + 1` +flags_value=$(func_parser_value "${lines[line_num]}") +# set flags +IFS=";" +flags_list=(${flags_value}) +for _flag in ${flags_list[*]}; do + cmd="export ${_flag}" + eval $cmd +done + +# set log_name +repo_name=$(get_repo_name ) +SAVE_LOG=${BENCHMARK_LOG_DIR:-$(pwd)} # */benchmark_log +mkdir -p "${SAVE_LOG}/benchmark_log/" +status_log="${SAVE_LOG}/benchmark_log/results.log" + +# The number of lines in which train params can be replaced. +line_python=3 +line_gpuid=4 +line_precision=6 +line_epoch=7 +line_batchsize=9 +line_profile=13 +line_eval_py=24 +line_export_py=30 + +func_sed_params "$FILENAME" "${line_eval_py}" "null" +func_sed_params "$FILENAME" "${line_export_py}" "null" +func_sed_params "$FILENAME" "${line_python}" "${python}" + +# if params +if [ ! -n "$PARAMS" ] ;then + # PARAMS input is not a word. + IFS="|" + batch_size_list=(${batch_size}) + fp_items_list=(${fp_items}) + device_num_list=(N1C4) + run_mode="DP" +else + # parser params from input: modeltype_bs${bs_item}_${fp_item}_${run_mode}_${device_num} + IFS="_" + params_list=(${PARAMS}) + model_type=${params_list[0]} + batch_size=${params_list[1]} + batch_size=`echo ${batch_size} | tr -cd "[0-9]" ` + precision=${params_list[2]} + # run_process_type=${params_list[3]} + run_mode=${params_list[3]} + device_num=${params_list[4]} + IFS=";" + + if [ ${precision} = "null" ];then + precision="fp32" + fi + + fp_items_list=($precision) + batch_size_list=($batch_size) + device_num_list=($device_num) +fi + +IFS="|" +for batch_size in ${batch_size_list[*]}; do + for precision in ${fp_items_list[*]}; do + for device_num in ${device_num_list[*]}; do + # sed batchsize and precision + func_sed_params "$FILENAME" "${line_precision}" "$precision" + func_sed_params "$FILENAME" "${line_batchsize}" "$MODE=$batch_size" + func_sed_params "$FILENAME" "${line_epoch}" "$MODE=$epoch" + gpu_id=$(set_gpu_id $device_num) + + if [ ${#gpu_id} -le 1 ];then + run_process_type="SingleP" + log_path="$SAVE_LOG/profiling_log" + mkdir -p $log_path + log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}_${device_num}_profiling" + func_sed_params "$FILENAME" "${line_gpuid}" "0" # sed used gpu_id + # set profile_option params + tmp=`sed -i "${line_profile}s/.*/${profile_option}/" "${FILENAME}"` + + # run test_train_inference_python.sh + cmd="bash test_tipc/test_train_inference_python.sh ${FILENAME} benchmark_train > ${log_path}/${log_name} 2>&1 " + echo $cmd + eval $cmd + eval "cat ${log_path}/${log_name}" + + # without profile + log_path="$SAVE_LOG/train_log" + speed_log_path="$SAVE_LOG/index" + mkdir -p $log_path + mkdir -p $speed_log_path + log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}_${device_num}_log" + speed_log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}_${device_num}_speed" + func_sed_params "$FILENAME" "${line_profile}" "null" # sed profile_id as null + cmd="bash test_tipc/test_train_inference_python.sh ${FILENAME} benchmark_train > ${log_path}/${log_name} 2>&1 " + echo $cmd + job_bt=`date '+%Y%m%d%H%M%S'` + eval $cmd + job_et=`date '+%Y%m%d%H%M%S'` + export model_run_time=$((${job_et}-${job_bt})) + eval "cat ${log_path}/${log_name}" + + # parser log + _model_name="${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}" + cmd="${python} ${BENCHMARK_ROOT}/scripts/analysis.py --filename ${log_path}/${log_name} \ + --speed_log_file '${speed_log_path}/${speed_log_name}' \ + --model_name ${_model_name} \ + --base_batch_size ${batch_size} \ + --run_mode ${run_mode} \ + --run_process_type ${run_process_type} \ + --fp_item ${precision} \ + --keyword ips: \ + --skip_steps 2 \ + --device_num ${device_num} \ + --speed_unit images/s \ + --convergence_key loss: " + echo $cmd + eval $cmd + last_status=${PIPESTATUS[0]} + status_check $last_status "${cmd}" "${status_log}" + else + IFS=";" + unset_env=`unset CUDA_VISIBLE_DEVICES` + run_process_type="MultiP" + log_path="$SAVE_LOG/train_log" + speed_log_path="$SAVE_LOG/index" + mkdir -p $log_path + mkdir -p $speed_log_path + log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}_${device_num}_log" + speed_log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}_${device_num}_speed" + func_sed_params "$FILENAME" "${line_gpuid}" "$gpu_id" # sed used gpu_id + func_sed_params "$FILENAME" "${line_profile}" "null" # sed --profile_option as null + cmd="bash test_tipc/test_train_inference_python.sh ${FILENAME} benchmark_train > ${log_path}/${log_name} 2>&1 " + echo $cmd + job_bt=`date '+%Y%m%d%H%M%S'` + eval $cmd + job_et=`date '+%Y%m%d%H%M%S'` + export model_run_time=$((${job_et}-${job_bt})) + eval "cat ${log_path}/${log_name}" + # parser log + _model_name="${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}" + + cmd="${python} ${BENCHMARK_ROOT}/scripts/analysis.py --filename ${log_path}/${log_name} \ + --speed_log_file '${speed_log_path}/${speed_log_name}' \ + --model_name ${_model_name} \ + --base_batch_size ${batch_size} \ + --run_mode ${run_mode} \ + --run_process_type ${run_process_type} \ + --fp_item ${precision} \ + --keyword ips: \ + --skip_steps 2 \ + --device_num ${device_num} \ + --speed_unit images/s \ + --convergence_key loss: " + echo $cmd + eval $cmd + last_status=${PIPESTATUS[0]} + status_check $last_status "${cmd}" "${status_log}" + fi + done + done +done diff --git a/test_tipc/configs/deformable_detr/deformable_detr_r50_1x_coco_train_infer_python.txt b/test_tipc/configs/deformable_detr/deformable_detr_r50_1x_coco_train_infer_python.txt new file mode 100644 index 000000000..c78b41204 --- /dev/null +++ b/test_tipc/configs/deformable_detr/deformable_detr_r50_1x_coco_train_infer_python.txt @@ -0,0 +1,57 @@ +===========================train_params=========================== +model_name:deformable_detr_r50_1x_coco +python:python3.7 +gpu_list:0|0,1 +use_gpu:True +auto_cast:null +epoch:lite_train_lite_infer=1|lite_train_whole_infer=1|whole_train_whole_infer=50 +save_dir:null +TrainReader.batch_size:lite_train_lite_infer=2|lite_train_whole_infer=2|whole_train_whole_infer=2 +pretrain_weights:https://paddledet.bj.bcebos.com/models/deformable_detr_r50_1x_coco.pdparams +trained_model_name:model_final.pdparams +train_infer_img_dir:./dataset/coco/test2017/ +filename:null +## +trainer:norm_train +norm_train:tools/train.py -c configs/deformable_detr/deformable_detr_r50_1x_coco.yml -o +pact_train:tools/train.py -c configs/deformable_detr/deformable_detr_r50_1x_coco.yml --slim_config _template_pact -o +fpgm_train:tools/train.py -c configs/deformable_detr/deformable_detr_r50_1x_coco.yml --slim_config _template_fpgm -o +distill_train:null +null:null +null:null +## +===========================eval_params=========================== +eval:tools/eval.py -c configs/deformable_detr/deformable_detr_r50_1x_coco.yml -o +null:null +## +===========================infer_params=========================== +--output_dir:./output_inference +weights:https://paddledet.bj.bcebos.com/models/deformable_detr_r50_1x_coco.pdparams +norm_export:tools/export_model.py -c configs/deformable_detr/deformable_detr_r50_1x_coco.yml -o +pact_export:tools/export_model.py -c configs/deformable_detr/deformable_detr_r50_1x_coco.yml --slim_config _template_pact -o +fpgm_export:tools/export_model.py -c configs/deformable_detr/deformable_detr_r50_1x_coco.yml --slim_config _template_fpgm -o +distill_export:null +export1:null +export2:null +kl_quant_export:tools/post_quant.py -c configs/deformable_detr/deformable_detr_r50_1x_coco.yml --slim_config _template_kl_quant -o +## +infer_mode:norm +infer_quant:False +inference:./deploy/python/infer.py +--device:gpu|cpu +--enable_mkldnn:True|False +--cpu_threads:1|6 +--batch_size:1|2 +--use_tensorrt:null +--run_mode:paddle|trt_fp32|trt_fp16 +--model_dir: +--image_dir:./dataset/coco/test2017/ +--save_log_path:null +--run_benchmark:False +--trt_max_shape:1600 +===========================train_benchmark_params========================== +batch_size:2 +fp_items:fp32|fp16 +epoch:1 +--profiler_options:batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile +flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98;FLAGS_conv_workspace_size_limit=4096 \ No newline at end of file diff --git a/test_tipc/configs/faster_rcnn/faster_rcnn_r50_fpn_1x_coco_train_infer_python.txt b/test_tipc/configs/faster_rcnn/faster_rcnn_r50_fpn_1x_coco_train_infer_python.txt index 370cb0507..86c12b880 100644 --- a/test_tipc/configs/faster_rcnn/faster_rcnn_r50_fpn_1x_coco_train_infer_python.txt +++ b/test_tipc/configs/faster_rcnn/faster_rcnn_r50_fpn_1x_coco_train_infer_python.txt @@ -48,4 +48,10 @@ inference:./deploy/python/infer.py --image_dir:./dataset/coco/test2017/ --save_log_path:null --run_benchmark:True ---trt_max_shape:1600 \ No newline at end of file +--trt_max_shape:1600 +===========================train_benchmark_params========================== +batch_size:2|8 +fp_items:fp32|fp16 +epoch:1 +--profiler_options:batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile +flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98;FLAGS_conv_workspace_size_limit=4096 \ No newline at end of file diff --git a/test_tipc/configs/fcos/fcos_r50_fpn_1x_coco_train_infer_python.txt b/test_tipc/configs/fcos/fcos_r50_fpn_1x_coco_train_infer_python.txt index 6c66f2830..36c2b37f5 100644 --- a/test_tipc/configs/fcos/fcos_r50_fpn_1x_coco_train_infer_python.txt +++ b/test_tipc/configs/fcos/fcos_r50_fpn_1x_coco_train_infer_python.txt @@ -48,4 +48,10 @@ inference:./deploy/python/infer.py --image_dir:./dataset/coco/test2017/ --save_log_path:null --run_benchmark:True ---trt_max_shape:1600 \ No newline at end of file +--trt_max_shape:1600 +===========================train_benchmark_params========================== +batch_size:2|8 +fp_items:fp32|fp16 +epoch:1 +--profiler_options:batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile +flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98;FLAGS_conv_workspace_size_limit=4096 \ No newline at end of file diff --git a/test_tipc/configs/gfl/gfl_r50_fpn_1x_coco_train_infer_python.txt b/test_tipc/configs/gfl/gfl_r50_fpn_1x_coco_train_infer_python.txt new file mode 100644 index 000000000..347a13fd0 --- /dev/null +++ b/test_tipc/configs/gfl/gfl_r50_fpn_1x_coco_train_infer_python.txt @@ -0,0 +1,57 @@ +===========================train_params=========================== +model_name:gfl_r50_fpn_1x_coco +python:python3.7 +gpu_list:0|0,1 +use_gpu:True +auto_cast:null +epoch:lite_train_lite_infer=1|lite_train_whole_infer=1|whole_train_whole_infer=12 +save_dir:null +TrainReader.batch_size:lite_train_lite_infer=2|lite_train_whole_infer=2|whole_train_whole_infer=2 +pretrain_weights:https://paddledet.bj.bcebos.com/models/gfl_r50_fpn_1x_coco.pdparams +trained_model_name:model_final.pdparams +train_infer_img_dir:./dataset/coco/test2017/ +filename:null +## +trainer:norm_train +norm_train:tools/train.py -c configs/gfl/gfl_r50_fpn_1x_coco.yml -o +pact_train:tools/train.py -c configs/gfl/gfl_r50_fpn_1x_coco.yml --slim_config _template_pact -o +fpgm_train:tools/train.py -c configs/gfl/gfl_r50_fpn_1x_coco.yml --slim_config _template_fpgm -o +distill_train:null +null:null +null:null +## +===========================eval_params=========================== +eval:tools/eval.py -c configs/gfl/gfl_r50_fpn_1x_coco.yml -o +null:null +## +===========================infer_params=========================== +--output_dir:./output_inference +weights:https://paddledet.bj.bcebos.com/models/gfl_r50_fpn_1x_coco.pdparams +norm_export:tools/export_model.py -c configs/gfl/gfl_r50_fpn_1x_coco.yml -o +pact_export:tools/export_model.py -c configs/gfl/gfl_r50_fpn_1x_coco.yml --slim_config _template_pact -o +fpgm_export:tools/export_model.py -c configs/gfl/gfl_r50_fpn_1x_coco.yml --slim_config _template_fpgm -o +distill_export:null +export1:null +export2:null +kl_quant_export:tools/post_quant.py -c configs/gfl/gfl_r50_fpn_1x_coco.yml --slim_config _template_kl_quant -o +## +infer_mode:norm +infer_quant:False +inference:./deploy/python/infer.py +--device:gpu|cpu +--enable_mkldnn:True|False +--cpu_threads:1|6 +--batch_size:1|2 +--use_tensorrt:null +--run_mode:paddle|trt_fp32|trt_fp16 +--model_dir: +--image_dir:./dataset/coco/test2017/ +--save_log_path:null +--run_benchmark:False +--trt_max_shape:1600 +===========================train_benchmark_params========================== +batch_size:2|8 +fp_items:fp32|fp16 +epoch:1 +--profiler_options:batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile +flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98;FLAGS_conv_workspace_size_limit=4096 \ No newline at end of file diff --git a/test_tipc/configs/keypoint/higherhrnet_hrnet_w32_512_train_infer_python.txt b/test_tipc/configs/keypoint/higherhrnet_hrnet_w32_512_train_infer_python.txt index b8af0121d..d7a756171 100644 --- a/test_tipc/configs/keypoint/higherhrnet_hrnet_w32_512_train_infer_python.txt +++ b/test_tipc/configs/keypoint/higherhrnet_hrnet_w32_512_train_infer_python.txt @@ -48,4 +48,10 @@ inference:./deploy/python/keypoint_infer.py --image_dir:./dataset/coco/test2017/ --save_log_path:null --run_benchmark:True ---trt_max_shape:1600 \ No newline at end of file +--trt_max_shape:1600 +===========================train_benchmark_params========================== +batch_size:20|24 +fp_items:fp32|fp16 +epoch:1 +--profiler_options:batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile +flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98;FLAGS_conv_workspace_size_limit=4096 \ No newline at end of file diff --git a/test_tipc/configs/keypoint/hrnet_w32_256x192_train_infer_python.txt b/test_tipc/configs/keypoint/hrnet_w32_256x192_train_infer_python.txt index daf12b1a5..cd3fd94f8 100644 --- a/test_tipc/configs/keypoint/hrnet_w32_256x192_train_infer_python.txt +++ b/test_tipc/configs/keypoint/hrnet_w32_256x192_train_infer_python.txt @@ -48,4 +48,10 @@ inference:./deploy/python/keypoint_infer.py --image_dir:./dataset/coco/test2017/ --save_log_path:null --run_benchmark:True -null:null \ No newline at end of file +null:null +===========================train_benchmark_params========================== +batch_size:64|160 +fp_items:fp32|fp16 +epoch:1 +--profiler_options:batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile +flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98;FLAGS_conv_workspace_size_limit=4096 \ No newline at end of file diff --git a/test_tipc/configs/mot/fairmot_dla34_30e_1088x608_train_infer_python.txt b/test_tipc/configs/mot/fairmot_dla34_30e_1088x608_train_infer_python.txt new file mode 100644 index 000000000..dab06fb9f --- /dev/null +++ b/test_tipc/configs/mot/fairmot_dla34_30e_1088x608_train_infer_python.txt @@ -0,0 +1,57 @@ +===========================train_params=========================== +model_name:fairmot_dla34_30e_1088x608 +python:python3.7 +gpu_list:0|0,1 +use_gpu:True +auto_cast:null +epoch:lite_train_lite_infer=1|lite_train_whole_infer=1|whole_train_whole_infer=30 +save_dir:null +TrainReader.batch_size:lite_train_lite_infer=2|lite_train_whole_infer=2|whole_train_whole_infer=6 +pretrain_weights:https://paddledet.bj.bcebos.com/models/mot/fairmot_dla34_30e_1088x608.pdparams +trained_model_name:model_final.pdparams +train_infer_img_dir:./dataset/coco/test2017/ +filename:null +## +trainer:norm_train +norm_train:tools/train.py -c configs/mot/fairmot/fairmot_dla34_30e_1088x608.yml -o +pact_train:tools/train.py -c configs/mot/fairmot/fairmot_dla34_30e_1088x608.yml --slim_config _template_pact -o +fpgm_train:tools/train.py -c configs/mot/fairmot/fairmot_dla34_30e_1088x608.yml --slim_config _template_fpgm -o +distill_train:null +null:null +null:null +## +===========================eval_params=========================== +eval:tools/eval.py -c configs/mot/fairmot/fairmot_dla34_30e_1088x608.yml -o +null:null +## +===========================infer_params=========================== +--output_dir:./output_inference +weights:https://paddledet.bj.bcebos.com/models/mot/fairmot_dla34_30e_1088x608.pdparams +norm_export:tools/export_model.py -c configs/mot/fairmot/fairmot_dla34_30e_1088x608.yml -o +pact_export:tools/export_model.py -c configs/mot/fairmot/fairmot_dla34_30e_1088x608.yml --slim_config _template_pact -o +fpgm_export:tools/export_model.py -c configs/mot/fairmot/fairmot_dla34_30e_1088x608.yml --slim_config _template_fpgm -o +distill_export:null +export1:null +export2:null +kl_quant_export:tools/post_quant.py -c configs/mot/fairmot/fairmot_dla34_30e_1088x608.yml --slim_config _template_kl_quant -o +## +infer_mode:norm +infer_quant:False +inference:./deploy/python/infer.py +--device:gpu|cpu +--enable_mkldnn:True|False +--cpu_threads:1|6 +--batch_size:1|2 +--use_tensorrt:null +--run_mode:paddle|trt_fp32|trt_fp16 +--model_dir: +--image_dir:./dataset/coco/test2017/ +--save_log_path:null +--run_benchmark:False +--trt_max_shape:1600 +===========================train_benchmark_params========================== +batch_size:6|22 +fp_items:fp32|fp16 +epoch:1 +--profiler_options:batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile +flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98;FLAGS_conv_workspace_size_limit=4096 \ No newline at end of file diff --git a/test_tipc/configs/mot/jde_darknet53_30e_1088x608_train_infer_python.txt b/test_tipc/configs/mot/jde_darknet53_30e_1088x608_train_infer_python.txt new file mode 100644 index 000000000..787af45f5 --- /dev/null +++ b/test_tipc/configs/mot/jde_darknet53_30e_1088x608_train_infer_python.txt @@ -0,0 +1,57 @@ +===========================train_params=========================== +model_name:jde_darknet53_30e_1088x608 +python:python3.7 +gpu_list:0|0,1 +use_gpu:True +auto_cast:null +epoch:lite_train_lite_infer=1|lite_train_whole_infer=1|whole_train_whole_infer=30 +save_dir:null +TrainReader.batch_size:lite_train_lite_infer=2|lite_train_whole_infer=2|whole_train_whole_infer=4 +pretrain_weights:https://paddledet.bj.bcebos.com/models/mot/jde_darknet53_30e_1088x608.pdparams +trained_model_name:model_final.pdparams +train_infer_img_dir:./dataset/coco/test2017/ +filename:null +## +trainer:norm_train +norm_train:tools/train.py -c configs/mot/jde/jde_darknet53_30e_1088x608.yml -o +pact_train:tools/train.py -c configs/mot/jde/jde_darknet53_30e_1088x608.yml --slim_config _template_pact -o +fpgm_train:tools/train.py -c configs/mot/jde/jde_darknet53_30e_1088x608.yml --slim_config _template_fpgm -o +distill_train:null +null:null +null:null +## +===========================eval_params=========================== +eval:tools/eval.py -c configs/mot/jde/jde_darknet53_30e_1088x608.yml -o +null:null +## +===========================infer_params=========================== +--output_dir:./output_inference +weights:https://paddledet.bj.bcebos.com/models/mot/jde_darknet53_30e_1088x608.pdparams +norm_export:tools/export_model.py -c configs/mot/jde/jde_darknet53_30e_1088x608.yml -o +pact_export:tools/export_model.py -c configs/mot/jde/jde_darknet53_30e_1088x608.yml --slim_config _template_pact -o +fpgm_export:tools/export_model.py -c configs/mot/jde/jde_darknet53_30e_1088x608.yml --slim_config _template_fpgm -o +distill_export:null +export1:null +export2:null +kl_quant_export:tools/post_quant.py -c configs/mot/jde/jde_darknet53_30e_1088x608.yml --slim_config _template_kl_quant -o +## +infer_mode:norm +infer_quant:False +inference:./deploy/python/infer.py +--device:gpu|cpu +--enable_mkldnn:True|False +--cpu_threads:1|6 +--batch_size:1|2 +--use_tensorrt:null +--run_mode:paddle|trt_fp32|trt_fp16 +--model_dir: +--image_dir:./dataset/coco/test2017/ +--save_log_path:null +--run_benchmark:False +--trt_max_shape:1600 +===========================train_benchmark_params========================== +batch_size:4|14 +fp_items:fp32|fp16 +epoch:1 +--profiler_options:batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile +flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98;FLAGS_conv_workspace_size_limit=4096 \ No newline at end of file diff --git a/test_tipc/configs/solov2/solov2_r50_fpn_1x_coco_train_infer_python.txt b/test_tipc/configs/solov2/solov2_r50_fpn_1x_coco_train_infer_python.txt index 7a72256f4..14f8cee5b 100644 --- a/test_tipc/configs/solov2/solov2_r50_fpn_1x_coco_train_infer_python.txt +++ b/test_tipc/configs/solov2/solov2_r50_fpn_1x_coco_train_infer_python.txt @@ -48,4 +48,10 @@ inference:./deploy/python/infer.py --image_dir:./dataset/coco/test2017/ --save_log_path:null --run_benchmark:True ---trt_max_shape:1600 \ No newline at end of file +--trt_max_shape:1600 +===========================train_benchmark_params========================== +batch_size:2|4 +fp_items:fp32|fp16 +epoch:1 +--profiler_options:batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile +flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98;FLAGS_conv_workspace_size_limit=4096 \ No newline at end of file diff --git a/test_tipc/docs/benchmark_train.md b/test_tipc/docs/benchmark_train.md new file mode 100644 index 000000000..82c3b07ab --- /dev/null +++ b/test_tipc/docs/benchmark_train.md @@ -0,0 +1,50 @@ +# TIPC Linux端Benchmark测试文档 + +该文档为Benchmark测试说明，Benchmark预测功能测试的主程序为`benchmark_train.sh`，用于验证监控模型训练的性能。 + +# 1. 测试流程 +## 1.1 准备数据和环境安装 +运行`test_tipc/prepare.sh`，完成训练数据准备和安装环境流程。 + +```shell +# 运行格式：bash test_tipc/prepare.sh train_benchmark.txt mode +bash test_tipc/prepare.sh test_tipc/configs/faster_rcnn/faster_rcnn_r50_fpn_1x_coco_train_infer_python.txt benchmark_train +``` + +## 1.2 功能测试 +执行`test_tipc/benchmark_train.sh`，完成模型训练和日志解析 + +```shell +# 运行格式：bash test_tipc/benchmark_train.sh train_benchmark.txt mode +bash test_tipc/benchmark_train.sh test_tipc/configs/faster_rcnn/faster_rcnn_r50_fpn_1x_coco_train_infer_python.txt benchmark_train +``` + +`test_tipc/benchmark_train.sh`支持根据传入的第三个参数实现只运行某一个训练配置，如下： +```shell +# 运行格式：bash test_tipc/benchmark_train.sh train_benchmark.txt mode +bash test_tipc/benchmark_train.sh test_tipc/configs/faster_rcnn/faster_rcnn_r50_fpn_1x_coco_train_infer_python.txt benchmark_train dynamic_bs2_fp32_DP_N1C1 +``` +dynamic_bs2_fp32_DP_N1C1为test_tipc/benchmark_train.sh传入的参数，格式如下： +`${modeltype}_${batch_size}_${fp_item}_${run_mode}_${device_num}` +包含的信息有：模型类型、batchsize大小、训练精度如fp32,fp16等、分布式运行模式以及分布式训练使用的机器信息如单机单卡（N1C1）。 + + +## 2. 日志输出 + +运行后将保存模型的训练日志和解析日志，使用 `test_tipc/configs/faster_rcnn/faster_rcnn_r50_fpn_1x_coco_train_infer_python.txt` 参数文件的训练日志解析结果是： + +``` +{"model_branch": "tipc_fuse_benchmark", "model_commit": "4cce901d231f7954468045cf96302505bd6be495", "model_name": "faster_rcnn_r50_fpn_1x_coco_bs2_fp32_SingleP_DP", "batch_size": 2, "fp_item": "fp32", "run_process_type": "SingleP", "run_mode": "DP", "convergence_value": "0.556966", "convergence_key": "loss:", "ips": 4.857, "speed_unit": "images/s", "device_num": "N1C1", "model_run_time": "590", "frame_commit": "6b0c57cf65945e97d87a8fba89c0a2fc18dd8544", "frame_version": "0.0.0"} +``` + +训练日志和日志解析结果保存在benchmark_log目录下，文件组织格式如下： +``` +train_log/ +├── index +│ └── PaddleDetection_faster_rcnn_r50_fpn_1x_coco_bs2_fp32_SingleP_DP_N1C1_speed +├── profiling_log +│ └── PaddleDetection_faster_rcnn_r50_fpn_1x_coco_bs2_fp32_SingleP_DP_N1C1_profiling +└── train_log + ├── PaddleDetection_faster_rcnn_r50_fpn_1x_coco_bs2_fp32_SingleP_DP_N1C1_log + └── PaddleDetection_faster_rcnn_r50_fpn_1x_coco_bs2_fp32_MultiP_DP_N1C4_log +``` diff --git a/test_tipc/prepare.sh b/test_tipc/prepare.sh index ca3328825..6bee7f061 100644 --- a/test_tipc/prepare.sh +++ b/test_tipc/prepare.sh @@ -56,6 +56,17 @@ elif [ ${MODE} = "cpp_infer" ];then fi fi cd ../../ +elif [ ${MODE} = "benchmark_train" ];then + pip install -U pip Cython + pip install -r requirements.txt + # prepare lite benchmark coco data + wget -nc -P ./dataset/coco/ https://paddledet.bj.bcebos.com/data/coco_benchmark.tar + cd ./dataset/coco/ && tar -xvf coco_benchmark.tar && mv -u coco_benchmark/* . + rm -rf coco_benchmark/ && cd ../../ + # prepare lite benchmark mot data + wget -nc -P ./dataset/mot/ https://paddledet.bj.bcebos.com/data/mot_benchmark.tar + cd ./dataset/mot/ && tar -xvf mot_benchmark.tar && mv -u mot_benchmark/* . + rm -rf mot_benchmark/ && cd ../../ else # download coco lite data wget -nc -P ./dataset/coco/ https://paddledet.bj.bcebos.com/data/tipc/coco_tipc.tar diff --git a/test_tipc/test_train_inference_python.sh b/test_tipc/test_train_inference_python.sh index 299f0248d..18711603a 100644 --- a/test_tipc/test_train_inference_python.sh +++ b/test_tipc/test_train_inference_python.sh @@ -29,7 +29,8 @@ pretrain_model_key=$(func_parser_key "${lines[9]}") pretrain_model_value=$(func_parser_value "${lines[9]}") train_model_name=$(func_parser_value "${lines[10]}") train_infer_img_dir=$(func_parser_value "${lines[11]}") -filename_key=$(func_parser_key "${lines[12]}") +train_param_key1=$(func_parser_key "${lines[12]}") +train_param_value1=$(func_parser_value "${lines[12]}") trainer_list=$(func_parser_value "${lines[14]}") norm_key=$(func_parser_key "${lines[15]}") @@ -173,6 +174,9 @@ if [ ${MODE} = "whole_infer" ] || [ ${MODE} = "klquant_whole_infer" ]; then IFS="|" infer_quant_flag=(${infer_is_quant_list}) for infer_mode in ${infer_mode_list[*]}; do + if [ ${infer_mode} = "null" ]; then + continue + fi if [ ${MODE} = "klquant_whole_infer" ] && [ ${infer_mode} != "kl_quant" ]; then continue fi @@ -188,12 +192,9 @@ if [ ${MODE} = "whole_infer" ] || [ ${MODE} = "klquant_whole_infer" ]; then kl_quant) run_export=${kl_quant_export} ;; *) echo "Undefined infer_mode!"; exit 1; esac - if [ ${run_export} = "null" ]; then - continue - fi set_export_weight=$(func_set_params "${export_weight_key}" "${export_weight_value}") set_save_export_dir=$(func_set_params "${save_export_key}" "${save_export_value}") - set_filename=$(func_set_params "${filename_key}" "${model_name}") + set_filename=$(func_set_params "filename" "${model_name}") export_cmd="${python} ${run_export} ${set_export_weight} ${set_filename} ${set_save_export_dir} " echo $export_cmd eval $export_cmd @@ -269,17 +270,18 @@ else set_epoch=$(func_set_params "${epoch_key}" "${epoch_num}") set_pretrain=$(func_set_params "${pretrain_model_key}" "${pretrain_model_value}") set_batchsize=$(func_set_params "${train_batch_key}" "${train_batch_value}") - set_filename=$(func_set_params "${filename_key}" "${model_name}") + set_filename=$(func_set_params "filename" "${model_name}") set_use_gpu=$(func_set_params "${train_use_gpu_key}" "${use_gpu}") + set_train_params1=$(func_set_params "${train_param_key1}" "${train_param_value1}") save_log="${LOG_PATH}/${trainer}_gpus_${gpu}_autocast_${autocast}" set_save_model=$(func_set_params "${save_model_key}" "${save_log}") if [ ${#gpu} -le 2 ];then # train with cpu or single gpu - cmd="${python} ${run_train} ${set_use_gpu} ${set_save_model} ${set_epoch} ${set_pretrain} ${set_batchsize} ${set_filename} ${set_autocast}" + cmd="${python} ${run_train} LearningRate.base_lr=0.0001 log_iter=1 ${set_use_gpu} ${set_save_model} ${set_epoch} ${set_pretrain} ${set_batchsize} ${set_filename} ${set_train_params1} ${set_autocast}" elif [ ${#ips} -le 26 ];then # train with multi-gpu - cmd="${python} -m paddle.distributed.launch --gpus=${gpu} ${run_train} ${set_use_gpu} ${set_save_model} ${set_epoch} ${set_pretrain} ${set_batchsize} ${set_filename} ${set_autocast}" + cmd="${python} -m paddle.distributed.launch --gpus=${gpu} ${run_train} log_iter=1 ${set_use_gpu} ${set_save_model} ${set_epoch} ${set_pretrain} ${set_batchsize} ${set_filename} ${set_train_params1} ${set_autocast}" else # train with multi-machine - cmd="${python} -m paddle.distributed.launch --ips=${ips} --gpus=${gpu} ${set_use_gpu} ${run_train} ${set_save_model} ${set_pretrain} ${set_epoch} ${set_batchsize} ${set_filename} ${set_autocast}" + cmd="${python} -m paddle.distributed.launch --ips=${ips} --gpus=${gpu} ${set_use_gpu} ${run_train} log_iter=1 ${set_save_model} ${set_epoch} ${set_pretrain} ${set_batchsize} ${set_filename} ${set_train_params1} ${set_autocast}" fi # run train eval $cmd diff --git a/tests/README.md b/tests/README.md deleted file mode 100644 index e1d4f8ffa..000000000 --- a/tests/README.md +++ /dev/null @@ -1,51 +0,0 @@ -# 全链条CI/CE脚本详细说明 - -``` -├── tests -│ ├── ppdet_params -│ │ ├── ppyolo_mbv3_large_coco_params.txt -│ │ ├── ppyolo_r50vd_dcn_1x_coco_params.txt -│ │ ├── ppyolov2_r50vd_dcn_365e_coco_params.txt -│ │ ├── yolov3_darknet53_270e_coco_params.txt -│ │ ├── ... -│ ├── prepare.sh -│ ├── README.md -│ ├── requirements.txt -│ ├── test.sh -``` - -## 脚本说明 - -### test.sh -主要运行脚本，以 `ppdet_params/xxx_params.txt` 作为参数配置，可完成相关测试方案 -### prepare.sh -相关数据准备脚本，以 `ppdet_params/xxx_params.txt` 作为参数配置，完成数据、预训练模型、预测模型的自动下载 - -## test.sh使用方法 -以 `yolov3_darknet53_270e_coco_params.txt` 为例，进入到PaddleDetection目录下 - -### 模式1： 少量数据训练，少量数据预测（lite_train_infer） -```bash -bash ./tests/prepare.sh ./tests/ppdet_params/yolov3_darknet53_270e_coco_params.txt lite_train_infer -bash ./tests/test.sh ./tests/ppdet_params/yolov3_darknet53_270e_coco_params.txt lite_train_infer -``` - -### 模式2： 少量数据训练，全量数据预测（whole_infer） -```bash -bash ./tests/prepare.sh ./tests/ppdet_params/yolov3_darknet53_270e_coco_params.txt whole_infer -bash ./tests/test.sh ./tests/ppdet_params/yolov3_darknet53_270e_coco_params.txt whole_infer -``` - -### 模式3： 全量数据训练，全量数据预测（whole_train_infer） -```bash -bash ./tests/prepare.sh ./tests/ppdet_params/yolov3_darknet53_270e_coco_params.txt whole_train_infer -bash ./tests/test.sh ./tests/ppdet_params/yolov3_darknet53_270e_coco_params.txt whole_train_infer -``` - -### 模式4： 不训练，全量数据预测（infer） -```bash -bash ./tests/prepare.sh ./tests/ppdet_params/yolov3_darknet53_270e_coco_params.txt infer -bash ./tests/test.sh ./tests/ppdet_params/yolov3_darknet53_270e_coco_params.txt infer -``` -**注：** -运行`prepare.sh`时，会清空`dataset/coco/`下所有文件 diff --git a/tests/ppdet_params/ppyolo_mbv3_large_coco_params.txt b/tests/ppdet_params/ppyolo_mbv3_large_coco_params.txt deleted file mode 100644 index e91713148..000000000 --- a/tests/ppdet_params/ppyolo_mbv3_large_coco_params.txt +++ /dev/null @@ -1,51 +0,0 @@ -===========================train_params=========================== -model_name:ppyolo_mbv3_large_coco -python:python3.7 -gpu_list:0 -use_gpu:True -auto_cast:null|amp -epoch:lite_train_infer=1|whole_train_infer=405 -save_dir:./output/ -TrainReader.batch_size:lite_train_infer=2|whole_train_infer=24 -pretrain_weights:null -train_model_name:ppyolo_mbv3_large_coco/model_final.pdparams -train_infer_img_dir:./tests/demo/ -filename:ppyolo_mbv3_large_coco -## -trainer:norm_train|pact_train|fpgm_train -norm_train:tools/train.py -c configs/ppyolo/ppyolo_mbv3_large_coco.yml -o -pact_train:tools/train.py -c configs/ppyolo/ppyolo_mbv3_large_coco.yml --slim_config configs/slim/quant/ppyolo_mbv3_large_qat.yml -o -fpgm_train:tools/train.py -c configs/ppyolo/ppyolo_mbv3_large_coco.yml --slim_config configs/slim/prune/ppyolo_mbv3_large_prune_fpgm.yml -o -distill_train:null -null:null -null:null -## -===========================eval_params=========================== -eval:tools/eval.py -c configs/ppyolo/ppyolo_mbv3_large_coco.yml -o -null:null -## -===========================infer_params=========================== ---output_dir:null -weights: -norm_export:tools/export_model.py -c configs/ppyolo/ppyolo_mbv3_large_coco.yml -o -quant_export:tools/export_model.py -c configs/ppyolo/ppyolo_mbv3_large_coco.yml --slim_config configs/slim/quant/ppyolo_mbv3_large_qat.yml -o -fpgm_export:tools/export_model.py -c configs/ppyolo/ppyolo_mbv3_large_coco.yml --slim_config configs/slim/prune/ppyolo_mbv3_large_prune_fpgm.yml -o -distill_export:null -kl_quant:tools/post_quant.py -c configs/ppyolo/ppyolo_mbv3_large_coco.yml --slim_config configs/slim/post_quant/ppyolo_mbv3_large_ptq.yml -o -export2:null -## -infer_model:ppyolo_mbv3_large_coco.pdparams|ppyolo_mbv3_large_qat.pdparams|ppyolo_mbv3_large_prune_fpgm.pdparams -infer_model_dir:./tests/weights -infer_quant:False|True|False -inference:deploy/python/infer.py ---device:gpu|cpu ---enable_mkldnn:False|True ---cpu_threads:1|4 ---batch_size:1|2 ---use_tensorrt:null ---run_mode:fluid|trt_fp32|trt_fp16|trt_int8 ---model_dir: ---image_dir:./dataset/coco/val2017/ ---save_log_path:null ---run_benchmark:True -null:null diff --git a/tests/ppdet_params/ppyolo_r50vd_dcn_1x_coco_params.txt b/tests/ppdet_params/ppyolo_r50vd_dcn_1x_coco_params.txt deleted file mode 100644 index d30124491..000000000 --- a/tests/ppdet_params/ppyolo_r50vd_dcn_1x_coco_params.txt +++ /dev/null @@ -1,51 +0,0 @@ -===========================train_params=========================== -model_name:ppyolo_r50vd_dcn_1x_coco -python:python3.7 -gpu_list:0 -use_gpu:True -auto_cast:null|amp -epoch:lite_train_infer=1|whole_train_infer=405 -save_dir:./output/ -TrainReader.batch_size:lite_train_infer=2|whole_train_infer=24 -pretrain_weights:null -train_model_name:ppyolo_r50vd_dcn_1x_coco/model_final.pdparams -train_infer_img_dir:./tests/demo/ -filename:ppyolo_r50vd_dcn_1x_coco -## -trainer:norm_train|pact_train|fpgm_train -norm_train:tools/train.py -c configs/ppyolo/ppyolo_r50vd_dcn_1x_coco.yml -o -pact_train:tools/train.py -c configs/ppyolo/ppyolo_r50vd_dcn_1x_coco.yml --slim_config configs/slim/quant/ppyolo_r50vd_qat_pact.yml -o -fpgm_train:tools/train.py -c configs/ppyolo/ppyolo_r50vd_dcn_1x_coco.yml --slim_config configs/slim/prune/ppyolo_r50vd_prune_fpgm.yml -o -distill_train:null -null:null -null:null -## -===========================eval_params=========================== -eval:tools/eval.py -c configs/ppyolo/ppyolo_r50vd_dcn_1x_coco.yml -o -null:null -## -===========================infer_params=========================== ---output_dir:null -weights: -norm_export:tools/export_model.py -c configs/ppyolo/ppyolo_r50vd_dcn_1x_coco.yml -o -quant_export:tools/export_model.py -c configs/ppyolo/ppyolo_r50vd_dcn_1x_coco.yml --slim_config configs/slim/quant/ppyolo_r50vd_qat_pact.yml -o -fpgm_export:tools/export_model.py -c configs/ppyolo/ppyolo_r50vd_dcn_1x_coco.yml --slim_config configs/slim/prune/ppyolo_r50vd_prune_fpgm.yml -o -distill_export:null -kl_quant:tools/post_quant.py -c configs/ppyolo/ppyolo_r50vd_dcn_1x_coco.yml --slim_config configs/slim/post_quant/ppyolo_r50vd_dcn_ptq.yml -o -export2:null -## -infer_model:ppyolo_r50vd_dcn_1x_coco.pdparams|ppyolo_r50vd_qat_pact.pdparams|ppyolo_r50vd_prune_fpgm.pdparams -infer_model_dir:./tests/weights -infer_quant:False|True|False -inference:deploy/python/infer.py ---device:gpu|cpu ---enable_mkldnn:False|True ---cpu_threads:1|4 ---batch_size:1|2 ---use_tensorrt:null ---run_mode:fluid|trt_fp32|trt_fp16|trt_int8 ---model_dir: ---image_dir:./dataset/coco/val2017/ ---save_log_path:null ---run_benchmark:True -null:null diff --git a/tests/ppdet_params/ppyolov2_r50vd_dcn_365e_coco_params.txt b/tests/ppdet_params/ppyolov2_r50vd_dcn_365e_coco_params.txt deleted file mode 100644 index 26e7e03cd..000000000 --- a/tests/ppdet_params/ppyolov2_r50vd_dcn_365e_coco_params.txt +++ /dev/null @@ -1,51 +0,0 @@ -===========================train_params=========================== -model_name:ppyolov2_r50vd_dcn_365e_coco -python:python3.7 -gpu_list:0 -use_gpu:True -auto_cast:null|amp -epoch:lite_train_infer=1|whole_train_infer=365 -save_dir:./output/ -TrainReader.batch_size:lite_train_infer=2|whole_train_infer=12 -pretrain_weights:./tests/weights/ppyolov2_r50vd_dcn_365e_coco.pdparams -train_model_name:ppyolov2_r50vd_dcn_365e_coco/model_final.pdparams -train_infer_img_dir:./tests/demo/ -filename:ppyolov2_r50vd_dcn_365e_coco -## -trainer:norm_train -norm_train:tools/train.py -c configs/ppyolo/ppyolov2_r50vd_dcn_365e_coco.yml -o -pact_train:tools/train.py -c configs/ppyolo/ppyolov2_r50vd_dcn_365e_coco.yml --slim_config configs/slim/quant/ppyolov2_r50vd_dcn_qat.yml -o -fpgm_train:tools/train.py -c configs/ppyolo/ppyolov2_r50vd_dcn_365e_coco.yml --slim_config configs/slim/prune/ppyolov2_r50vd_dcn_prune_fpgm.yml -o -distill_train:null -null:null -null:null -## -===========================eval_params=========================== -eval:tools/eval.py -c configs/ppyolo/ppyolov2_r50vd_dcn_365e_coco.yml -o -null:null -## -===========================infer_params=========================== ---output_dir:null -weights: -norm_export:tools/export_model.py -c configs/ppyolo/ppyolov2_r50vd_dcn_365e_coco.yml -o -quant_export:tools/export_model.py -c configs/ppyolo/ppyolov2_r50vd_dcn_365e_coco.yml --slim_config configs/slim/quant/ppyolov2_r50vd_dcn_qat.yml -o -fpgm_export:tools/export_model.py -c configs/ppyolo/ppyolov2_r50vd_dcn_365e_coco.yml --slim_config configs/slim/prune/ppyolov2_r50vd_dcn_prune_fpgm.yml -o -distill_export:null -export1:null -export2:null -## -infer_model:./tests/weights/ppyolov2_r50vd_dcn_365e_coco.pdparams -infer_export:True -infer_quant:False -inference:deploy/python/infer.py ---device:gpu|cpu ---enable_mkldnn:False|True ---cpu_threads:1|4 ---batch_size:1|2 ---use_tensorrt:null ---run_mode:fluid|trt_fp32|trt_fp16|trt_int8 ---model_dir: ---image_dir:./dataset/coco/val2017/ ---save_log_path:null ---run_benchmark:True -null:null diff --git a/tests/ppdet_params/yolov3_darknet53_270e_coco_params.txt b/tests/ppdet_params/yolov3_darknet53_270e_coco_params.txt deleted file mode 100644 index a4be4ae5d..000000000 --- a/tests/ppdet_params/yolov3_darknet53_270e_coco_params.txt +++ /dev/null @@ -1,51 +0,0 @@ -===========================train_params=========================== -model_name:yolov3_darknet53_270e_coco -python:python3.7 -gpu_list:0 -use_gpu:True -auto_cast:null|amp -epoch:lite_train_infer=1|whole_train_infer=270 -save_dir:./output/ -TrainReader.batch_size:lite_train_infer=2|whole_train_infer=8 -pretrain_weights:null -train_model_name:yolov3_darknet53_270e_coco/model_final.pdparams -train_infer_img_dir:./tests/demo/ -filename:yolov3_darknet53_270e_coco -## -trainer:norm_train|pact_train|fpgm_train -norm_train:tools/train.py -c configs/yolov3/yolov3_darknet53_270e_coco.yml -o -pact_train:tools/train.py -c configs/yolov3/yolov3_darknet53_270e_coco.yml --slim_config configs/slim/quant/yolov3_darknet_qat.yml -o -fpgm_train:tools/train.py -c configs/yolov3/yolov3_darknet53_270e_coco.yml --slim_config configs/slim/prune/yolov3_darknet_prune_fpgm.yml -o -distill_train:null -null:null -null:null -## -===========================eval_params=========================== -eval:tools/eval.py -c configs/yolov3/yolov3_darknet53_270e_coco.yml -o -null:null -## -===========================infer_params=========================== ---output_dir:null -weights: -norm_export:tools/export_model.py -c configs/yolov3/yolov3_darknet53_270e_coco.yml -o -quant_export:tools/export_model.py -c configs/yolov3/yolov3_darknet53_270e_coco.yml --slim_config configs/slim/quant/yolov3_darknet_qat.yml -o -fpgm_export:tools/export_model.py -c configs/yolov3/yolov3_darknet53_270e_coco.yml --slim_config configs/slim/prune/yolov3_darknet_prune_fpgm.yml -o -distill_export:null -kl_quant:tools/post_quant.py -c configs/yolov3/yolov3_darknet53_270e_coco.yml --slim_config configs/slim/post_quant/yolov3_darknet53_ptq.yml -o -export2:null -## -infer_model:yolov3_darknet53_270e_coco.pdparams|yolov3_darknet_coco_qat.pdparams|yolov3_darknet_prune_fpgm.pdparams -infer_model_dir:./tests/weights -infer_quant:False|True|False -inference:deploy/python/infer.py ---device:gpu|cpu ---enable_mkldnn:False|True ---cpu_threads:1|4 ---batch_size:1|2 ---use_tensorrt:null ---run_mode:fluid|trt_fp32|trt_fp16|trt_int8 ---model_dir: ---image_dir:./dataset/coco/val2017/ ---save_log_path:null ---run_benchmark:True -null:null diff --git a/tests/prepare.sh b/tests/prepare.sh deleted file mode 100644 index 96dc514e2..000000000 --- a/tests/prepare.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/bash -FILENAME=$1 -# MODE be one of ['lite_train_infer' 'whole_infer' 'whole_train_infer', 'infer'] -MODE=$2 - -dataline=$(cat ${FILENAME}) - -# parser params -IFS=$'\n' -lines=(${dataline}) -function func_parser_key(){ - strs=$1 - IFS=":" - array=(${strs}) - tmp=${array[0]} - echo ${tmp} -} -function func_parser_value(){ - strs=$1 - IFS=":" - array=(${strs}) - tmp=${array[1]} - echo ${tmp} -} -IFS=$'\n' -# The training params -model_name=$(func_parser_value "${lines[1]}") -python=$(func_parser_value "${lines[2]}") -trainer_list=$(func_parser_value "${lines[14]}") - -# MODE be one of ['lite_train_infer' 'whole_infer' 'whole_train_infer'] -MODE=$2 - -if [ ${MODE} = "lite_train_infer" ];then - mv ./dataset/coco/download_coco.py . && rm -rf ./dataset/coco/* && mv ./download_coco.py ./dataset/coco/ - # prepare lite train data - wget -nc -P ./dataset/coco/ https://paddledet.bj.bcebos.com/data/coco_ce.tar - cd ./dataset/coco/ && tar -xvf coco_ce.tar && mv -u coco_ce/* . - rm -rf coco_ce/ - # prepare lite train infer_img_dir - cd ../../ && mkdir -p ./tests/demo/ - cp -u ./demo/road554.png ./demo/orange_71.jpg ./tests/demo/ - if [[ ${model_name} = "ppyolov2_r50vd_dcn_365e_coco" ]]; then - wget -nc -P ./tests/weights/ https://paddledet.bj.bcebos.com/models/ppyolov2_r50vd_dcn_365e_coco.pdparams - fi -elif [ ${MODE} = "whole_train_infer" ];then - mv ./dataset/coco/download_coco.py . && rm -rf ./dataset/coco/* && mv ./download_coco.py ./dataset/coco/ - # prepare whole train data - eval "${python} ./dataset/coco/download_coco.py" - # prepare whole train infer_img_dir - mkdir -p ./tests/demo/ - cp -u dataset/coco/val2017/* ./tests/demo/ -elif [ ${MODE} = "whole_infer" ];then - mv ./dataset/coco/download_coco.py . && rm -rf ./dataset/coco/* && mv ./download_coco.py ./dataset/coco/ - # prepare whole infer data - wget -nc -P ./dataset/coco/ https://paddledet.bj.bcebos.com/data/coco_ce.tar - cd ./dataset/coco/ && tar -xvf coco_ce.tar && mv -u coco_ce/* . - rm -rf coco_ce/ - # prepare whole infer infer_img_dir - cd ../../ && mkdir -p ./tests/demo/ - cp -u dataset/coco/val2017/* ./tests/demo/ -else - mv ./dataset/coco/download_coco.py . && rm -rf ./dataset/coco/* && mv ./download_coco.py ./dataset/coco/ - # prepare infer data - wget -nc -P ./dataset/coco/ https://paddledet.bj.bcebos.com/data/coco_ce.tar - cd ./dataset/coco/ && tar -xvf coco_ce.tar && mv -u coco_ce/* . - rm -rf coco_ce/ && cd ../../ - if [[ ${model_name} = "yolov3_darknet53_270e_coco" ]]; then - wget -nc -P ./tests/weights/ https://paddledet.bj.bcebos.com/models/yolov3_darknet53_270e_coco.pdparams - wget -nc -P ./tests/weights/ https://paddledet.bj.bcebos.com/models/slim/yolov3_darknet_coco_qat.pdparams - wget -nc -P ./tests/weights/ https://paddledet.bj.bcebos.com/models/slim/yolov3_darknet_prune_fpgm.pdparams - elif [[ ${model_name} = "ppyolo_r50vd_dcn_1x_coco" ]]; then - wget -nc -P ./tests/weights/ https://paddledet.bj.bcebos.com/models/ppyolo_r50vd_dcn_1x_coco.pdparams - wget -nc -P ./tests/weights/ https://paddledet.bj.bcebos.com/models/slim/ppyolo_r50vd_qat_pact.pdparams - wget -nc -P ./tests/weights/ https://paddledet.bj.bcebos.com/models/slim/ppyolo_r50vd_prune_fpgm.pdparams - elif [[ ${model_name} = "ppyolo_mbv3_large_coco" ]]; then - wget -nc -P ./tests/weights/ https://paddledet.bj.bcebos.com/models/ppyolo_mbv3_large_coco.pdparams - wget -nc -P ./tests/weights/ https://paddledet.bj.bcebos.com/models/slim/ppyolo_mbv3_large_qat.pdparams - wget -nc -P ./tests/weights/ https://paddledet.bj.bcebos.com/models/slim/ppyolo_mbv3_large_prune_fpgm.pdparams - elif [[ ${model_name} = "ppyolov2_r50vd_dcn_365e_coco" ]]; then - wget -nc -P ./tests/weights/ https://paddledet.bj.bcebos.com/models/ppyolov2_r50vd_dcn_365e_coco.pdparams - elif [[ ${model_name} = "mask_rcnn_r50_fpn_1x_coco" ]]; then - wget -nc -P ./tests/weights/ https://paddledet.bj.bcebos.com/models/mask_rcnn_r50_fpn_1x_coco.pdparams - elif [[ ${model_name} = "solov2_r50_fpn_1x_coco" ]]; then - wget -nc -P ./tests/weights/ https://paddledet.bj.bcebos.com/models/solov2_r50_fpn_1x_coco.pdparams - else - sleep 2 - fi -fi diff --git a/tests/requirements.txt b/tests/requirements.txt deleted file mode 100644 index 8dc76aceb..000000000 --- a/tests/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -pynvml -psutil -GPUtil -paddleslim diff --git a/tests/test.sh b/tests/test.sh deleted file mode 100644 index c41c25d59..000000000 --- a/tests/test.sh +++ /dev/null @@ -1,383 +0,0 @@ -#!/bin/bash -FILENAME=$1 -# MODE be one of ['lite_train_infer' 'whole_infer' 'whole_train_infer', 'infer'] -MODE=$2 - -dataline=$(cat ${FILENAME}) - -# parser params -IFS=$'\n' -lines=(${dataline}) - -function func_parser_key(){ - strs=$1 - IFS=":" - array=(${strs}) - tmp=${array[0]} - echo ${tmp} -} -function func_parser_value(){ - strs=$1 - IFS=":" - array=(${strs}) - tmp=${array[1]} - echo ${tmp} -} -function func_set_params(){ - key=$1 - value=$2 - if [ ${key} = "null" ];then - echo " " - elif [[ ${value} = "null" ]] || [[ ${value} = " " ]] || [ ${#value} -le 0 ];then - echo " " - else - echo "${key}=${value}" - fi -} -function func_parser_params(){ - strs=$1 - IFS=":" - array=(${strs}) - key=${array[0]} - tmp=${array[1]} - IFS="|" - res="" - for _params in ${tmp[*]}; do - IFS="=" - array=(${_params}) - mode=${array[0]} - value=${array[1]} - if [[ ${mode} = ${MODE} ]]; then - IFS="|" - #echo $(func_set_params "${mode}" "${value}") - echo $value - break - fi - IFS="|" - done - echo ${res} -} -function status_check(){ - last_status=$1 # the exit code - run_command=$2 - run_log=$3 - if [ $last_status -eq 0 ]; then - echo -e "\033[33m Run successfully with command - ${run_command}! \033[0m" | tee -a ${run_log} - else - echo -e "\033[33m Run failed with command - ${run_command}! \033[0m" | tee -a ${run_log} - fi -} - -IFS=$'\n' -# The training params -model_name=$(func_parser_value "${lines[1]}") -python=$(func_parser_value "${lines[2]}") -gpu_list=$(func_parser_value "${lines[3]}") -train_use_gpu_key=$(func_parser_key "${lines[4]}") -train_use_gpu_value=$(func_parser_value "${lines[4]}") -autocast_list=$(func_parser_value "${lines[5]}") -autocast_key=$(func_parser_key "${lines[5]}") -epoch_key=$(func_parser_key "${lines[6]}") -epoch_num=$(func_parser_params "${lines[6]}") -save_model_key=$(func_parser_key "${lines[7]}") -train_batch_key=$(func_parser_key "${lines[8]}") -train_batch_value=$(func_parser_params "${lines[8]}") -pretrain_model_key=$(func_parser_key "${lines[9]}") -pretrain_model_value=$(func_parser_value "${lines[9]}") -train_model_name=$(func_parser_value "${lines[10]}") -train_infer_img_dir=$(func_parser_value "${lines[11]}") -train_param_key1=$(func_parser_key "${lines[12]}") -train_param_value1=$(func_parser_value "${lines[12]}") - -trainer_list=$(func_parser_value "${lines[14]}") -trainer_norm=$(func_parser_key "${lines[15]}") -norm_trainer=$(func_parser_value "${lines[15]}") -pact_key=$(func_parser_key "${lines[16]}") -pact_trainer=$(func_parser_value "${lines[16]}") -fpgm_key=$(func_parser_key "${lines[17]}") -fpgm_trainer=$(func_parser_value "${lines[17]}") -distill_key=$(func_parser_key "${lines[18]}") -distill_trainer=$(func_parser_value "${lines[18]}") -trainer_key1=$(func_parser_key "${lines[19]}") -trainer_value1=$(func_parser_value "${lines[19]}") -trainer_key2=$(func_parser_key "${lines[20]}") -trainer_value2=$(func_parser_value "${lines[20]}") - -eval_py=$(func_parser_value "${lines[23]}") -eval_key1=$(func_parser_key "${lines[24]}") -eval_value1=$(func_parser_value "${lines[24]}") - -save_infer_key=$(func_parser_key "${lines[27]}") -export_weight=$(func_parser_key "${lines[28]}") -norm_export=$(func_parser_value "${lines[29]}") -pact_export=$(func_parser_value "${lines[30]}") -fpgm_export=$(func_parser_value "${lines[31]}") -distill_export=$(func_parser_value "${lines[32]}") -export_key1=$(func_parser_key "${lines[33]}") -export_value1=$(func_parser_value "${lines[33]}") -export_key2=$(func_parser_key "${lines[34]}") -export_value2=$(func_parser_value "${lines[34]}") - -# parser inference model -infer_model_name_list=$(func_parser_value "${lines[36]}") -infer_model_dir=$(func_parser_value "${lines[37]}") -infer_is_quant=$(func_parser_value "${lines[38]}") -# parser inference -inference_py=$(func_parser_value "${lines[39]}") -device_key=$(func_parser_key "${lines[40]}") -device_list=$(func_parser_value "${lines[40]}") -use_mkldnn_key=$(func_parser_key "${lines[41]}") -use_mkldnn_list=$(func_parser_value "${lines[41]}") -cpu_threads_key=$(func_parser_key "${lines[42]}") -cpu_threads_list=$(func_parser_value "${lines[42]}") -batch_size_key=$(func_parser_key "${lines[43]}") -batch_size_list=$(func_parser_value "${lines[43]}") -use_trt_key=$(func_parser_key "${lines[44]}") -use_trt_list=$(func_parser_value "${lines[44]}") -precision_key=$(func_parser_key "${lines[45]}") -precision_list=$(func_parser_value "${lines[45]}") -infer_model_key=$(func_parser_key "${lines[46]}") -image_dir_key=$(func_parser_key "${lines[47]}") -infer_img_dir=$(func_parser_value "${lines[47]}") -save_log_key=$(func_parser_key "${lines[48]}") -benchmark_key=$(func_parser_key "${lines[49]}") -benchmark_value=$(func_parser_value "${lines[49]}") -infer_key1=$(func_parser_key "${lines[50]}") -infer_value1=$(func_parser_value "${lines[50]}") - -LOG_PATH="./tests/output" -mkdir -p ${LOG_PATH} -status_log="${LOG_PATH}/results.log" - - -function func_inference(){ - IFS='|' - _python=$1 - _script=$2 - _model_dir=$3 - _log_path=$4 - _img_dir=$5 - _flag_quant=$6 - # inference - for device in ${device_list[*]}; do - if [ ${device} = "False" ] || [ ${device} = "cpu" ]; then - for use_mkldnn in ${use_mkldnn_list[*]}; do - if [ ${use_mkldnn} = "False" ] && [ ${_flag_quant} = "True" ]; then - continue - fi - for threads in ${cpu_threads_list[*]}; do - for batch_size in ${batch_size_list[*]}; do - _save_log_path="${_log_path}/infer_cpu_usemkldnn_${use_mkldnn}_threads_${threads}_batchsize_${batch_size}.log" - set_infer_data=$(func_set_params "${image_dir_key}" "${_img_dir}") - set_benchmark=$(func_set_params "${benchmark_key}" "${benchmark_value}") - set_batchsize=$(func_set_params "${batch_size_key}" "${batch_size}") - set_cpu_threads=$(func_set_params "${cpu_threads_key}" "${threads}") - set_model_dir=$(func_set_params "${infer_model_key}" "${_model_dir}") - set_infer_params1=$(func_set_params "${infer_key1}" "${infer_value1}") - command="${_python} ${_script} ${device_key}=${device} ${use_mkldnn_key}=${use_mkldnn} ${set_cpu_threads} ${set_model_dir} ${set_batchsize} ${set_infer_data} ${set_benchmark} ${set_infer_params1} > ${_save_log_path} 2>&1 " - eval $command - last_status=${PIPESTATUS[0]} - eval "cat ${_save_log_path}" - status_check $last_status "${command}" "${status_log}" - done - done - done - elif [ ${device} = "True" ] || [ ${device} = "gpu" ]; then - for use_trt in ${use_trt_list[*]}; do - for precision in ${precision_list[*]}; do - if [[ ${precision} != "fluid" ]]; then - if [[ ${_flag_quant} = "False" ]] && [[ ${precision} = "trt_int8" ]]; then - continue - fi - if [[ ${_flag_quant} = "True" ]] && [[ ${precision} != "trt_int8" ]]; then - continue - fi - fi - for batch_size in ${batch_size_list[*]}; do - _save_log_path="${_log_path}/infer_gpu_usetrt_${use_trt}_precision_${precision}_batchsize_${batch_size}.log" - set_infer_data=$(func_set_params "${image_dir_key}" "${_img_dir}") - set_benchmark=$(func_set_params "${benchmark_key}" "${benchmark_value}") - set_batchsize=$(func_set_params "${batch_size_key}" "${batch_size}") - set_tensorrt=$(func_set_params "${use_trt_key}" "${use_trt}") - set_precision=$(func_set_params "${precision_key}" "${precision}") - set_model_dir=$(func_set_params "${infer_model_key}" "${_model_dir}") - set_infer_params1=$(func_set_params "${infer_key1}" "${infer_value1}") - command="${_python} ${_script} ${device_key}=${device} ${set_tensorrt} ${set_precision} ${set_model_dir} ${set_batchsize} ${set_infer_data} ${set_benchmark} ${set_infer_params1} > ${_save_log_path} 2>&1 " - eval $command - last_status=${PIPESTATUS[0]} - eval "cat ${_save_log_path}" - status_check $last_status "${command}" "${status_log}" - - done - done - done - else - echo "Does not support hardware other than CPU and GPU Currently!" - fi - done -} - -if [ ${MODE} = "infer" ]; then - GPUID=$3 - if [ ${#GPUID} -le 0 ];then - env=" " - else - env="export CUDA_VISIBLE_DEVICES=${GPUID}" - fi - # set CUDA_VISIBLE_DEVICES - eval $env - export Count=0 - IFS="|" - infer_quant_flag=(${infer_is_quant}) - set_train_params1=$(func_set_params "${train_param_key1}" "${train_param_value1}") - set_save_infer_key=$(func_set_params "${save_infer_key}" "${infer_model_dir}") - infer_model="${infer_model_dir}/${train_param_value1}" - for infer_model_name in ${infer_model_name_list[*]}; do - # run export - case ${Count} in - 0) run_export=${norm_export} ;; - 1) run_export=${pact_export} ;; - 2) run_export=${fpgm_export} ;; - *) echo "Undefined run_export"; exit 1; - esac - set_export_weight=$(func_set_params "${export_weight}" "${infer_model_dir}/${infer_model_name}") - export_cmd="${python} ${run_export} ${set_export_weight} ${set_train_params1} ${set_save_infer_key}" - eval $export_cmd - status_export=$? - if [ ${status_export} = 0 ];then - status_check $status_export "${export_cmd}" "${status_log}" - fi - #run inference - is_quant=${infer_quant_flag[Count]} - func_inference "${python}" "${inference_py}" "${infer_model}" "${LOG_PATH}" "${infer_img_dir}" ${is_quant} - Count=$(($Count + 1)) - done - - # kl quant - if [ ${export_key1} = "kl_quant" ]; then - # run kl quant - kl_cmd="${python} ${export_value1} ${set_train_params1} ${set_save_infer_key}" - eval $kl_cmd - status_export=$? - if [ ${status_export} = 0 ];then - status_check $status_export "${kl_cmd}" "${status_log}" - fi - # run inference - is_quant=True - func_inference "${python}" "${inference_py}" "${infer_model}" "${LOG_PATH}" "${infer_img_dir}" ${is_quant} - fi - -else - IFS="|" - export Count=0 - USE_GPU_KEY=(${train_use_gpu_value}) - for gpu in ${gpu_list[*]}; do - use_gpu=${USE_GPU_KEY[Count]} - Count=$(($Count + 1)) - if [ ${gpu} = "-1" ];then - env="" - elif [ ${#gpu} -le 1 ];then - env="export CUDA_VISIBLE_DEVICES=${gpu}" - eval ${env} - elif [ ${#gpu} -le 15 ];then - IFS="," - array=(${gpu}) - env="export CUDA_VISIBLE_DEVICES=${array[0]}" - IFS="|" - else - IFS=";" - array=(${gpu}) - ips=${array[0]} - gpu=${array[1]} - IFS="|" - env=" " - fi - for autocast in ${autocast_list[*]}; do - for trainer in ${trainer_list[*]}; do - flag_quant=False - if [ ${trainer} = ${pact_key} ]; then - run_train=${pact_trainer} - run_export=${pact_export} - flag_quant=True - if [ ${autocast} = "amp" ]; then - continue - fi - elif [ ${trainer} = "${fpgm_key}" ]; then - run_train=${fpgm_trainer} - run_export=${fpgm_export} - elif [ ${trainer} = "${distill_key}" ]; then - run_train=${distill_trainer} - run_export=${distill_export} - elif [ ${trainer} = ${trainer_key1} ]; then - run_train=${trainer_value1} - run_export=${export_value1} - elif [[ ${trainer} = ${trainer_key2} ]]; then - run_train=${trainer_value2} - run_export=${export_value2} - else - run_train=${norm_trainer} - run_export=${norm_export} - fi - - if [ ${run_train} = "null" ]; then - continue - fi - - if [ ${autocast} = "amp" ]; then - set_autocast="--fp16" - else - set_autocast=$(func_set_params "${autocast_key}" "${autocast}") - fi - set_epoch=$(func_set_params "${epoch_key}" "${epoch_num}") - set_pretrain=$(func_set_params "${pretrain_model_key}" "${pretrain_model_value}") - set_batchsize=$(func_set_params "${train_batch_key}" "${train_batch_value}") - set_train_params1=$(func_set_params "${train_param_key1}" "${train_param_value1}") - set_use_gpu=$(func_set_params "${train_use_gpu_key}" "${use_gpu}") - save_log="${LOG_PATH}/${trainer}_gpus_${gpu}_autocast_${autocast}" - - # load pretrain from norm training if current trainer is pact or fpgm trainer - if [ ${trainer} = ${pact_key} ] || [ ${trainer} = ${fpgm_key} ]; then - set_pretrain=$(func_set_params "${pretrain_model_key}" "${save_log}/${train_model_name}") - fi - - set_save_model=$(func_set_params "${save_model_key}" "${save_log}") - if [ ${#gpu} -le 2 ];then # train with cpu or single gpu - cmd="${python} ${run_train} ${set_use_gpu} ${set_save_model} ${set_epoch} ${set_pretrain} ${set_batchsize} ${set_train_params1} ${set_autocast} " - elif [ ${#gpu} -le 15 ];then # train with multi-gpu - cmd="${python} -m paddle.distributed.launch --gpus=${gpu} ${run_train} ${set_save_model} ${set_epoch} ${set_pretrain} ${set_batchsize} ${set_train_params1} ${set_autocast}" - else # train with multi-machine - cmd="${python} -m paddle.distributed.launch --ips=${ips} --gpus=${gpu} ${run_train} ${set_save_model} ${set_pretrain} ${set_epoch} ${set_batchsize} ${set_train_params1} ${set_autocast}" - fi - # run train - # eval "unset CUDA_VISIBLE_DEVICES" - eval $cmd - status_check $? "${cmd}" "${status_log}" - - set_eval_trained_weight=$(func_set_params "weights" "${save_log}/${train_model_name}") - # run eval - if [ ${eval_py} != "null" ]; then - set_eval_params1=$(func_set_params "${eval_key1}" "${eval_value1}") - eval_cmd="${python} ${eval_py} ${set_eval_trained_weight} ${set_use_gpu} ${set_eval_params1}" - eval $eval_cmd - status_check $? "${eval_cmd}" "${status_log}" - fi - # run export model - if [ ${run_export} != "null" ]; then - # run export model - save_infer_path="${save_log}" - set_export_weight=$(func_set_params "${export_weight}" "${save_log}/${train_model_name}") - set_save_infer_key=$(func_set_params "${save_infer_key}" "${save_infer_path}") - export_cmd="${python} ${run_export} ${set_export_weight} ${set_train_params1} ${set_save_infer_key}" - eval $export_cmd - status_check $? "${export_cmd}" "${status_log}" - - #run inference - eval $env - save_infer_path="${save_log}/${train_param_value1}" - func_inference "${python}" "${inference_py}" "${save_infer_path}" "${LOG_PATH}" "${train_infer_img_dir}" "${flag_quant}" - # eval "unset CUDA_VISIBLE_DEVICES" - fi - done # done with: for trainer in ${trainer_list[*]}; do - done # done with: for autocast in ${autocast_list[*]}; do - done # done with: for gpu in ${gpu_list[*]}; do -fi # end if [ ${MODE} = "infer" ]; then -- GitLab