[TIPC] add tipc benchmark for msvsr (#672)

* add tipc benchmark for msvsr

* update tipc readme img

ppgan/engine/trainer.py

@@ -32,6 +32,7 @@ from ..utils.profiler import add_profiler_step
 
 
 class IterLoader:
+
     def __init__(self, dataloader):
         self._dataloader = dataloader
         self.iter_loader = iter(self._dataloader)
@@ -79,6 +80,7 @@ class Trainer:
     #                     |                                    ||
     #         save checkpoint (model.nets)                     \/
     """
+
     def __init__(self, cfg):
         # base config
         self.logger = logging.getLogger(__name__)
@@ -181,6 +183,22 @@ class Trainer:
 
         iter_loader = IterLoader(self.train_dataloader)
 
+        # use amp
+        if self.cfg.amp:
+            self.logger.info('use AMP to train. AMP level = {}'.format(
+                self.cfg.amp_level))
+            assert self.cfg.model.name == 'MultiStageVSRModel', "AMP only support msvsr model"
+            scaler = paddle.amp.GradScaler(init_loss_scaling=1024)
+            # need to decorate model and optim if amp_level == 'O2'
+            if self.cfg.amp_level == 'O2':
+                # msvsr has only one generator and one optimizer
+                self.model.nets['generator'], self.optimizers[
+                    'optim'] = paddle.amp.decorate(
+                        models=self.model.nets['generator'],
+                        optimizers=self.optimizers['optim'],
+                        level='O2',
+                        save_dtype='float32')
+
         # set model.is_train = True
         self.model.setup_train_mode(is_train=True)
         while self.current_iter < (self.total_iters + 1):
@@ -195,7 +213,12 @@ class Trainer:
             # unpack data from dataset and apply preprocessing
             # data input should be dict
             self.model.setup_input(data)
-            self.model.train_iter(self.optimizers)
+
+            if self.cfg.amp:
+                self.model.train_iter_amp(self.optimizers, scaler,
+                                          self.cfg.amp_level)  # amp train
+            else:
+                self.model.train_iter(self.optimizers)  # norm train
 
             batch_cost_averager.record(
                 time.time() - step_start_time,

ppgan/models/msvsr_model.py

@@ -30,6 +30,7 @@ class MultiStageVSRModel(BaseSRModel):
     Paper:
         PP-MSVSR: Multi-Stage Video Super-Resolution, 2021
     """
+
     def __init__(self, generator, fix_iter, pixel_criterion=None):
         """Initialize the PP-MSVSR class.
 
@@ -96,6 +97,48 @@ class MultiStageVSRModel(BaseSRModel):
 
         self.current_iter += 1
 
+    # amp train with brute force implementation, maybe decorator can simplify this
+    def train_iter_amp(self, optims=None, scaler=None, amp_level='O1'):
+        optims['optim'].clear_grad()
+        if self.fix_iter:
+            if self.current_iter == 1:
+                print('Train MSVSR with fixed spynet for', self.fix_iter,
+                      'iters.')
+                for name, param in self.nets['generator'].named_parameters():
+                    if 'spynet' in name:
+                        param.trainable = False
+            elif self.current_iter >= self.fix_iter + 1 and self.flag:
+                print('Train all the parameters.')
+                for name, param in self.nets['generator'].named_parameters():
+                    param.trainable = True
+                    if 'spynet' in name:
+                        param.optimize_attr['learning_rate'] = 0.25
+                self.flag = False
+                for net in self.nets.values():
+                    net.find_unused_parameters = False
+
+        # put loss computation in amp context
+        with paddle.amp.auto_cast(enable=True, level=amp_level):
+            output = self.nets['generator'](self.lq)
+            if isinstance(output, (list, tuple)):
+                out_stage2, output = output
+                loss_pix_stage2 = self.pixel_criterion(out_stage2, self.gt)
+                self.losses['loss_pix_stage2'] = loss_pix_stage2
+            self.visual_items['output'] = output[:, 0, :, :, :]
+            # pixel loss
+            loss_pix = self.pixel_criterion(output, self.gt)
+            self.losses['loss_pix'] = loss_pix
+
+            self.loss = sum(_value for _key, _value in self.losses.items()
+                            if 'loss_pix' in _key)
+        scaled_loss = scaler.scale(self.loss)
+        self.losses['loss'] = scaled_loss
+
+        scaled_loss.backward()
+        scaler.minimize(optims['optim'], scaled_loss)
+
+        self.current_iter += 1
+
     def test_iter(self, metrics=None):
         self.gt = self.gt.cpu()
         self.nets['generator'].eval()

ppgan/utils/options.py

@@ -45,9 +45,9 @@ def parse_args():
                         default=False,
                         help='skip validation during training')
     # config options
-    parser.add_argument("-o", 
-                        "--opt", 
-                        nargs='+', 
+    parser.add_argument("-o",
+                        "--opt",
+                        nargs='+',
                         help="set configuration options")
 
     #for inference
@@ -60,19 +60,31 @@ def parse_args():
                         help="path to reference images")
     parser.add_argument("--model_path", default=None, help="model for loading")
 
-    # for profiler                
-    parser.add_argument('-p',
-                        '--profiler_options',
-                        type=str,
-                        default=None,
-                        help='The option of profiler, which should be in format \"key1=value1;key2=value2;key3=value3\".'
+    # for profiler
+    parser.add_argument(
+        '-p',
+        '--profiler_options',
+        type=str,
+        default=None,
+        help=
+        'The option of profiler, which should be in format \"key1=value1;key2=value2;key3=value3\".'
     )
     # fix random numbers by setting seed
     parser.add_argument('--seed',
                         type=int,
                         default=None,
-                        help='fix random numbers by setting seed\".'
-    )
+                        help='fix random numbers by setting seed\".')
+
+    # add for amp training
+    parser.add_argument('--amp',
+                        action='store_true',
+                        default=False,
+                        help='whether to enable amp training')
+    parser.add_argument('--amp_level',
+                        type=str,
+                        default='O1',
+                        choices=['O1', 'O2'],
+                        help='level of amp training; O2 represent pure fp16')
     args = parser.parse_args()
 
     return args

ppgan/utils/setup.py

@@ -19,6 +19,7 @@ import numpy as np
 import random
 from .logger import setup_logger
 
+
 def setup(args, cfg):
     if args.evaluate_only:
         cfg.is_train = False
@@ -44,10 +45,13 @@ def setup(args, cfg):
         paddle.set_device('gpu')
     else:
         paddle.set_device('cpu')
-    
+
     if args.seed:
         paddle.seed(args.seed)
         random.seed(args.seed)
-        np.random.seed(args.seed)    
+        np.random.seed(args.seed)
         paddle.framework.random._manual_program_seed(args.seed)
-    
+
+    # add amp and amp_level args into cfg
+    cfg['amp'] = args.amp
+    cfg['amp_level'] = args.amp_level

test_tipc/readme.md → test_tipc/README.md

@@ -57,9 +57,8 @@ test_tipc/
 
 ### 测试流程
 使用本工具，可以测试不同功能的支持情况，以及预测结果是否对齐，测试流程如下：
-<div align="center">
-    <img src="docs/test.png" width="800">
-</div>
+
+![img](https://user-images.githubusercontent.com/79366697/185377097-a0f852a8-2d78-45ae-84ba-ae71b799d738.png)
 
 1. 运行prepare.sh准备测试所需数据和模型；
 2. 运行要测试的功能对应的测试脚本`test_*.sh`，产出log，由log可以看到不同配置是否运行成功；
@@ -72,4 +71,4 @@ test_tipc/
 <a name="more"></a>
 #### 更多教程
 各功能测试中涉及混合精度、裁剪、量化等训练相关，及mkldnn、Tensorrt等多种预测相关参数配置，请点击下方相应链接了解更多细节和使用教程：  
-[test_train_inference_python 使用](docs/test_train_inference_python.md)  
+- [test_train_inference_python 使用](docs/test_train_inference_python.md): 测试基于Python的模型训练、评估、推理等基本功能

test_tipc/benchmark_train.sh

@@ -4,15 +4,15 @@ source test_tipc/common_func.sh
 # set env
 python=python
 export model_branch=`git symbolic-ref HEAD 2>/dev/null | cut -d"/" -f 3`
-export model_commit=$(git log|head -n1|awk '{print $2}') 
+export model_commit=$(git log|head -n1|awk '{print $2}')
 export str_tmp=$(echo `pip list|grep paddlepaddle-gpu|awk -F ' ' '{print $2}'`)
 export frame_version=${str_tmp%%.post*}
 export frame_commit=$(echo `${python} -c "import paddle;print(paddle.version.commit)"`)
 
-# run benchmark sh 
+# run benchmark sh
 # Usage:
 # bash run_benchmark_train.sh config.txt params
-# or 
+# or
 # bash run_benchmark_train.sh config.txt
 
 function func_parser_params(){
@@ -100,6 +100,7 @@ for _flag in ${flags_list[*]}; do
 done
 
 # set log_name
+BENCHMARK_ROOT=./ # self-test only
 repo_name=$(get_repo_name )
 SAVE_LOG=${BENCHMARK_LOG_DIR:-$(pwd)}   # */benchmark_log
 mkdir -p "${SAVE_LOG}/benchmark_log/"
@@ -149,11 +150,11 @@ else
 fi
 
 IFS="|"
-for batch_size in ${batch_size_list[*]}; do 
+for batch_size in ${batch_size_list[*]}; do
     for precision in ${fp_items_list[*]}; do
         for device_num in ${device_num_list[*]}; do
             # sed batchsize and precision
-            #func_sed_params "$FILENAME" "${line_precision}" "$precision"
+            func_sed_params "$FILENAME" "${line_precision}" "$precision"
             func_sed_params "$FILENAME" "${line_batchsize}" "$MODE=$batch_size"
             func_sed_params "$FILENAME" "${line_epoch}" "$MODE=$epoch"
             gpu_id=$(set_gpu_id $device_num)
@@ -162,7 +163,7 @@ for batch_size in ${batch_size_list[*]}; do
                 log_path="$SAVE_LOG/profiling_log"
                 mkdir -p $log_path
                 log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_mode}_${device_num}_profiling"
-                func_sed_params "$FILENAME" "${line_gpuid}" "0"  # sed used gpu_id 
+                func_sed_params "$FILENAME" "${line_gpuid}" "0"  # sed used gpu_id
                 # set profile_option params
                 tmp=`sed -i "${line_profile}s/.*/${profile_option}/" "${FILENAME}"`
 
@@ -214,7 +215,7 @@ for batch_size in ${batch_size_list[*]}; do
                 mkdir -p $speed_log_path
                 log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_mode}_${device_num}_log"
                 speed_log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_mode}_${device_num}_speed"
-                func_sed_params "$FILENAME" "${line_gpuid}" "$gpu_id"  # sed used gpu_id 
+                func_sed_params "$FILENAME" "${line_gpuid}" "$gpu_id"  # sed used gpu_id
                 func_sed_params "$FILENAME" "${line_profile}" "null"  # sed --profile_option as null
                 cmd="bash test_tipc/test_train_inference_python.sh ${FILENAME} benchmark_train > ${log_path}/${log_name} 2>&1 "
                 echo $cmd
@@ -244,4 +245,4 @@ for batch_size in ${batch_size_list[*]}; do
             fi
         done
     done
-done 
+done
\ No newline at end of file

test_tipc/configs/msvsr/train_amp_infer_python.txt

+===========================train_params===========================
+model_name:msvsr
+python:python3.7
+gpu_list:0
+##
+auto_cast:null
+total_iters:lite_train_lite_infer=10|lite_train_whole_infer=10|whole_train_whole_infer=200
+output_dir:./output/
+dataset.train.batch_size:lite_train_lite_infer=1|whole_train_whole_infer=1
+pretrained_model:null
+train_model_name:msvsr_reds*/*checkpoint.pdparams
+train_infer_img_dir:./data/msvsr_reds/test
+null:null
+##
+trainer:amp_train
+amp_train:tools/main.py --amp --amp_level O1 -c configs/msvsr_reds.yaml --seed 123 -o dataset.train.num_workers=0 log_config.interval=1 snapshot_config.interval=5 dataset.train.dataset.num_frames=2
+pact_train:null
+fpgm_train:null
+distill_train:null
+null:null
+null:null
+##
+===========================eval_params===========================
+eval:null
+null:null
+##
+===========================infer_params===========================
+--output_dir:./output/
+load:null
+norm_export:tools/export_model.py -c configs/msvsr_reds.yaml  --inputs_size="1,2,3,180,320"  --model_name inference --load
+quant_export:null
+fpgm_export:null
+distill_export:null
+export1:null
+export2:null
+inference_dir:inference
+train_model:./inference/msvsr/multistagevsrmodel_generator
+infer_export:null
+infer_quant:False
+inference:tools/inference.py --model_type msvsr -c configs/msvsr_reds.yaml --seed 123 -o dataset.test.num_frames=2 --output_path test_tipc/output/
+--device:cpu
+null:null
+null:null
+null:null
+null:null
+null:null
+--model_path:
+null:null
+null:null
+--benchmark:True
+null:null
+===========================infer_benchmark_params==========================
+random_infer_input:[{float32,[2,3,180,320]}]

test_tipc/configs/msvsr/train_infer_python.txt

@@ -13,22 +13,22 @@ train_infer_img_dir:./data/msvsr_reds/test
 null:null
 ##
 trainer:norm_train
-norm_train:tools/main.py -c configs/msvsr_reds.yaml --seed 123 -o dataset.train.num_workers=0 log_config.interval=1 snapshot_config.interval=5 dataset.train.dataset.num_frames=2 
+norm_train:tools/main.py -c configs/msvsr_reds.yaml --seed 123 -o log_config.interval=2 snapshot_config.interval=50 dataset.train.dataset.num_frames=15
 pact_train:null
 fpgm_train:null
 distill_train:null
 null:null
 null:null
 ##
-===========================eval_params=========================== 
+===========================eval_params===========================
 eval:null
 null:null
 ##
 ===========================infer_params===========================
 --output_dir:./output/
 load:null
-norm_export:tools/export_model.py -c configs/msvsr_reds.yaml  --inputs_size="1,2,3,180,320"  --model_name inference --load 
-quant_export:null 
+norm_export:tools/export_model.py -c configs/msvsr_reds.yaml  --inputs_size="1,2,3,180,320"  --model_name inference --load
+quant_export:null
 fpgm_export:null
 distill_export:null
 export1:null
@@ -49,5 +49,11 @@ null:null
 null:null
 --benchmark:True
 null:null
+===========================train_benchmark_params==========================
+batch_size:4
+fp_items:fp32
+total_iters:60
+--profiler_options:batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile
+flags:null
 ===========================infer_benchmark_params==========================
 random_infer_input:[{float32,[2,3,180,320]}]

test_tipc/docs/benchmark_train.md

@@ -9,7 +9,7 @@
 
 ```shell
 # 运行格式：bash test_tipc/prepare.sh  train_benchmark.txt  mode
-bash test_tipc/prepare.sh test_tipc/configs/basicvsr/train_benchmark.txt benchmark_train
+bash test_tipc/prepare.sh test_tipc/configs/msvsr/train_infer_python.txt benchmark_train
 ```
 
 ## 1.2 功能测试
@@ -17,13 +17,13 @@ bash test_tipc/prepare.sh test_tipc/configs/basicvsr/train_benchmark.txt benchma
 
 ```shell
 # 运行格式：bash test_tipc/benchmark_train.sh train_benchmark.txt mode
-bash test_tipc/benchmark_train.sh test_tipc/configs/basicvsr/train_infer_python.txt benchmark_train
+bash test_tipc/benchmark_train.sh test_tipc/configs/msvsr/train_infer_python.txt benchmark_train
 ```
 
 `test_tipc/benchmark_train.sh`支持根据传入的第三个参数实现只运行某一个训练配置，如下：
 ```shell
 # 运行格式：bash test_tipc/benchmark_train.sh train_benchmark.txt mode
-bash test_tipc/benchmark_train.sh test_tipc/configs/basicvsr/train_infer_python.txt benchmark_train  dynamic_bs4_fp32_DP_N1C1
+bash test_tipc/benchmark_train.sh test_tipc/configs/msvsr/train_infer_python.txt benchmark_train  dynamic_bs4_fp32_DP_N1C1
 ```
 dynamic_bs4_fp32_DP_N1C1为test_tipc/benchmark_train.sh传入的参数，格式如下：
 `${modeltype}_${batch_size}_${fp_item}_${run_mode}_${device_num}`
@@ -42,11 +42,11 @@ dynamic_bs4_fp32_DP_N1C1为test_tipc/benchmark_train.sh传入的参数，格式
 ```
 train_log/
 ├── index
-│   ├── PaddleGAN_basicvsr_bs4_fp32_SingleP_DP_N1C1_speed
-│   └── PaddleGAN_basicvsr_bs4_fp32_SingleP_DP_N1C4_speed
+│   ├── PaddleGAN_msvsr_bs4_fp32_SingleP_DP_N1C1_speed
+│   └── PaddleGAN_msvsr_bs4_fp32_SingleP_DP_N1C4_speed
 ├── profiling_log
-│   └── PaddleGAN_basicvsr_bs4_fp32_SingleP_DP_N1C1_profiling
+│   └── PaddleGAN_msvsr_bs4_fp32_SingleP_DP_N1C1_profiling
 └── train_log
-    ├── PaddleGAN_basicvsr_bs4_fp32_SingleP_DP_N1C1_log
-    └── PaddleGAN_basicvsr_bs4_fp32_MultiP_DP_N1C4_log
+    ├── PaddleGAN_msvsr_bs4_fp32_SingleP_DP_N1C1_log
+    └── PaddleGAN_msvsr_bs4_fp32_MultiP_DP_N1C4_log
 ```

test_tipc/prepare.sh

@@ -172,5 +172,10 @@ elif [ ${MODE} = "whole_infer" ];then
         mkdir -p ./data/singan
         mv ./data/SinGAN-official_images/Images/stone.png ./data/singan
     fi
-
+elif [ ${MODE} = "benchmark_train" ];then
+    if [ ${model_name} = "msvsr" ]; then
+        rm -rf ./data/reds*
+        wget -nc -P ./data/ https://paddlegan.bj.bcebos.com/datasets/reds_lite.tar --no-check-certificate
+        cd ./data/ && tar xf reds_lite.tar && cd ../
+    fi
 fi

test_tipc/test_train_inference_python.sh

@@ -48,11 +48,11 @@ norm_export=$(func_parser_value "${lines[29]}")
 
 inference_dir=$(func_parser_value "${lines[35]}")
 
-# parser inference model 
+# parser inference model
 infer_model_dir_list=$(func_parser_value "${lines[36]}")
 infer_export_list=$(func_parser_value "${lines[37]}")
 infer_is_quant=$(func_parser_value "${lines[38]}")
-# parser inference 
+# parser inference
 inference_py=$(func_parser_value "${lines[39]}")
 use_gpu_key=$(func_parser_key "${lines[40]}")
 use_gpu_list=$(func_parser_value "${lines[40]}")
@@ -85,7 +85,7 @@ function func_inference(){
     _log_path=$4
     _img_dir=$5
     _flag_quant=$6
-    # inference 
+    # inference
     for use_gpu in ${use_gpu_list[*]}; do
         if [ ${use_gpu} = "False" ] || [ ${use_gpu} = "cpu" ]; then
             for use_mkldnn in ${use_mkldnn_list[*]}; do
@@ -96,7 +96,7 @@ function func_inference(){
                     for batch_size in ${batch_size_list[*]}; do
                         for precision in ${precision_list[*]}; do
                             set_precision=$(func_set_params "${precision_key}" "${precision}")
-                            
+
                             _save_log_path="${_log_path}/python_infer_cpu_usemkldnn_${use_mkldnn}_threads_${threads}_precision_${precision}_batchsize_${batch_size}.log"
                             set_infer_data=$(func_set_params "${image_dir_key}" "${_img_dir}")
                             set_benchmark=$(func_set_params "${benchmark_key}" "${benchmark_value}")
@@ -118,7 +118,7 @@ function func_inference(){
                 for precision in ${precision_list[*]}; do
                     if [[ ${_flag_quant} = "False" ]] && [[ ${precision} =~ "int8" ]]; then
                         continue
-                    fi 
+                    fi
                     if [[ ${precision} =~ "fp16" || ${precision} =~ "int8" ]] && [ ${use_trt} = "False" ]; then
                         continue
                     fi
@@ -139,7 +139,7 @@ function func_inference(){
                         last_status=${PIPESTATUS[0]}
                         eval "cat ${_save_log_path}"
                         status_check $last_status "${command}" "${status_log}"
-                        
+
                     done
                 done
             done
@@ -169,7 +169,7 @@ if [ ${MODE} = "whole_infer" ]; then
             set_export_weight=$(func_set_params "${export_weight}" "${infer_model}")
             set_save_infer_key="${save_infer_key} ${save_infer_dir}"
             export_cmd="${python} ${infer_run_exports[Count]} ${set_export_weight} ${set_save_infer_key}"
-            echo ${infer_run_exports[Count]} 
+            echo ${infer_run_exports[Count]}
             echo  $export_cmd
             eval $export_cmd
             status_export=$?
@@ -207,17 +207,17 @@ else
             IFS="|"
             env=" "
         fi
-        for autocast in ${autocast_list[*]}; do 
-            if [ ${autocast} = "amp" ]; then
-                set_amp_config="Global.use_amp=True Global.scale_loss=1024.0 Global.use_dynamic_loss_scaling=True"
+        for autocast in ${autocast_list[*]}; do
+            if [ ${autocast} = "fp16" ]; then
+                set_amp_config="--amp"
             else
                 set_amp_config=" "
-            fi          
-            for trainer in ${trainer_list[*]}; do 
+            fi
+            for trainer in ${trainer_list[*]}; do
                 flag_quant=False
                 run_train=${norm_trainer}
                 run_export=${norm_export}
-                
+
                 if [ ${run_train} = "null" ]; then
                     continue
                 fi
@@ -239,11 +239,11 @@ else
                 fi
                 set_save_model=$(func_set_params "${save_model_key}" "${save_log}")
                 if [ ${#gpu} -le 2 ];then  # train with cpu or single gpu
-                    cmd="${python} ${run_train} ${set_use_gpu}  ${set_save_model} ${set_train_params1} ${set_epoch} ${set_pretrain} ${set_autocast} ${set_batchsize} ${set_amp_config} "
+                    cmd="${python} ${run_train} ${set_use_gpu}  ${set_save_model} ${set_train_params1} ${set_epoch} ${set_pretrain} ${set_batchsize} ${set_amp_config}"
                 elif [ ${#ips} -le 26 ];then  # train with multi-gpu
-                    cmd="${python} -m paddle.distributed.launch --gpus=${gpu} ${run_train} ${set_use_gpu} ${set_save_model} ${set_train_params1} ${set_epoch} ${set_pretrain} ${set_autocast} ${set_batchsize} ${set_amp_config}"
+                    cmd="${python} -m paddle.distributed.launch --gpus=${gpu} ${run_train} ${set_use_gpu} ${set_save_model} ${set_train_params1} ${set_epoch} ${set_pretrain} ${set_batchsize} ${set_amp_config}"
                 else     # train with multi-machine
-                    cmd="${python} -m paddle.distributed.launch --ips=${ips} --gpus=${gpu} ${run_train} ${set_use_gpu} ${set_save_model} ${set_train_params1} ${set_pretrain} ${set_epoch} ${set_autocast} ${set_batchsize} ${set_amp_config}"
+                    cmd="${python} -m paddle.distributed.launch --ips=${ips} --gpus=${gpu} ${run_train} ${set_use_gpu} ${set_save_model} ${set_train_params1} ${set_pretrain} ${set_epoch} ${set_batchsize} ${set_amp_config}"
                 fi
                 # run train
                 eval "unset CUDA_VISIBLE_DEVICES"
@@ -253,17 +253,17 @@ else
                 status_check $? "${cmd}" "${status_log}"
 
                 set_eval_pretrain=$(func_set_params "${pretrain_model_key}" "${save_log}/${train_model_name}")
-                # save norm trained models to set pretrain for pact training and fpgm training 
-                
-                # run eval 
+                # save norm trained models to set pretrain for pact training and fpgm training
+
+                # run eval
                 if [ ${eval_py} != "null" ]; then
                     set_eval_params1=$(func_set_params "${eval_key1}" "${eval_value1}")
-                    eval_cmd="${python} ${eval_py} ${set_eval_pretrain} ${set_use_gpu} ${set_eval_params1}" 
+                    eval_cmd="${python} ${eval_py} ${set_eval_pretrain} ${set_use_gpu} ${set_eval_params1}"
                     eval $eval_cmd
                     status_check $? "${eval_cmd}" "${status_log}"
                 fi
                 # run export model
-                if [ ${run_export} != "null" ]; then 
+                if [ ${run_export} != "null" ]; then
                     # run export model
                     save_infer_path="${save_log}"
                     set_export_weight="${save_log}/${train_model_name}"
@@ -272,7 +272,7 @@ else
                     export_cmd="${python} ${run_export}  ${set_export_weight_path} ${set_save_infer_key}"
                     eval "$export_cmd"
                     status_check $? "${export_cmd}" "${status_log}"
-                    
+
                     #run inference
                     eval $env
                     save_infer_path="${save_log}"
@@ -282,11 +282,10 @@ else
                         infer_model_dir=${save_infer_path}
                     fi
                     func_inference "${python}" "${inference_py}" "${infer_model_dir}" "${LOG_PATH}" "${train_infer_img_dir}" "${flag_quant}"
-                    
+
                     eval "unset CUDA_VISIBLE_DEVICES"
                 fi
-            done  # done with:    for trainer in ${trainer_list[*]}; do 
-        done      # done with:    for autocast in ${autocast_list[*]}; do 
+            done  # done with:    for trainer in ${trainer_list[*]}; do
+        done      # done with:    for autocast in ${autocast_list[*]}; do
     done          # done with:    for gpu in ${gpu_list[*]}; do
 fi  # end if [ ${MODE} = "infer" ]; then
-