Merge pull request #1469 from hysunflower/update_scripts_benchmark

Update scripts benchmark

benchmark/prepare_data.sh

 #!/bin/bash
 dataset_url=$1
 
+package_check_list=(imageio tqdm Cython pycocotools tb_paddle scipy pandas wget h5py sklearn opencv-python visualdl)
+for package in ${package_check_list[@]}; do
+    if python -c "import ${package}" >/dev/null 2>&1; then
+        echo "${package} have already installed"
+    else
+        echo "${package} NOT FOUND"
+        pip install ${package}
+        echo "${package} installed"
+    fi
+done
+
 cd dataset
 rm -rf ILSVRC2012
 wget -nc ${dataset_url}

benchmark/run_all.sh

@@ -5,20 +5,25 @@
 # pip install ...
 # 2 拷贝该模型需要数据、预训练模型
 # 3 批量运行（如不方便批量，1，2需放到单个模型中）
- 
-model_mode_list=(MobileNetV1 MobileNetV2 MobileNetV3_large_x1_0 EfficientNetB0 ShuffleNetV2_x1_0 DenseNet121 HRNet_W48_C SwinTransformer_tiny_patch4_window7_224 alt_gvt_base)
+model_mode_list=(MobileNetV1 MobileNetV2 MobileNetV3_large_x1_0 ShuffleNetV2_x1_0 HRNet_W48_C SwinTransformer_tiny_patch4_window7_224 alt_gvt_base)    # benchmark 监控模型列表
+#model_mode_list=(MobileNetV1 MobileNetV2 MobileNetV3_large_x1_0 EfficientNetB0 ShuffleNetV2_x1_0 DenseNet121 HRNet_W48_C SwinTransformer_tiny_patch4_window7_224 alt_gvt_base)   # 该脚本支持列表
 fp_item_list=(fp32)
-bs_list=(32 64 96 128)
+#bs_list=(32 64 96 128)
 for model_mode in ${model_mode_list[@]}; do
       for fp_item in ${fp_item_list[@]}; do
+          if [ ${model_mode} = MobileNetV3_large_x1_0 ] || [ ${model_mode} = ShuffleNetV2_x1_0 ]; then
+              bs_list=(256)
+          else
+              bs_list=(64)
+          fi
           for bs_item in ${bs_list[@]};do
 	    echo "index is speed, 1gpus, begin, ${model_name}"
 	    run_mode=sp
-	    CUDA_VISIBLE_DEVICES=0 bash benchmark/run_benchmark.sh ${run_mode} ${bs_item} ${fp_item} 10 ${model_mode}     #  (5min)
+	    CUDA_VISIBLE_DEVICES=0 bash benchmark/run_benchmark.sh ${run_mode} ${bs_item} ${fp_item} 1 ${model_mode} | tee ${log_path}/clas_${model_mode}_${run_mode}_bs${bs_item}_${fp_item}_1gpus 2>&1    #  (5min)
 	    sleep 10
             echo "index is speed, 8gpus, run_mode is multi_process, begin, ${model_name}"
             run_mode=mp
-            CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 bash benchmark/run_benchmark.sh ${run_mode} ${bs_item} ${fp_item} 10 ${model_mode} 
+            CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 bash benchmark/run_benchmark.sh ${run_mode} ${bs_item} ${fp_item} 1 ${model_mode}| tee ${log_path}/clas_${model_mode}_${run_mode}_bs${bs_item}_${fp_item}_8gpus8p 2>&1
             sleep 10
             done
       done

benchmark/run_benchmark.sh

@@ -6,10 +6,19 @@ function _set_params(){
     run_mode=${1:-"sp"}          # 单卡sp|多卡mp
     batch_size=${2:-"64"}
     fp_item=${3:-"fp32"}        # fp32|fp16
-    epochs=${4:-"10"}       # 可选，如果需要修改代码提前中断
+    epochs=${4:-"2"}       # 可选，如果需要修改代码提前中断
     model_name=${5:-"model_name"}
     run_log_path="${TRAIN_LOG_DIR:-$(pwd)}/benchmark"  # TRAIN_LOG_DIR 后续QA设置该参数
- 
+
+    index=1
+    mission_name="图像分类"           # 模型所属任务名称，具体可参考scripts/config.ini                                （必填）
+    direction_id=0                    # 任务所属方向，0：CV，1：NLP，2：Rec。                                          (必填)
+    skip_steps=8                      # 解析日志，有些模型前几个step耗时长，需要跳过                                  (必填)
+    keyword="ips:"              # 解析日志，筛选出数据所在行的关键字                                            (必填)
+    keyword_loss="loss:"       #选填
+    model_mode=-1                      # 解析日志，具体参考scripts/analysis.py.                                        (必填)
+    ips_unit="images/s"
+    base_batch_size=$batch_size
 #   以下不用修改   
     device=${CUDA_VISIBLE_DEVICES//,/ }
     arr=(${device})
@@ -26,7 +35,7 @@ function _train(){
         model_config=`find ppcls/configs/ImageNet -name ${model_name}_fp16.yaml` 
     fi
 
-    train_cmd="-c ${model_config} -o DataLoader.Train.sampler.batch_size=${batch_size} -o Global.epochs=${epochs}"   
+    train_cmd="-c ${model_config} -o DataLoader.Train.sampler.batch_size=${batch_size} -o Global.epochs=${epochs}  -o Global.eval_during_train=False -o Global.print_batch_step=2"
     case ${run_mode} in
     sp) train_cmd="python -u tools/train.py ${train_cmd}" ;;
     mp)
@@ -36,7 +45,7 @@ function _train(){
     esac
     rm -rf mylog
 # 以下不用修改
-    timeout 15m ${train_cmd} > ${log_file} 2>&1
+    timeout 5m ${train_cmd} > ${log_file} 2>&1
     if [ $? -ne 0 ];then
         echo -e "${model_name}, FAIL"
         export job_fail_flag=1
@@ -51,6 +60,8 @@ function _train(){
         cp mylog/workerlog.0 ${log_file}
     fi
 }
- 
+
+source ${BENCHMARK_ROOT}/scripts/run_model.sh # 在该脚本中会对符合benchmark规范的log使用analysis.py 脚本进行性能数据解析;该脚本在连调时可从benchmark repo中下载https://github.com/PaddlePaddle/benchmark/blob/master/scripts/run_model.sh;如果不联调只想要产出训练log可以注掉本行,提交时需打开
 _set_params $@
-_train
+_run
+#_train