prepare.sh 11.1 KB
Newer Older
1 2 3 4 5 6
#!/bin/bash
source test_tipc/utils_func.sh

FILENAME=$1
# MODE be one of ['lite_train_lite_infer' 'lite_train_whole_infer'
#                 'whole_train_whole_infer', 'whole_infer', 'klquant_whole_infer',
7
#                 'cpp_infer', 'serving_infer',  'lite_infer', 'paddle2onnx_infer']
8 9 10 11 12 13 14 15 16 17 18
MODE=$2

# parse params
dataline=$(cat ${FILENAME})
IFS=$'\n'
lines=(${dataline})

# The training params
model_name=$(func_parser_value "${lines[1]}")
python=$(func_parser_value "${lines[2]}")

19
if [ ${MODE} = "whole_train_whole_infer" ];then
20 21 22 23
    mv ./dataset/coco/download_coco.py . && rm -rf ./dataset/coco/* && mv ./download_coco.py ./dataset/coco/
    # prepare whole training data
    eval "${python} ./dataset/coco/download_coco.py"
elif [ ${MODE} = "cpp_infer" ];then
24
    # download coco lite data
25
    wget -nc -P ./dataset/coco/ https://paddledet.bj.bcebos.com/data/tipc/coco_tipc.tar --no-check-certificate
26 27 28
    cd ./dataset/coco/ && tar -xvf coco_tipc.tar && mv -n coco_tipc/* .
    rm -rf coco_tipc/ && cd ../../
    # download wider_face lite data
29
    wget -nc -P ./dataset/wider_face/ https://paddledet.bj.bcebos.com/data/tipc/wider_tipc.tar --no-check-certificate
30 31 32
    cd ./dataset/wider_face/ && tar -xvf wider_tipc.tar && mv -n wider_tipc/* .
    rm -rf wider_tipc/ && cd ../../
    # download spine lite data
33
    wget -nc -P ./dataset/spine_coco/ https://paddledet.bj.bcebos.com/data/tipc/spine_tipc.tar --no-check-certificate
34 35 36 37 38
    cd ./dataset/spine_coco/ && tar -xvf spine_tipc.tar && mv -n spine_tipc/* .
    rm -rf spine_tipc/ && cd ../../
    if [[ ${model_name} =~ "s2anet" ]]; then
        cd ./ppdet/ext_op && eval "${python} setup.py install"
        cd ../../
39 40 41 42
    elif [[ ${model_name} =~ "tinypose" ]]; then
        wget -nc -P ./output_inference/ https://bj.bcebos.com/v1/paddledet/models/keypoint/picodet_s_320_pedestrian.tar --no-check-certificate
        cd ./output_inference/ && tar -xvf picodet_s_320_pedestrian.tar
        cd ../
43
    fi
44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60
    # download KL model
    if [[ ${model_name} = "picodet_lcnet_1_5x_416_coco_KL" ]]; then
        wget -nc -P ./output_inference/picodet_lcnet_1_5x_416_coco_KL/ https://bj.bcebos.com/v1/paddledet/data/tipc/models/picodet_lcnet_1_5x_416_coco_ptq.tar --no-check-certificate
        cd ./output_inference/picodet_lcnet_1_5x_416_coco_KL/ && tar -xvf picodet_lcnet_1_5x_416_coco_ptq.tar && mv -n picodet_lcnet_1_5x_416_coco_ptq/* .
        cd ../../
    elif [[ ${model_name} = "ppyoloe_crn_s_300e_coco_KL" ]]; then
        wget -nc -P ./output_inference/ppyoloe_crn_s_300e_coco_KL/ https://bj.bcebos.com/v1/paddledet/data/tipc/models/ppyoloe_crn_s_300e_coco_ptq.tar --no-check-certificate
        cd ./output_inference/ppyoloe_crn_s_300e_coco_KL/ && tar -xvf ppyoloe_crn_s_300e_coco_ptq.tar && mv -n ppyoloe_crn_s_300e_coco_ptq/* .
        cd ../../
    elif [[ ${model_name} = "ppyolo_mbv3_large_coco_KL" ]]; then
        wget -nc -P ./output_inference/ppyolo_mbv3_large_coco_KL/ https://bj.bcebos.com/v1/paddledet/data/tipc/models/ppyolo_mbv3_large_ptq.tar --no-check-certificate
        cd ./output_inference/ppyolo_mbv3_large_coco_KL/ && tar -xvf ppyolo_mbv3_large_ptq.tar && mv -n ppyolo_mbv3_large_ptq/* .
        cd ../../
    elif [[ ${model_name} = "mask_rcnn_r50_fpn_1x_coco_KL" ]]; then
        wget -nc -P ./output_inference/mask_rcnn_r50_fpn_1x_coco_KL/ https://bj.bcebos.com/v1/paddledet/data/tipc/models/mask_rcnn_r50_fpn_1x_coco_ptq.tar --no-check-certificate
        cd ./output_inference/mask_rcnn_r50_fpn_1x_coco_KL/ && tar -xvf mask_rcnn_r50_fpn_1x_coco_ptq.tar && mv -n mask_rcnn_r50_fpn_1x_coco_ptq/* .
        cd ../../
61 62 63 64
    elif [[ ${model_name} = "tinypose_128x96_KL" ]]; then
        wget -nc -P ./output_inference/tinypose_128x96_KL/ https://bj.bcebos.com/v1/paddledet/data/tipc/models/tinypose_128x96_ptq.tar --no-check-certificate
        cd ./output_inference/tinypose_128x96_KL/ && tar -xvf tinypose_128x96_ptq.tar && mv -n tinypose_128x96_ptq/* .
        cd ../../
65
    fi
66
    # download mot lite data
67
    wget -nc -P ./dataset/mot/ https://paddledet.bj.bcebos.com/data/tipc/mot_tipc.tar --no-check-certificate
68 69 70
    cd ./dataset/mot/ && tar -xvf mot_tipc.tar && mv -n mot_tipc/* .
    rm -rf mot_tipc/ && cd ../../

71 72 73 74 75 76 77 78
    opencv_dir=$(func_parser_value "${lines[15]}")
    # prepare opencv
    cd ./deploy/cpp
    if [ ${opencv_dir} = "default" ] || [ ${opencv_dir} = "null" ]; then
        if [ -d "deps/opencv-3.4.16_gcc8.2_ffmpeg/" ]; then
            echo "################### Opencv already exists, skip downloading. ###################"
        else
            mkdir -p $(pwd)/deps && cd $(pwd)/deps
79
            wget -c https://paddledet.bj.bcebos.com/data/opencv-3.4.16_gcc8.2_ffmpeg.tar.gz --no-check-certificate
80 81 82 83 84
            tar -xvf opencv-3.4.16_gcc8.2_ffmpeg.tar.gz && cd ../
            echo "################### Finish downloading opencv. ###################"
        fi
    fi
    cd ../../
85
elif [ ${MODE} = "benchmark_train" ];then
86 87
    pip install -U pip
    pip install Cython
88
    pip install -r requirements.txt
89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107
    if [[ ${model_name} =~ "higherhrnet" ]] || [[ ${model_name} =~ "hrnet" ]] || [[ ${model_name} =~ "tinypose" ]];then
        wget -nc -P ./dataset/ https://bj.bcebos.com/v1/paddledet/data/coco.tar --no-check-certificate
        cd ./dataset/ && tar -xf coco.tar
        ls ./coco/
        cd ../
    else
        # prepare lite benchmark coco data
        wget -nc -P ./dataset/coco/ https://paddledet.bj.bcebos.com/data/coco_benchmark.tar --no-check-certificate
        cd ./dataset/coco/ && tar -xf coco_benchmark.tar
        mv -u coco_benchmark/* ./
        ls ./
        cd ../../
        # prepare lite benchmark mot data
        wget -nc -P ./dataset/mot/ https://paddledet.bj.bcebos.com/data/mot_benchmark.tar --no-check-certificate
        cd ./dataset/mot/ && tar -xf mot_benchmark.tar
        mv -u mot_benchmark/* ./
        ls ./
        cd ../../
    fi
108
elif [ ${MODE} = "paddle2onnx_infer" ];then
109 110 111
    # install paddle2onnx
    ${python} -m pip install paddle2onnx
    ${python} -m pip install onnx onnxruntime
112
elif [ ${MODE} = "serving_infer" ];then
113
    unset https_proxy http_proxy
S
shangliang Xu 已提交
114 115 116 117
    # download coco lite data
    wget -nc -P ./dataset/coco/ https://paddledet.bj.bcebos.com/data/tipc/coco_tipc.tar --no-check-certificate
    cd ./dataset/coco/ && tar -xvf coco_tipc.tar && mv -n coco_tipc/* .
    rm -rf coco_tipc/ && cd ../../
118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138
    # download KL model
    if [[ ${model_name} = "picodet_lcnet_1_5x_416_coco_KL" ]]; then
        wget -nc -P ./output_inference/picodet_lcnet_1_5x_416_coco_KL/ https://bj.bcebos.com/v1/paddledet/data/tipc/models/picodet_lcnet_1_5x_416_coco_ptq.tar --no-check-certificate
        cd ./output_inference/picodet_lcnet_1_5x_416_coco_KL/ && tar -xvf picodet_lcnet_1_5x_416_coco_ptq.tar && mv -n picodet_lcnet_1_5x_416_coco_ptq/* .
        cd ../../
        eval "${python} -m paddle_serving_client.convert --dirname output_inference/picodet_lcnet_1_5x_416_coco_KL/ --model_filename model.pdmodel --params_filename model.pdiparams --serving_server output_inference/picodet_lcnet_1_5x_416_coco_KL/serving_server --serving_client output_inference/picodet_lcnet_1_5x_416_coco_KL/serving_client"
    elif [[ ${model_name} = "ppyoloe_crn_s_300e_coco_KL" ]]; then
        wget -nc -P ./output_inference/ppyoloe_crn_s_300e_coco_KL/ https://bj.bcebos.com/v1/paddledet/data/tipc/models/ppyoloe_crn_s_300e_coco_ptq.tar --no-check-certificate
        cd ./output_inference/ppyoloe_crn_s_300e_coco_KL/ && tar -xvf ppyoloe_crn_s_300e_coco_ptq.tar && mv -n ppyoloe_crn_s_300e_coco_ptq/* .
        cd ../../
        eval "${python} -m paddle_serving_client.convert --dirname output_inference/ppyoloe_crn_s_300e_coco_KL/ --model_filename model.pdmodel --params_filename model.pdiparams --serving_server output_inference/ppyoloe_crn_s_300e_coco_KL/serving_server --serving_client output_inference/ppyoloe_crn_s_300e_coco_KL/serving_client"
    elif [[ ${model_name} = "ppyolo_mbv3_large_coco_KL" ]]; then
        wget -nc -P ./output_inference/ppyolo_mbv3_large_coco_KL/ https://bj.bcebos.com/v1/paddledet/data/tipc/models/ppyolo_mbv3_large_ptq.tar --no-check-certificate
        cd ./output_inference/ppyolo_mbv3_large_coco_KL/ && tar -xvf ppyolo_mbv3_large_ptq.tar && mv -n ppyolo_mbv3_large_ptq/* .
        cd ../../
        eval "${python} -m paddle_serving_client.convert --dirname output_inference/ppyolo_mbv3_large_coco_KL/ --model_filename model.pdmodel --params_filename model.pdiparams --serving_server output_inference/ppyolo_mbv3_large_coco_KL/serving_server --serving_client output_inference/ppyolo_mbv3_large_coco_KL/serving_client"
    elif [[ ${model_name} = "mask_rcnn_r50_fpn_1x_coco_KL" ]]; then
        wget -nc -P ./output_inference/mask_rcnn_r50_fpn_1x_coco_KL/ https://bj.bcebos.com/v1/paddledet/data/tipc/models/mask_rcnn_r50_fpn_1x_coco_ptq.tar --no-check-certificate
        cd ./output_inference/mask_rcnn_r50_fpn_1x_coco_KL/ && tar -xvf mask_rcnn_r50_fpn_1x_coco_ptq.tar && mv -n mask_rcnn_r50_fpn_1x_coco_ptq/* .
        cd ../../
        eval "${python} -m paddle_serving_client.convert --dirname output_inference/mask_rcnn_r50_fpn_1x_coco_KL/ --model_filename model.pdmodel --params_filename model.pdiparams --serving_server output_inference/mask_rcnn_r50_fpn_1x_coco_KL/serving_server --serving_client output_inference/mask_rcnn_r50_fpn_1x_coco_KL/serving_client"
139 140 141 142 143
    elif [[ ${model_name} = "tinypose_128x96_KL" ]]; then
        wget -nc -P ./output_inference/tinypose_128x96_KL/ https://bj.bcebos.com/v1/paddledet/data/tipc/models/tinypose_128x96_ptq.tar --no-check-certificate
        cd ./output_inference/tinypose_128x96_KL/ && tar -xvf tinypose_128x96_ptq.tar && mv -n tinypose_128x96_ptq/* .
        cd ../../
        eval "${python} -m paddle_serving_client.convert --dirname output_inference/tinypose_128x96_KL/ --model_filename model.pdmodel --params_filename model.pdiparams --serving_server output_inference/tinypose_128x96_KL/serving_server --serving_client output_inference/tinypose_128x96_KL/serving_client"
144
    fi
145
else
146
    # download coco lite data
147
    wget -nc -P ./dataset/coco/ https://paddledet.bj.bcebos.com/data/tipc/coco_tipc.tar --no-check-certificate
148
    cd ./dataset/coco/ && tar -xvf coco_tipc.tar && mv -n coco_tipc/* .
149
    rm -rf coco_tipc/ && cd ../../
150
    # download wider_face lite data
151
    wget -nc -P ./dataset/wider_face/ https://paddledet.bj.bcebos.com/data/tipc/wider_tipc.tar --no-check-certificate
152 153 154
    cd ./dataset/wider_face/ && tar -xvf wider_tipc.tar && mv -n wider_tipc/* .
    rm -rf wider_tipc/ && cd ../../
    # download spine_coco lite data
155 156
    wget -nc -P ./dataset/spine_coco/ https://paddledet.bj.bcebos.com/data/tipc/spine_coco_tipc.tar --no-check-certificate
    cd ./dataset/spine_coco/ && tar -xvf spine_coco_tipc.tar && mv -n spine_coco_tipc/* .
157 158 159 160
    rm -rf spine_tipc/ && cd ../../
    if [[ ${model_name} =~ "s2anet" ]]; then
        cd ./ppdet/ext_op && eval "${python} setup.py install"
        cd ../../
L
LokeZhou 已提交
161 162 163
    elif [[ ${model_name} =~ "ppyoloe_r_crn_s_3x_spine_coco" ]]; then
        cd ./ppdet/ext_op && eval "${python} setup.py install"
        cd ../../
164
    fi
165
    # download mot lite data
166
    wget -nc -P ./dataset/mot/ https://paddledet.bj.bcebos.com/data/tipc/mot_tipc.tar --no-check-certificate
167 168
    cd ./dataset/mot/ && tar -xvf mot_tipc.tar && mv -n mot_tipc/* .
    rm -rf mot_tipc/ && cd ../../
169
fi