rm tests/

tests/compare_results.py

-import numpy as np
-import os
-import subprocess
-import json
-import argparse
-import glob
-
-
-def init_args():
-    parser = argparse.ArgumentParser()
-    # params for testing assert allclose
-    parser.add_argument("--atol", type=float, default=1e-3)
-    parser.add_argument("--rtol", type=float, default=1e-3)
-    parser.add_argument("--gt_file", type=str, default="")
-    parser.add_argument("--log_file", type=str, default="")
-    parser.add_argument("--precision", type=str, default="fp32")
-    return parser
-
-
-def parse_args():
-    parser = init_args()
-    return parser.parse_args()
-
-
-def run_shell_command(cmd):
-    p = subprocess.Popen(
-        cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
-    out, err = p.communicate()
-
-    if p.returncode == 0:
-        return out.decode('utf-8')
-    else:
-        return None
-
-
-def parser_results_from_log_by_name(log_path, names_list):
-    if not os.path.exists(log_path):
-        raise ValueError("The log file {} does not exists!".format(log_path))
-
-    if names_list is None or len(names_list) < 1:
-        return []
-
-    parser_results = {}
-    for name in names_list:
-        cmd = "grep {} {}".format(name, log_path)
-        outs = run_shell_command(cmd)
-        outs = outs.split("\n")[0]
-        result = outs.split("{}".format(name))[-1]
-        result = json.loads(result)
-        parser_results[name] = result
-    return parser_results
-
-
-def load_gt_from_file(gt_file):
-    if not os.path.exists(gt_file):
-        raise ValueError("The log file {} does not exists!".format(gt_file))
-    with open(gt_file, 'r') as f:
-        data = f.readlines()
-        f.close()
-    parser_gt = {}
-    for line in data:
-        image_name, result = line.strip("\n").split("\t")
-        result = json.loads(result)
-        parser_gt[image_name] = result
-    return parser_gt
-
-
-def load_gt_from_txts(gt_file):
-    gt_list = glob.glob(gt_file)
-    gt_collection = {}
-    for gt_f in gt_list:
-        gt_dict = load_gt_from_file(gt_f)
-        basename = os.path.basename(gt_f)
-        if "fp32" in basename:
-            gt_collection["fp32"] = [gt_dict, gt_f]
-        elif "fp16" in basename:
-            gt_collection["fp16"] = [gt_dict, gt_f]
-        elif "int8" in basename:
-            gt_collection["int8"] = [gt_dict, gt_f]
-        else:
-            continue
-    return gt_collection
-
-
-def collect_predict_from_logs(log_path, key_list):
-    log_list = glob.glob(log_path)
-    pred_collection = {}
-    for log_f in log_list:
-        pred_dict = parser_results_from_log_by_name(log_f, key_list)
-        key = os.path.basename(log_f)
-        pred_collection[key] = pred_dict
-
-    return pred_collection
-
-
-def testing_assert_allclose(dict_x, dict_y, atol=1e-7, rtol=1e-7):
-    for k in dict_x:
-        np.testing.assert_allclose(
-            np.array(dict_x[k]), np.array(dict_y[k]), atol=atol, rtol=rtol)
-
-
-if __name__ == "__main__":
-    # Usage:
-    # python3.7 tests/compare_results.py --gt_file=./tests/results/*.txt  --log_file=./tests/output/infer_*.log
-
-    args = parse_args()
-
-    gt_collection = load_gt_from_txts(args.gt_file)
-    key_list = gt_collection["fp32"][0].keys()
-
-    pred_collection = collect_predict_from_logs(args.log_file, key_list)
-    for filename in pred_collection.keys():
-        if "fp32" in filename:
-            gt_dict, gt_filename = gt_collection["fp32"]
-        elif "fp16" in filename:
-            gt_dict, gt_filename = gt_collection["fp16"]
-        elif "int8" in filename:
-            gt_dict, gt_filename = gt_collection["int8"]
-        else:
-            continue
-        pred_dict = pred_collection[filename]
-
-        try:
-            testing_assert_allclose(
-                gt_dict, pred_dict, atol=args.atol, rtol=args.rtol)
-            print(
-                "Assert allclose passed! The results of {} and {} are consistent!".
-                format(filename, gt_filename))
-        except Exception as E:
-            print(E)
-            raise ValueError(
-                "The results of {} and the results of {} are inconsistent!".
-                format(filename, gt_filename))

tests/ocr_det_params.txt

-===========================train_params===========================
-model_name:ocr_det
-python:python3.7
-gpu_list:0|0,1
-Global.use_gpu:True|True
-Global.auto_cast:null
-Global.epoch_num:lite_train_infer=1|whole_train_infer=300
-Global.save_model_dir:./output/
-Train.loader.batch_size_per_card:lite_train_infer=2|whole_train_infer=4
-Global.pretrained_model:null
-train_model_name:latest
-train_infer_img_dir:./train_data/icdar2015/text_localization/ch4_test_images/
-null:null
-##
-trainer:norm_train|pact_train
-norm_train:tools/train.py -c configs/det/det_mv3_db.yml -o Global.pretrained_model=./pretrain_models/MobileNetV3_large_x0_5_pretrained
-pact_train:deploy/slim/quantization/quant.py -c configs/det/det_mv3_db.yml -o
-fpgm_train:deploy/slim/prune/sensitivity_anal.py -c configs/det/det_mv3_db.yml -o Global.pretrained_model=./pretrain_models/det_mv3_db_v2.0_train/best_accuracy
-distill_train:null
-null:null
-null:null
-##
-===========================eval_params=========================== 
-eval:tools/eval.py -c configs/det/det_mv3_db.yml -o
-null:null
-##
-===========================infer_params===========================
-Global.save_inference_dir:./output/
-Global.pretrained_model:
-norm_export:tools/export_model.py -c configs/det/det_mv3_db.yml -o 
-quant_export:deploy/slim/quantization/export_model.py -c configs/det/det_mv3_db.yml -o 
-fpgm_export:deploy/slim/prune/export_prune_model.py -c configs/det/det_mv3_db.yml -o 
-distill_export:null
-export1:null
-export2:null
-##
-infer_model:./inference/ch_ppocr_mobile_v2.0_det_infer/
-infer_export:null
-infer_quant:False
-inference:tools/infer/predict_det.py
---use_gpu:True|False
---enable_mkldnn:True|False
---cpu_threads:1|6
---rec_batch_num:1
---use_tensorrt:False|True
---precision:fp32|fp16|int8
---det_model_dir:
---image_dir:./inference/ch_det_data_50/all-sum-510/
---save_log_path:null
---benchmark:True
-null:null
-===========================cpp_infer_params===========================
-use_opencv:True
-infer_model:./inference/ch_ppocr_mobile_v2.0_det_infer/
-infer_quant:False
-inference:./deploy/cpp_infer/build/ppocr det
---use_gpu:True|False
---enable_mkldnn:True|False
---cpu_threads:1|6
---rec_batch_num:1
---use_tensorrt:False|True
---precision:fp32|fp16
---det_model_dir:
---image_dir:./inference/ch_det_data_50/all-sum-510/
---save_log_path:null
---benchmark:True
-

tests/ocr_det_server_params.txt

-===========================train_params===========================
-model_name:ocr_server_det
-python:python3.7
-gpu_list:0|0,1
-Global.use_gpu:True|True
-Global.auto_cast:null
-Global.epoch_num:lite_train_infer=2|whole_train_infer=300
-Global.save_model_dir:./output/
-Train.loader.batch_size_per_card:lite_train_infer=2|whole_train_infer=4
-Global.pretrained_model:null
-train_model_name:latest
-train_infer_img_dir:./train_data/icdar2015/text_localization/ch4_test_images/
-null:null
-##
-trainer:norm_train|pact_train
-norm_train:tools/train.py -c configs/det/det_r50_vd_db.yml -o Global.pretrained_model=""
-pact_train:null
-fpgm_train:null
-distill_train:null
-null:null
-null:null
-##
-===========================eval_params=========================== 
-eval:tools/eval.py -c configs/det/det_mv3_db.yml -o
-null:null
-##
-===========================infer_params===========================
-Global.save_inference_dir:./output/
-Global.pretrained_model:
-norm_export:tools/export_model.py -c configs/det/det_r50_vd_db.yml -o 
-quant_export:null 
-fpgm_export:null
-distill_export:null
-export1:null
-export2:null
-##
-infer_model:./inference/ch_ppocr_server_v2.0_det_infer/
-infer_export:null
-infer_quant:False
-inference:tools/infer/predict_det.py
---use_gpu:True|False
---enable_mkldnn:True|False
---cpu_threads:1|6
---rec_batch_num:1
---use_tensorrt:False|True
---precision:fp32|fp16|int8
---det_model_dir:
---image_dir:./inference/ch_det_data_50/all-sum-510/
---save_log_path:null
---benchmark:True
-null:null
-

tests/ocr_rec_params.txt

-===========================train_params===========================
-model_name:ocr_rec
-python:python3.7
-gpu_list:0|2,3
-Global.use_gpu:True|True
-Global.auto_cast:null
-Global.epoch_num:lite_train_infer=2|whole_train_infer=300
-Global.save_model_dir:./output/
-Train.loader.batch_size_per_card:lite_train_infer=128|whole_train_infer=128
-Global.pretrained_model:null
-train_model_name:latest
-train_infer_img_dir:./train_data/ic15_data/train
-null:null
-##
-trainer:norm_train|pact_train
-norm_train:tools/train.py -c configs/rec/rec_icdar15_train.yml -o
-pact_train:deploy/slim/quantization/quant.py -c configs/rec/rec_icdar15_train.yml -o
-fpgm_train:null
-distill_train:null
-null:null
-null:null
-##
-===========================eval_params===========================
-eval:tools/eval.py -c configs/rec/rec_icdar15_train.yml -o
-null:null
-##
-===========================infer_params===========================
-Global.save_inference_dir:./output/
-Global.pretrained_model:
-norm_export:tools/export_model.py -c configs/rec/rec_icdar15_train.yml -o
-quant_export:deploy/slim/quantization/export_model.py -c configs/rec/rec_icdar15_train.yml -o
-fpgm_export:null
-distill_export:null
-export1:null
-export2:null
-##
-infer_model:./inference/ch_ppocr_mobile_v2.0_rec_infer/
-infer_export:null
-infer_quant:False
-inference:tools/infer/predict_rec.py
---use_gpu:True|False
---enable_mkldnn:True|False
---cpu_threads:1|6
---rec_batch_num:1
---use_tensorrt:True|False
---precision:fp32|fp16|int8
---rec_model_dir:
---image_dir:./inference/rec_inference
---save_log_path:./test/output/
---benchmark:True
-null:null

tests/prepare.sh

-#!/bin/bash
-FILENAME=$1
-# MODE be one of ['lite_train_infer' 'whole_infer' 'whole_train_infer', 'infer', 'cpp_infer']
-MODE=$2
-
-dataline=$(cat ${FILENAME})
-
-# parser params
-IFS=$'\n'
-lines=(${dataline})
-function func_parser_key(){
-    strs=$1
-    IFS=":"
-    array=(${strs})
-    tmp=${array[0]}
-    echo ${tmp}
-}
-function func_parser_value(){
-    strs=$1
-    IFS=":"
-    array=(${strs})
-    tmp=${array[1]}
-    echo ${tmp}
-}
-IFS=$'\n'
-# The training params
-model_name=$(func_parser_value "${lines[1]}")
-
-trainer_list=$(func_parser_value "${lines[14]}")
-
-# MODE be one of ['lite_train_infer' 'whole_infer' 'whole_train_infer']
-MODE=$2
-
-if [ ${MODE} = "lite_train_infer" ];then
-    # pretrain lite train data
-    wget -nc -P  ./pretrain_models/ https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/MobileNetV3_large_x0_5_pretrained.pdparams
-    wget -nc -P ./pretrain_models/  https://paddleocr.bj.bcebos.com/dygraph_v2.0/en/det_mv3_db_v2.0_train.tar
-    cd ./pretrain_models/ && tar xf det_mv3_db_v2.0_train.tar && cd ../
-    rm -rf ./train_data/icdar2015
-    rm -rf ./train_data/ic15_data
-    wget -nc -P ./train_data/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/icdar2015_lite.tar
-    wget -nc -P ./train_data/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ic15_data.tar # todo change to bcebos
-    wget -nc -P ./deploy/slim/prune https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/sen.pickle
-    
-    cd ./train_data/ && tar xf icdar2015_lite.tar && tar xf ic15_data.tar
-    ln -s ./icdar2015_lite ./icdar2015
-    cd ../
-elif [ ${MODE} = "whole_train_infer" ];then
-    wget -nc -P  ./pretrain_models/ https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/MobileNetV3_large_x0_5_pretrained.pdparams
-    rm -rf ./train_data/icdar2015
-    rm -rf ./train_data/ic15_data
-    wget -nc -P ./train_data/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/icdar2015.tar
-    wget -nc -P ./train_data/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ic15_data.tar
-    cd ./train_data/ && tar xf icdar2015.tar && tar xf ic15_data.tar && cd ../
-elif [ ${MODE} = "whole_infer" ];then
-    wget -nc -P  ./pretrain_models/ https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/MobileNetV3_large_x0_5_pretrained.pdparams
-    rm -rf ./train_data/icdar2015
-    rm -rf ./train_data/ic15_data
-    wget -nc -P ./train_data/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/icdar2015_infer.tar
-    wget -nc -P ./train_data/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ic15_data.tar
-    cd ./train_data/ && tar xf icdar2015_infer.tar && tar xf ic15_data.tar
-    ln -s ./icdar2015_infer ./icdar2015
-    cd ../
-elif [ ${MODE} = "infer" ] || [ ${MODE} = "cpp_infer" ];then
-    if [ ${model_name} = "ocr_det" ]; then
-        eval_model_name="ch_ppocr_mobile_v2.0_det_infer"
-        rm -rf ./train_data/icdar2015
-        wget -nc -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ch_det_data_50.tar
-        wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_det_infer.tar
-        cd ./inference && tar xf ${eval_model_name}.tar && tar xf ch_det_data_50.tar && cd ../
-    elif [ ${model_name} = "ocr_server_det" ]; then
-        wget -nc -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_server_v2.0_det_infer.tar
-        wget -nc -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ch_det_data_50.tar
-        cd ./inference && tar xf ch_ppocr_server_v2.0_det_infer.tar && tar xf ch_det_data_50.tar && cd ../
-    else 
-        rm -rf ./train_data/ic15_data
-        eval_model_name="ch_ppocr_mobile_v2.0_rec_infer"
-        wget -nc -P ./train_data/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ic15_data.tar
-        wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_rec_infer.tar
-        cd ./inference && tar xf ${eval_model_name}.tar && tar xf ic15_data.tar && cd ../
-    fi 
-fi
-
-if [ ${MODE} = "cpp_infer" ];then
-    cd deploy/cpp_infer
-    use_opencv=$(func_parser_value "${lines[52]}")
-    if [ ${use_opencv} = "True" ]; then
-        echo "################### build opencv ###################"
-        rm -rf 3.4.7.tar.gz opencv-3.4.7/
-        wget https://github.com/opencv/opencv/archive/3.4.7.tar.gz
-        tar -xf 3.4.7.tar.gz
-
-        cd opencv-3.4.7/
-        install_path=$(pwd)/opencv-3.4.7/opencv3
-
-        rm -rf build
-        mkdir build
-        cd build
-
-        cmake .. \
-            -DCMAKE_INSTALL_PREFIX=${install_path} \
-            -DCMAKE_BUILD_TYPE=Release \
-            -DBUILD_SHARED_LIBS=OFF \
-            -DWITH_IPP=OFF \
-            -DBUILD_IPP_IW=OFF \
-            -DWITH_LAPACK=OFF \
-            -DWITH_EIGEN=OFF \
-            -DCMAKE_INSTALL_LIBDIR=lib64 \
-            -DWITH_ZLIB=ON \
-            -DBUILD_ZLIB=ON \
-            -DWITH_JPEG=ON \
-            -DBUILD_JPEG=ON \
-            -DWITH_PNG=ON \
-            -DBUILD_PNG=ON \
-            -DWITH_TIFF=ON \
-            -DBUILD_TIFF=ON
-
-        make -j
-        make install
-        cd ../
-        echo "################### build opencv finished ###################"
-    fi
-
-
-    echo "################### build PaddleOCR demo ####################"
-    if [ ${use_opencv} = "True" ]; then
-        OPENCV_DIR=$(pwd)/opencv-3.4.7/opencv3/
-    else
-        OPENCV_DIR=''
-    fi
-    LIB_DIR=$(pwd)/Paddle/build/paddle_inference_install_dir/
-    CUDA_LIB_DIR=$(dirname `find /usr -name libcudart.so`)
-    CUDNN_LIB_DIR=$(dirname `find /usr -name libcudnn.so`)
-    
-    BUILD_DIR=build
-    rm -rf ${BUILD_DIR}
-    mkdir ${BUILD_DIR}
-    cd ${BUILD_DIR}
-    cmake .. \
-        -DPADDLE_LIB=${LIB_DIR} \
-        -DWITH_MKL=ON \
-        -DWITH_GPU=OFF \
-        -DWITH_STATIC_LIB=OFF \
-        -DWITH_TENSORRT=OFF \
-        -DOPENCV_DIR=${OPENCV_DIR} \
-        -DCUDNN_LIB=${CUDNN_LIB_DIR} \
-        -DCUDA_LIB=${CUDA_LIB_DIR} \
-        -DTENSORRT_DIR=${TENSORRT_DIR} \
-
-    make -j
-    echo "################### build PaddleOCR demo finished ###################"
-fi
\ No newline at end of file

tests/readme.md

-
-# 介绍
-
-test.sh和params.txt文件配合使用，完成OCR轻量检测和识别模型从训练到预测的流程测试。
-
-# 安装依赖
-- 安装PaddlePaddle >= 2.0
-- 安装PaddleOCR依赖
-    ```
-    pip3 install  -r ../requirements.txt
-    ```
-- 安装autolog
-    ```
-    git clone https://github.com/LDOUBLEV/AutoLog
-    cd AutoLog
-    pip3 install -r requirements.txt
-    python3 setup.py bdist_wheel
-    pip3 install ./dist/auto_log-1.0.0-py3-none-any.whl
-    cd ../
-    ```
-
-# 目录介绍
-
-```bash
-tests/
-├── ocr_det_params.txt   # 测试OCR检测模型的参数配置文件
-├── ocr_rec_params.txt   # 测试OCR识别模型的参数配置文件
-└── prepare.sh           # 完成test.sh运行所需要的数据和模型下载
-└── test.sh              # 根据
-```
-
-# 使用方法
-test.sh包含四种运行模式，每种模式的运行数据不同，分别用于测试速度和精度，分别是：
-- 模式1 lite_train_infer，使用少量数据训练，用于快速验证训练到预测的走通流程，不验证精度和速度；
-```
-bash test/prepare.sh ./tests/ocr_det_params.txt 'lite_train_infer'
-bash tests/test.sh ./tests/ocr_det_params.txt 'lite_train_infer'
-```
-- 模式2 whole_infer，使用少量数据训练，一定量数据预测，用于验证训练后的模型执行预测，预测速度是否合理；
-```
-bash tests/prepare.sh ./tests/ocr_det_params.txt 'whole_infer'
-bash tests/test.sh ./tests/ocr_det_params.txt 'whole_infer'
-```
-
-- 模式3 infer 不训练，全量数据预测，走通开源模型评估、动转静，检查inference model预测时间和精度;
-```
-bash tests/prepare.sh ./tests/ocr_det_params.txt 'infer'
-用法1:
-bash tests/test.sh ./tests/ocr_det_params.txt 'infer'
-用法2: 指定GPU卡预测，第三个传入参数为GPU卡号
-bash tests/test.sh ./tests/ocr_det_params.txt 'infer' '1'
-```
-
-模式4: whole_train_infer , CE： 全量数据训练，全量数据预测，验证模型训练精度，预测精度，预测速度
-```
-bash tests/prepare.sh ./tests/ocr_det_params.txt 'whole_train_infer'
-bash tests/test.sh ./tests/ocr_det_params.txt 'whole_train_infer'
-```