diff --git a/deploy/cpp_infer/src/main.cpp b/deploy/cpp_infer/src/main.cpp index 5e5c851517d5efaa75f54b7a156563a4a42880d5..139493a2aae1050a96cb978745b0620f052ed22d 100644 --- a/deploy/cpp_infer/src/main.cpp +++ b/deploy/cpp_infer/src/main.cpp @@ -39,8 +39,8 @@ DEFINE_bool(use_gpu, false, "Infering with GPU or CPU."); DEFINE_int32(gpu_id, 0, "Device id of GPU to execute."); DEFINE_int32(gpu_mem, 4000, "GPU id when infering with GPU."); -DEFINE_int32(cpu_math_library_num_threads, 10, "Num of threads with CPU."); -DEFINE_bool(use_mkldnn, false, "Whether use mkldnn with CPU."); +DEFINE_int32(cpu_threads, 10, "Num of threads with CPU."); +DEFINE_bool(enable_mkldnn, false, "Whether use mkldnn with CPU."); DEFINE_bool(use_tensorrt, false, "Whether use tensorrt."); DEFINE_string(precision, "fp32", "Precision be one of fp32/fp16/int8"); DEFINE_bool(benchmark, true, "Whether use benchmark."); @@ -60,6 +60,7 @@ DEFINE_string(cls_model_dir, "", "Path of cls inference model."); DEFINE_double(cls_thresh, 0.9, "Threshold of cls_thresh."); // recognition related DEFINE_string(rec_model_dir, "", "Path of rec inference model."); +DEFINE_int32(rec_batch_num, 1, "rec_batch_num."); DEFINE_string(char_list_file, "../../ppocr/utils/ppocr_keys_v1.txt", "Path of dictionary."); @@ -78,8 +79,8 @@ void PrintBenchmarkLog(std::string model_name, LOG(INFO) << "ir_optim: " << "True"; LOG(INFO) << "enable_memory_optim: " << "True"; LOG(INFO) << "enable_tensorrt: " << FLAGS_use_tensorrt; - LOG(INFO) << "enable_mkldnn: " << (FLAGS_use_mkldnn ? "True" : "False"); - LOG(INFO) << "cpu_math_library_num_threads: " << FLAGS_cpu_math_library_num_threads; + LOG(INFO) << "enable_mkldnn: " << (FLAGS_enable_mkldnn ? "True" : "False"); + LOG(INFO) << "cpu_math_library_num_threads: " << FLAGS_cpu_threads; LOG(INFO) << "----------------------- Data info -----------------------"; LOG(INFO) << "batch_size: " << batch_size; LOG(INFO) << "input_shape: " << input_shape; @@ -110,8 +111,8 @@ static bool PathExists(const std::string& path){ int main_det(std::vector cv_all_img_names) { std::vector time_info = {0, 0, 0}; DBDetector det(FLAGS_det_model_dir, FLAGS_use_gpu, FLAGS_gpu_id, - FLAGS_gpu_mem, FLAGS_cpu_math_library_num_threads, - FLAGS_use_mkldnn, FLAGS_max_side_len, FLAGS_det_db_thresh, + FLAGS_gpu_mem, FLAGS_cpu_threads, + FLAGS_enable_mkldnn, FLAGS_max_side_len, FLAGS_det_db_thresh, FLAGS_det_db_box_thresh, FLAGS_det_db_unclip_ratio, FLAGS_use_polygon_score, FLAGS_visualize, FLAGS_use_tensorrt, FLAGS_precision); @@ -144,8 +145,8 @@ int main_det(std::vector cv_all_img_names) { int main_rec(std::vector cv_all_img_names) { std::vector time_info = {0, 0, 0}; CRNNRecognizer rec(FLAGS_rec_model_dir, FLAGS_use_gpu, FLAGS_gpu_id, - FLAGS_gpu_mem, FLAGS_cpu_math_library_num_threads, - FLAGS_use_mkldnn, FLAGS_char_list_file, + FLAGS_gpu_mem, FLAGS_cpu_threads, + FLAGS_enable_mkldnn, FLAGS_char_list_file, FLAGS_use_tensorrt, FLAGS_precision); for (int i = 0; i < cv_all_img_names.size(); ++i) { @@ -175,8 +176,8 @@ int main_rec(std::vector cv_all_img_names) { int main_system(std::vector cv_all_img_names) { DBDetector det(FLAGS_det_model_dir, FLAGS_use_gpu, FLAGS_gpu_id, - FLAGS_gpu_mem, FLAGS_cpu_math_library_num_threads, - FLAGS_use_mkldnn, FLAGS_max_side_len, FLAGS_det_db_thresh, + FLAGS_gpu_mem, FLAGS_cpu_threads, + FLAGS_enable_mkldnn, FLAGS_max_side_len, FLAGS_det_db_thresh, FLAGS_det_db_box_thresh, FLAGS_det_db_unclip_ratio, FLAGS_use_polygon_score, FLAGS_visualize, FLAGS_use_tensorrt, FLAGS_precision); @@ -184,14 +185,14 @@ int main_system(std::vector cv_all_img_names) { Classifier *cls = nullptr; if (FLAGS_use_angle_cls) { cls = new Classifier(FLAGS_cls_model_dir, FLAGS_use_gpu, FLAGS_gpu_id, - FLAGS_gpu_mem, FLAGS_cpu_math_library_num_threads, - FLAGS_use_mkldnn, FLAGS_cls_thresh, + FLAGS_gpu_mem, FLAGS_cpu_threads, + FLAGS_enable_mkldnn, FLAGS_cls_thresh, FLAGS_use_tensorrt, FLAGS_precision); } CRNNRecognizer rec(FLAGS_rec_model_dir, FLAGS_use_gpu, FLAGS_gpu_id, - FLAGS_gpu_mem, FLAGS_cpu_math_library_num_threads, - FLAGS_use_mkldnn, FLAGS_char_list_file, + FLAGS_gpu_mem, FLAGS_cpu_threads, + FLAGS_enable_mkldnn, FLAGS_char_list_file, FLAGS_use_tensorrt, FLAGS_precision); auto start = std::chrono::system_clock::now(); diff --git a/tests/ocr_det_params.txt b/tests/ocr_det_params.txt index 6aff66c6aa8591c9f48c81cf857809f956a3cda2..6230cfc9bf328677dd93abc84fba49d98b7ce342 100644 --- a/tests/ocr_det_params.txt +++ b/tests/ocr_det_params.txt @@ -49,4 +49,18 @@ inference:tools/infer/predict_det.py --save_log_path:null --benchmark:True null:null +===========================cpp_infer_params=========================== +infer_model:./inference/ch_ppocr_mobile_v2.0_det_infer/ +infer_quant:False +inference:./deploy/cpp_infer/build/ppocr det +--use_gpu:True|False +--enable_mkldnn:True|False +--cpu_threads:1|6 +--rec_batch_num:1 +--use_tensorrt:False|True +--precision:fp32|fp16 +--det_model_dir: +--image_dir:./inference/ch_det_data_50/all-sum-510/ +--save_log_path:null +--benchmark:True diff --git a/tests/prepare.sh b/tests/prepare.sh index d27a051cb0a7effc50305db8e2268ad36492d6cb..6b54e5c1001f316dd4a1fcd8f675647bd572e0e8 100644 --- a/tests/prepare.sh +++ b/tests/prepare.sh @@ -1,6 +1,6 @@ #!/bin/bash FILENAME=$1 -# MODE be one of ['lite_train_infer' 'whole_infer' 'whole_train_infer', 'infer'] +# MODE be one of ['lite_train_infer' 'whole_infer' 'whole_train_infer', 'infer', 'cpp_infer'] MODE=$2 dataline=$(cat ${FILENAME}) @@ -58,11 +58,11 @@ elif [ ${MODE} = "whole_infer" ];then cd ./train_data/ && tar xf icdar2015_infer.tar && tar xf ic15_data.tar ln -s ./icdar2015_infer ./icdar2015 cd ../ -else +elif [ ${MODE} = "infer" ] || [ ${MODE} = "cpp_infer" ];then if [ ${model_name} = "ocr_det" ]; then eval_model_name="ch_ppocr_mobile_v2.0_det_infer" rm -rf ./train_data/icdar2015 - wget -nc -P ./train_data https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ch_det_data_50.tar + wget -nc -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ch_det_data_50.tar wget -nc -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_det_infer.tar cd ./inference && tar xf ${eval_model_name}.tar && tar xf ch_det_data_50.tar && cd ../ else @@ -74,3 +74,89 @@ else fi fi +if [ ${MODE} = "cpp_infer" ];then + ################### build opencv ################### + cd deploy/cpp_infer + rm -rf 3.4.7.tar.gz opencv-3.4.7/ + wget https://github.com/opencv/opencv/archive/3.4.7.tar.gz + tar -xf 3.4.7.tar.gz + + cd opencv-3.4.7/ + install_path=$(pwd)/opencv-3.4.7/opencv3 + + rm -rf build + mkdir build + cd build + + cmake .. \ + -DCMAKE_INSTALL_PREFIX=${install_path} \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=OFF \ + -DWITH_IPP=OFF \ + -DBUILD_IPP_IW=OFF \ + -DWITH_LAPACK=OFF \ + -DWITH_EIGEN=OFF \ + -DCMAKE_INSTALL_LIBDIR=lib64 \ + -DWITH_ZLIB=ON \ + -DBUILD_ZLIB=ON \ + -DWITH_JPEG=ON \ + -DBUILD_JPEG=ON \ + -DWITH_PNG=ON \ + -DBUILD_PNG=ON \ + -DWITH_TIFF=ON \ + -DBUILD_TIFF=ON + + make -j + make install + cd ../ + ################### build opencv finished ################### + + +# ################### build paddle inference ################### +# rm -rf Paddle +# git clone https://github.com/PaddlePaddle/Paddle.git + +# cd Paddle +# rm -rf build +# mkdir build +# cd build + +# cmake .. \ +# -DWITH_CONTRIB=OFF \ +# -DWITH_MKL=ON \ +# -DWITH_MKLDNN=ON \ +# -DWITH_TESTING=OFF \ +# -DCMAKE_BUILD_TYPE=Release \ +# -DWITH_INFERENCE_API_TEST=OFF \ +# -DON_INFER=ON \ +# -DWITH_PYTHON=ON +# make -j +# make inference_lib_dist +# cd ../ +# ################### build paddle inference finished ################### + + + ################### build PaddleOCR demo ################### + OPENCV_DIR=$(pwd)/opencv-3.4.7/opencv3/ + LIB_DIR=$(pwd)/Paddle/build/paddle_inference_install_dir/ + CUDA_LIB_DIR=/usr/local/cuda/lib64/ + CUDNN_LIB_DIR=/usr/lib/x86_64-linux-gnu/ + + BUILD_DIR=build + rm -rf ${BUILD_DIR} + mkdir ${BUILD_DIR} + cd ${BUILD_DIR} + cmake .. \ + -DPADDLE_LIB=${LIB_DIR} \ + -DWITH_MKL=ON \ + -DWITH_GPU=OFF \ + -DWITH_STATIC_LIB=OFF \ + -DWITH_TENSORRT=OFF \ + -DOPENCV_DIR=${OPENCV_DIR} \ + -DCUDNN_LIB=${CUDNN_LIB_DIR} \ + -DCUDA_LIB=${CUDA_LIB_DIR} \ + -DTENSORRT_DIR=${TENSORRT_DIR} \ + + make -j + ################### build PaddleOCR demo finished ################### +fi \ No newline at end of file diff --git a/tests/test.sh b/tests/test.sh index 9888e0faabb13b00acdf41ad154ba0a0e7ec2b63..659c01491561f36690403bbff3fbff51083384b9 100644 --- a/tests/test.sh +++ b/tests/test.sh @@ -1,6 +1,6 @@ #!/bin/bash FILENAME=$1 -# MODE be one of ['lite_train_infer' 'whole_infer' 'whole_train_infer', 'infer'] +# MODE be one of ['lite_train_infer' 'whole_infer' 'whole_train_infer', 'infer', 'cpp_infer'] MODE=$2 dataline=$(cat ${FILENAME}) @@ -145,6 +145,35 @@ benchmark_value=$(func_parser_value "${lines[49]}") infer_key1=$(func_parser_key "${lines[50]}") infer_value1=$(func_parser_value "${lines[50]}") +# parser cpp inference model +cpp_infer_model_dir_list=$(func_parser_value "${lines[52]}") +cpp_infer_is_quant=$(func_parser_value "${lines[53]}") +# parser cpp inference +inference_cmd=$(func_parser_value "${lines[54]}") +cpp_use_gpu_key=$(func_parser_key "${lines[55]}") +cpp_use_gpu_list=$(func_parser_value "${lines[55]}") +cpp_use_mkldnn_key=$(func_parser_key "${lines[56]}") +cpp_use_mkldnn_list=$(func_parser_value "${lines[56]}") +cpp_cpu_threads_key=$(func_parser_key "${lines[57]}") +cpp_cpu_threads_list=$(func_parser_value "${lines[57]}") +cpp_batch_size_key=$(func_parser_key "${lines[58]}") +cpp_batch_size_list=$(func_parser_value "${lines[58]}") +cpp_use_trt_key=$(func_parser_key "${lines[59]}") +cpp_use_trt_list=$(func_parser_value "${lines[59]}") +cpp_precision_key=$(func_parser_key "${lines[60]}") +cpp_precision_list=$(func_parser_value "${lines[60]}") +cpp_infer_model_key=$(func_parser_key "${lines[61]}") +cpp_image_dir_key=$(func_parser_key "${lines[62]}") +cpp_infer_img_dir=$(func_parser_value "${lines[62]}") +cpp_save_log_key=$(func_parser_key "${lines[63]}") +cpp_benchmark_key=$(func_parser_key "${lines[64]}") +cpp_benchmark_value=$(func_parser_value "${lines[64]}") + +echo $inference_cmd +echo $cpp_cpu_threads_key $cpp_cpu_threads_list +echo $cpp_precision_key $cpp_precision_list +echo $cpp_benchmark_key $cpp_benchmark_value + LOG_PATH="./tests/output" mkdir -p ${LOG_PATH} status_log="${LOG_PATH}/results.log" @@ -218,6 +247,71 @@ function func_inference(){ done } +function func_cpp_inference(){ + IFS='|' + _script=$1 + _model_dir=$2 + _log_path=$3 + _img_dir=$4 + _flag_quant=$5 + # inference + for use_gpu in ${cpp_use_gpu_list[*]}; do + if [ ${use_gpu} = "False" ] || [ ${use_gpu} = "cpu" ]; then + for use_mkldnn in ${cpp_use_mkldnn_list[*]}; do + if [ ${use_mkldnn} = "False" ] && [ ${_flag_quant} = "True" ]; then + continue + fi + for threads in ${cpp_cpu_threads_list[*]}; do + for batch_size in ${cpp_batch_size_list[*]}; do + _save_log_path="${_log_path}/cpp_infer_cpu_usemkldnn_${use_mkldnn}_threads_${threads}_batchsize_${batch_size}.log" + set_infer_data=$(func_set_params "${cpp_image_dir_key}" "${_img_dir}") + set_benchmark=$(func_set_params "${cpp_benchmark_key}" "${cpp_benchmark_value}") + set_batchsize=$(func_set_params "${cpp_batch_size_key}" "${batch_size}") + set_cpu_threads=$(func_set_params "${cpp_cpu_threads_key}" "${threads}") + set_model_dir=$(func_set_params "${cpp_infer_model_key}" "${_model_dir}") + command="${_script} ${cpp_use_gpu_key}=${use_gpu} ${cpp_use_mkldnn_key}=${use_mkldnn} ${set_cpu_threads} ${set_model_dir} ${set_batchsize} ${set_infer_data} ${set_benchmark} > ${_save_log_path} 2>&1 " + eval $command + last_status=${PIPESTATUS[0]} + eval "cat ${_save_log_path}" + status_check $last_status "${command}" "${status_log}" + done + done + done + elif [ ${use_gpu} = "True" ] || [ ${use_gpu} = "gpu" ]; then + for use_trt in ${cpp_use_trt_list[*]}; do + for precision in ${cpp_precision_list[*]}; do + if [[ ${_flag_quant} = "False" ]] && [[ ${precision} =~ "int8" ]]; then + continue + fi + if [[ ${precision} =~ "fp16" || ${precision} =~ "int8" ]] && [ ${use_trt} = "False" ]; then + continue + fi + if [[ ${use_trt} = "False" || ${precision} =~ "int8" ]] && [ ${_flag_quant} = "True" ]; then + continue + fi + for batch_size in ${cpp_batch_size_list[*]}; do + _save_log_path="${_log_path}/cpp_infer_gpu_usetrt_${use_trt}_precision_${precision}_batchsize_${batch_size}.log" + set_infer_data=$(func_set_params "${cpp_image_dir_key}" "${_img_dir}") + set_benchmark=$(func_set_params "${cpp_benchmark_key}" "${cpp_benchmark_value}") + set_batchsize=$(func_set_params "${cpp_batch_size_key}" "${batch_size}") + set_tensorrt=$(func_set_params "${cpp_use_trt_key}" "${use_trt}") + set_precision=$(func_set_params "${cpp_precision_key}" "${precision}") + set_model_dir=$(func_set_params "${cpp_infer_model_key}" "${_model_dir}") + command="${_script} ${cpp_use_gpu_key}=${use_gpu} ${set_tensorrt} ${set_precision} ${set_model_dir} ${set_batchsize} ${set_infer_data} ${set_benchmark} > ${_save_log_path} 2>&1 " + eval $command + last_status=${PIPESTATUS[0]} + eval "cat ${_save_log_path}" + status_check $last_status "${command}" "${status_log}" + + done + done + done + else + echo "Does not support hardware other than CPU and GPU Currently!" + fi + done +} + if [ ${MODE} = "infer" ]; then GPUID=$3 if [ ${#GPUID} -le 0 ];then @@ -252,6 +346,25 @@ if [ ${MODE} = "infer" ]; then Count=$(($Count + 1)) done +elif [ ${MODE} = "cpp_infer" ]; then + GPUID=$3 + if [ ${#GPUID} -le 0 ];then + env=" " + else + env="export CUDA_VISIBLE_DEVICES=${GPUID}" + fi + # set CUDA_VISIBLE_DEVICES + eval $env + export Count=0 + IFS="|" + infer_quant_flag=(${cpp_infer_is_quant}) + for infer_model in ${cpp_infer_model_dir_list[*]}; do + #run inference + is_quant=${infer_quant_flag[Count]} + func_cpp_inference "${inference_cmd}" "${infer_model}" "${LOG_PATH}" "${cpp_infer_img_dir}" ${is_quant} + Count=$(($Count + 1)) + done + else IFS="|" export Count=0