Merge pull request #6721 from LDOUBLEV/dyg_pts

support PP-OCRv3 for lite demo (arm8)

deploy/lite/config.txt

@@ -5,3 +5,4 @@ det_db_unclip_ratio  1.6
 det_db_use_dilate 0
 det_use_polygon_score 1
 use_direction_classify  1
+rec_image_height  32
\ No newline at end of file

deploy/lite/crnn_process.cc

@@ -19,25 +19,27 @@
 
 const std::vector<int> rec_image_shape{3, 32, 320};
 
-cv::Mat CrnnResizeImg(cv::Mat img, float wh_ratio) {
+cv::Mat CrnnResizeImg(cv::Mat img, float wh_ratio, int rec_image_height) {
   int imgC, imgH, imgW;
   imgC = rec_image_shape[0];
+  imgH = rec_image_height;
   imgW = rec_image_shape[2];
-  imgH = rec_image_shape[1];
 
-  imgW = int(32 * wh_ratio);
+  imgW = int(imgH * wh_ratio);
 
-  float ratio = static_cast<float>(img.cols) / static_cast<float>(img.rows);
+  float ratio = float(img.cols) / float(img.rows);
   int resize_w, resize_h;
+
   if (ceilf(imgH * ratio) > imgW)
     resize_w = imgW;
   else
-    resize_w = static_cast<int>(ceilf(imgH * ratio));
-  cv::Mat resize_img;
+    resize_w = int(ceilf(imgH * ratio));
+
   cv::resize(img, resize_img, cv::Size(resize_w, imgH), 0.f, 0.f,
              cv::INTER_LINEAR);
-
-  return resize_img;
+  cv::copyMakeBorder(resize_img, resize_img, 0, 0, 0,
+                     int(imgW - resize_img.cols), cv::BORDER_CONSTANT,
+                     {127, 127, 127});
 }
 
 std::vector<std::string> ReadDict(std::string path) {

deploy/lite/crnn_process.h

@@ -26,7 +26,7 @@
 #include "opencv2/imgcodecs.hpp"
 #include "opencv2/imgproc.hpp"
 
-cv::Mat CrnnResizeImg(cv::Mat img, float wh_ratio);
+cv::Mat CrnnResizeImg(cv::Mat img, float wh_ratio, int rec_image_height);
 
 std::vector<std::string> ReadDict(std::string path);
 

deploy/lite/ocr_db_crnn.cc

@@ -162,7 +162,8 @@ void RunRecModel(std::vector<std::vector<std::vector<int>>> boxes, cv::Mat img,
                  std::vector<std::string> charactor_dict,
                  std::shared_ptr<PaddlePredictor> predictor_cls,
                  int use_direction_classify,
-                 std::vector<double> *times) {
+                 std::vector<double> *times,
+                 int rec_image_height) {
   std::vector<float> mean = {0.5f, 0.5f, 0.5f};
   std::vector<float> scale = {1 / 0.5f, 1 / 0.5f, 1 / 0.5f};
 
@@ -183,7 +184,7 @@ void RunRecModel(std::vector<std::vector<std::vector<int>>> boxes, cv::Mat img,
     float wh_ratio =
         static_cast<float>(crop_img.cols) / static_cast<float>(crop_img.rows);
 
-    resize_img = CrnnResizeImg(crop_img, wh_ratio);
+    resize_img = CrnnResizeImg(crop_img, wh_ratio, rec_image_height);
     resize_img.convertTo(resize_img, CV_32FC3, 1 / 255.f);
 
     const float *dimg = reinterpret_cast<const float *>(resize_img.data);
@@ -444,7 +445,7 @@ void system(char **argv){
   //// load config from txt file
   auto Config = LoadConfigTxt(det_config_path);
   int use_direction_classify = int(Config["use_direction_classify"]);
-
+  int rec_image_height = int(Config["rec_image_height"]);
   auto charactor_dict = ReadDict(dict_path);
   charactor_dict.insert(charactor_dict.begin(), "#"); // blank char for ctc
   charactor_dict.push_back(" ");
@@ -590,12 +591,16 @@ void rec(int argc, char **argv) {
   std::string batchsize = argv[6];
   std::string img_dir = argv[7];
   std::string dict_path = argv[8];
+  std::string config_path = argv[9];
 
   if (strcmp(argv[4], "FP32") != 0 && strcmp(argv[4], "INT8") != 0) {
       std::cerr << "Only support FP32 or INT8." << std::endl;
       exit(1);
   }
 
+  auto Config = LoadConfigTxt(config_path);
+  int rec_image_height = int(Config["rec_image_height"]);
+
   std::vector<cv::String> cv_all_img_names;
   cv::glob(img_dir, cv_all_img_names);
 
@@ -630,7 +635,7 @@ void rec(int argc, char **argv) {
     std::vector<float> rec_text_score;
     std::vector<double> times;
     RunRecModel(boxes, srcimg, rec_predictor, rec_text, rec_text_score,
-                charactor_dict, cls_predictor, 0, &times);
+                charactor_dict, cls_predictor, 0, &times, rec_image_height);
   
     //// print recognized text
     for (int i = 0; i < rec_text.size(); i++) {

deploy/lite/readme.md

@@ -34,7 +34,7 @@ For the compilation process of different development environments, please refer
 ### 1.2 Prepare Paddle-Lite library
 
 There are two ways to obtain the Paddle-Lite library：
-- 1. Download directly, the download link of the Paddle-Lite library is as follows：
+- 1. [Recommended] Download directly, the download link of the Paddle-Lite library is as follows：
 
       | Platform | Paddle-Lite library download link |
       |---|---|
@@ -43,7 +43,9 @@ There are two ways to obtain the Paddle-Lite library：
 
       Note: 1. The above Paddle-Lite library is compiled from the Paddle-Lite 2.10 branch. For more information about Paddle-Lite 2.10, please refer to [link](https://github.com/PaddlePaddle/Paddle-Lite/releases/tag/v2.10).
 
-- 2. [Recommended] Compile Paddle-Lite to get the prediction library. The compilation method of Paddle-Lite is as follows：
+    **Note: It is recommended to use paddlelite>=2.10 version of the prediction library, other prediction library versions [download link](https://github.com/PaddlePaddle/Paddle-Lite/tags)**
+
+- 2. Compile Paddle-Lite to get the prediction library. The compilation method of Paddle-Lite is as follows：
 ```
 git clone https://github.com/PaddlePaddle/Paddle-Lite.git
 cd Paddle-Lite
@@ -104,21 +106,17 @@ If you directly use the model in the above table for deployment, you can skip th
 
 If the model to be deployed is not in the above table, you need to follow the steps below to obtain the optimized model.
 
-The `opt` tool can be obtained by compiling Paddle Lite.
+- Step 1: Refer to [document](https://www.paddlepaddle.org.cn/lite/v2.10/user_guides/opt/opt_python.html) to install paddlelite, which is used to convert paddle inference model to paddlelite required for running nb model
 ```
-git clone https://github.com/PaddlePaddle/Paddle-Lite.git
-cd Paddle-Lite
-git checkout release/v2.10
-./lite/tools/build.sh build_optimize_tool
+pip install paddlelite==2.10 # The paddlelite version should be the same as the prediction library version
 ```
-
-After the compilation is complete, the opt file is located under build.opt/lite/api/, You can view the operating options and usage of opt in the following ways:
-
+After installation, the following commands can view the help information
 ```
-cd build.opt/lite/api/
-./opt
+paddle_lite_opt
 ```
 
+Introduction to paddle_lite_opt parameters:
+
 |Options|Description|
 |---|---|
 |--model_dir|The path of the PaddlePaddle model to be optimized (non-combined form)|
@@ -131,6 +129,8 @@ cd build.opt/lite/api/
 
 `--model_dir` is suitable for the non-combined mode of the model to be optimized, and the inference model of PaddleOCR is the combined mode, that is, the model structure and model parameters are stored in a single file.
 
+- Step 2: Use paddle_lite_opt to convert the inference model to the mobile model format.
+
 The following takes the ultra-lightweight Chinese model of PaddleOCR as an example to introduce the use of the compiled opt file to complete the conversion of the inference model to the Paddle-Lite optimized model
 
 ```
@@ -240,6 +240,7 @@ det_db_thresh  0.3        # Used to filter the binarized image of DB prediction,
 det_db_box_thresh  0.5    # DDB post-processing filter box threshold, if there is a missing box detected, it can be reduced as appropriate
 det_db_unclip_ratio  1.6  # Indicates the compactness of the text box, the smaller the value, the closer the text box to the text
 use_direction_classify  0  # Whether to use the direction classifier, 0 means not to use, 1 means to use
+rec_image_height  32      # The height of the input image of the recognition model, the PP-OCRv3 model needs to be set to 48, and the PP-OCRv2 model needs to be set to 32
 ```
 
  5. Run Model on phone
@@ -258,8 +259,15 @@ After the above steps are completed, you can use adb to push the file to the pho
  cd /data/local/tmp/debug
  export LD_LIBRARY_PATH=${PWD}:$LD_LIBRARY_PATH
  # The use of ocr_db_crnn is:
- # ./ocr_db_crnn Detection model file Orientation classifier model file Recognition model file Test image path Dictionary file path
- ./ocr_db_crnn ch_PP-OCRv2_det_slim_opt.nb  ch_PP-OCRv2_rec_slim_opt.nb  ch_ppocr_mobile_v2.0_cls_opt.nb  ./11.jpg  ppocr_keys_v1.txt
+ # ./ocr_db_crnn Mode Detection model file Orientation classifier model file Recognition model file  Hardware  Precision  Threads Batchsize  Test image path Dictionary file path
+ ./ocr_db_crnn system ch_PP-OCRv2_det_slim_opt.nb  ch_PP-OCRv2_rec_slim_opt.nb  ch_ppocr_mobile_v2.0_cls_slim_opt.nb  arm8 INT8 10 1  ./11.jpg  config.txt  ppocr_keys_v1.txt  True
+# precision can be INT8 for quantitative model or FP32 for normal model.
+
+# Only using detection model
+./ocr_db_crnn  det ch_PP-OCRv2_det_slim_opt.nb arm8 INT8 10 1 ./11.jpg  config.txt
+
+# Only using recognition model
+./ocr_db_crnn  rec ch_PP-OCRv2_rec_slim_opt.nb arm8 INT8 10 1 word_1.jpg ppocr_keys_v1.txt config.txt
  ```
 
 If you modify the code, you need to recompile and push to the phone.
@@ -283,3 +291,7 @@ A2: Replace the .jpg test image under ./debug with the image you want to test, a
 Q3: How to package it into the mobile APP?
 
 A3: This demo aims to provide the core algorithm part that can run OCR on mobile phones. Further, PaddleOCR/deploy/android_demo is an example of encapsulating this demo into a mobile app for reference.
+
+Q4: When running the demo, an error is reported `Error: This model is not supported, because kernel for 'io_copy' is not supported by Paddle-Lite.`
+
+A4: The problem is that the installed paddlelite version does not match the downloaded prediction library version. Make sure that the paddleliteopt tool matches your prediction library version, and try to switch to the nb model again.

deploy/lite/readme_ch.md

@@ -32,7 +32,7 @@ Paddle Lite是飞桨轻量化推理引擎，为手机、IOT端提供高效推理
 ### 1.2 准备预测库
 
 预测库有两种获取方式：
-- 1. 直接下载，预测库下载链接如下：
+- 1. [推荐]直接下载，预测库下载链接如下：
 
       | 平台 | 预测库下载链接 |
       |---|---|
@@ -41,7 +41,9 @@ Paddle Lite是飞桨轻量化推理引擎，为手机、IOT端提供高效推理
 
       注：1. 上述预测库为PaddleLite 2.10分支编译得到，有关PaddleLite 2.10 详细信息可参考 [链接](https://github.com/PaddlePaddle/Paddle-Lite/releases/tag/v2.10) 。
 
-- 2. [推荐]编译Paddle-Lite得到预测库，Paddle-Lite的编译方式如下：
+**注：建议使用paddlelite>=2.10版本的预测库，其他预测库版本[下载链接](https://github.com/PaddlePaddle/Paddle-Lite/tags)**
+
+- 2. 编译Paddle-Lite得到预测库，Paddle-Lite的编译方式如下：
 ```
 git clone https://github.com/PaddlePaddle/Paddle-Lite.git
 cd Paddle-Lite
@@ -102,22 +104,16 @@ Paddle-Lite 提供了多种策略来自动优化原始的模型，其中包括
 
 如果要部署的模型不在上述表格中，则需要按照如下步骤获得优化后的模型。
 
-模型优化需要Paddle-Lite的opt可执行文件，可以通过编译Paddle-Lite源码获得，编译步骤如下：
+- 步骤1：参考[文档](https://www.paddlepaddle.org.cn/lite/v2.10/user_guides/opt/opt_python.html)安装paddlelite，用于转换paddle inference model为paddlelite运行所需的nb模型
 ```
-# 如果准备环境时已经clone了Paddle-Lite，则不用重新clone Paddle-Lite
-git clone https://github.com/PaddlePaddle/Paddle-Lite.git
-cd Paddle-Lite
-git checkout release/v2.10
-# 启动编译
-./lite/tools/build.sh build_optimize_tool
+pip install paddlelite==2.10  # paddlelite版本要与预测库版本一致
 ```
-
-编译完成后，opt文件位于`build.opt/lite/api/`下，可通过如下方式查看opt的运行选项和使用方式；
+安装完后，如下指令可以查看帮助信息
 ```
-cd build.opt/lite/api/
-./opt
+paddle_lite_opt
 ```
 
+paddle_lite_opt 参数介绍：
 |选项|说明|
 |---|---|
 |--model_dir|待优化的PaddlePaddle模型（非combined形式）的路径|
@@ -130,6 +126,8 @@ cd build.opt/lite/api/
 
 `--model_dir`适用于待优化的模型是非combined方式，PaddleOCR的inference模型是combined方式，即模型结构和模型参数使用单独一个文件存储。
 
+- 步骤2：使用paddle_lite_opt将inference模型转换成移动端模型格式。
+
 下面以PaddleOCR的超轻量中文模型为例，介绍使用编译好的opt文件完成inference模型到Paddle-Lite优化模型的转换。
 
 ```
@@ -148,7 +146,7 @@ wget  https://paddleocr.bj.bcebos.com/dygraph_v2.0/slim/ch_ppocr_mobile_v2.0_cls
 
 转换成功后，inference模型目录下会多出`.nb`结尾的文件，即是转换成功的模型文件。
 
-注意：使用paddle-lite部署时，需要使用opt工具优化后的模型。 opt 工具的输入模型是paddle保存的inference模型
+注意：使用paddle-lite部署时，需要使用opt工具优化后的模型。 opt工具的输入模型是paddle保存的inference模型
 
 <a name="2.2与手机联调"></a>
 ### 2.2 与手机联调
@@ -234,13 +232,14 @@ ppocr_keys_v1.txt   # 中文字典
 ...
 ```
 
-2.  `config.txt` 包含了检测器、分类器的超参数，如下：
+2.  `config.txt` 包含了检测器、分类器、识别器的超参数，如下：
 ```
 max_side_len  960         # 输入图像长宽大于960时，等比例缩放图像，使得图像最长边为960
 det_db_thresh  0.3        # 用于过滤DB预测的二值化图像，设置为0.-0.3对结果影响不明显
-det_db_box_thresh  0.5    # DB后处理过滤box的阈值，如果检测存在漏框情况，可酌情减小
+det_db_box_thresh  0.5    # 检测器后处理过滤box的阈值，如果检测存在漏框情况，可酌情减小
 det_db_unclip_ratio  1.6  # 表示文本框的紧致程度，越小则文本框更靠近文本
 use_direction_classify  0  # 是否使用方向分类器，0表示不使用，1表示使用
+rec_image_height  32      # 识别模型输入图像的高度，PP-OCRv3模型设置为48，PP-OCRv2模型需要设置为32
 ```
 
  5. 启动调试
@@ -259,8 +258,14 @@ use_direction_classify  0  # 是否使用方向分类器，0表示不使用，1
  cd /data/local/tmp/debug
  export LD_LIBRARY_PATH=${PWD}:$LD_LIBRARY_PATH
  # 开始使用，ocr_db_crnn可执行文件的使用方式为:
- # ./ocr_db_crnn  检测模型文件 方向分类器模型文件  识别模型文件  测试图像路径  字典文件路径
- ./ocr_db_crnn ch_PP-OCRv2_det_slim_opt.nb  ch_PP-OCRv2_rec_slim_opt.nb  ch_ppocr_mobile_v2.0_cls_slim_opt.nb  ./11.jpg  ppocr_keys_v1.txt
+ # ./ocr_db_crnn 预测模式  检测模型文件 方向分类器模型文件  识别模型文件 运行硬件 运行精度 线程数  batchsize  测试图像路径  参数配置路径  字典文件路径 是否使用benchmark参数
+ ./ocr_db_crnn system  ch_PP-OCRv2_det_slim_opt.nb  ch_PP-OCRv2_rec_slim_opt.nb  ch_ppocr_mobile_v2.0_cls_slim_opt.nb  arm8 INT8 10 1  ./11.jpg  config.txt  ppocr_keys_v1.txt  True
+
+# 仅使用文本检测模型，使用方式如下：
+./ocr_db_crnn  det ch_PP-OCRv2_det_slim_opt.nb arm8 INT8 10 1 ./11.jpg  config.txt
+
+# 仅使用文本识别模型，使用方式如下：
+./ocr_db_crnn  rec ch_PP-OCRv2_rec_slim_opt.nb arm8 INT8 10 1 word_1.jpg ppocr_keys_v1.txt config.txt
  ```
 
  如果对代码做了修改，则需要重新编译并push到手机上。
@@ -284,3 +289,7 @@ A2：替换debug下的.jpg测试图像为你想要测试的图像，adb push 到
 Q3：如何封装到手机APP中？
 
 A3：此demo旨在提供能在手机上运行OCR的核心算法部分，PaddleOCR/deploy/android_demo是将这个demo封装到手机app的示例，供参考
+
+Q4：运行demo时遇到报错`Error: This model is not supported, because kernel for 'io_copy' is not supported by Paddle-Lite.`
+
+A4：问题是安装的paddlelite版本和下载的预测库版本不匹配，确保paddleliteopt工具和你的预测库版本匹配，重新转nb模型试试。