models/image_classification_mobilenet/demo.py

@@ -31,13 +31,16 @@ parser.add_argument('--backend_target', '-bt', type=int, default=0,
                         {:d}: TIM-VX + NPU,
                         {:d}: CANN + NPU
                     '''.format(*[x for x in range(len(backend_target_pairs))]))
+parser.add_argument('--top_k', type=int, default=1,
+                    help='Usage: Get top k predictions.')
 args = parser.parse_args()
 
 if __name__ == '__main__':
     backend_id = backend_target_pairs[args.backend_target][0]
     target_id = backend_target_pairs[args.backend_target][1]
+    top_k = args.top_k
     # Instantiate MobileNet
-    model = MobileNet(modelPath=args.model, backendId=backend_id, targetId=target_id)
+    model = MobileNet(modelPath=args.model, topK=top_k, backendId=backend_id, targetId=target_id)
 
     # Read image and get a 224x224 crop from a 256x256 resized
     image = cv.imread(args.input)

models/image_classification_mobilenet/mobilenet.py

@@ -6,10 +6,11 @@ class MobileNet:
     Works with MobileNet V1 & V2.
     '''
 
-    def __init__(self, modelPath, topK=1, backendId=0, targetId=0):
+    def __init__(self, modelPath, topK=1, loadLabel=True, backendId=0, targetId=0):
         self.model_path = modelPath
         assert topK >= 1
         self.top_k = topK
+        self.load_label = loadLabel
         self.backend_id = backendId
         self.target_id = targetId
 
@@ -64,7 +65,7 @@ class MobileNet:
         for o in output_blob:
             class_id_list = o.argsort()[::-1][:self.top_k]
             batched_class_id_list.append(class_id_list)
-        if len(self._labels) > 0:
+        if len(self._labels) > 0 and self.load_label:
             batched_predicted_labels = []
             for class_id_list in batched_class_id_list:
                 predicted_labels = []

models/image_classification_ppresnet/demo.py

@@ -37,13 +37,16 @@ parser.add_argument('--backend_target', '-bt', type=int, default=0,
                         {:d}: TIM-VX + NPU,
                         {:d}: CANN + NPU
                     '''.format(*[x for x in range(len(backend_target_pairs))]))
+parser.add_argument('--top_k', type=int, default=1,
+                    help='Usage: Get top k predictions.')
 args = parser.parse_args()
 
 if __name__ == '__main__':
     backend_id = backend_target_pairs[args.backend_target][0]
     target_id = backend_target_pairs[args.backend_target][1]
+    top_k = args.top_k
     # Instantiate ResNet
-    model = PPResNet(modelPath=args.model, backendId=backend_id, targetId=target_id)
+    model = PPResNet(modelPath=args.model, topK=top_k, backendId=backend_id, targetId=target_id)
 
     # Read image and get a 224x224 crop from a 256x256 resized
     image = cv.imread(args.input)

models/image_classification_ppresnet/ppresnet.py

@@ -9,10 +9,11 @@ import numpy as np
 import cv2 as cv
 
 class PPResNet:
-    def __init__(self, modelPath, topK=1, backendId=0, targetId=0):
+    def __init__(self, modelPath, topK=1, loadLabel=True, backendId=0, targetId=0):
         self._modelPath = modelPath
         assert topK >= 1
         self._topK = topK
+        self._load_label = loadLabel
         self._backendId = backendId
         self._targetId = targetId
 
@@ -69,7 +70,7 @@ class PPResNet:
         for ob in outputBlob:
             class_id_list = ob.argsort()[::-1][:self._topK]
             batched_class_id_list.append(class_id_list)
-        if len(self._labels) > 0:
+        if len(self._labels) > 0 and self._load_label:
             batched_predicted_labels = []
             for class_id_list in batched_class_id_list:
                 predicted_labels = []

models/object_detection_yolox/CMakeLists.txt

+cmake_minimum_required(VERSION 3.24)
+set(project_name "opencv_zoo_object_detection_yolox")
+
+PROJECT (${project_name})
+
+set(OPENCV_VERSION "4.7.0")
+set(OPENCV_INSTALLATION_PATH "" CACHE PATH "Where to look for OpenCV installation")
+find_package(OpenCV ${OPENCV_VERSION} REQUIRED HINTS ${OPENCV_INSTALLATION_PATH})
+# Find OpenCV, you may need to set OpenCV_DIR variable
+# to the absolute path to the directory containing OpenCVConfig.cmake file
+# via the command line or GUI
+
+file(GLOB SourceFile
+    "demo.cpp")
+# If the package has been found, several variables will
+# be set, you can find the full list with descriptions
+# in the OpenCVConfig.cmake file.
+# Print some message showing some of them
+message(STATUS "OpenCV library status:")
+message(STATUS "    config: ${OpenCV_DIR}")
+message(STATUS "    version: ${OpenCV_VERSION}")
+message(STATUS "    libraries: ${OpenCV_LIBS}")
+message(STATUS "    include path: ${OpenCV_INCLUDE_DIRS}")
+
+# Declare the executable target built from your sources
+add_executable(${project_name} ${SourceFile})
+
+# Link your application with OpenCV libraries
+target_link_libraries(${project_name} PRIVATE ${OpenCV_LIBS})

models/object_detection_yolox/README.md

@@ -13,6 +13,8 @@ Note:
 
 ## Demo
 
+### Python
+
 Run the following command to try the demo: 
 ```shell
 # detect on camera input
@@ -24,6 +26,23 @@ Note:
 - image result saved as "result.jpg"
 - this model requires `opencv-python>=4.8.0`
 
+### C++
+
+Install latest OpenCV and CMake >= 3.24.0 to get started with:
+
+```shell
+# A typical and default installation path of OpenCV is /usr/local
+cmake -B build -D OPENCV_INSTALLATION_PATH=/path/to/opencv/installation .
+cmake --build build
+
+# detect on camera input
+./build/opencv_zoo_object_detection_yolox
+# detect on an image
+./build/opencv_zoo_object_detection_yolox -m=/path/to/model -i=/path/to/image -v
+# get help messages
+./build/opencv_zoo_object_detection_yolox -h
+```
+
 
 ## Results
 

models/object_detection_yolox/demo.cpp

+#include <vector>
+#include <string>
+#include <utility>
+
+#include <opencv2/opencv.hpp>
+
+using namespace std;
+using namespace cv;
+using namespace dnn;
+
+vector< pair<dnn::Backend, dnn::Target> > backendTargetPairs = {
+        std::make_pair<dnn::Backend, dnn::Target>(dnn::DNN_BACKEND_OPENCV, dnn::DNN_TARGET_CPU),
+        std::make_pair<dnn::Backend, dnn::Target>(dnn::DNN_BACKEND_CUDA, dnn::DNN_TARGET_CUDA),
+        std::make_pair<dnn::Backend, dnn::Target>(dnn::DNN_BACKEND_CUDA, dnn::DNN_TARGET_CUDA_FP16),
+        std::make_pair<dnn::Backend, dnn::Target>(dnn::DNN_BACKEND_TIMVX, dnn::DNN_TARGET_NPU),
+        std::make_pair<dnn::Backend, dnn::Target>(dnn::DNN_BACKEND_CANN, dnn::DNN_TARGET_NPU) };
+
+vector<string> labelYolox = {
+        "person", "bicycle", "car", "motorcycle", "airplane", "bus",
+        "train", "truck", "boat", "traffic light", "fire hydrant",
+        "stop sign", "parking meter", "bench", "bird", "cat", "dog",
+        "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe",
+        "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee",
+        "skis", "snowboard", "sports ball", "kite", "baseball bat",
+        "baseball glove", "skateboard", "surfboard", "tennis racket",
+        "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl",
+        "banana", "apple", "sandwich", "orange", "broccoli", "carrot",
+        "hot dog", "pizza", "donut", "cake", "chair", "couch",
+        "potted plant", "bed", "dining table", "toilet", "tv", "laptop",
+        "mouse", "remote", "keyboard", "cell phone", "microwave",
+        "oven", "toaster", "sink", "refrigerator", "book", "clock",
+        "vase", "scissors", "teddy bear", "hair drier", "toothbrush" };
+
+class YoloX {
+private:
+    Net net;
+    string modelPath;
+    Size inputSize;
+    float confThreshold;
+    float nmsThreshold;
+    float objThreshold;
+    dnn::Backend backendId;
+    dnn::Target targetId;
+    int num_classes;
+    vector<int> strides;
+    Mat expandedStrides;
+    Mat grids;
+
+public:
+    YoloX(string modPath, float confThresh = 0.35, float nmsThresh = 0.5, float objThresh = 0.5, dnn::Backend bId = DNN_BACKEND_DEFAULT, dnn::Target tId = DNN_TARGET_CPU) :
+        modelPath(modPath), confThreshold(confThresh),
+        nmsThreshold(nmsThresh), objThreshold(objThresh),
+        backendId(bId), targetId(tId)
+    {
+        this->num_classes = int(labelYolox.size());
+        this->net = readNet(modelPath);
+        this->inputSize = Size(640, 640);
+        this->strides = vector<int>{ 8, 16, 32 };
+        this->net.setPreferableBackend(this->backendId);
+        this->net.setPreferableTarget(this->targetId);
+        this->generateAnchors();
+    }
+
+    void setBackendAndTarget(dnn::Backend bId, dnn::Target tId)
+    {
+        this->backendId = bId;
+        this->targetId = tId;
+        this->net.setPreferableBackend(this->backendId);
+        this->net.setPreferableTarget(this->targetId);
+    }
+
+    Mat preprocess(Mat img)
+    {
+        Mat blob;
+        Image2BlobParams paramYolox;
+        paramYolox.datalayout = DNN_LAYOUT_NCHW;
+        paramYolox.ddepth = CV_32F;
+        paramYolox.mean = Scalar::all(0);
+        paramYolox.scalefactor = Scalar::all(1);
+        paramYolox.size = Size(img.cols, img.rows);
+        paramYolox.swapRB = true;
+
+        blob = blobFromImageWithParams(img, paramYolox);
+        return blob;
+    }
+
+   Mat infer(Mat srcimg)
+    {
+        Mat inputBlob = this->preprocess(srcimg);
+
+        this->net.setInput(inputBlob);
+        vector<Mat> outs;
+        this->net.forward(outs, this->net.getUnconnectedOutLayersNames());
+
+        Mat predictions = this->postprocess(outs[0]);
+        return predictions;
+    }
+
+    Mat postprocess(Mat outputs)
+    {
+        Mat dets = outputs.reshape(0,outputs.size[1]);
+        Mat col01;
+        add(dets.colRange(0, 2), this->grids, col01);
+        Mat col23;
+        exp(dets.colRange(2, 4), col23);
+        vector<Mat> col = { col01, col23 };
+        Mat boxes;
+        hconcat(col, boxes);
+        float* ptr = this->expandedStrides.ptr<float>(0);
+        for (int r = 0; r < boxes.rows; r++, ptr++)
+        {
+            boxes.rowRange(r, r + 1) = *ptr * boxes.rowRange(r, r + 1);
+        }
+        // get boxes
+        Mat boxes_xyxy(boxes.rows, boxes.cols, CV_32FC1, Scalar(1));
+        Mat scores = dets.colRange(5, dets.cols).clone();
+        vector<float> maxScores(dets.rows);
+        vector<int> maxScoreIdx(dets.rows);
+        vector<Rect2d> boxesXYXY(dets.rows);
+
+        for (int r = 0; r < boxes_xyxy.rows; r++, ptr++)
+        {
+            boxes_xyxy.at<float>(r, 0) = boxes.at<float>(r, 0) - boxes.at<float>(r, 2) / 2.f;
+            boxes_xyxy.at<float>(r, 1) = boxes.at<float>(r, 1) - boxes.at<float>(r, 3) / 2.f;
+            boxes_xyxy.at<float>(r, 2) = boxes.at<float>(r, 0) + boxes.at<float>(r, 2) / 2.f;
+            boxes_xyxy.at<float>(r, 3) = boxes.at<float>(r, 1) + boxes.at<float>(r, 3) / 2.f;
+            // get scores and class indices
+            scores.rowRange(r, r + 1) = scores.rowRange(r, r + 1) * dets.at<float>(r, 4);
+            double minVal, maxVal;
+            Point maxIdx;
+            minMaxLoc(scores.rowRange(r, r+1), &minVal, &maxVal, nullptr, &maxIdx);
+            maxScoreIdx[r] = maxIdx.x;
+            maxScores[r] = float(maxVal);
+            boxesXYXY[r].x = boxes_xyxy.at<float>(r, 0);
+            boxesXYXY[r].y = boxes_xyxy.at<float>(r, 1);
+            boxesXYXY[r].width = boxes_xyxy.at<float>(r, 2);
+            boxesXYXY[r].height = boxes_xyxy.at<float>(r, 3);
+        }
+
+        vector< int > keep;
+        NMSBoxesBatched(boxesXYXY, maxScores, maxScoreIdx, this->confThreshold, this->nmsThreshold, keep);
+        Mat candidates(int(keep.size()), 6, CV_32FC1);
+        int row = 0;
+        for (auto idx : keep)
+        {
+            boxes_xyxy.rowRange(idx, idx + 1).copyTo(candidates(Rect(0, row, 4, 1)));
+            candidates.at<float>(row, 4) = maxScores[idx];
+            candidates.at<float>(row, 5) = float(maxScoreIdx[idx]);
+            row++;
+        }
+        if (keep.size() == 0)
+            return Mat();
+        return candidates;
+        
+    }
+
+
+    void generateAnchors()
+    {
+        vector< tuple<int, int, int> > nb;
+        int total = 0;
+
+        for (auto v : this->strides)
+        {
+            int w = this->inputSize.width / v;
+            int h = this->inputSize.height / v;
+            nb.push_back(tuple<int, int, int>(w * h, w, v));
+            total += w * h;
+        }
+        this->grids = Mat(total, 2, CV_32FC1);
+        this->expandedStrides = Mat(total, 1, CV_32FC1);
+        float* ptrGrids = this->grids.ptr<float>(0);
+        float* ptrStrides = this->expandedStrides.ptr<float>(0);
+        int pos = 0;
+        for (auto le : nb)
+        {
+            int r = get<1>(le);
+            for (int i = 0; i < get<0>(le); i++, pos++)
+            {
+                *ptrGrids++ = float(i % r);
+                *ptrGrids++ = float(i / r);
+                *ptrStrides++ = float((get<2>(le)));
+            }
+        }
+    }
+};
+
+std::string keys =
+"{ help  h          |                                               | Print help message. }"
+"{ model m          | object_detection_yolox_2022nov.onnx           | Usage: Path to the model, defaults to object_detection_yolox_2022nov.onnx  }"
+"{ input i          |                                               | Path to input image or video file. Skip this argument to capture frames from a camera.}"
+"{ confidence       | 0.5                                           | Class confidence }"
+"{ obj              | 0.5                                           | Enter object threshold }"
+"{ nms              | 0.5                                           | Enter nms IOU threshold }"
+"{ save s           | true                                          | Specify to save results. This flag is invalid when using camera. }"
+"{ vis v            | 1                                             | Specify to open a window for result visualization. This flag is invalid when using camera. }"
+"{ backend bt       | 0                                             | Choose one of computation backends: "
+"0: (default) OpenCV implementation + CPU, "
+"1: CUDA + GPU (CUDA), "
+"2: CUDA + GPU (CUDA FP16), "
+"3: TIM-VX + NPU, "
+"4: CANN + NPU}";
+
+pair<Mat, double> letterBox(Mat srcimg, Size targetSize = Size(640, 640))
+{
+    Mat paddedImg(targetSize.height, targetSize.width, CV_32FC3, Scalar::all(114.0));
+    Mat resizeImg;
+
+    double ratio = min(targetSize.height / double(srcimg.rows), targetSize.width / double(srcimg.cols));
+    resize(srcimg, resizeImg, Size(int(srcimg.cols * ratio), int(srcimg.rows * ratio)), INTER_LINEAR);
+    resizeImg.copyTo(paddedImg(Rect(0, 0, int(srcimg.cols * ratio), int(srcimg.rows * ratio))));
+    return pair<Mat, double>(paddedImg, ratio);
+}
+
+Mat unLetterBox(Mat bbox, double letterboxScale)
+{
+    return bbox / letterboxScale;
+}
+
+Mat visualize(Mat dets, Mat srcimg, double letterbox_scale, double fps = -1)
+{
+    Mat resImg = srcimg.clone();
+
+    if (fps > 0)
+        putText(resImg, format("FPS: %.2f", fps), Size(10, 25), FONT_HERSHEY_SIMPLEX, 1, Scalar(0, 0, 255), 2);
+
+    for (int row = 0; row < dets.rows; row++)
+    {
+        Mat boxF = unLetterBox(dets(Rect(0, row, 4, 1)), letterbox_scale);
+        Mat box;
+        boxF.convertTo(box, CV_32S);
+        float score = dets.at<float>(row, 4);
+        int clsId = int(dets.at<float>(row, 5));
+
+        int x0 = box.at<int>(0, 0);
+        int y0 = box.at<int>(0, 1);
+        int x1 = box.at<int>(0, 2);
+        int y1 = box.at<int>(0, 3);
+
+        string text = format("%s : %f", labelYolox[clsId].c_str(), score * 100);
+        int font = FONT_HERSHEY_SIMPLEX;
+        int baseLine = 0;
+        Size txtSize = getTextSize(text, font, 0.4, 1, &baseLine);
+        rectangle(resImg, Point(x0, y0), Point(x1, y1), Scalar(0, 255, 0), 2);
+        rectangle(resImg, Point(x0, y0 + 1), Point(x0 + txtSize.width + 1, y0 + int(1.5 * txtSize.height)), Scalar(255, 255, 255), -1);
+        putText(resImg, text, Point(x0, y0 + txtSize.height), font, 0.4, Scalar(0, 0, 0), 1);
+    }
+
+    return resImg;
+}
+
+int main(int argc, char** argv)
+{
+    CommandLineParser parser(argc, argv, keys);
+
+    parser.about("Use this script to run Yolox deep learning networks in opencv_zoo using OpenCV.");
+    if (parser.has("help"))
+    {
+        parser.printMessage();
+        return 0;
+    }
+
+    string model = parser.get<String>("model");
+    float confThreshold = parser.get<float>("confidence");
+    float objThreshold = parser.get<float>("obj");
+    float nmsThreshold = parser.get<float>("nms");
+    bool vis = parser.get<bool>("vis");
+    bool save = parser.get<bool>("save");
+    int backendTargetid = parser.get<int>("backend");
+
+    if (model.empty())
+    {
+        CV_Error(Error::StsError, "Model file " + model + " not found");
+    }
+
+    YoloX modelNet(model, confThreshold, nmsThreshold, objThreshold,
+        backendTargetPairs[backendTargetid].first, backendTargetPairs[backendTargetid].second);
+    //! [Open a video file or an image file or a camera stream]
+    VideoCapture cap;
+    if (parser.has("input"))
+        cap.open(samples::findFile(parser.get<String>("input")));
+    else
+        cap.open(0);
+    if (!cap.isOpened())
+        CV_Error(Error::StsError, "Cannot opend video or file");
+    Mat frame, inputBlob;
+    double letterboxScale;
+
+    static const std::string kWinName = model;
+    int nbInference = 0;
+    while (waitKey(1) < 0)
+    {
+        cap >> frame;
+        if (frame.empty())
+        {
+            cout << "Frame is empty" << endl;
+            waitKey();
+            break;
+        }
+        pair<Mat, double> w = letterBox(frame);
+        inputBlob = get<0>(w);
+        letterboxScale  = get<1>(w);
+        TickMeter tm;
+        tm.start();
+        Mat predictions = modelNet.infer(inputBlob);
+        tm.stop();
+        cout << "Inference time: " << tm.getTimeMilli() << " ms\n";
+        Mat img = visualize(predictions, frame, letterboxScale, tm.getFPS());
+        if (vis)
+        {
+            imshow(kWinName, img);
+        }
+    }
+    return 0;
+}

models/person_detection_mediapipe/CMakeLists.txt

+cmake_minimum_required(VERSION 3.24.0)
+set(project_name "opencv_zoo_person_detection_mediapipe")
+
+PROJECT (${project_name})
+
+set(OPENCV_VERSION "4.7.0")
+set(OPENCV_INSTALLATION_PATH "" CACHE PATH "Where to look for OpenCV installation")
+find_package(OpenCV ${OPENCV_VERSION} REQUIRED HINTS ${OPENCV_INSTALLATION_PATH})
+# Find OpenCV, you may need to set OpenCV_DIR variable
+# to the absolute path to the directory containing OpenCVConfig.cmake file
+# via the command line or GUI
+
+file(GLOB SourceFile
+    "demo.cpp")
+# If the package has been found, several variables will
+# be set, you can find the full list with descriptions
+# in the OpenCVConfig.cmake file.
+# Print some message showing some of them
+message(STATUS "OpenCV library status:")
+message(STATUS "    config: ${OpenCV_DIR}")
+message(STATUS "    version: ${OpenCV_VERSION}")
+message(STATUS "    libraries: ${OpenCV_LIBS}")
+message(STATUS "    include path: ${OpenCV_INCLUDE_DIRS}")
+
+# Declare the executable target built from your sources
+add_executable(${project_name} ${SourceFile})
+
+# Link your application with OpenCV libraries
+target_link_libraries(${project_name} PRIVATE ${OpenCV_LIBS})

models/person_detection_mediapipe/README.md

@@ -9,6 +9,8 @@ SSD Anchors are generated from [GenMediaPipePalmDectionSSDAnchors](https://githu
 
 ## Demo
 
+### Python
+
 Run the following commands to try the demo:
 
 ```bash
@@ -21,6 +23,23 @@ python demo.py -i /path/to/image -v
 python demo.py --help
 ```
 
+### C++
+
+Install latest OpenCV and CMake >= 3.24.0 to get started with:
+
+```shell
+# A typical and default installation path of OpenCV is /usr/local
+cmake -B build -D OPENCV_INSTALLATION_PATH=/path/to/opencv/installation .
+cmake --build build
+
+# detect on camera input
+./build/opencv_zoo_person_detection_mediapipe
+# detect on an image
+./build/opencv_zoo_person_detection_mediapipe -m=/path/to/model -i=/path/to/image -v
+# get help messages
+./build/opencv_zoo_person_detection_mediapipe -h
+```
+
 ### Example outputs
 
 ![webcam demo](./example_outputs/mppersondet_demo.webp)

models/text_recognition_crnn/CMakeLists.txt

+cmake_minimum_required(VERSION 3.24)
+set(project_name "opencv_zoo_text_recognition_crnn")
+
+PROJECT (${project_name})
+
+set(OPENCV_VERSION "4.7.0")
+set(OPENCV_INSTALLATION_PATH "" CACHE PATH "Where to look for OpenCV installation")
+find_package(OpenCV ${OPENCV_VERSION} REQUIRED HINTS ${OPENCV_INSTALLATION_PATH})
+# Find OpenCV, you may need to set OpenCV_DIR variable
+# to the absolute path to the directory containing OpenCVConfig.cmake file
+# via the command line or GUI
+
+file(GLOB SourceFile
+    "demo.cpp")
+# If the package has been found, several variables will
+# be set, you can find the full list with descriptions
+# in the OpenCVConfig.cmake file.
+# Print some message showing some of them
+message(STATUS "OpenCV library status:")
+message(STATUS "    config: ${OpenCV_DIR}")
+message(STATUS "    version: ${OpenCV_VERSION}")
+message(STATUS "    libraries: ${OpenCV_LIBS}")
+message(STATUS "    include path: ${OpenCV_INCLUDE_DIRS}")
+
+# Declare the executable target built from your sources
+add_executable(${project_name} ${SourceFile})
+
+# Link your application with OpenCV libraries
+target_link_libraries(${project_name} PRIVATE ${OpenCV_LIBS})

models/text_recognition_crnn/README.md

@@ -36,6 +36,8 @@ Note:
   - Try `text_recognition_CRNN_CH_2021sep.onnx` with `charset_94_CH.txt`
   - Try `text_recognition_CRNN_CN_2021sep.onnx` with `charset_3944_CN.txt`.
 
+### Python
+
 Run the demo detecting English:
 
 ```shell
@@ -52,13 +54,38 @@ Run the demo detecting Chinese:
 
 ```shell
 # detect on camera input
-python demo.py --model text_recognition_CRNN_CN_2021nov.onnx --charset charset_3944_CN.txt
+python demo.py --model text_recognition_CRNN_CN_2021nov.onnx
 # detect on an image
-python demo.py --input /path/to/image --model text_recognition_CRNN_CN_2021nov.onnx --charset charset_3944_CN.txt
+python demo.py --input /path/to/image --model text_recognition_CRNN_CN_2021nov.onnx
 
 # get help regarding various parameters
 python demo.py --help
 ```
+### C++
+
+Install latest OpenCV and CMake >= 3.24.0 to get started with:
+
+```shell
+# detect on camera input
+./build/opencv_zoo_text_recognition_crnn 
+# detect on an image
+./build/opencv_zoo_text_recognition_crnn --input /path/to/image -v
+
+# get help regarding various parameters
+./build/opencv_zoo_text_recognition_crnn --help
+```
+
+Run the demo detecting Chinese:
+
+```shell
+# detect on camera input
+./build/opencv_zoo_text_recognition_crnn  --model=text_recognition_CRNN_CN_2021nov.onnx --charset=charset_3944_CN.txt
+# detect on an image
+./build/opencv_zoo_text_recognition_crnn  --input=/path/to/image --model=text_recognition_CRNN_CN_2021nov.onnx --charset=charset_3944_CN.txt
+
+# get help regarding various parameters
+./build/opencv_zoo_text_recognition_crnn  --help
+
 
 ### Examples
 

models/text_recognition_crnn/demo.cpp

+#include <iostream>
+#include <codecvt>
+
+
+#include <opencv2/dnn.hpp>
+#include <opencv2/imgproc.hpp>
+#include <opencv2/highgui.hpp>
+
+#include "charset_32_94_3944.h"
+
+using namespace std;
+using namespace cv;
+using namespace dnn;
+
+vector< pair<cv::dnn::Backend, cv::dnn::Target> > backendTargetPairs = {
+        std::make_pair<cv::dnn::Backend, cv::dnn::Target>(dnn::DNN_BACKEND_OPENCV, dnn::DNN_TARGET_CPU),
+        std::make_pair<cv::dnn::Backend, cv::dnn::Target>(dnn::DNN_BACKEND_CUDA, dnn::DNN_TARGET_CUDA),
+        std::make_pair<cv::dnn::Backend, cv::dnn::Target>(dnn::DNN_BACKEND_CUDA, dnn::DNN_TARGET_CUDA_FP16),
+        std::make_pair<cv::dnn::Backend, cv::dnn::Target>(dnn::DNN_BACKEND_TIMVX, dnn::DNN_TARGET_NPU),
+        std::make_pair<cv::dnn::Backend, cv::dnn::Target>(dnn::DNN_BACKEND_CANN, dnn::DNN_TARGET_NPU)};
+        
+vector<u16string> loadCharset(string);
+
+std::string keys =
+"{ help  h           |                                              | Print help message. }"
+"{ model m           | text_recognition_CRNN_EN_2021sep.onnx        | Usage: Set model type, defaults to text_recognition_CRNN_EN_2021sep.onnx }"
+"{ input i           |                                              | Usage: Path to input image or video file. Skip this argument to capture frames from a camera.}"
+"{ width             | 736                                          | Usage: Resize input image to certain width, default = 736. It should be multiple by 32.}"
+"{ height            | 736                                          | Usage: Resize input image to certain height, default = 736. It should be multiple by 32.}"
+"{ binary_threshold  | 0.3                                          | Usage: Threshold of the binary map, default = 0.3.}"
+"{ polygon_threshold | 0.5                                          | Usage: Threshold of polygons, default = 0.5.}"
+"{ max_candidates    | 200                                          | Usage: Set maximum number of polygon candidates, default = 200.}"
+"{ unclip_ratio      | 2.0                                          | Usage: The unclip ratio of the detected text region, which determines the output size, default = 2.0.}"
+"{ save s            | 1                                            | Usage: Specify to save file with results (i.e. bounding box, confidence level). Invalid in case of camera input.}"
+"{ viz v             | 1                                            | Usage: Specify to open a new window to show results.}"
+"{ backend bt        | 0                                            | Choose one of computation backends: "
+"0: (default) OpenCV implementation + CPU, "
+"1: CUDA + GPU (CUDA), "
+"2: CUDA + GPU (CUDA FP16), "
+"3: TIM-VX + NPU, "
+"4: CANN + NPU}";
+
+
+class DB {
+public:
+
+    DB(string modPath, Size inSize = Size(736, 736), float binThresh = 0.3,
+        float polyThresh = 0.5, int maxCand = 200, double unRatio = 2.0,
+        dnn::Backend bId = DNN_BACKEND_DEFAULT, dnn::Target tId = DNN_TARGET_CPU) : modelPath(modPath), inputSize(inSize), binaryThreshold(binThresh),
+        polygonThreshold(polyThresh), maxCandidates(maxCand), unclipRatio(unRatio),
+        backendId(bId), targetId(tId)
+    {
+        this->model = TextDetectionModel_DB(readNet(modelPath));
+        this->model.setPreferableBackend(backendId);
+        this->model.setPreferableTarget(targetId);
+
+        this->model.setBinaryThreshold(binaryThreshold);
+        this->model.setPolygonThreshold(polygonThreshold);
+        this->model.setUnclipRatio(unclipRatio);
+        this->model.setMaxCandidates(maxCandidates);
+
+        this->model.setInputParams(1.0 / 255.0, inputSize, Scalar(122.67891434, 116.66876762, 104.00698793));
+    }
+    pair< vector<vector<Point>>, vector<float> > infer(Mat image) {
+        CV_Assert(image.rows == this->inputSize.height && "height of input image != net input size ");
+        CV_Assert(image.cols == this->inputSize.width && "width of input image != net input size ");
+        vector<vector<Point>> pt;
+        vector<float> confidence;
+        this->model.detect(image, pt, confidence);
+        return make_pair< vector<vector<Point>> &, vector< float > &>(pt, confidence);
+    }
+
+private:
+    string modelPath;
+    TextDetectionModel_DB model;
+    Size inputSize;
+    float binaryThreshold;
+    float polygonThreshold;
+    int maxCandidates;
+    double unclipRatio;
+    dnn::Backend backendId;
+    dnn::Target targetId;
+
+};
+
+
+
+class CRNN {
+private:
+    string modelPath;
+    dnn::Backend backendId;
+    dnn::Target targetId;
+    Net model;
+    vector<u16string> charset;
+    Size inputSize;
+    Mat targetVertices;
+
+public:
+    CRNN(string modPath, dnn::Backend bId = DNN_BACKEND_DEFAULT, dnn::Target tId = DNN_TARGET_CPU) : modelPath(modPath), backendId(bId), targetId(tId) {
+
+        this->model = readNet(this->modelPath);
+        this->model.setPreferableBackend(this->backendId);
+        this->model.setPreferableTarget(this->targetId);
+        // load charset by the name of model
+        if (this->modelPath.find("_EN_") != string::npos)
+            this->charset = loadCharset("CHARSET_EN_36");
+        else if (this->modelPath.find("_CH_") != string::npos)
+            this->charset = loadCharset("CHARSET_CH_94");
+        else if (this->modelPath.find("_CN_") != string::npos)
+            this->charset = loadCharset("CHARSET_CN_3944");
+        else
+            CV_Error(-1, "Charset not supported! Exiting ...");
+
+        this->inputSize = Size(100, 32); // Fixed
+        this->targetVertices = Mat(4, 1, CV_32FC2);
+        this->targetVertices.row(0) = Vec2f(0, this->inputSize.height - 1);
+        this->targetVertices.row(1) = Vec2f(0, 0);
+        this->targetVertices.row(2) = Vec2f(this->inputSize.width - 1, 0);
+        this->targetVertices.row(3) = Vec2f(this->inputSize.width - 1, this->inputSize.height - 1);
+    }
+
+    Mat preprocess(Mat image, Mat rbbox)
+	{
+        // Remove conf, reshape and ensure all is np.float32
+        Mat vertices;
+        rbbox.reshape(2, 4).convertTo(vertices, CV_32FC2);
+
+        Mat rotationMatrix = getPerspectiveTransform(vertices, this->targetVertices);
+        Mat cropped;
+        warpPerspective(image, cropped, rotationMatrix, this->inputSize);
+
+        // 'CN' can detect digits (0\~9), upper/lower-case letters (a\~z and A\~Z), and some special characters
+        // 'CH' can detect digits (0\~9), upper/lower-case le6tters (a\~z and A\~Z), some Chinese characters and some special characters
+        if (this->modelPath.find("CN") == string::npos && this->modelPath.find("CH") == string::npos)
+            cvtColor(cropped, cropped, COLOR_BGR2GRAY);
+        Mat blob = blobFromImage(cropped, 1 / 127.5, this->inputSize, Scalar::all(127.5));
+        return blob;
+    }
+
+    u16string infer(Mat image, Mat rbbox)
+	{
+        // Preprocess
+        Mat inputBlob = this->preprocess(image, rbbox);
+
+        //  Forward
+        this->model.setInput(inputBlob);
+        Mat outputBlob = this->model.forward();
+
+        // Postprocess
+        u16string results = this->postprocess(outputBlob);
+
+        return results;
+    }
+
+    u16string postprocess(Mat outputBlob)
+	{
+        // Decode charaters from outputBlob
+        Mat character = outputBlob.reshape(1, outputBlob.size[0]);
+        u16string    text(u"");
+        for (int i = 0; i < character.rows; i++)
+        {
+            double minVal, maxVal;
+            Point maxIdx;
+            minMaxLoc(character.row(i), &minVal, &maxVal, nullptr, &maxIdx);
+            if (maxIdx.x != 0)
+                text += charset[maxIdx.x - 1];
+            else
+                text += u"-";
+        }
+        // adjacent same letters as well as background text must be removed to get the final output
+        u16string    textFilter(u"");
+
+        for (int i = 0; i < text.size(); i++)
+            if (text[i] != u'-' && !(i > 0 && text[i] == text[i - 1]))
+                textFilter += text[i];
+        return textFilter;
+    }
+};
+
+
+Mat visualize(Mat image, pair< vector<vector<Point>>, vector<float> >&results, double fps=-1, Scalar boxColor=Scalar(0, 255, 0), Scalar textColor=Scalar(0, 0, 255), bool isClosed=true, int thickness=2)
+{
+    Mat output;
+    image.copyTo(output);
+    if (fps > 0)
+        putText(output, format("FPS: %.2f", fps), Point(0, 15), FONT_HERSHEY_SIMPLEX, 0.5, textColor);
+    polylines(output, results.first, isClosed, boxColor, thickness);
+    return output;
+}
+
+int main(int argc, char** argv)
+{
+    CommandLineParser parser(argc, argv, keys);
+
+    parser.about("An End-to-End Trainable Neural Network for Image-based Sequence Recognition and Its Application to Scene Text Recognition (https://arxiv.org/abs/1507.05717)");
+    if (parser.has("help"))
+    {
+        parser.printMessage();
+        return 0;
+    }
+
+    int backendTargetid = parser.get<int>("backend");
+    String modelPath = parser.get<String>("model");
+
+    if (modelPath.empty())
+    {
+        CV_Error(Error::StsError, "Model file " + modelPath + " not found");
+    }
+
+    Size inpSize(parser.get<int>("width"), parser.get<int>("height"));
+    float binThresh = parser.get<float>("binary_threshold");
+    float polyThresh = parser.get<float>("polygon_threshold");
+    int maxCand = parser.get<int>("max_candidates");
+    double unRatio = parser.get<float>("unclip_ratio");
+    bool save = parser.get<bool>("save");
+    bool viz = parser.get<float>("viz");
+
+    DB detector("../text_detection_db/text_detection_DB_IC15_resnet18_2021sep.onnx", inpSize, binThresh, polyThresh, maxCand, unRatio, backendTargetPairs[backendTargetid].first, backendTargetPairs[backendTargetid].second);
+    CRNN recognizer(modelPath, backendTargetPairs[backendTargetid].first, backendTargetPairs[backendTargetid].second);
+    //! [Open a video file or an image file or a camera stream]
+    VideoCapture cap;
+    if (parser.has("input"))
+        cap.open(parser.get<String>("input"));
+    else
+        cap.open(0);
+    if (!cap.isOpened())
+        CV_Error(Error::StsError, "Cannot opend video or file");
+    Mat originalImage;
+    static const std::string kWinName = modelPath;
+    while (waitKey(1) < 0)
+    {
+        cap >> originalImage;
+        if (originalImage.empty())
+        {
+            cout << "Frame is empty" << endl;
+            waitKey();
+            break;
+        }
+        int originalW = originalImage.cols;
+        int originalH = originalImage.rows;
+        double scaleHeight = originalH / double(inpSize.height);
+        double scaleWidth = originalW / double(inpSize.width);
+        Mat image;
+        resize(originalImage, image, inpSize);
+
+        // inference of text detector
+        TickMeter tm;
+        tm.start();
+        pair< vector<vector<Point>>, vector<float> > results = detector.infer(image);
+        tm.stop();
+        if (results.first.size() > 0 && results.second.size() > 0)
+        {
+            u16string texts;
+            auto score=results.second.begin();
+            for (auto box : results.first)
+            {
+                Mat result = Mat(box).reshape(2, 4);
+                texts = texts + u"'" + recognizer.infer(image, result) + u"'";
+            }
+            std::wstring_convert<std::codecvt_utf8<char16_t>, char16_t> converter;
+            std::cout << converter.to_bytes(texts) << std::endl;
+        }
+        auto x = results.first;
+        // Scale the results bounding box
+        for (auto &pts : results.first)
+        {
+            for (int i = 0; i < 4; i++)
+            {
+                pts[i].x = int(pts[i].x * scaleWidth);
+                pts[i].y = int(pts[i].y * scaleHeight);
+            }
+        }
+        originalImage = visualize(originalImage, results, tm.getFPS());
+        tm.reset();
+        if (parser.has("input"))
+        {
+            if (save)
+            {
+                cout << "Result image saved to result.jpg\n";
+                imwrite("result.jpg", originalImage);
+            }
+            if (viz)
+                imshow(kWinName, originalImage);
+        }
+        else
+            imshow(kWinName, originalImage);
+  
+    }
+    return 0;
+}

tools/eval/README.md

@@ -5,7 +5,7 @@ Make sure you have the following packages installed:
 ```shell
 pip install tqdm
 pip install scikit-learn
-pip install scipy
+pip install scipy==1.8.1
 ```
 
 Generally speaking, evaluation can be done with the following command:
@@ -27,7 +27,7 @@ Supported datasets:
 
 ### Prepare data
 
-Please visit https://image-net.org/ to download the ImageNet dataset and [the labels from caffe](http://dl.caffe.berkeleyvision.org/caffe_ilsvrc12.tar.gz). Organize files as follow:
+Please visit https://image-net.org/ to download the ImageNet dataset (only need images in `ILSVRC/Data/CLS-LOC/val`) and [the labels from caffe](http://dl.caffe.berkeleyvision.org/caffe_ilsvrc12.tar.gz). Organize files as follow:
 
 ```shell
 $ tree -L 2 /path/to/imagenet

tools/eval/eval.py

@@ -22,35 +22,41 @@ args = parser.parse_args()
 
 models = dict(
     mobilenetv1=dict(
-        name="MobileNetV1",
+        name="MobileNet",
         topic="image_classification",
         modelPath=os.path.join(root_dir, "models/image_classification_mobilenet/image_classification_mobilenetv1_2022apr.onnx"),
-        topK=5),
+        topK=5,
+        loadLabel=False),
     mobilenetv1_q=dict(
-        name="MobileNetV1",
+        name="MobileNet",
         topic="image_classification",
-        modelPath=os.path.join(root_dir, "models/image_classification_mobilenet/image_classification_mobilenetv1_2022apr-int8-quantized.onnx"),
-        topK=5),
+        modelPath=os.path.join(root_dir, "models/image_classification_mobilenet/image_classification_mobilenetv1_2022apr_int8.onnx"),
+        topK=5,
+        loadLabel=False),
     mobilenetv2=dict(
-        name="MobileNetV2",
+        name="MobileNet",
         topic="image_classification",
         modelPath=os.path.join(root_dir, "models/image_classification_mobilenet/image_classification_mobilenetv2_2022apr.onnx"),
-        topK=5),
+        topK=5,
+        loadLabel=False),
     mobilenetv2_q=dict(
-        name="MobileNetV2",
+        name="MobileNet",
         topic="image_classification",
-        modelPath=os.path.join(root_dir, "models/image_classification_mobilenet/image_classification_mobilenetv2_2022apr-int8-quantized.onnx"),
-        topK=5),
+        modelPath=os.path.join(root_dir, "models/image_classification_mobilenet/image_classification_mobilenetv2_2022apr_int8.onnx"),
+        topK=5,
+        loadLabel=False),
     ppresnet=dict(
         name="PPResNet",
         topic="image_classification",
         modelPath=os.path.join(root_dir, "models/image_classification_ppresnet/image_classification_ppresnet50_2022jan.onnx"),
-        topK=5),
+        topK=5,
+        loadLabel=False),
     ppresnet_q=dict(
         name="PPResNet",
         topic="image_classification",
-        modelPath=os.path.join(root_dir, "models/image_classification_ppresnet/image_classification_ppresnet50_2022jan-act_int8-wt_int8-quantized.onnx"),
-        topK=5),
+        modelPath=os.path.join(root_dir, "models/image_classification_ppresnet/image_classification_ppresnet50_2022jan_int8.onnx"),
+        topK=5,
+        loadLabel=False),
     yunet=dict(
         name="YuNet",
         topic="face_detection",
@@ -72,19 +78,23 @@ models = dict(
     sface_q=dict(
         name="SFace",
         topic="face_recognition",
-        modelPath=os.path.join(root_dir, "models/face_recognition_sface/face_recognition_sface_2021dec-act_int8-wt_int8-quantized.onnx")),
-    crnn=dict(
+        modelPath=os.path.join(root_dir, "models/face_recognition_sface/face_recognition_sface_2021dec_int8.onnx")),
+    crnn_en=dict(
         name="CRNN",
         topic="text_recognition",
         modelPath=os.path.join(root_dir, "models/text_recognition_crnn/text_recognition_CRNN_EN_2021sep.onnx")),
+    crnn_en_q=dict(
+        name="CRNN",
+        topic="text_recognition",
+        modelPath=os.path.join(root_dir, "models/text_recognition_crnn/text_recognition_CRNN_EN_2022oct_int8.onnx")),
     pphumanseg=dict(
         name="PPHumanSeg",
         topic="human_segmentation",
-        modelPath=os.path.join(root_dir, "models/human_segmentation_pphumanseg/human_segmentation_pphumanseg_2021oct.onnx")),
+        modelPath=os.path.join(root_dir, "models/human_segmentation_pphumanseg/human_segmentation_pphumanseg_2023mar.onnx")),
     pphumanseg_q=dict(
         name="PPHumanSeg",
         topic="human_segmentation",
-        modelPath=os.path.join(root_dir, "models/human_segmentation_pphumanseg/human_segmentation_pphumanseg_2021oct-act_int8-wt_int8-quantized.onnx")),
+        modelPath=os.path.join(root_dir, "models/human_segmentation_pphumanseg/human_segmentation_pphumanseg_2023mar_int8.onnx")),
 )
 
 datasets = dict(