Merge pull request #19318 from TolyaTalamanov:at/python-generic-infer

[G-API] Python ROI generic inference

* Python generic infer overloads

* Move wrappers to appropriate file

modules/gapi/include/opencv2/gapi/infer.hpp

@@ -636,11 +636,6 @@ infer2(const std::string& tag,
     return cv::GInferListOutputs{std::move(call)};
 }
 
-GAPI_EXPORTS_W inline cv::GInferOutputs infer(const String& name, const cv::GInferInputs& inputs)
-{
-    return infer<Generic>(name, inputs);
-}
-
 } // namespace gapi
 } // namespace cv
 

modules/gapi/misc/python/pyopencv_gapi.hpp

@@ -34,6 +34,7 @@ using GArray_Size    = cv::GArray<cv::Size>;
 using GArray_Rect    = cv::GArray<cv::Rect>;
 using GArray_Scalar  = cv::GArray<cv::Scalar>;
 using GArray_Mat     = cv::GArray<cv::Mat>;
+using GArray_GMat    = cv::GArray<cv::GMat>;
 
 // FIXME: Python wrapper generate code without namespace std,
 // so it cause error: "string wasn't declared"
@@ -58,7 +59,7 @@ bool pyopencv_to(PyObject* obj, GRunArgs& value, const ArgInfo& info)
     return pyopencv_to_generic_vec(obj, value, info);
 }
 
-template <>
+template<>
 PyObject* pyopencv_from(const cv::detail::OpaqueRef& o)
 {
     switch (o.getKind())
@@ -201,6 +202,7 @@ static PyObject* extract_proto_args(PyObject* py_args, PyObject* kw)
 
     GProtoArgs args;
     Py_ssize_t size = PyTuple_Size(py_args);
+    args.reserve(size);
     for (int i = 0; i < size; ++i)
     {
         PyObject* item = PyTuple_GetItem(py_args, i);
@@ -318,12 +320,9 @@ static cv::GRunArg extract_run_arg(const cv::GTypeInfo& info, PyObject* item)
                     reinterpret_cast<pyopencv_gapi_wip_IStreamSource_t*>(item)->v;
                 return source;
             }
-            else
-            {
-                cv::Mat obj;
-                pyopencv_to_with_check(item, obj, "Failed to obtain cv::Mat");
-                return obj;
-            }
+            cv::Mat obj;
+            pyopencv_to_with_check(item, obj, "Failed to obtain cv::Mat");
+            return obj;
         }
         case cv::GShape::GSCALAR:
         {

modules/gapi/misc/python/python_bridge.hpp

@@ -68,6 +68,32 @@ enum ArgType {
     CV_GMAT,
 };
 
+GAPI_EXPORTS_W inline cv::GInferOutputs infer(const String& name, const cv::GInferInputs& inputs)
+{
+    return infer<Generic>(name, inputs);
+}
+
+GAPI_EXPORTS_W inline GInferOutputs infer(const std::string& name,
+                                          const cv::GOpaque<cv::Rect>& roi,
+                                          const GInferInputs& inputs)
+{
+    return infer<Generic>(name, roi, inputs);
+}
+
+GAPI_EXPORTS_W inline GInferListOutputs infer(const std::string& name,
+                                              const cv::GArray<cv::Rect>& rois,
+                                              const GInferInputs& inputs)
+{
+    return infer<Generic>(name, rois, inputs);
+}
+
+GAPI_EXPORTS_W inline GInferListOutputs infer2(const std::string& name,
+                                               const cv::GMat in,
+                                               const GInferListInputs& inputs)
+{
+    return infer2<Generic>(name, in, inputs);
+}
+
 } // namespace gapi
 
 namespace detail {

modules/gapi/misc/python/shadow_gapi.hpp

@@ -27,6 +27,14 @@ namespace cv
        GAPI_WRAP void setInput(const std::string& name, const cv::GFrame& value);
    };
 
+   class GAPI_EXPORTS_W_SIMPLE GInferListInputs
+   {
+   public:
+       GAPI_WRAP GInferListInputs();
+       GAPI_WRAP void setInput(const std::string& name, const cv::GArray<cv::GMat>& value);
+       GAPI_WRAP void setInput(const std::string& name, const cv::GArray<cv::Rect>& value);
+   };
+
    class GAPI_EXPORTS_W_SIMPLE GInferOutputs
    {
    public:
@@ -34,6 +42,13 @@ namespace cv
        GAPI_WRAP cv::GMat at(const std::string& name);
    };
 
+   class GAPI_EXPORTS_W_SIMPLE GInferListOutputs
+   {
+   public:
+       GAPI_WRAP GInferListOutputs();
+       GAPI_WRAP cv::GArray<cv::GMat> at(const std::string& name);
+   };
+
    namespace detail
    {
        struct GAPI_EXPORTS_W_SIMPLE ExtractArgsCallback { };

modules/gapi/misc/python/test/test_gapi_core.py

@@ -13,7 +13,7 @@ pkgs = [
           ('cpu'    , cv.gapi.core.cpu.kernels()),
           ('fluid'  , cv.gapi.core.fluid.kernels())
           # ('plaidml', cv.gapi.core.plaidml.kernels())
-      ]
+       ]
 
 
 class gapi_core_test(NewOpenCVTests):
@@ -127,7 +127,6 @@ class gapi_core_test(NewOpenCVTests):
             self.assertEqual(expected_thresh, actual_thresh[0],
                              'Failed on ' + pkg_name + ' backend')
 
-
     def test_kmeans(self):
         # K-means params
         count    = 100
@@ -154,10 +153,12 @@ class gapi_core_test(NewOpenCVTests):
         self.assertEqual(centers.shape[0], K);
         self.assertTrue(centers.size != 0);
 
+
     def generate_random_points(self, sz):
         arr = np.random.random(sz).astype(np.float32).T
         return list(zip(arr[0], arr[1]))
 
+
     def test_kmeans_2d(self):
         # K-means 2D params
         count     = 100

modules/gapi/misc/python/test/test_gapi_infer.py

@@ -9,13 +9,145 @@ from tests_common import NewOpenCVTests
 
 class test_gapi_infer(NewOpenCVTests):
 
-    def test_getAvailableTargets(self):
-        targets = cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_OPENCV)
-        self.assertTrue(cv.dnn.DNN_TARGET_CPU in targets)
+    def infer_reference_network(self, model_path, weights_path, img):
+        net = cv.dnn.readNetFromModelOptimizer(model_path, weights_path)
+        net.setPreferableBackend(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE)
+        net.setPreferableTarget(cv.dnn.DNN_TARGET_CPU)
+
+        blob = cv.dnn.blobFromImage(img)
+
+        net.setInput(blob)
+        return net.forward(net.getUnconnectedOutLayersNames())
+
+
+    def make_roi(self, img, roi):
+        return img[roi[1]:roi[1] + roi[3], roi[0]:roi[0] + roi[2], ...]
 
 
     def test_age_gender_infer(self):
+        # NB: Check IE
+        if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE):
+            return
+
+        root_path    = '/omz_intel_models/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013'
+        model_path   = self.find_file(root_path + '.xml',   [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')])
+        weights_path = self.find_file(root_path + '.bin',   [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')])
+        device_id    = 'CPU'
+
+        img_path  = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')])
+        img       = cv.resize(cv.imread(img_path), (62,62))
+
+        # OpenCV DNN
+        dnn_age, dnn_gender = self.infer_reference_network(model_path, weights_path, img)
+
+        # OpenCV G-API
+        g_in   = cv.GMat()
+        inputs = cv.GInferInputs()
+        inputs.setInput('data', g_in)
+
+        outputs  = cv.gapi.infer("net", inputs)
+        age_g    = outputs.at("age_conv3")
+        gender_g = outputs.at("prob")
+
+        comp = cv.GComputation(cv.GIn(g_in), cv.GOut(age_g, gender_g))
+        pp = cv.gapi.ie.params("net", model_path, weights_path, device_id)
+
+        gapi_age, gapi_gender = comp.apply(cv.gin(img), args=cv.compile_args(cv.gapi.networks(pp)))
+
+        # Check
+        self.assertEqual(0.0, cv.norm(dnn_gender, gapi_gender, cv.NORM_INF))
+        self.assertEqual(0.0, cv.norm(dnn_age, gapi_age, cv.NORM_INF))
+
+
+    def test_age_gender_infer_roi(self):
+        # NB: Check IE
+        if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE):
+            return
+
+        root_path    = '/omz_intel_models/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013'
+        model_path   = self.find_file(root_path + '.xml',   [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')])
+        weights_path = self.find_file(root_path + '.bin',   [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')])
+        device_id    = 'CPU'
+
+        img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')])
+        img = cv.imread(img_path)
+        roi      = (10, 10, 62, 62)
+
+        # OpenCV DNN
+        dnn_age, dnn_gender = self.infer_reference_network(model_path,
+                                                           weights_path,
+                                                           self.make_roi(img, roi))
+
+        # OpenCV G-API
+        g_in   = cv.GMat()
+        g_roi  = cv.GOpaqueT(cv.gapi.CV_RECT)
+        inputs = cv.GInferInputs()
+        inputs.setInput('data', g_in)
+
+        outputs  = cv.gapi.infer("net", g_roi, inputs)
+        age_g    = outputs.at("age_conv3")
+        gender_g = outputs.at("prob")
+
+        comp = cv.GComputation(cv.GIn(g_in, g_roi), cv.GOut(age_g, gender_g))
+        pp = cv.gapi.ie.params("net", model_path, weights_path, device_id)
+
+        gapi_age, gapi_gender = comp.apply(cv.gin(img, roi), args=cv.compile_args(cv.gapi.networks(pp)))
+
+        # Check
+        self.assertEqual(0.0, cv.norm(dnn_gender, gapi_gender, cv.NORM_INF))
+        self.assertEqual(0.0, cv.norm(dnn_age, gapi_age, cv.NORM_INF))
+
+
+    def test_age_gender_infer_roi_list(self):
+        # NB: Check IE
+        if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE):
+            return
+
+        root_path    = '/omz_intel_models/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013'
+        model_path   = self.find_file(root_path + '.xml',   [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')])
+        weights_path = self.find_file(root_path + '.bin',   [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')])
+        device_id    = 'CPU'
+
+        rois = [(10, 15, 62, 62), (23, 50, 62, 62), (14, 100, 62, 62), (80, 50, 62, 62)]
+        img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')])
+        img = cv.imread(img_path)
+
+        # OpenCV DNN
+        dnn_age_list    = []
+        dnn_gender_list = []
+        for roi in rois:
+            age, gender = self.infer_reference_network(model_path,
+                                                       weights_path,
+                                                       self.make_roi(img, roi))
+            dnn_age_list.append(age)
+            dnn_gender_list.append(gender)
+
+        # OpenCV G-API
+        g_in   = cv.GMat()
+        g_rois = cv.GArrayT(cv.gapi.CV_RECT)
+        inputs = cv.GInferInputs()
+        inputs.setInput('data', g_in)
+
+        outputs  = cv.gapi.infer("net", g_rois, inputs)
+        age_g    = outputs.at("age_conv3")
+        gender_g = outputs.at("prob")
+
+        comp = cv.GComputation(cv.GIn(g_in, g_rois), cv.GOut(age_g, gender_g))
+        pp = cv.gapi.ie.params("net", model_path, weights_path, device_id)
+
+        gapi_age_list, gapi_gender_list = comp.apply(cv.gin(img, rois),
+                                                     args=cv.compile_args(cv.gapi.networks(pp)))
+
+        # Check
+        for gapi_age, gapi_gender, dnn_age, dnn_gender in zip(gapi_age_list,
+                                                              gapi_gender_list,
+                                                              dnn_age_list,
+                                                              dnn_gender_list):
+            self.assertEqual(0.0, cv.norm(dnn_gender, gapi_gender, cv.NORM_INF))
+            self.assertEqual(0.0, cv.norm(dnn_age, gapi_age, cv.NORM_INF))
+
 
+    def test_age_gender_infer2_roi(self):
         # NB: Check IE
         if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE):
             return
@@ -23,9 +155,59 @@ class test_gapi_infer(NewOpenCVTests):
         root_path    = '/omz_intel_models/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013'
         model_path   = self.find_file(root_path + '.xml',   [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')])
         weights_path = self.find_file(root_path + '.bin',   [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')])
-        img_path     = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')])
         device_id    = 'CPU'
-        img          = cv.resize(cv.imread(img_path), (62,62))
+
+        rois = [(10, 15, 62, 62), (23, 50, 62, 62), (14, 100, 62, 62), (80, 50, 62, 62)]
+        img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')])
+        img = cv.imread(img_path)
+
+        # OpenCV DNN
+        dnn_age_list    = []
+        dnn_gender_list = []
+        for roi in rois:
+            age, gender = self.infer_reference_network(model_path,
+                                                       weights_path,
+                                                       self.make_roi(img, roi))
+            dnn_age_list.append(age)
+            dnn_gender_list.append(gender)
+
+        # OpenCV G-API
+        g_in   = cv.GMat()
+        g_rois = cv.GArrayT(cv.gapi.CV_RECT)
+        inputs = cv.GInferListInputs()
+        inputs.setInput('data', g_rois)
+
+        outputs  = cv.gapi.infer2("net", g_in, inputs)
+        age_g    = outputs.at("age_conv3")
+        gender_g = outputs.at("prob")
+
+        comp = cv.GComputation(cv.GIn(g_in, g_rois), cv.GOut(age_g, gender_g))
+        pp = cv.gapi.ie.params("net", model_path, weights_path, device_id)
+
+        gapi_age_list, gapi_gender_list = comp.apply(cv.gin(img, rois),
+                                                     args=cv.compile_args(cv.gapi.networks(pp)))
+
+        # Check
+        for gapi_age, gapi_gender, dnn_age, dnn_gender in zip(gapi_age_list,
+                                                              gapi_gender_list,
+                                                              dnn_age_list,
+                                                              dnn_gender_list):
+            self.assertEqual(0.0, cv.norm(dnn_gender, gapi_gender, cv.NORM_INF))
+            self.assertEqual(0.0, cv.norm(dnn_age, gapi_age, cv.NORM_INF))
+
+
+
+    def test_person_detection_retail_0013(self):
+        # NB: Check IE
+        if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE):
+            return
+
+        root_path    = '/omz_intel_models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013'
+        model_path   = self.find_file(root_path + '.xml',   [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')])
+        weights_path = self.find_file(root_path + '.bin',   [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')])
+        img_path     = self.find_file('gpu/lbpcascade/er.png', [os.environ.get('OPENCV_TEST_DATA_PATH')])
+        device_id    = 'CPU'
+        img          = cv.resize(cv.imread(img_path), (544, 320))
 
         # OpenCV DNN
         net = cv.dnn.readNetFromModelOptimizer(model_path, weights_path)
@@ -34,26 +216,46 @@ class test_gapi_infer(NewOpenCVTests):
 
         blob = cv.dnn.blobFromImage(img)
 
+        def parseSSD(detections, size):
+            h, w = size
+            bboxes = []
+            detections = detections.reshape(-1, 7)
+            for sample_id, class_id, confidence, xmin, ymin, xmax, ymax in detections:
+                if confidence >= 0.5:
+                    x      = int(xmin * w)
+                    y      = int(ymin * h)
+                    width  = int(xmax * w - x)
+                    height = int(ymax * h - y)
+                    bboxes.append((x, y, width, height))
+
+            return bboxes
+
         net.setInput(blob)
-        dnn_age, dnn_gender = net.forward(net.getUnconnectedOutLayersNames())
+        dnn_detections = net.forward()
+        dnn_boxes = parseSSD(np.array(dnn_detections), img.shape[:2])
 
         # OpenCV G-API
         g_in   = cv.GMat()
         inputs = cv.GInferInputs()
         inputs.setInput('data', g_in)
 
-        outputs  = cv.gapi.infer("net", inputs)
-        age_g    = outputs.at("age_conv3")
-        gender_g = outputs.at("prob")
+        g_sz       = cv.gapi.streaming.size(g_in)
+        outputs    = cv.gapi.infer("net", inputs)
+        detections = outputs.at("detection_out")
+        bboxes     = cv.gapi.parseSSD(detections, g_sz, 0.5, False, False)
 
-        comp = cv.GComputation(cv.GIn(g_in), cv.GOut(age_g, gender_g))
+        comp = cv.GComputation(cv.GIn(g_in), cv.GOut(bboxes))
         pp = cv.gapi.ie.params("net", model_path, weights_path, device_id)
 
         gapi_age, gapi_gender = comp.apply(cv.gin(img), args=cv.compile_args(cv.gapi.networks(pp)))
 
-        # Check
-        self.assertEqual(0.0, cv.norm(dnn_gender, gapi_gender, cv.NORM_INF))
-        self.assertEqual(0.0, cv.norm(dnn_age, gapi_age, cv.NORM_INF))
+        gapi_boxes = comp.apply(cv.gin(img.astype(np.float32)),
+                                args=cv.compile_args(cv.gapi.networks(pp)))
+
+        # Comparison
+        self.assertEqual(0.0, cv.norm(np.array(dnn_boxes).flatten(),
+                                      np.array(gapi_boxes).flatten(),
+                                      cv.NORM_INF))
 
 
     def test_person_detection_retail_0013(self):