Merge pull request #20785 from smirnov-alexey:as/oak_backend

GAPI: Add OAK backend

* Initial tests and cmake integration

* Add a public header and change tests

* Stub initial empty template for the OAK backend

* WIP

* WIP

* WIP

* WIP

* Runtime dai hang debug

* Refactoring

* Fix hang and debug frame data

* Fix frame size

* Fix data size issue

* Move test code to sample

* tmp refactoring

* WIP: Code refactoring except for the backend

* WIP: Add non-camera sample

* Fix samples

* Backend refactoring wip

* Backend rework wip

* Backend rework wip

* Remove mat encoder

* Fix namespace

* Minor backend fixes

* Fix hetero sample and refactor backend

* Change linking logic in the backend

* Fix oak sample

* Fix working with ins/outs in OAK island

* Trying to fix nv12 problem

* Make both samples work

* Small refactoring

* Remove meta args

* WIP refactoring kernel API

* Change in/out args API for kernels

* Fix build

* Fix cmake warning

* Partially address review comments

* Partially address review comments

* Address remaining comments

* Add memory ownership

* Change pointer-to-pointer to reference-to-pointer

* Remove unnecessary reference wrappers

* Apply review comments

* Check that graph contains only one OAK island

* Minor refactoring

* Address review comments

modules/gapi/CMakeLists.txt

@@ -45,6 +45,7 @@ file(GLOB gapi_ext_hdrs
     "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/fluid/*.hpp"
     "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/gpu/*.hpp"
     "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/infer/*.hpp"
+    "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/oak/*.hpp"
     "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/ocl/*.hpp"
     "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/own/*.hpp"
     "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${name}/plaidml/*.hpp"
@@ -127,6 +128,11 @@ set(gapi_srcs
     src/backends/fluid/gfluidcore.cpp
 	  src/backends/fluid/gfluidcore_func.dispatch.cpp
 
+    # OAK Backend (optional)
+    src/backends/oak/goak.cpp
+    src/backends/oak/goakbackend.cpp
+    src/backends/oak/goak_media_adapter.cpp
+
     # OCL Backend (currently built-in)
     src/backends/ocl/goclbackend.cpp
     src/backends/ocl/goclkernel.cpp
@@ -272,6 +278,14 @@ if(HAVE_FREETYPE)
   ocv_target_include_directories(${the_module} PRIVATE ${FREETYPE_INCLUDE_DIRS})
 endif()
 
+if(HAVE_OAK)
+  ocv_target_compile_definitions(${the_module} PRIVATE -DHAVE_OAK)
+  if(TARGET opencv_test_gapi)
+    ocv_target_compile_definitions(opencv_test_gapi PRIVATE -DHAVE_OAK)
+  endif()
+  ocv_target_link_libraries(${the_module} PRIVATE depthai::core)
+endif()
+
 if(HAVE_PLAIDML)
   ocv_target_compile_definitions(${the_module} PRIVATE -DHAVE_PLAIDML)
   if(TARGET opencv_test_gapi)
@@ -350,3 +364,13 @@ if(HAVE_GAPI_ONEVPL)
     endif()
   endif()
 endif()
+
+if(HAVE_OAK)
+  # FIXME: consider better solution
+  if(TARGET example_gapi_oak_rgb_camera_encoding)
+    ocv_target_compile_definitions(example_gapi_oak_rgb_camera_encoding PRIVATE -DHAVE_OAK)
+  endif()
+  if(TARGET example_gapi_oak_small_hetero_pipeline)
+    ocv_target_compile_definitions(example_gapi_oak_small_hetero_pipeline PRIVATE -DHAVE_OAK)
+  endif()
+endif()

modules/gapi/cmake/init.cmake

 OCV_OPTION(WITH_ADE "Enable ADE framework (required for Graph API module)" ON)
 
-OCV_OPTION(WITH_FREETYPE "Enable FreeType framework" OFF)
-OCV_OPTION(WITH_PLAIDML  "Include PlaidML2 support"  OFF)
+OCV_OPTION(WITH_FREETYPE "Enable FreeType framework"     OFF)
+OCV_OPTION(WITH_PLAIDML  "Include PlaidML2 support"      OFF)
+OCV_OPTION(WITH_OAK      "Include OpenCV AI Kit support" OFF)
 
 if(NOT WITH_ADE)
   return()
@@ -39,3 +40,10 @@ if(WITH_GAPI_ONEVPL)
         set(HAVE_GAPI_ONEVPL TRUE)
     endif()
 endif()
+
+if(WITH_OAK)
+  find_package(depthai QUIET)
+  if(depthai_FOUND)
+      set(HAVE_OAK TRUE)
+  endif()
+endif()

modules/gapi/include/opencv2/gapi/oak/oak.hpp

+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+//
+// Copyright (C) 2021 Intel Corporation
+
+#ifndef OPENCV_GAPI_OAK_HPP
+#define OPENCV_GAPI_OAK_HPP
+
+#include <opencv2/gapi/garg.hpp>       // IStreamSource
+#include <opencv2/gapi/gkernel.hpp>    // GKernelPackage
+#include <opencv2/gapi/gstreaming.hpp> // GOptRunArgsP
+
+namespace cv {
+namespace gapi {
+namespace oak {
+
+// FIXME: copypasted from dai library
+struct EncoderConfig {
+    /**
+     * Rate control mode specifies if constant or variable bitrate should be used (H264 / H265)
+     */
+    enum class RateControlMode: int { CBR, VBR };
+
+    /**
+     * Encoding profile, H264, H265 or MJPEG
+     */
+    enum class Profile: int { H264_BASELINE, H264_HIGH, H264_MAIN, H265_MAIN, MJPEG };
+    /**
+     * Specifies prefered bitrate (kb) of compressed output bitstream
+     */
+    std::int32_t bitrate = 8000;
+    /**
+     * Every x number of frames a keyframe will be inserted
+     */
+    std::int32_t keyframeFrequency = 30;
+    /**
+     * Specifies maximum bitrate (kb) of compressed output bitstream
+     */
+    std::int32_t maxBitrate = 8000;
+    /**
+     * Specifies number of B frames to be inserted
+     */
+    std::int32_t numBFrames = 0;
+    /**
+     * This options specifies how many frames are available in this nodes pool (can help if
+     * receiver node is slow at consuming
+     */
+    std::uint32_t numFramesPool = 4;
+    /**
+     * Encoding profile, H264, H265 or MJPEG
+     */
+    Profile profile = Profile::H265_MAIN;
+    /**
+     * Value between 0-100% (approximates quality)
+     */
+    std::int32_t quality = 80;
+    /**
+     * Lossless mode ([M]JPEG only)
+     */
+    bool lossless = false;
+    /**
+     * Rate control mode specifies if constant or variable bitrate should be used (H264 / H265)
+     */
+    RateControlMode rateCtrlMode = RateControlMode::CBR;
+    /**
+     * Input and compressed output frame width
+     */
+    std::int32_t width = 1920;
+    /**
+     * Input and compressed output frame height
+     */
+    std::int32_t height = 1080;
+    /**
+     * Frame rate
+     */
+    float frameRate = 30.0f;
+};
+
+G_API_OP(GEncFrame, <GArray<uint8_t>(GFrame, EncoderConfig)>, "org.opencv.oak.enc_frame") {
+    static GArrayDesc outMeta(const GFrameDesc&, const EncoderConfig&) {
+        return cv::empty_array_desc();
+    }
+};
+
+G_API_OP(GSobelXY, <GFrame(GFrame, const cv::Mat&, const cv::Mat&)>, "org.opencv.oak.sobelxy") {
+    static GFrameDesc outMeta(const GFrameDesc& in, const cv::Mat&, const cv::Mat&) {
+        return in;
+    }
+};
+
+GAPI_EXPORTS GArray<uint8_t> encode(const GFrame& in, const EncoderConfig&);
+
+GAPI_EXPORTS GFrame sobelXY(const GFrame& in,
+                            const cv::Mat& hk,
+                            const cv::Mat& vk);
+
+// OAK backend & kernels ////////////////////////////////////////////////////////
+GAPI_EXPORTS cv::gapi::GBackend backend();
+GAPI_EXPORTS cv::gapi::GKernelPackage kernels();
+
+// Camera object ///////////////////////////////////////////////////////////////
+
+struct GAPI_EXPORTS ColorCameraParams {};
+
+class GAPI_EXPORTS ColorCamera: public cv::gapi::wip::IStreamSource {
+    cv::MediaFrame m_dummy;
+
+    virtual bool pull(cv::gapi::wip::Data &data) override;
+    virtual GMetaArg descr_of() const override;
+
+public:
+    ColorCamera();
+};
+
+} // namespace oak
+} // namespace gapi
+
+namespace detail {
+template<> struct CompileArgTag<gapi::oak::ColorCameraParams> {
+    static const char* tag() { return "gapi.oak.colorCameraParams"; }
+};
+
+template<> struct CompileArgTag<gapi::oak::EncoderConfig> {
+    static const char* tag() { return "gapi.oak.encoderConfig"; }
+};
+} // namespace detail
+
+} // namespace cv
+
+#endif // OPENCV_GAPI_OAK_HPP

modules/gapi/samples/oak_rgb_camera_encoding.cpp

+#include <fstream>
+
+#include <opencv2/gapi.hpp>
+#include <opencv2/gapi/core.hpp>
+#include <opencv2/gapi/gframe.hpp>
+
+#include <opencv2/gapi/oak/oak.hpp>
+#include <opencv2/gapi/streaming/format.hpp> // BGR accessor
+
+#include <opencv2/highgui.hpp> // CommandLineParser
+
+const std::string keys =
+    "{ h help  |              | Print this help message }"
+    "{ output  | output.h265  | Path to the output .h265 video file }";
+
+#ifdef HAVE_OAK
+
+int main(int argc, char *argv[]) {
+    cv::CommandLineParser cmd(argc, argv, keys);
+    if (cmd.has("help")) {
+        cmd.printMessage();
+        return 0;
+    }
+
+    const std::string output_name = cmd.get<std::string>("output");
+
+    cv::gapi::oak::EncoderConfig cfg;
+    cfg.profile = cv::gapi::oak::EncoderConfig::Profile::H265_MAIN;
+
+    cv::GFrame in;
+    cv::GArray<uint8_t> encoded = cv::gapi::oak::encode(in, cfg);
+
+    auto args = cv::compile_args(cv::gapi::oak::ColorCameraParams{}, cv::gapi::oak::kernels());
+
+    auto pipeline = cv::GComputation(cv::GIn(in), cv::GOut(encoded)).compileStreaming(std::move(args));
+
+    // Graph execution /////////////////////////////////////////////////////////
+    pipeline.setSource(cv::gapi::wip::make_src<cv::gapi::oak::ColorCamera>());
+    pipeline.start();
+
+    std::vector<uint8_t> out_h265_data;
+
+    std::ofstream out_h265_file;
+    out_h265_file.open(output_name, std::ofstream::out | std::ofstream::binary | std::ofstream::trunc);
+
+    // Pull 300 frames from the camera
+    uint32_t frames = 300;
+    uint32_t pulled = 0;
+
+    while (pipeline.pull(cv::gout(out_h265_data))) {
+        if (out_h265_file.is_open()) {
+            out_h265_file.write(reinterpret_cast<const char*>(out_h265_data.data()),
+                                                              out_h265_data.size());
+        }
+        if (pulled++ == frames) {
+            pipeline.stop();
+            break;
+        }
+    }
+
+    std::cout << "Pipeline finished: " << output_name << " file has been written." << std::endl;
+}
+#else // HAVE_OAK
+
+int main() {
+    GAPI_Assert(false && "Built without OAK support");
+    return -1;
+}
+
+#endif // HAVE_OAK

modules/gapi/samples/oak_small_hetero_pipeline.cpp

+#include <opencv2/gapi.hpp>
+#include <opencv2/gapi/core.hpp>
+#include <opencv2/gapi/cpu/core.hpp>
+#include <opencv2/gapi/gframe.hpp>
+#include <opencv2/gapi/media.hpp>
+
+#include <opencv2/gapi/oak/oak.hpp>
+#include <opencv2/gapi/streaming/format.hpp> // BGR accessor
+
+#include <opencv2/highgui.hpp> // CommandLineParser
+
+const std::string keys =
+    "{ h help  |              | Print this help message }"
+    "{ output  | output.png   | Path to the output file }";
+
+#ifdef HAVE_OAK
+
+int main(int argc, char *argv[]) {
+    cv::CommandLineParser cmd(argc, argv, keys);
+    if (cmd.has("help")) {
+        cmd.printMessage();
+        return 0;
+    }
+
+    const std::string output_name = cmd.get<std::string>("output");
+
+    std::vector<int> h = {1, 0, -1,
+                          2, 0, -2,
+                          1, 0, -1};
+    std::vector<int> v = { 1,  2,  1,
+                           0,  0,  0,
+                          -1, -2, -1};
+    cv::Mat hk(3, 3, CV_32SC1, h.data());
+    cv::Mat vk(3, 3, CV_32SC1, v.data());
+
+    // Heterogeneous pipeline:
+    // OAK camera -> Sobel -> streaming accessor (CPU)
+    cv::GFrame in;
+    cv::GFrame sobel = cv::gapi::oak::sobelXY(in, hk, vk);
+    // Default camera and then sobel work only with nv12 format
+    cv::GMat out = cv::gapi::streaming::Y(sobel);
+
+    auto args = cv::compile_args(cv::gapi::oak::ColorCameraParams{},
+                                 cv::gapi::oak::kernels());
+
+    auto pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out)).compileStreaming(std::move(args));
+
+    // Graph execution /////////////////////////////////////////////////////////
+    cv::Mat out_mat(1920, 1080, CV_8UC1);
+
+    pipeline.setSource(cv::gapi::wip::make_src<cv::gapi::oak::ColorCamera>());
+    pipeline.start();
+
+    // pull 1 frame
+    pipeline.pull(cv::gout(out_mat));
+
+    cv::imwrite(output_name, out_mat);
+
+    std::cout << "Pipeline finished: " << output_name << " file has been written." << std::endl;
+}
+
+#else // HAVE_OAK
+
+int main() {
+    GAPI_Assert(false && "Built without OAK support");
+    return -1;
+}
+
+#endif // HAVE_OAK

modules/gapi/src/backends/oak/goak.cpp

+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+//
+// Copyright (C) 2021 Intel Corporation
+
+#include <opencv2/gapi/oak/oak.hpp>
+#include <opencv2/gapi/cpu/gcpukernel.hpp>
+
+#include "oak_media_adapter.hpp"
+
+#include <thread>
+#include <chrono>
+
+namespace cv {
+namespace gapi {
+namespace oak {
+
+GArray<uint8_t> encode(const GFrame& in, const EncoderConfig& cfg) {
+    return GEncFrame::on(in, cfg);
+}
+
+GFrame sobelXY(const GFrame& in, const cv::Mat& hk, const cv::Mat& vk) {
+    return GSobelXY::on(in, hk, vk);
+}
+
+// This is a dummy oak::ColorCamera class that just makes our pipelining
+// machinery work. The real data comes from the physical camera which
+// is handled by DepthAI library.
+ColorCamera::ColorCamera()
+    : m_dummy(cv::MediaFrame::Create<cv::gapi::oak::OAKMediaAdapter>()) {
+}
+
+bool ColorCamera::pull(cv::gapi::wip::Data &data) {
+    // FIXME: Avoid passing this formal frame to the pipeline
+    std::this_thread::sleep_for(std::chrono::milliseconds(10));
+    data = m_dummy;
+    return true;
+}
+
+cv::GMetaArg ColorCamera::descr_of() const {
+    return cv::GMetaArg{cv::descr_of(m_dummy)};
+}
+
+} // namespace oak
+} // namespace gapi
+} // namespace cv

modules/gapi/src/backends/oak/goak_media_adapter.cpp

+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+//
+// Copyright (C) 2021 Intel Corporation
+
+#include "oak_media_adapter.hpp"
+
+namespace cv {
+namespace gapi {
+namespace oak {
+
+OAKMediaAdapter::OAKMediaAdapter(cv::Size sz, cv::MediaFormat fmt, std::vector<uint8_t>&& buffer) {
+    GAPI_Assert(fmt == cv::MediaFormat::NV12 && "OAKMediaAdapter only supports NV12 format for now");
+    m_sz = sz;
+    m_fmt = fmt;
+    m_buffer = buffer;
+}
+
+MediaFrame::View OAKMediaAdapter::OAKMediaAdapter::access(MediaFrame::Access) {
+    uint8_t* y_ptr = m_buffer.data();
+    uint8_t* uv_ptr = m_buffer.data() + static_cast<long>(m_buffer.size() / 3 * 2);
+    return MediaFrame::View{cv::MediaFrame::View::Ptrs{y_ptr, uv_ptr},
+                            cv::MediaFrame::View::Strides{static_cast<long unsigned int>(m_sz.width),
+                                                          static_cast<long unsigned int>(m_sz.width)}};
+}
+
+cv::GFrameDesc OAKMediaAdapter::OAKMediaAdapter::meta() const { return {m_fmt, m_sz}; }
+
+} // namespace oak
+} // namespace gapi
+} // namespace cv

modules/gapi/src/backends/oak/oak_media_adapter.hpp

+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+//
+// Copyright (C) 2021 Intel Corporation
+
+#ifndef OPENCV_GAPI_OAK_MEDIA_ADAPTER_HPP
+#define OPENCV_GAPI_OAK_MEDIA_ADAPTER_HPP
+
+#include <memory>
+
+#include <opencv2/gapi/media.hpp>
+
+namespace cv {
+namespace gapi {
+namespace oak {
+
+class GAPI_EXPORTS OAKMediaAdapter final : public cv::MediaFrame::IAdapter {
+public:
+    OAKMediaAdapter() = default;
+    OAKMediaAdapter(cv::Size sz, cv::MediaFormat fmt, std::vector<uint8_t>&& buffer);
+    cv::GFrameDesc meta() const override;
+    cv::MediaFrame::View access(cv::MediaFrame::Access) override;
+    ~OAKMediaAdapter() = default;
+private:
+    cv::Size m_sz;
+    cv::MediaFormat m_fmt;
+    std::vector<uint8_t> m_buffer;
+};
+
+} // namespace oak
+} // namespace gapi
+} // namespace cv
+
+#endif // OPENCV_GAPI_OAK_MEDIA_ADAPTER_HPP

modules/gapi/test/oak/gapi_tests_oak.cpp

+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+//
+// Copyright (C) 2021 Intel Corporation
+
+#include "../test_precomp.hpp"
+
+#ifdef HAVE_OAK
+
+#include <opencv2/gapi/oak/oak.hpp>
+
+namespace opencv_test
+{
+
+// FIXME: consider a better solution
+TEST(OAK, Available)
+{
+    cv::GFrame in;
+    auto out = cv::gapi::oak::encode(in, {});
+    auto args = cv::compile_args(cv::gapi::oak::ColorCameraParams{}, cv::gapi::oak::kernels());
+    auto pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out)).compileStreaming(std::move(args));
+}
+} // opencv_test
+
+#endif // HAVE_OAK