Merge pull request #18600 from TolyaTalamanov:at/implement-render-using-stateful

[G-API] Implement render using stateful kernel

* Implement render using stateful kernel

* Move freetype to backends folder

* Fix freetype compilation

* Fix freetype smoke test

* Fix comments

* Refactoring

modules/gapi/CMakeLists.txt

@@ -77,7 +77,6 @@ set(gapi_srcs
     src/api/render.cpp
     src/api/render_ocv.cpp
     src/api/ginfer.cpp
-    src/api/ft_render.cpp
     src/api/media.cpp
     src/api/rmat.cpp
 
@@ -131,8 +130,8 @@ set(gapi_srcs
     src/backends/ie/giebackend/giewrapper.cpp
 
     # Render Backend.
-    src/backends/render/grenderocvbackend.cpp
     src/backends/render/grenderocv.cpp
+    src/backends/render/ft_render.cpp
 
     #PlaidML Backend
     src/backends/plaidml/gplaidmlcore.cpp

modules/gapi/src/api/render_ocv.cpp

@@ -2,7 +2,7 @@
 #include <opencv2/gapi/render/render.hpp> // Kernel API's
 
 #include "api/render_ocv.hpp"
-#include "api/ft_render.hpp"
+#include "backends/render/ft_render.hpp"
 
 namespace cv
 {
@@ -146,12 +146,8 @@ struct EmptyConverter
 template <typename ColorConverter>
 void drawPrimitivesOCV(cv::Mat& in,
                        const cv::gapi::wip::draw::Prims& prims,
-                       cv::gapi::wip::draw::FTTextRender* ftpr)
+                       std::shared_ptr<cv::gapi::wip::draw::FTTextRender>& ftpr)
 {
-#ifndef HAVE_FREETYPE
-    cv::util::suppress_unused_warning(ftpr);
-#endif
-
     using namespace cv::gapi::wip::draw;
 
     ColorConverter converter;
@@ -177,7 +173,6 @@ void drawPrimitivesOCV(cv::Mat& in,
 
             case Prim::index_of<FText>():
             {
-#ifdef HAVE_FREETYPE
                 const auto& ftp  = cv::util::get<FText>(p);
                 const auto color = converter.cvtColor(ftp.color);
 
@@ -196,9 +191,6 @@ void drawPrimitivesOCV(cv::Mat& in,
                 cv::Point tl(ftp.org.x, ftp.org.y - mask.size().height + baseline);
 
                 blendTextMask(in, mask, tl, color);
-#else
-                cv::util::throw_error(std::runtime_error("FreeType not found !"));
-#endif
                 break;
             }
 
@@ -251,16 +243,16 @@ void drawPrimitivesOCV(cv::Mat& in,
     }
 }
 
-void drawPrimitivesOCVBGR(cv::Mat &in,
-                          const cv::gapi::wip::draw::Prims &prims,
-                          cv::gapi::wip::draw::FTTextRender* ftpr)
+void drawPrimitivesOCVBGR(cv::Mat                                                  &in,
+                          const cv::gapi::wip::draw::Prims                         &prims,
+                          std::shared_ptr<cv::gapi::wip::draw::FTTextRender> &ftpr)
 {
     drawPrimitivesOCV<EmptyConverter>(in, prims, ftpr);
 }
 
-void drawPrimitivesOCVYUV(cv::Mat &in,
-                          const cv::gapi::wip::draw::Prims &prims,
-                          cv::gapi::wip::draw::FTTextRender* ftpr)
+void drawPrimitivesOCVYUV(cv::Mat                                                  &in,
+                          const cv::gapi::wip::draw::Prims                         &prims,
+                          std::shared_ptr<cv::gapi::wip::draw::FTTextRender> &ftpr)
 {
     drawPrimitivesOCV<BGR2YUVConverter>(in, prims, ftpr);
 }

modules/gapi/src/api/render_ocv.hpp

 #include <vector>
 #include "render_priv.hpp"
-#include "ft_render.hpp"
+#include "backends/render/ft_render.hpp"
 
 #ifndef OPENCV_RENDER_OCV_HPP
 #define OPENCV_RENDER_OCV_HPP
@@ -15,8 +15,8 @@ namespace draw
 {
 
 // FIXME only for tests
-void GAPI_EXPORTS drawPrimitivesOCVYUV(cv::Mat& yuv, const Prims& prims, cv::gapi::wip::draw::FTTextRender* mc);
-void GAPI_EXPORTS drawPrimitivesOCVBGR(cv::Mat& bgr, const Prims& prims, cv::gapi::wip::draw::FTTextRender* mc);
+void GAPI_EXPORTS drawPrimitivesOCVYUV(cv::Mat& yuv, const Prims& prims, std::shared_ptr<cv::gapi::wip::draw::FTTextRender>& mc);
+void GAPI_EXPORTS drawPrimitivesOCVBGR(cv::Mat& bgr, const Prims& prims, std::shared_ptr<cv::gapi::wip::draw::FTTextRender>& mc);
 
 } // namespace draw
 } // namespace wip

modules/gapi/src/api/ft_render.cpp → modules/gapi/src/backends/render/ft_render.cpp

@@ -5,11 +5,11 @@
 // Copyright (C) 2019 Intel Corporation
 
 #include "precomp.hpp"
+#include "ft_render.hpp"
 
 #ifdef HAVE_FREETYPE
 
-#include "api/ft_render.hpp"
-#include "api/ft_render_priv.hpp"
+#include "ft_render_priv.hpp"
 
 #include <opencv2/gapi/util/throw.hpp>
 #include <opencv2/gapi/own/assert.hpp>
@@ -166,6 +166,11 @@ void cv::gapi::wip::draw::FTTextRender::Priv::putText(cv::Mat& mat,
                     "Failed to load char");
         FT_Bitmap *bitmap = &(m_face->glyph->bitmap);
 
+        // FIXME: Skip glyph, if size is 0
+        if (bitmap->rows == 0 || bitmap->width == 0) {
+            continue;
+        }
+
         cv::Mat glyph(bitmap->rows, bitmap->width, CV_8UC1, bitmap->buffer, bitmap->pitch);
 
         int left    = m_face->glyph->bitmap_left;
@@ -211,4 +216,21 @@ void cv::gapi::wip::draw::FTTextRender::putText(cv::Mat& mat,
     m_priv->putText(mat, text, org, fh);
 }
 
+#else
+
+cv::Size cv::gapi::wip::draw::FTTextRender::getTextSize(const std::wstring&, int, int*)
+{
+    cv::util::throw_error(std::runtime_error("Freetype not found"));
+}
+
+void cv::gapi::wip::draw::FTTextRender::putText(cv::Mat&, const std::wstring&, const cv::Point&, int)
+{
+    cv::util::throw_error(std::runtime_error("Freetype not found"));
+}
+
+cv::gapi::wip::draw::FTTextRender::FTTextRender(const std::string&)
+{
+    cv::util::throw_error(std::runtime_error("Freetype not found"));
+}
+
 #endif // HAVE_FREETYPE

modules/gapi/src/api/ft_render.hpp → modules/gapi/src/backends/render/ft_render.hpp

@@ -23,8 +23,6 @@ namespace wip
 namespace draw
 {
 
-#ifdef HAVE_FREETYPE
-
 class GAPI_EXPORTS FTTextRender
 {
 public:
@@ -38,12 +36,6 @@ private:
     std::shared_ptr<Priv> m_priv;
 };
 
-#else
-
-class GAPI_EXPORTS FTTextRender {};
-
-#endif // HAVE_FREETYPE
-
 } // namespace draw
 } // namespace wip
 } // namespace gapi

modules/gapi/src/api/ft_render_priv.hpp → modules/gapi/src/backends/render/ft_render_priv.hpp

@@ -10,7 +10,7 @@
 #ifndef OPENCV_FT_RENDER_PRIV_HPP
 #define OPENCV_FT_RENDER_PRIV_HPP
 
-#include "api/ft_render.hpp"
+#include "ft_render.hpp"
 
 #include <ft2build.h>
 #include FT_FREETYPE_H

modules/gapi/src/backends/render/grenderocv.cpp

 #include <opencv2/imgproc.hpp>
 
 #include "api/render_ocv.hpp"
-#include "backends/render/grenderocv.hpp"
 
 #include <opencv2/gapi/cpu/gcpukernel.hpp>
+#include <opencv2/gapi/fluid/core.hpp>
 
-GAPI_RENDER_OCV_KERNEL(RenderBGROCVImpl, cv::gapi::wip::draw::GRenderBGR)
+struct RenderOCVState
+{
+    std::shared_ptr<cv::gapi::wip::draw::FTTextRender> ftpr;
+};
+
+GAPI_OCV_KERNEL_ST(RenderBGROCVImpl, cv::gapi::wip::draw::GRenderBGR, RenderOCVState)
 {
     static void run(const cv::Mat& in,
                     const cv::gapi::wip::draw::Prims& prims,
-                    cv::gapi::wip::draw::FTTextRender* ftpr,
-                    cv::Mat& out)
+                    cv::Mat& out,
+                    RenderOCVState& state)
     {
         // NB: If in and out cv::Mats are the same object
         // we can avoid copy and render on out cv::Mat
@@ -19,18 +24,33 @@ GAPI_RENDER_OCV_KERNEL(RenderBGROCVImpl, cv::gapi::wip::draw::GRenderBGR)
             in.copyTo(out);
         }
 
-        cv::gapi::wip::draw::drawPrimitivesOCVBGR(out, prims, ftpr);
+        cv::gapi::wip::draw::drawPrimitivesOCVBGR(out, prims, state.ftpr);
+    }
+
+    static void setup(const cv::GMatDesc& /* in */,
+                      const cv::GArrayDesc& /* prims */,
+                      std::shared_ptr<RenderOCVState>& state,
+                      const cv::GCompileArgs& args)
+    {
+        using namespace cv::gapi::wip::draw;
+        auto opt_freetype_font = cv::gapi::getCompileArg<freetype_font>(args);
+        state = std::make_shared<RenderOCVState>();
+
+        if (opt_freetype_font.has_value())
+        {
+            state->ftpr = std::make_shared<FTTextRender>(opt_freetype_font->path);
+        }
     }
 };
 
-GAPI_RENDER_OCV_KERNEL(RenderNV12OCVImpl, cv::gapi::wip::draw::GRenderNV12)
+GAPI_OCV_KERNEL_ST(RenderNV12OCVImpl, cv::gapi::wip::draw::GRenderNV12, RenderOCVState)
 {
     static void run(const cv::Mat& in_y,
                     const cv::Mat& in_uv,
                     const cv::gapi::wip::draw::Prims& prims,
-                    cv::gapi::wip::draw::FTTextRender* ftpr,
                     cv::Mat& out_y,
-                    cv::Mat& out_uv)
+                    cv::Mat& out_uv,
+                    RenderOCVState& state)
     {
         // NB: If in and out cv::Mats are the same object
         // we can avoid copy and render on out cv::Mat
@@ -67,7 +87,7 @@ GAPI_RENDER_OCV_KERNEL(RenderNV12OCVImpl, cv::gapi::wip::draw::GRenderNV12)
         cv::resize(in_uv, upsample_uv, in_uv.size() * 2, cv::INTER_LINEAR);
         cv::merge(std::vector<cv::Mat>{in_y, upsample_uv}, yuv);
 
-        cv::gapi::wip::draw::drawPrimitivesOCVYUV(yuv, prims, ftpr);
+        cv::gapi::wip::draw::drawPrimitivesOCVYUV(yuv, prims, state.ftpr);
 
         // YUV -> NV12
         cv::Mat out_u, out_v, uv_plane;
@@ -76,6 +96,22 @@ GAPI_RENDER_OCV_KERNEL(RenderNV12OCVImpl, cv::gapi::wip::draw::GRenderNV12)
         cv::merge(std::vector<cv::Mat>{chs[1], chs[2]}, uv_plane);
         cv::resize(uv_plane, out_uv, uv_plane.size() / 2, cv::INTER_LINEAR);
     }
+
+    static void setup(const cv::GMatDesc&   /* in_y  */,
+                      const cv::GMatDesc&   /* in_uv */,
+                      const cv::GArrayDesc& /* prims */,
+                      std::shared_ptr<RenderOCVState>& state,
+                      const cv::GCompileArgs& args)
+    {
+        using namespace cv::gapi::wip::draw;
+        auto has_freetype_font = cv::gapi::getCompileArg<freetype_font>(args);
+        state = std::make_shared<RenderOCVState>();
+
+        if (has_freetype_font)
+        {
+            state->ftpr = std::make_shared<FTTextRender>(has_freetype_font->path);
+        }
+    }
 };
 
 cv::gapi::GKernelPackage cv::gapi::render::ocv::kernels()

modules/gapi/src/backends/render/grenderocv.hpp

-// This file is part of OpenCV project.
-// It is subject to the license terms in the LICENSE file found in the top-level directory
-// of this distribution and at http://opencv.org/license.html.
-//
-// Copyright (C) 2019 Intel Corporation
-
-#ifndef OPENCV_GAPI_GRENDEROCV_HPP
-#define OPENCV_GAPI_GRENDEROCV_HPP
-
-#include <opencv2/gapi/cpu/gcpukernel.hpp>
-#include "api/render_priv.hpp"
-#include "api/ft_render.hpp"
-
-namespace cv
-{
-namespace gapi
-{
-namespace render
-{
-namespace ocv
-{
-
-GAPI_EXPORTS cv::gapi::GBackend backend();
-
-template<typename, typename>
-struct add_type_to_tuple;
-
-template<typename P, typename ...Ts>
-struct add_type_to_tuple<P, std::tuple<Ts...>>
-{
-    using type = std::tuple<Ts..., P>;
-};
-
-template<class Impl, class K>
-class GRenderKernelImpl: public cv::detail::OCVCallHelper<Impl, typename K::InArgs, typename K::OutArgs>,
-                         public cv::detail::KernelTag
-{
-    using InArgs = typename add_type_to_tuple<cv::gapi::wip::draw::FTTextRender*, typename K::InArgs>::type;
-    using P      = detail::OCVCallHelper<Impl, InArgs, typename K::OutArgs>;
-
-public:
-    using API = K;
-
-    static cv::gapi::GBackend backend()  { return cv::gapi::render::ocv::backend(); }
-    static cv::GCPUKernel     kernel()   { return GCPUKernel(&P::call);             }
-};
-
-#define GAPI_RENDER_OCV_KERNEL(Name, API) struct Name: public cv::gapi::render::ocv::GRenderKernelImpl<Name, API>
-
-} // namespace ocv
-} // namespace render
-} // namespace gapi
-} // namespace cv
-
-#endif // OPENCV_GAPI_GRENDEROCV_HPP

modules/gapi/src/backends/render/grenderocvbackend.cpp

-// This file is part of OpenCV project.
-// It is subject to the license terms in the LICENSE file found in the top-level directory
-// of this distribution and at http://opencv.org/license.html.
-//
-// Copyright (C) 2018-2020 Intel Corporation
-
-#include "precomp.hpp"
-
-#include <functional>
-#include <unordered_set>
-
-#include <ade/util/algorithm.hpp>
-
-#include <ade/util/range.hpp>
-#include <ade/util/zip_range.hpp>
-#include <ade/util/chain_range.hpp>
-#include <ade/typed_graph.hpp>
-
-#include <opencv2/gapi/gcommon.hpp>
-#include <opencv2/gapi/garray.hpp>
-#include <opencv2/gapi/util/any.hpp>
-#include <opencv2/gapi/gtype_traits.hpp>
-
-#include "compiler/gobjref.hpp"
-#include "compiler/gmodel.hpp"
-
-#include "api/gbackend_priv.hpp" // FIXME: Make it part of Backend SDK!
-#include "api/render_ocv.hpp"
-
-#include "backends/render/grenderocvbackend.hpp"
-
-#include <opencv2/gapi/render/render.hpp>
-#include "api/ocv_mask_creator.hpp"
-#include "api/ft_render.hpp"
-
-
-using GRenderModel = ade::TypedGraph
-    < cv::gimpl::render::ocv::RenderUnit
-    >;
-
-// FIXME: Same issue with Typed and ConstTyped
-using GConstRenderModel = ade::ConstTypedGraph
-    < cv::gimpl::render::ocv::RenderUnit
-    >;
-
-cv::gimpl::render::ocv::GRenderExecutable::GRenderExecutable(const ade::Graph &g,
-                                                             const std::vector<ade::NodeHandle> &nodes,
-                                                             std::unique_ptr<cv::gapi::wip::draw::FTTextRender>&& ftpr)
-    : m_g(g), m_gm(m_g), m_ftpr(std::move(ftpr)) {
-        GConstRenderModel gcm(m_g);
-
-        auto is_op = [&](ade::NodeHandle nh) {
-            return m_gm.metadata(nh).get<NodeType>().t == NodeType::OP;
-        };
-
-        auto it = ade::util::find_if(nodes, is_op);
-
-        GAPI_Assert(it != nodes.end());
-        this_nh = *it;
-
-        if (!std::none_of(std::next(it), nodes.end(), is_op)) {
-            util::throw_error(std::logic_error("Multi-node rendering is not supported!"));
-        }
-}
-
-void cv::gimpl::render::ocv::GRenderExecutable::run(std::vector<InObj>  &&input_objs,
-                                                    std::vector<OutObj> &&output_objs) {
-    GConstRenderModel gcm(m_g);
-
-    for (auto& it : input_objs)   magazine::bindInArg (m_res, it.first, it.second);
-    for (auto& it : output_objs)  magazine::bindOutArg(m_res, it.first, it.second);
-
-    const auto &op = m_gm.metadata(this_nh).get<Op>();
-
-    // Initialize kernel's execution context:
-    // - Input parameters
-    GCPUContext context;
-    context.m_args.reserve(op.args.size());
-    using namespace std::placeholders;
-    ade::util::transform(op.args,
-                          std::back_inserter(context.m_args),
-                          std::bind(&GRenderExecutable::packArg, this, _1));
-
-    // - Output parameters.
-    for (const auto &out_it : ade::util::indexed(op.outs)) {
-        // FIXME: Can the same GArg type resolution mechanism be reused here?
-        const auto out_port  = ade::util::index(out_it);
-        const auto out_desc  = ade::util::value(out_it);
-        context.m_results[out_port] = magazine::getObjPtr(m_res, out_desc);
-    }
-
-    auto k = gcm.metadata(this_nh).get<RenderUnit>().k;
-
-    context.m_args.emplace_back(m_ftpr.get());
-
-    k.m_runF(context);
-
-    for (auto &it : output_objs) magazine::writeBack(m_res, it.first, it.second);
-
-    // In/Out args clean-up is mandatory now with RMat
-    for (auto &it : input_objs) magazine::unbind(m_res, it.first);
-    for (auto &it : output_objs) magazine::unbind(m_res, it.first);
-}
-
-cv::GArg cv::gimpl::render::ocv::GRenderExecutable::packArg(const cv::GArg &arg) {
-    // No API placeholders allowed at this point
-    // FIXME: this check has to be done somewhere in compilation stage.
-    GAPI_Assert(   arg.kind != cv::detail::ArgKind::GMAT
-                && arg.kind != cv::detail::ArgKind::GSCALAR
-                && arg.kind != cv::detail::ArgKind::GARRAY);
-
-    if (arg.kind != cv::detail::ArgKind::GOBJREF) {
-        util::throw_error(std::logic_error("Render supports G-types ONLY!"));
-    }
-    GAPI_Assert(arg.kind == cv::detail::ArgKind::GOBJREF);
-
-    const cv::gimpl::RcDesc &ref = arg.get<cv::gimpl::RcDesc>();
-    switch (ref.shape)
-    {
-    case GShape::GMAT:   return GArg(m_res.slot<cv::Mat>()[ref.id]);
-    case GShape::GARRAY: return GArg(m_res.slot<cv::detail::VectorRef>().at(ref.id));
-    default:
-        util::throw_error(std::logic_error("Unsupported GShape type"));
-        break;
-    }
-}
-
-namespace {
-    class GRenderBackendImpl final: public cv::gapi::GBackend::Priv {
-        virtual void unpackKernel(ade::Graph &gr,
-                                  const ade::NodeHandle &op_node,
-                                  const cv::GKernelImpl &impl) override {
-            GRenderModel rm(gr);
-            auto render_impl = cv::util::any_cast<cv::GCPUKernel>(impl.opaque);
-            rm.metadata(op_node).set(cv::gimpl::render::ocv::RenderUnit{render_impl});
-        }
-
-        virtual EPtr compile(const ade::Graph &graph,
-                             const cv::GCompileArgs& args,
-                             const std::vector<ade::NodeHandle> &nodes) const override {
-
-            using namespace cv::gapi::wip::draw;
-            auto has_freetype_font = cv::gapi::getCompileArg<freetype_font>(args);
-            std::unique_ptr<FTTextRender> ftpr;
-            if (has_freetype_font)
-            {
-#ifndef HAVE_FREETYPE
-                throw std::runtime_error("Freetype not found");
-#else
-                ftpr.reset(new FTTextRender(has_freetype_font.value().path));
-#endif
-            }
-            return EPtr{new cv::gimpl::render::ocv::GRenderExecutable(graph, nodes, std::move(ftpr))};
-        }
-    };
-}
-
-cv::gapi::GBackend cv::gapi::render::ocv::backend() {
-    static cv::gapi::GBackend this_backend(std::make_shared<GRenderBackendImpl>());
-    return this_backend;
-}

modules/gapi/src/backends/render/grenderocvbackend.hpp

-// This file is part of OpenCV project.
-// It is subject to the license terms in the LICENSE file found in the top-level directory
-// of this distribution and at http://opencv.org/license.html.
-//
-// Copyright (C) 2019 Intel Corporation
-
-#ifndef OPENCV_GAPI_GRENDEROCVBACKEND_HPP
-#define OPENCV_GAPI_GRENDEROCVBACKEND_HPP
-
-#include <opencv2/gapi/garg.hpp>
-#include <opencv2/gapi/gproto.hpp>
-#include <opencv2/gapi/render/render.hpp>
-
-#include "api/gorigin.hpp"
-#include "backends/common/gbackend.hpp"
-#include "compiler/gislandmodel.hpp"
-
-#include "backends/render/grenderocv.hpp"
-
-#include <opencv2/gapi/cpu/gcpukernel.hpp>
-
-namespace cv
-{
-namespace gimpl
-{
-namespace render
-{
-namespace ocv
-{
-
-struct RenderUnit
-{
-    static const char *name() { return "RenderUnit"; }
-    GCPUKernel k;
-};
-
-class GRenderExecutable final: public GIslandExecutable
-{
-    const ade::Graph &m_g;
-    GModel::ConstGraph m_gm;
-    std::unique_ptr<cv::gapi::wip::draw::FTTextRender> m_ftpr;
-
-    // The only executable stuff in this graph
-    // (assuming it is always single-op)
-    ade::NodeHandle this_nh;
-
-    //// Actual data of all resources in graph (both internal and external)
-    Mag m_res;
-
-    //// Execution helpers
-    GArg packArg(const GArg &arg);
-
-public:
-    GRenderExecutable(const ade::Graph                   &graph,
-                      const std::vector<ade::NodeHandle> &nodes,
-                      std::unique_ptr<cv::gapi::wip::draw::FTTextRender>&& ftpr);
-
-    virtual inline bool canReshape() const override { return false; }
-
-    virtual inline void reshape(ade::Graph&, const GCompileArgs&) override {
-        GAPI_Assert(false); // Not implemented yet
-    }
-
-    virtual void run(std::vector<InObj>  &&input_objs,
-                     std::vector<OutObj> &&output_objs) override;
-};
-
-} // namespace ocv
-} // namespace render
-} // namespace gimpl
-} // namespace cv
-
-#endif // OPENCV_GAPI_GRENDEROCVBACKEND_HPP

modules/gapi/test/render/ftp_render_test.cpp

@@ -13,7 +13,7 @@
 
 #include <opencv2/core/utils/configuration.private.hpp>
 
-#include "api/ft_render.hpp"
+#include "backends/render/ft_render.hpp"
 
 namespace opencv_test
 {

modules/gapi/test/render/gapi_render_tests_ocv.cpp

@@ -95,7 +95,6 @@ TEST_P(RenderNV12OCVTestFTexts, AccuracyTest)
                                 cv::compile_args(cv::gapi::wip::draw::freetype_font{
                                 "/usr/share/fonts/truetype/wqy/wqy-microhei.ttc"
                                 })));
-
 }
 
 static std::wstring to_wstring(const char* bytes)