Merge pull request #18584 from smirnov-alexey:as/rmat_s11n

[G-API]: Introduce RMat serialization API

* Introduce RMat serialization API

* Fix RunArgs deserialization

* Address review comments

* Export operators for GRunArg serialization

* Fix warning and add handling for RMat in bind()

* Update CMakeLists.txt

* G-API: RMat S11N -- probably fix the Windows warning

modules/gapi/include/opencv2/gapi/rmat.hpp

@@ -10,6 +10,16 @@
 #include <opencv2/gapi/gmat.hpp>
 #include <opencv2/gapi/own/exports.hpp>
 
+// Forward declaration
+namespace cv {
+namespace gapi {
+namespace s11n {
+    struct IOStream;
+    struct IIStream;
+} // namespace s11n
+} // namespace gapi
+} // namespace cv
+
 namespace cv {
 
 // "Remote Mat", a general class which provides an abstraction layer over the data
@@ -90,6 +100,12 @@ public:
         // the view when accessed for writing, to ensure that the data from the view
         // is transferred to the device when the view is destroyed
         virtual View access(Access) = 0;
+        virtual void serialize(cv::gapi::s11n::IOStream&) {
+            GAPI_Assert(false && "Generic serialize method should never be called for RMat adapter");
+        }
+        virtual void deserialize(cv::gapi::s11n::IIStream&) {
+            GAPI_Assert(false && "Generic deserialize method should never be called for RMat adapter");
+        }
     };
     using AdapterP = std::shared_ptr<Adapter>;
 
@@ -113,6 +129,10 @@ public:
         return dynamic_cast<T*>(m_adapter.get());
     }
 
+    void serialize(cv::gapi::s11n::IOStream& os) const {
+        m_adapter->serialize(os);
+    }
+
 private:
     AdapterP m_adapter = nullptr;
 };

modules/gapi/include/opencv2/gapi/s11n.hpp

@@ -12,6 +12,7 @@
 #include <unordered_map>
 #include <opencv2/gapi/s11n/base.hpp>
 #include <opencv2/gapi/gcomputation.hpp>
+#include <opencv2/gapi/rmat.hpp>
 
 namespace cv {
 namespace gapi {
@@ -25,6 +26,9 @@ namespace detail {
 
     template<typename... Types>
     cv::GCompileArgs getCompileArgs(const std::vector<char> &p);
+
+    template<typename RMatAdapterType>
+    cv::GRunArgs getRunArgsWithRMats(const std::vector<char> &p);
 } // namespace detail
 
 GAPI_EXPORTS std::vector<char> serialize(const cv::GComputation &c);
@@ -59,6 +63,12 @@ typename std::enable_if<std::is_same<T, GCompileArgs>::value, GCompileArgs>::
 type deserialize(const std::vector<char> &p) {
     return detail::getCompileArgs<Types...>(p);
 }
+
+template<typename T, typename RMatAdapterType> inline
+typename std::enable_if<std::is_same<T, GRunArgs>::value, GRunArgs>::
+type deserialize(const std::vector<char> &p) {
+    return detail::getRunArgsWithRMats<RMatAdapterType>(p);
+}
 } // namespace gapi
 } // namespace cv
 
@@ -123,6 +133,27 @@ GAPI_EXPORTS IIStream& operator>> (IIStream& is,       cv::Scalar &s);
 GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::Mat &m);
 GAPI_EXPORTS IIStream& operator>> (IIStream& is,       cv::Mat &m);
 
+// FIXME: for GRunArgs serailization
+#if !defined(GAPI_STANDALONE)
+GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::UMat &);
+GAPI_EXPORTS IIStream& operator>> (IIStream& is,       cv::UMat &);
+#endif // !defined(GAPI_STANDALONE)
+
+GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::RMat &r);
+GAPI_EXPORTS IIStream& operator>> (IIStream& is,       cv::RMat &r);
+
+GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::gapi::wip::IStreamSource::Ptr &);
+GAPI_EXPORTS IIStream& operator>> (IIStream& is,       cv::gapi::wip::IStreamSource::Ptr &);
+
+GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::detail::VectorRef &);
+GAPI_EXPORTS IIStream& operator>> (IIStream& is,       cv::detail::VectorRef &);
+
+GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::detail::OpaqueRef &);
+GAPI_EXPORTS IIStream& operator>> (IIStream& is,       cv::detail::OpaqueRef &);
+
+GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::MediaFrame &);
+GAPI_EXPORTS IIStream& operator>> (IIStream& is,       cv::MediaFrame &);
+
 // Generic STL types ////////////////////////////////////////////////////////////////
 template<typename K, typename V>
 IOStream& operator<< (IOStream& os, const std::map<K, V> &m) {
@@ -184,6 +215,52 @@ IIStream& operator>> (IIStream& is, std::vector<T> &ts) {
     }
     return is;
 }
+
+// Generic: variant serialization
+namespace detail {
+template<typename V>
+IOStream& put_v(IOStream&, const V&, std::size_t) {
+    GAPI_Assert(false && "variant>>: requested index is invalid");
+};
+template<typename V, typename X, typename... Xs>
+IOStream& put_v(IOStream& os, const V& v, std::size_t x) {
+    return (x == 0u)
+        ? os << cv::util::get<X>(v)
+        : put_v<V, Xs...>(os, v, x-1);
+}
+template<typename V>
+IIStream& get_v(IIStream&, V&, std::size_t, std::size_t) {
+    GAPI_Assert(false && "variant<<: requested index is invalid");
+}
+template<typename V, typename X, typename... Xs>
+IIStream& get_v(IIStream& is, V& v, std::size_t i, std::size_t gi) {
+    if (i == gi) {
+        X x{};
+        is >> x;
+        v = V{std::move(x)};
+        return is;
+    } else return get_v<V, Xs...>(is, v, i+1, gi);
+}
+} // namespace detail
+
+template<typename... Ts>
+IOStream& operator<< (IOStream& os, const cv::util::variant<Ts...> &v) {
+    os << static_cast<uint32_t>(v.index());
+    return detail::put_v<cv::util::variant<Ts...>, Ts...>(os, v, v.index());
+}
+template<typename... Ts>
+IIStream& operator>> (IIStream& is, cv::util::variant<Ts...> &v) {
+    int idx = -1;
+    is >> idx;
+    GAPI_Assert(idx >= 0 && idx < (int)sizeof...(Ts));
+    return detail::get_v<cv::util::variant<Ts...>, Ts...>(is, v, 0u, idx);
+}
+
+// FIXME: consider a better solution
+template<typename... Ts>
+void getRunArgByIdx (IIStream& is, cv::util::variant<Ts...> &v, uint32_t idx) {
+    is = detail::get_v<cv::util::variant<Ts...>, Ts...>(is, v, 0u, idx);
+}
 } // namespace s11n
 
 namespace detail
@@ -204,11 +281,27 @@ static GCompileArg exec(cv::gapi::s11n::IIStream& is, const std::string& tag) {
             cv::gapi::s11n::detail::S11N<T>::deserialize(is)
         };
     }
-
     return deserialize_arg<std::tuple<Types...>>::exec(is, tag);
 }
 };
 
+template<typename T> struct deserialize_runarg;
+
+template<typename RMatAdapterType>
+struct deserialize_runarg {
+static GRunArg exec(cv::gapi::s11n::IIStream& is, uint32_t idx) {
+    if (idx == GRunArg::index_of<RMat>()) {
+        auto ptr = std::make_shared<RMatAdapterType>();
+        ptr->deserialize(is);
+        return GRunArg { RMat(std::move(ptr)) };
+    } else { // non-RMat arg - use default deserialization
+        GRunArg arg;
+        getRunArgByIdx(is, arg, idx);
+        return arg;
+    }
+}
+};
+
 template<typename... Types>
 cv::GCompileArgs getCompileArgs(const std::vector<char> &p) {
     std::unique_ptr<cv::gapi::s11n::IIStream> pIs = cv::gapi::s11n::detail::getInStream(p);
@@ -225,6 +318,23 @@ cv::GCompileArgs getCompileArgs(const std::vector<char> &p) {
 
     return args;
 }
+
+template<typename RMatAdapterType>
+cv::GRunArgs getRunArgsWithRMats(const std::vector<char> &p) {
+    std::unique_ptr<cv::gapi::s11n::IIStream> pIs = cv::gapi::s11n::detail::getInStream(p);
+    cv::gapi::s11n::IIStream& is = *pIs;
+    cv::GRunArgs args;
+
+    uint32_t sz = 0;
+    is >> sz;
+    for (uint32_t i = 0; i < sz; ++i) {
+        uint32_t idx = 0;
+        is >> idx;
+        args.push_back(cv::gapi::detail::deserialize_runarg<RMatAdapterType>::exec(is, idx));
+    }
+
+    return args;
+}
 } // namespace detail
 } // namespace gapi
 } // namespace cv

modules/gapi/src/api/s11n.cpp

@@ -79,6 +79,9 @@ cv::GRunArgsP cv::gapi::bind(cv::GRunArgs &results)
         case T::index_of<cv::detail::OpaqueRef>() :
             outputs.emplace_back(cv::util::get<cv::detail::OpaqueRef>(res_obj));
             break;
+        case cv::GRunArg::index_of<cv::RMat>() :
+            outputs.emplace_back((cv::RMat*)(&(cv::util::get<cv::RMat>(res_obj))));
+            break;
         default:
             GAPI_Assert(false && "This value type is not supported!"); // ...maybe because of STANDALONE mode.
             break;
@@ -112,6 +115,9 @@ cv::GRunArg cv::gapi::bind(cv::GRunArgP &out)
     case T::index_of<cv::Scalar*>() :
         return cv::GRunArg(*cv::util::get<cv::Scalar*>(out));
 
+    case T::index_of<cv::RMat*>() :
+        return cv::GRunArg(*cv::util::get<cv::RMat*>(out));
+
     default:
         // ...maybe our types were extended
         GAPI_Assert(false && "This value type is UNKNOWN!");

modules/gapi/src/backends/common/serialization.cpp

@@ -165,12 +165,12 @@ IOStream& operator<< (IOStream& os, const cv::Scalar &s) {
 IIStream& operator>> (IIStream& is, cv::Scalar& s) {
     return is >> s.val[0] >> s.val[1] >> s.val[2] >> s.val[3];
 }
-IOStream& operator<< (IOStream& os, const cv::RMat&) {
-    util::throw_error(std::logic_error("Serialization of RMat is not supported"));
+IOStream& operator<< (IOStream& os, const cv::RMat& mat) {
+    mat.serialize(os);
     return os;
 }
 IIStream& operator>> (IIStream& is, cv::RMat&) {
-    util::throw_error(std::logic_error("Serialization of RMat is not supported"));
+    util::throw_error(std::logic_error("operator>> for RMat should never be called"));
     return is;
 }
 

modules/gapi/src/backends/common/serialization.hpp

@@ -88,26 +88,6 @@ GAPI_EXPORTS IIStream& operator>> (IIStream& is,       cv::GArrayDesc &);
 GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::GFrameDesc &);
 GAPI_EXPORTS IIStream& operator>> (IIStream& is,       cv::GFrameDesc &);
 
-#if !defined(GAPI_STANDALONE)
-GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::UMat &);
-GAPI_EXPORTS IIStream& operator>> (IIStream& is,       cv::UMat &);
-#endif // !defined(GAPI_STANDALONE)
-
-GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::RMat &r);
-GAPI_EXPORTS IIStream& operator>> (IIStream& is,       cv::RMat &r);
-
-GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::gapi::wip::IStreamSource::Ptr &);
-GAPI_EXPORTS IIStream& operator>> (IIStream& is,       cv::gapi::wip::IStreamSource::Ptr &);
-
-GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::detail::VectorRef &);
-GAPI_EXPORTS IIStream& operator>> (IIStream& is,       cv::detail::VectorRef &);
-
-GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::detail::OpaqueRef &);
-GAPI_EXPORTS IIStream& operator>> (IIStream& is,       cv::detail::OpaqueRef &);
-
-GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::MediaFrame &);
-GAPI_EXPORTS IIStream& operator>> (IIStream& is,       cv::MediaFrame &);
-
 GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::gimpl::RcDesc &rc);
 GAPI_EXPORTS IIStream& operator>> (IIStream& is,       cv::gimpl::RcDesc &rc);
 
@@ -178,46 +158,6 @@ GAPI_EXPORTS void serialize( IOStream& os
 GAPI_EXPORTS GSerialized deserialize(IIStream& is);
 GAPI_EXPORTS void reconstruct(const GSerialized &s, ade::Graph &g);
 
-// Generic: variant serialization //////////////////////////////////////////////
-namespace detail { // FIXME: breaks old code
-template<typename V>
-IOStream& put_v(IOStream&, const V&, std::size_t) {
-    GAPI_Assert(false && "variant>>: requested index is invalid");
-};
-template<typename V, typename X, typename... Xs>
-IOStream& put_v(IOStream& os, const V& v, std::size_t x) {
-    return (x == 0u)
-        ? os << cv::util::get<X>(v)
-        : put_v<V, Xs...>(os, v, x-1);
-}
-template<typename V>
-IIStream& get_v(IIStream&, V&, std::size_t, std::size_t) {
-    GAPI_Assert(false && "variant<<: requested index is invalid");
-}
-template<typename V, typename X, typename... Xs>
-IIStream& get_v(IIStream& is, V& v, std::size_t i, std::size_t gi) {
-    if (i == gi) {
-        X x{};
-        is >> x;
-        v = std::move(x);
-        return is;
-    } else return get_v<V, Xs...>(is, v, i+1, gi);
-}
-} // namespace detail FIXME: breaks old code
-
-template<typename... Ts>
-IOStream& operator<< (IOStream& os, const cv::util::variant<Ts...> &v) {
-    os << (uint32_t)v.index();
-    return detail::put_v<cv::util::variant<Ts...>, Ts...>(os, v, v.index());
-}
-template<typename... Ts>
-IIStream& operator>> (IIStream& is, cv::util::variant<Ts...> &v) {
-    int idx = -1;
-    is >> idx;
-    GAPI_Assert(idx >= 0 && idx < (int)sizeof...(Ts));
-    return detail::get_v<cv::util::variant<Ts...>, Ts...>(is, v, 0u, idx);
-}
-
 // FIXME: Basic Stream implementaions //////////////////////////////////////////
 
 // Basic in-memory stream implementations.

modules/gapi/test/s11n/gapi_s11n_tests.cpp

 #include "../test_precomp.hpp"
 
 #include "backends/common/serialization.hpp"
+#include <opencv2/gapi/rmat.hpp>
 
 namespace {
     struct MyCustomType {
@@ -45,6 +46,35 @@ template<> struct CompileArgTag<MyCustomType> {
 } // namespace detail
 } // namespace cv
 
+namespace {
+class MyRMatAdapter : public cv::RMat::Adapter {
+    cv::Mat m_mat;
+    int m_value;
+    std::string m_str;
+public:
+    MyRMatAdapter() = default;
+    MyRMatAdapter(cv::Mat m, int value, const std::string& str)
+        : m_mat(m), m_value(value), m_str(str)
+    {}
+    virtual cv::RMat::View access(cv::RMat::Access access) override {
+        if (access == cv::RMat::Access::W) {
+            return cv::RMat::View(cv::descr_of(m_mat), m_mat.data, m_mat.step);
+        } else {
+            return cv::RMat::View(cv::descr_of(m_mat), m_mat.data, m_mat.step);
+        }
+    }
+    virtual cv::GMatDesc desc() const override { return cv::descr_of(m_mat); }
+    virtual void serialize(cv::gapi::s11n::IOStream& os) override {
+        os << m_value << m_str;
+    }
+    virtual void deserialize(cv::gapi::s11n::IIStream& is) override {
+        is >> m_value >> m_str;
+    }
+    int getVal() { return m_value; }
+    std::string getStr() { return m_str; }
+};
+}
+
 namespace opencv_test {
 
 struct S11N_Basic: public ::testing::Test {
@@ -460,6 +490,39 @@ TEST_F(S11N_Basic, Test_Bind_RunArgs_MatScalar) {
     }
 }
 
+TEST_F(S11N_Basic, Test_RunArg_RMat) {
+    cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3);
+    cv::RMat rmat = cv::make_rmat<MyRMatAdapter>(mat, 42, "It actually works");
+    auto v = cv::GRunArgs{ cv::GRunArg{ rmat } };
+
+    const std::vector<char> sargsin = cv::gapi::serialize(v);
+    cv::GRunArgs out = cv::gapi::deserialize<cv::GRunArgs, MyRMatAdapter>(sargsin);
+    cv::RMat out_mat = cv::util::get<cv::RMat>(out[0]);
+    auto adapter = out_mat.get<MyRMatAdapter>();
+    EXPECT_EQ(42, adapter->getVal());
+    EXPECT_EQ("It actually works", adapter->getStr());
+}
+
+TEST_F(S11N_Basic, Test_RunArg_RMat_Scalar_Mat) {
+    cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3);
+    cv::RMat rmat = cv::make_rmat<MyRMatAdapter>(mat, 42, "It actually works");
+    cv::Scalar sc(111);
+    auto v = cv::GRunArgs{ cv::GRunArg{ rmat }, cv::GRunArg{ sc }, cv::GRunArg{ mat } };
+
+    const std::vector<char> sargsin = cv::gapi::serialize(v);
+    cv::GRunArgs out = cv::gapi::deserialize<cv::GRunArgs, MyRMatAdapter>(sargsin);
+    cv::RMat out_rmat = cv::util::get<cv::RMat>(out[0]);
+    auto adapter = out_rmat.get<MyRMatAdapter>();
+    EXPECT_EQ(42, adapter->getVal());
+    EXPECT_EQ("It actually works", adapter->getStr());
+
+    cv::Scalar out_sc = cv::util::get<cv::Scalar>(out[1]);
+    EXPECT_EQ(sc, out_sc);
+
+    cv::Mat out_mat = cv::util::get<cv::Mat>(out[2]);
+    EXPECT_EQ(0, cv::norm(mat, out_mat));
+}
+
 namespace {
     template <cv::detail::OpaqueKind K, typename T>
     bool verifyOpaqueKind(T&& in) {