G-API: Integrated cv::MediaFrame as I/O type + CPU backend

modules/gapi/include/opencv2/gapi/cpu/gcpukernel.hpp

@@ -164,7 +164,7 @@ template<> struct get_in<cv::GMatP>
 };
 template<> struct get_in<cv::GFrame>
 {
-    static cv::Mat    get(GCPUContext &ctx, int idx) { return get_in<cv::GMat>::get(ctx, idx); }
+    static cv::MediaFrame get(GCPUContext &ctx, int idx) { return ctx.inArg<cv::MediaFrame>(idx); }
 };
 template<> struct get_in<cv::GScalar>
 {

modules/gapi/include/opencv2/gapi/garg.hpp

@@ -13,6 +13,7 @@
 
 #include <opencv2/gapi/opencv_includes.hpp>
 #include <opencv2/gapi/own/mat.hpp>
+#include <opencv2/gapi/media.hpp>
 
 #include <opencv2/gapi/util/any.hpp>
 #include <opencv2/gapi/util/variant.hpp>
@@ -21,6 +22,7 @@
 #include <opencv2/gapi/gscalar.hpp>
 #include <opencv2/gapi/garray.hpp>
 #include <opencv2/gapi/gopaque.hpp>
+#include <opencv2/gapi/gframe.hpp>
 #include <opencv2/gapi/gtype_traits.hpp>
 #include <opencv2/gapi/gmetaarg.hpp>
 #include <opencv2/gapi/streaming/source.hpp>
@@ -100,7 +102,8 @@ using GRunArg  = util::variant<
     cv::Mat,
     cv::Scalar,
     cv::detail::VectorRef,
-    cv::detail::OpaqueRef
+    cv::detail::OpaqueRef,
+    cv::MediaFrame
     >;
 using GRunArgs = std::vector<GRunArg>;
 

modules/gapi/include/opencv2/gapi/gframe.hpp

@@ -42,7 +42,7 @@ private:
 };
 /** @} */
 
-enum class MediaFormat
+enum class MediaFormat: int
 {
     BGR = 0,
     NV12,
@@ -56,6 +56,8 @@ struct GAPI_EXPORTS GFrameDesc
 {
     MediaFormat fmt;
     cv::Size size;
+
+    bool operator== (const GFrameDesc &) const;
 };
 static inline GFrameDesc empty_gframe_desc() { return GFrameDesc{}; }
 /** @} */

modules/gapi/include/opencv2/gapi/gkernel.hpp

@@ -95,11 +95,12 @@ namespace detail
     template<typename T> struct MetaType;
     template<> struct MetaType<cv::GMat>    { using type = GMatDesc; };
     template<> struct MetaType<cv::GMatP>   { using type = GMatDesc; };
-    template<> struct MetaType<cv::GFrame>  { using type = GMatDesc; };
+    template<> struct MetaType<cv::GFrame>  { using type = GFrameDesc; };
     template<> struct MetaType<cv::GScalar> { using type = GScalarDesc; };
     template<typename U> struct MetaType<cv::GArray<U> >  { using type = GArrayDesc; };
     template<typename U> struct MetaType<cv::GOpaque<U> > { using type = GOpaqueDesc; };
     template<typename T> struct MetaType    { using type = T; }; // opaque args passed as-is
+    // FIXME: Move it to type traits?
 
     // 2. Hacky test based on MetaType to check if we operate on G-* type or not
     template<typename T> using is_nongapi_type = std::is_same<T, typename MetaType<T>::type>;

modules/gapi/include/opencv2/gapi/gmetaarg.hpp

@@ -18,6 +18,7 @@
 #include <opencv2/gapi/gscalar.hpp>
 #include <opencv2/gapi/garray.hpp>
 #include <opencv2/gapi/gopaque.hpp>
+#include <opencv2/gapi/gframe.hpp>
 
 namespace cv
 {
@@ -38,6 +39,7 @@ using GMetaArg = util::variant
     , GScalarDesc
     , GArrayDesc
     , GOpaqueDesc
+    , GFrameDesc
     >;
 GAPI_EXPORTS std::ostream& operator<<(std::ostream& os, const GMetaArg &);
 

modules/gapi/include/opencv2/gapi/gtype_traits.hpp

@@ -17,6 +17,7 @@
 #include <opencv2/gapi/gopaque.hpp>
 #include <opencv2/gapi/gframe.hpp>
 #include <opencv2/gapi/streaming/source.hpp>
+#include <opencv2/gapi/media.hpp>
 #include <opencv2/gapi/gcommon.hpp>
 
 namespace cv
@@ -67,7 +68,7 @@ namespace detail
     template<>           struct GTypeTraits<cv::GFrame>
     {
         static constexpr const ArgKind kind = ArgKind::GFRAME;
-        static constexpr const GShape shape = GShape::GMAT;
+        static constexpr const GShape shape = GShape::GFRAME;
         static constexpr const OpaqueKind op_kind = OpaqueKind::CV_UNKNOWN;
     };
     template<>           struct GTypeTraits<cv::GScalar>
@@ -125,6 +126,7 @@ namespace detail
     template<>           struct GTypeOf<cv::Scalar>            { using type = cv::GScalar;   };
     template<typename U> struct GTypeOf<std::vector<U> >       { using type = cv::GArray<U>; };
     template<typename U> struct GTypeOf                        { using type = cv::GOpaque<U>;};
+    template<>           struct GTypeOf<cv::MediaFrame>        { using type = cv::GFrame;    };
     // FIXME: This is not quite correct since IStreamSource may produce not only Mat but also Scalar
     // and vector data. TODO: Extend the type dispatching on these types too.
     template<>           struct GTypeOf<cv::gapi::wip::IStreamSource::Ptr> { using type = cv::GMat;};

modules/gapi/include/opencv2/gapi/media.hpp

@@ -27,7 +27,7 @@ public:
     explicit MediaFrame(AdapterPtr &&);
     template<class T, class... Args> static cv::MediaFrame Create(Args&&...);
 
-    View access(Access);
+    View access(Access) const;
     cv::GFrameDesc desc() const;
 
 private:

modules/gapi/src/api/gbackend.cpp

@@ -150,6 +150,10 @@ void bindInArg(Mag& mag, const RcDesc &rc, const GRunArg &arg, HandleRMat handle
         mag.template slot<cv::detail::OpaqueRef>()[rc.id] = util::get<cv::detail::OpaqueRef>(arg);
         break;
 
+    case GShape::GFRAME:
+        mag.template slot<cv::MediaFrame>()[rc.id] = util::get<cv::MediaFrame>(arg);
+        break;
+
     default:
         util::throw_error(std::logic_error("Unsupported GShape type"));
     }
@@ -216,6 +220,7 @@ void resetInternalData(Mag& mag, const Data &d)
         break;
 
     case GShape::GMAT:
+    case GShape::GFRAME:
         // Do nothing here - FIXME unify with initInternalData?
         break;
 
@@ -236,6 +241,7 @@ cv::GRunArg getArg(const Mag& mag, const RcDesc &ref)
     //   (and constructed by either bindIn/Out or resetInternal)
     case GShape::GARRAY:  return GRunArg(mag.template slot<cv::detail::VectorRef>().at(ref.id));
     case GShape::GOPAQUE: return GRunArg(mag.template slot<cv::detail::OpaqueRef>().at(ref.id));
+    case GShape::GFRAME:  return GRunArg(mag.template slot<cv::MediaFrame>().at(ref.id));
     default:
         util::throw_error(std::logic_error("Unsupported GShape type"));
         break;
@@ -329,6 +335,12 @@ void unbind(Mag& mag, const RcDesc &rc)
         mag.slot<cv::RMat>().erase(rc.id);
         break;
 
+    case GShape::GFRAME:
+        // MediaFrame can also be associated with external memory,
+        // so requires a special handling here.
+        mag.slot<cv::MediaFrame>().erase(rc.id);
+        break;
+
     default:
         GAPI_Assert(false);
     }

modules/gapi/src/api/gframe.cpp

@@ -13,16 +13,11 @@
 
 // cv::GFrame public implementation //////////////////////////////////////////////
 cv::GFrame::GFrame()
-    : m_priv(new GOrigin(GShape::GMAT, GNode::Param())) {
+    : m_priv(new GOrigin(GShape::GFRAME, GNode::Param())) {
-    // N.B.: The shape here is still GMAT as currently cv::Mat is used
-    // as an underlying host type. Will be changed to GFRAME once
-    // GExecutor & GStreamingExecutor & selected backends will be extended
-    // to support cv::MediaFrame.
 }
 
 cv::GFrame::GFrame(const GNode &n, std::size_t out)
-    : m_priv(new GOrigin(GShape::GMAT, n, out)) {
+    : m_priv(new GOrigin(GShape::GFRAME, n, out)) {
-    // N.B.: GMAT is here for the same reason as above ^
 }
 
 cv::GOrigin& cv::GFrame::priv() {
@@ -34,7 +29,19 @@ const cv::GOrigin& cv::GFrame::priv() const {
 }
 
 namespace cv {
-std::ostream& operator<<(std::ostream& os, const cv::GFrameDesc &) {
+
+bool GFrameDesc::operator== (const GFrameDesc &rhs) const {
+    return fmt == rhs.fmt && size == rhs.size;
+}
+
+std::ostream& operator<<(std::ostream& os, const cv::GFrameDesc &d) {
+    os << '[';
+    switch (d.fmt) {
+    case MediaFormat::BGR:  os << "BGR"; break;
+    case MediaFormat::NV12: os << "NV12"; break;
+    default: GAPI_Assert(false && "Invalid media format");
+    }
+    os << ' ' << d.size << ']';
     return os;
 }
 

modules/gapi/src/api/gproto.cpp

@@ -122,6 +122,9 @@ cv::GMetaArg cv::descr_of(const cv::GRunArg &arg)
         case GRunArg::index_of<cv::RMat>():
             return cv::GMetaArg(cv::util::get<cv::RMat>(arg).desc());
 
+        case GRunArg::index_of<cv::MediaFrame>():
+            return cv::GMetaArg(cv::util::get<cv::MediaFrame>(arg).desc());
+
         default: util::throw_error(std::logic_error("Unsupported GRunArg type"));
     }
 }
@@ -133,6 +136,7 @@ cv::GMetaArgs cv::descr_of(const cv::GRunArgs &args)
     return metas;
 }
 
+// FIXME: Is it tested for all types?
 cv::GMetaArg cv::descr_of(const cv::GRunArgP &argp)
 {
     switch (argp.index())
@@ -148,6 +152,7 @@ cv::GMetaArg cv::descr_of(const cv::GRunArgP &argp)
     }
 }
 
+// FIXME: Is it tested for all types??
 bool cv::can_describe(const GMetaArg& meta, const GRunArgP& argp)
 {
     switch (argp.index())
@@ -164,6 +169,7 @@ bool cv::can_describe(const GMetaArg& meta, const GRunArgP& argp)
     }
 }
 
+// FIXME: Is it tested for all types??
 bool cv::can_describe(const GMetaArg& meta, const GRunArg& arg)
 {
     switch (arg.index())
@@ -179,6 +185,7 @@ bool cv::can_describe(const GMetaArg& meta, const GRunArg& arg)
     case GRunArg::index_of<cv::gapi::wip::IStreamSource::Ptr>(): return util::holds_alternative<GMatDesc>(meta); // FIXME(?) may be not the best option
     case GRunArg::index_of<cv::RMat>():              return util::holds_alternative<GMatDesc>(meta) &&
                                                             util::get<GMatDesc>(meta).canDescribe(cv::util::get<cv::RMat>(arg));
+    case GRunArg::index_of<cv::MediaFrame>():        return meta == cv::GMetaArg(util::get<cv::MediaFrame>(arg).desc());
     default: util::throw_error(std::logic_error("Unsupported GRunArg type"));
     }
 }
@@ -192,6 +199,8 @@ bool cv::can_describe(const GMetaArgs &metas, const GRunArgs &args)
                      });
 }
 
+// FIXME: Is it tested for all types?
+// FIXME: Where does this validation happen??
 void cv::validate_input_arg(const GRunArg& arg)
 {
     // FIXME: It checks only Mat argument
@@ -248,6 +257,11 @@ std::ostream& operator<<(std::ostream& os, const cv::GMetaArg &arg)
     case cv::GMetaArg::index_of<cv::GOpaqueDesc>():
         os << util::get<cv::GOpaqueDesc>(arg);
         break;
+
+    case cv::GMetaArg::index_of<cv::GFrameDesc>():
+        os << util::get<cv::GFrameDesc>(arg);
+        break;
+
     default:
         GAPI_Assert(false);
     }

modules/gapi/src/api/media.cpp

@@ -22,7 +22,7 @@ cv::GFrameDesc cv::MediaFrame::desc() const {
     return m->adapter->meta();
 }
 
-cv::MediaFrame::View cv::MediaFrame::access(Access code) {
+cv::MediaFrame::View cv::MediaFrame::access(Access code) const {
     return m->adapter->access(code);
 }
 

modules/gapi/src/backends/common/gbackend.hpp

@@ -60,11 +60,18 @@ namespace magazine {
     };
 
 } // namespace magazine
+
+using Mag = magazine::Class< cv::Mat
+                           , cv::Scalar
+                           , cv::detail::VectorRef
+                           , cv::detail::OpaqueRef
+                           , cv::RMat
+                           , cv::RMat::View
+                           , cv::MediaFrame
 #if !defined(GAPI_STANDALONE)
-using Mag = magazine::Class<cv::Mat, cv::UMat, cv::Scalar, cv::detail::VectorRef, cv::detail::OpaqueRef, cv::RMat, cv::RMat::View>;
+                           , cv::UMat
-#else
-using Mag = magazine::Class<cv::Mat, cv::Scalar, cv::detail::VectorRef, cv::detail::OpaqueRef, cv::RMat, cv::RMat::View>;
 #endif
+                           >;
 
 namespace magazine
 {

modules/gapi/src/backends/common/serialization.cpp

@@ -174,6 +174,17 @@ IIStream& operator>> (IIStream& is, cv::RMat&) {
     return is;
 }
 
+IOStream& operator<< (IOStream& os, const cv::MediaFrame &) {
+    // Stub
+    GAPI_Assert(false && "cv::MediaFrame serialization is not supported!");
+    return os;
+}
+IIStream& operator>> (IIStream& is, cv::MediaFrame &) {
+    // Stub
+    GAPI_Assert(false && "cv::MediaFrame serialization is not supported!");
+    return is;
+}
+
 namespace
 {
 
@@ -555,6 +566,12 @@ IIStream& operator>> (IIStream& is, cv::GMatDesc &d) {
     return is >> d.depth >> d.chan >> d.size >> d.planar >> d.dims;
 }
 
+IOStream& operator<< (IOStream& os, const cv::GFrameDesc &d) {
+    return put_enum(os, d.fmt) << d.size;
+}
+IIStream& operator>> (IIStream& is,       cv::GFrameDesc &d) {
+    return get_enum(is, d.fmt) >> d.size;
+}
 
 IOStream& operator<< (IOStream& os, const cv::gimpl::RcDesc &rc) {
     // FIXME: HostCtor is not serialized!

modules/gapi/src/backends/common/serialization.hpp

@@ -83,6 +83,9 @@ GAPI_EXPORTS IIStream& operator>> (IIStream& is,       cv::GOpaqueDesc &);
 GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::GArrayDesc &);
 GAPI_EXPORTS IIStream& operator>> (IIStream& is,       cv::GArrayDesc &);
 
+GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::GFrameDesc &);
+GAPI_EXPORTS IIStream& operator>> (IIStream& is,       cv::GFrameDesc &);
+
 #if !defined(GAPI_STANDALONE)
 GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::UMat &);
 GAPI_EXPORTS IIStream& operator>> (IIStream& is,       cv::UMat &);
@@ -100,6 +103,9 @@ GAPI_EXPORTS IIStream& operator>> (IIStream& is,       cv::detail::VectorRef &);
 GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::detail::OpaqueRef &);
 GAPI_EXPORTS IIStream& operator>> (IIStream& is,       cv::detail::OpaqueRef &);
 
+GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::MediaFrame &);
+GAPI_EXPORTS IIStream& operator>> (IIStream& is,       cv::MediaFrame &);
+
 GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::gimpl::RcDesc &rc);
 GAPI_EXPORTS IIStream& operator>> (IIStream& is,       cv::gimpl::RcDesc &rc);
 

modules/gapi/src/backends/cpu/gcpubackend.cpp

@@ -128,9 +128,10 @@ cv::GArg cv::gimpl::GCPUExecutable::packArg(const GArg &arg)
     // No API placeholders allowed at this point
     // FIXME: this check has to be done somewhere in compilation stage.
     GAPI_Assert(   arg.kind != cv::detail::ArgKind::GMAT
-              && arg.kind != cv::detail::ArgKind::GSCALAR
+                && arg.kind != cv::detail::ArgKind::GSCALAR
-              && arg.kind != cv::detail::ArgKind::GARRAY
+                && arg.kind != cv::detail::ArgKind::GARRAY
-              && arg.kind != cv::detail::ArgKind::GOPAQUE);
+                && arg.kind != cv::detail::ArgKind::GOPAQUE
+                && arg.kind != cv::detail::ArgKind::GFRAME);
 
     if (arg.kind != cv::detail::ArgKind::GOBJREF)
     {
@@ -150,6 +151,7 @@ cv::GArg cv::gimpl::GCPUExecutable::packArg(const GArg &arg)
     //   (and constructed by either bindIn/Out or resetInternal)
     case GShape::GARRAY:  return GArg(m_res.slot<cv::detail::VectorRef>().at(ref.id));
     case GShape::GOPAQUE: return GArg(m_res.slot<cv::detail::OpaqueRef>().at(ref.id));
+    case GShape::GFRAME:  return GArg(m_res.slot<cv::MediaFrame>().at(ref.id));
     default:
         util::throw_error(std::logic_error("Unsupported GShape type"));
         break;

modules/gapi/src/compiler/gcompiled.cpp

@@ -57,6 +57,11 @@ void cv::GCompiled::Priv::checkArgs(const cv::gimpl::GRuntimeArgs &args) const
         // FIXME: Add details on what is actually wrong
     }
     validate_input_args(args.inObjs);
+    // FIXME: Actually, the passed parameter vector is never checked
+    // against its shapes - so if you compile with GScalarDesc passed
+    // for GMat argument, you will get your compilation right (!!)
+    // Probably it was there but somehow that olds checks (if they
+    // exist) are bypassed now.
 }
 
 bool cv::GCompiled::Priv::canReshape() const
@@ -97,6 +102,7 @@ cv::GCompiled::operator bool() const
 
 void cv::GCompiled::operator() (GRunArgs &&ins, GRunArgsP &&outs)
 {
+    // FIXME: Check that <ins> matches the protocol!!!
     // FIXME: Check that <outs> matches the protocol
     m_priv->run(cv::gimpl::GRuntimeArgs{std::move(ins),std::move(outs)});
 }

modules/gapi/src/compiler/gcompiler.cpp

@@ -311,9 +311,11 @@ void cv::gimpl::GCompiler::validateInputMeta()
         // FIXME: Auto-generate methods like this from traits:
         case GProtoArg::index_of<cv::GMat>():
         case GProtoArg::index_of<cv::GMatP>():
-        case GProtoArg::index_of<cv::GFrame>():
             return util::holds_alternative<cv::GMatDesc>(meta);
 
+        case GProtoArg::index_of<cv::GFrame>():
+            return util::holds_alternative<cv::GFrameDesc>(meta);
+
         case GProtoArg::index_of<cv::GScalar>():
             return util::holds_alternative<cv::GScalarDesc>(meta);
 

modules/gapi/src/compiler/passes/dump_dot.cpp

@@ -33,6 +33,7 @@ void dumpDot(const ade::Graph &g, std::ostream& os)
         {cv::GShape::GSCALAR, "GScalar"},
         {cv::GShape::GARRAY,  "GArray"},
         {cv::GShape::GOPAQUE, "GOpaque"},
+        {cv::GShape::GFRAME,  "GFrame"},
     };
 
     auto format_op_label  = [&gr](ade::NodeHandle nh) -> std::string {

modules/gapi/test/gapi_frame_tests.cpp

@@ -14,47 +14,21 @@
 namespace opencv_test {
 
 G_API_OP(GBlurFrame, <GMat(GFrame)>, "test.blur_frame") {
-    static GMatDesc outMeta(GMatDesc in) {
+    static GMatDesc outMeta(GFrameDesc in) {
-        return in;
+        return cv::GMatDesc(CV_8U,3,in.size);
     }
 };
 
 GAPI_OCV_KERNEL(OCVBlurFrame, GBlurFrame) {
-    static void run(const cv::Mat& in, cv::Mat& out) {
+    static void run(const cv::MediaFrame &in, cv::Mat& out) {
-        cv::blur(in, out, cv::Size{3,3});
+        GAPI_Assert(in.desc().fmt == cv::MediaFormat::BGR);
+        cv::MediaFrame::View view = in.access(cv::MediaFrame::Access::R);
+        cv::blur(cv::Mat(in.desc().size, CV_8UC3, view.ptr[0], view.stride[0]),
+                 out,
+                 cv::Size{3,3});
     }
 };
 
-struct GFrameTest : public ::testing::Test {
-    cv::Size sz{32,32};
-    cv::Mat in_mat;
-    cv::Mat out_mat;
-    cv::Mat out_mat_ocv;
-
-    GFrameTest()
-        : in_mat(cv::Mat(sz, CV_8UC1))
-        , out_mat(cv::Mat::zeros(sz, CV_8UC1))
-        , out_mat_ocv(cv::Mat::zeros(sz, CV_8UC1)) {
-        cv::randn(in_mat, cv::Scalar::all(127.0f), cv::Scalar::all(40.f));
-        cv::blur(in_mat, out_mat_ocv, cv::Size{3,3});
-    }
-
-    void check() {
-        EXPECT_EQ(0, cvtest::norm(out_mat, out_mat_ocv, NORM_INF));
-    }
-};
-
-TEST_F(GFrameTest, Input) {
-    cv::GFrame in;
-    auto out = GBlurFrame::on(in);
-    cv::GComputation c(cv::GIn(in), cv::GOut(out));
-
-    auto pkg = cv::gapi::kernels<OCVBlurFrame>();
-    c.apply(cv::gin(in_mat), cv::gout(out_mat), cv::compile_args(pkg));
-
-    check();
-}
-
 ////////////////////////////////////////////////////////////////////////////////
 // cv::MediaFrame tests
 namespace {
@@ -108,6 +82,7 @@ struct MediaFrame_BGR: public MediaFrame_Test {
     M bgr;
     MediaFrame_BGR()
         : bgr(M::eye(240, 320, CV_8UC3)) {
+        cv::randn(bgr, cv::Scalar::all(127.0f), cv::Scalar::all(40.f));
         frame = MF::Create<TestMediaBGR>(bgr);
     }
 };
@@ -128,6 +103,23 @@ TEST_F(MediaFrame_BGR, Access) {
     EXPECT_EQ(bgr.step,  view2.stride[0]);
 }
 
+TEST_F(MediaFrame_BGR, Input) {
+    // Run the OpenCV code
+    cv::Mat out_mat_ocv, out_mat_gapi;
+    cv::blur(bgr, out_mat_ocv, cv::Size{3,3});
+
+    // Run the G-API code
+    cv::GFrame in;
+    cv::GMat out = GBlurFrame::on(in);
+    cv::GComputation(cv::GIn(in), cv::GOut(out))
+        .apply(cv::gin(frame),
+               cv::gout(out_mat_gapi),
+               cv::compile_args(cv::gapi::kernels<OCVBlurFrame>()));
+
+    // Compare
+    EXPECT_EQ(0, cvtest::norm(out_mat_ocv, out_mat_gapi, NORM_INF));
+}
+
 struct MediaFrame_NV12: public MediaFrame_Test {
     cv::Size sz;
     cv::Mat buf, y, uv;