mirror/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2025-07-30 17:37:05 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

G-API: Integrated cv::MediaFrame as I/O type + CPU backend

Browse Source
This commit is contained in:
Dmitry Matveev 2020-10-02 21:48:54 +03:00
parent aabeb8a18e
commit 050c960dfc
19 changed files with 132 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
modules/gapi/include/opencv2/gapi/cpu/gcpukernel.hpp
View File

@ -164,7 +164,7 @@ template<> struct get_in<cv::GMatP>
};
template<> struct get_in<cv::GFrame>
{
static cv::Mat get(GCPUContext &ctx, int idx) { return get_in<cv::GMat>::get(ctx, idx); }
static cv::MediaFrame get(GCPUContext &ctx, int idx) { return ctx.inArg<cv::MediaFrame>(idx); }
};
template<> struct get_in<cv::GScalar>
{

5
modules/gapi/include/opencv2/gapi/garg.hpp
View File

@ -13,6 +13,7 @@
#include <opencv2/gapi/opencv_includes.hpp>
#include <opencv2/gapi/own/mat.hpp>
#include <opencv2/gapi/media.hpp>
#include <opencv2/gapi/util/any.hpp>
#include <opencv2/gapi/util/variant.hpp>
@ -21,6 +22,7 @@
#include <opencv2/gapi/gscalar.hpp>
#include <opencv2/gapi/garray.hpp>
#include <opencv2/gapi/gopaque.hpp>
#include <opencv2/gapi/gframe.hpp>
#include <opencv2/gapi/gtype_traits.hpp>
#include <opencv2/gapi/gmetaarg.hpp>
#include <opencv2/gapi/streaming/source.hpp>
@ -100,7 +102,8 @@ using GRunArg = util::variant<
cv::Mat,
cv::Scalar,
cv::detail::VectorRef,
cv::detail::OpaqueRef
cv::detail::OpaqueRef,
cv::MediaFrame
>;
using GRunArgs = std::vector<GRunArg>;

4
modules/gapi/include/opencv2/gapi/gframe.hpp
View File

@ -42,7 +42,7 @@ private:
};
/** @} */
enum class MediaFormat
enum class MediaFormat: int
{
BGR = 0,
NV12,
@ -56,6 +56,8 @@ struct GAPI_EXPORTS GFrameDesc
{
MediaFormat fmt;
cv::Size size;
bool operator== (const GFrameDesc &) const;
};
static inline GFrameDesc empty_gframe_desc() { return GFrameDesc{}; }
/** @} */

3
modules/gapi/include/opencv2/gapi/gkernel.hpp
View File

@ -95,11 +95,12 @@ namespace detail
template<typename T> struct MetaType;
template<> struct MetaType<cv::GMat> { using type = GMatDesc; };
template<> struct MetaType<cv::GMatP> { using type = GMatDesc; };
template<> struct MetaType<cv::GFrame> { using type = GMatDesc; };
template<> struct MetaType<cv::GFrame> { using type = GFrameDesc; };
template<> struct MetaType<cv::GScalar> { using type = GScalarDesc; };
template<typename U> struct MetaType<cv::GArray<U> > { using type = GArrayDesc; };
template<typename U> struct MetaType<cv::GOpaque<U> > { using type = GOpaqueDesc; };
template<typename T> struct MetaType { using type = T; }; // opaque args passed as-is
// FIXME: Move it to type traits?
// 2. Hacky test based on MetaType to check if we operate on G-* type or not
template<typename T> using is_nongapi_type = std::is_same<T, typename MetaType<T>::type>;

2
modules/gapi/include/opencv2/gapi/gmetaarg.hpp
View File

@ -18,6 +18,7 @@
#include <opencv2/gapi/gscalar.hpp>
#include <opencv2/gapi/garray.hpp>
#include <opencv2/gapi/gopaque.hpp>
#include <opencv2/gapi/gframe.hpp>
namespace cv
{
@ -38,6 +39,7 @@ using GMetaArg = util::variant
, GScalarDesc
, GArrayDesc
, GOpaqueDesc
, GFrameDesc
>;
GAPI_EXPORTS std::ostream& operator<<(std::ostream& os, const GMetaArg &);

4
modules/gapi/include/opencv2/gapi/gtype_traits.hpp
View File

@ -17,6 +17,7 @@
#include <opencv2/gapi/gopaque.hpp>
#include <opencv2/gapi/gframe.hpp>
#include <opencv2/gapi/streaming/source.hpp>
#include <opencv2/gapi/media.hpp>
#include <opencv2/gapi/gcommon.hpp>
namespace cv
@ -67,7 +68,7 @@ namespace detail
template<> struct GTypeTraits<cv::GFrame>
{
static constexpr const ArgKind kind = ArgKind::GFRAME;
static constexpr const GShape shape = GShape::GMAT;
static constexpr const GShape shape = GShape::GFRAME;
static constexpr const OpaqueKind op_kind = OpaqueKind::CV_UNKNOWN;
};
template<> struct GTypeTraits<cv::GScalar>
@ -125,6 +126,7 @@ namespace detail
template<> struct GTypeOf<cv::Scalar> { using type = cv::GScalar; };
template<typename U> struct GTypeOf<std::vector<U> > { using type = cv::GArray<U>; };
template<typename U> struct GTypeOf { using type = cv::GOpaque<U>;};
template<> struct GTypeOf<cv::MediaFrame> { using type = cv::GFrame; };
// FIXME: This is not quite correct since IStreamSource may produce not only Mat but also Scalar
// and vector data. TODO: Extend the type dispatching on these types too.
template<> struct GTypeOf<cv::gapi::wip::IStreamSource::Ptr> { using type = cv::GMat;};

2
modules/gapi/include/opencv2/gapi/media.hpp
View File

@ -27,7 +27,7 @@ public:
explicit MediaFrame(AdapterPtr &&);
template<class T, class... Args> static cv::MediaFrame Create(Args&&...);
View access(Access);
View access(Access) const;
cv::GFrameDesc desc() const;
private:

12
modules/gapi/src/api/gbackend.cpp
View File

@ -150,6 +150,10 @@ void bindInArg(Mag& mag, const RcDesc &rc, const GRunArg &arg, HandleRMat handle
mag.template slot<cv::detail::OpaqueRef>()[rc.id] = util::get<cv::detail::OpaqueRef>(arg);
break;
case GShape::GFRAME:
mag.template slot<cv::MediaFrame>()[rc.id] = util::get<cv::MediaFrame>(arg);
break;
default:
util::throw_error(std::logic_error("Unsupported GShape type"));
}
@ -216,6 +220,7 @@ void resetInternalData(Mag& mag, const Data &d)
break;
case GShape::GMAT:
case GShape::GFRAME:
// Do nothing here - FIXME unify with initInternalData?
break;
@ -236,6 +241,7 @@ cv::GRunArg getArg(const Mag& mag, const RcDesc &ref)
// (and constructed by either bindIn/Out or resetInternal)
case GShape::GARRAY: return GRunArg(mag.template slot<cv::detail::VectorRef>().at(ref.id));
case GShape::GOPAQUE: return GRunArg(mag.template slot<cv::detail::OpaqueRef>().at(ref.id));
case GShape::GFRAME: return GRunArg(mag.template slot<cv::MediaFrame>().at(ref.id));
default:
util::throw_error(std::logic_error("Unsupported GShape type"));
break;
@ -329,6 +335,12 @@ void unbind(Mag& mag, const RcDesc &rc)
mag.slot<cv::RMat>().erase(rc.id);
break;
case GShape::GFRAME:
// MediaFrame can also be associated with external memory,
// so requires a special handling here.
mag.slot<cv::MediaFrame>().erase(rc.id);
break;
default:
GAPI_Assert(false);
}

23
modules/gapi/src/api/gframe.cpp
View File

@ -13,16 +13,11 @@
// cv::GFrame public implementation //////////////////////////////////////////////
cv::GFrame::GFrame()
: m_priv(new GOrigin(GShape::GMAT, GNode::Param())) {
// N.B.: The shape here is still GMAT as currently cv::Mat is used
// as an underlying host type. Will be changed to GFRAME once
// GExecutor & GStreamingExecutor & selected backends will be extended
// to support cv::MediaFrame.
: m_priv(new GOrigin(GShape::GFRAME, GNode::Param())) {
}
cv::GFrame::GFrame(const GNode &n, std::size_t out)
: m_priv(new GOrigin(GShape::GMAT, n, out)) {
// N.B.: GMAT is here for the same reason as above ^
: m_priv(new GOrigin(GShape::GFRAME, n, out)) {
}
cv::GOrigin& cv::GFrame::priv() {
@ -34,7 +29,19 @@ const cv::GOrigin& cv::GFrame::priv() const {
}
namespace cv {
std::ostream& operator<<(std::ostream& os, const cv::GFrameDesc &) {
bool GFrameDesc::operator== (const GFrameDesc &rhs) const {
return fmt == rhs.fmt && size == rhs.size;
}
std::ostream& operator<<(std::ostream& os, const cv::GFrameDesc &d) {
os << '[';
switch (d.fmt) {
case MediaFormat::BGR: os << "BGR"; break;
case MediaFormat::NV12: os << "NV12"; break;
default: GAPI_Assert(false && "Invalid media format");
}
os << ' ' << d.size << ']';
return os;
}

14
modules/gapi/src/api/gproto.cpp
View File

@ -122,6 +122,9 @@ cv::GMetaArg cv::descr_of(const cv::GRunArg &arg)
case GRunArg::index_of<cv::RMat>():
return cv::GMetaArg(cv::util::get<cv::RMat>(arg).desc());
case GRunArg::index_of<cv::MediaFrame>():
return cv::GMetaArg(cv::util::get<cv::MediaFrame>(arg).desc());
default: util::throw_error(std::logic_error("Unsupported GRunArg type"));
}
}
@ -133,6 +136,7 @@ cv::GMetaArgs cv::descr_of(const cv::GRunArgs &args)
return metas;
}
// FIXME: Is it tested for all types?
cv::GMetaArg cv::descr_of(const cv::GRunArgP &argp)
{
switch (argp.index())
@ -148,6 +152,7 @@ cv::GMetaArg cv::descr_of(const cv::GRunArgP &argp)
}
}
// FIXME: Is it tested for all types??
bool cv::can_describe(const GMetaArg& meta, const GRunArgP& argp)
{
switch (argp.index())
@ -164,6 +169,7 @@ bool cv::can_describe(const GMetaArg& meta, const GRunArgP& argp)
}
}
// FIXME: Is it tested for all types??
bool cv::can_describe(const GMetaArg& meta, const GRunArg& arg)
{
switch (arg.index())
@ -179,6 +185,7 @@ bool cv::can_describe(const GMetaArg& meta, const GRunArg& arg)
case GRunArg::index_of<cv::gapi::wip::IStreamSource::Ptr>(): return util::holds_alternative<GMatDesc>(meta); // FIXME(?) may be not the best option
case GRunArg::index_of<cv::RMat>(): return util::holds_alternative<GMatDesc>(meta) &&
util::get<GMatDesc>(meta).canDescribe(cv::util::get<cv::RMat>(arg));
case GRunArg::index_of<cv::MediaFrame>(): return meta == cv::GMetaArg(util::get<cv::MediaFrame>(arg).desc());
default: util::throw_error(std::logic_error("Unsupported GRunArg type"));
}
}
@ -192,6 +199,8 @@ bool cv::can_describe(const GMetaArgs &metas, const GRunArgs &args)
});
}
// FIXME: Is it tested for all types?
// FIXME: Where does this validation happen??
void cv::validate_input_arg(const GRunArg& arg)
{
// FIXME: It checks only Mat argument
@ -248,6 +257,11 @@ std::ostream& operator<<(std::ostream& os, const cv::GMetaArg &arg)
case cv::GMetaArg::index_of<cv::GOpaqueDesc>():
os << util::get<cv::GOpaqueDesc>(arg);
break;
case cv::GMetaArg::index_of<cv::GFrameDesc>():
os << util::get<cv::GFrameDesc>(arg);
break;
default:
GAPI_Assert(false);
}

2
modules/gapi/src/api/media.cpp
View File

@ -22,7 +22,7 @@ cv::GFrameDesc cv::MediaFrame::desc() const {
return m->adapter->meta();
}
cv::MediaFrame::View cv::MediaFrame::access(Access code) {
cv::MediaFrame::View cv::MediaFrame::access(Access code) const {
return m->adapter->access(code);
}

13
modules/gapi/src/backends/common/gbackend.hpp
View File

@ -60,11 +60,18 @@ namespace magazine {
};
} // namespace magazine
using Mag = magazine::Class< cv::Mat
, cv::Scalar
, cv::detail::VectorRef
, cv::detail::OpaqueRef
, cv::RMat
, cv::RMat::View
, cv::MediaFrame
#if !defined(GAPI_STANDALONE)
using Mag = magazine::Class<cv::Mat, cv::UMat, cv::Scalar, cv::detail::VectorRef, cv::detail::OpaqueRef, cv::RMat, cv::RMat::View>;
#else
using Mag = magazine::Class<cv::Mat, cv::Scalar, cv::detail::VectorRef, cv::detail::OpaqueRef, cv::RMat, cv::RMat::View>;
, cv::UMat
#endif
>;
namespace magazine
{

17
modules/gapi/src/backends/common/serialization.cpp
View File

@ -174,6 +174,17 @@ IIStream& operator>> (IIStream& is, cv::RMat&) {
return is;
}
IOStream& operator<< (IOStream& os, const cv::MediaFrame &) {
// Stub
GAPI_Assert(false && "cv::MediaFrame serialization is not supported!");
return os;
}
IIStream& operator>> (IIStream& is, cv::MediaFrame &) {
// Stub
GAPI_Assert(false && "cv::MediaFrame serialization is not supported!");
return is;
}
namespace
{
@ -555,6 +566,12 @@ IIStream& operator>> (IIStream& is, cv::GMatDesc &d) {
return is >> d.depth >> d.chan >> d.size >> d.planar >> d.dims;
}
IOStream& operator<< (IOStream& os, const cv::GFrameDesc &d) {
return put_enum(os, d.fmt) << d.size;
}
IIStream& operator>> (IIStream& is, cv::GFrameDesc &d) {
return get_enum(is, d.fmt) >> d.size;
}
IOStream& operator<< (IOStream& os, const cv::gimpl::RcDesc &rc) {
// FIXME: HostCtor is not serialized!

6
modules/gapi/src/backends/common/serialization.hpp
View File

@ -83,6 +83,9 @@ GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::GOpaqueDesc &);
GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::GArrayDesc &);
GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::GArrayDesc &);
GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::GFrameDesc &);
GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::GFrameDesc &);
#if !defined(GAPI_STANDALONE)
GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::UMat &);
GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::UMat &);
@ -100,6 +103,9 @@ GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::detail::VectorRef &);
GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::detail::OpaqueRef &);
GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::detail::OpaqueRef &);
GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::MediaFrame &);
GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::MediaFrame &);
GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::gimpl::RcDesc &rc);
GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::gimpl::RcDesc &rc);

8
modules/gapi/src/backends/cpu/gcpubackend.cpp
View File

@ -128,9 +128,10 @@ cv::GArg cv::gimpl::GCPUExecutable::packArg(const GArg &arg)
// No API placeholders allowed at this point
// FIXME: this check has to be done somewhere in compilation stage.
GAPI_Assert( arg.kind != cv::detail::ArgKind::GMAT
&& arg.kind != cv::detail::ArgKind::GSCALAR
&& arg.kind != cv::detail::ArgKind::GARRAY
&& arg.kind != cv::detail::ArgKind::GOPAQUE);
&& arg.kind != cv::detail::ArgKind::GSCALAR
&& arg.kind != cv::detail::ArgKind::GARRAY
&& arg.kind != cv::detail::ArgKind::GOPAQUE
&& arg.kind != cv::detail::ArgKind::GFRAME);
if (arg.kind != cv::detail::ArgKind::GOBJREF)
{
@ -150,6 +151,7 @@ cv::GArg cv::gimpl::GCPUExecutable::packArg(const GArg &arg)
// (and constructed by either bindIn/Out or resetInternal)
case GShape::GARRAY: return GArg(m_res.slot<cv::detail::VectorRef>().at(ref.id));
case GShape::GOPAQUE: return GArg(m_res.slot<cv::detail::OpaqueRef>().at(ref.id));
case GShape::GFRAME: return GArg(m_res.slot<cv::MediaFrame>().at(ref.id));
default:
util::throw_error(std::logic_error("Unsupported GShape type"));
break;

6
modules/gapi/src/compiler/gcompiled.cpp
View File

@ -57,6 +57,11 @@ void cv::GCompiled::Priv::checkArgs(const cv::gimpl::GRuntimeArgs &args) const
// FIXME: Add details on what is actually wrong
}
validate_input_args(args.inObjs);
// FIXME: Actually, the passed parameter vector is never checked
// against its shapes - so if you compile with GScalarDesc passed
// for GMat argument, you will get your compilation right (!!)
// Probably it was there but somehow that olds checks (if they
// exist) are bypassed now.
}
bool cv::GCompiled::Priv::canReshape() const
@ -97,6 +102,7 @@ cv::GCompiled::operator bool() const
void cv::GCompiled::operator() (GRunArgs &&ins, GRunArgsP &&outs)
{
// FIXME: Check that <ins> matches the protocol!!!
// FIXME: Check that <outs> matches the protocol
m_priv->run(cv::gimpl::GRuntimeArgs{std::move(ins),std::move(outs)});
}

4
modules/gapi/src/compiler/gcompiler.cpp
View File

@ -311,9 +311,11 @@ void cv::gimpl::GCompiler::validateInputMeta()
// FIXME: Auto-generate methods like this from traits:
case GProtoArg::index_of<cv::GMat>():
case GProtoArg::index_of<cv::GMatP>():
case GProtoArg::index_of<cv::GFrame>():
return util::holds_alternative<cv::GMatDesc>(meta);
case GProtoArg::index_of<cv::GFrame>():
return util::holds_alternative<cv::GFrameDesc>(meta);
case GProtoArg::index_of<cv::GScalar>():
return util::holds_alternative<cv::GScalarDesc>(meta);

1
modules/gapi/src/compiler/passes/dump_dot.cpp
View File

@ -33,6 +33,7 @@ void dumpDot(const ade::Graph &g, std::ostream& os)
{cv::GShape::GSCALAR, "GScalar"},
{cv::GShape::GARRAY, "GArray"},
{cv::GShape::GOPAQUE, "GOpaque"},
{cv::GShape::GFRAME, "GFrame"},
};
auto format_op_label = [&gr](ade::NodeHandle nh) -> std::string {

60
modules/gapi/test/gapi_frame_tests.cpp
View File

@ -14,47 +14,21 @@
namespace opencv_test {
G_API_OP(GBlurFrame, <GMat(GFrame)>, "test.blur_frame") {
static GMatDesc outMeta(GMatDesc in) {
return in;
static GMatDesc outMeta(GFrameDesc in) {
return cv::GMatDesc(CV_8U,3,in.size);
}
};
GAPI_OCV_KERNEL(OCVBlurFrame, GBlurFrame) {
static void run(const cv::Mat& in, cv::Mat& out) {
cv::blur(in, out, cv::Size{3,3});
static void run(const cv::MediaFrame &in, cv::Mat& out) {
GAPI_Assert(in.desc().fmt == cv::MediaFormat::BGR);
cv::MediaFrame::View view = in.access(cv::MediaFrame::Access::R);
cv::blur(cv::Mat(in.desc().size, CV_8UC3, view.ptr[0], view.stride[0]),
out,
cv::Size{3,3});
}
};
struct GFrameTest : public ::testing::Test {
cv::Size sz{32,32};
cv::Mat in_mat;
cv::Mat out_mat;
cv::Mat out_mat_ocv;
GFrameTest()
: in_mat(cv::Mat(sz, CV_8UC1))
, out_mat(cv::Mat::zeros(sz, CV_8UC1))
, out_mat_ocv(cv::Mat::zeros(sz, CV_8UC1)) {
cv::randn(in_mat, cv::Scalar::all(127.0f), cv::Scalar::all(40.f));
cv::blur(in_mat, out_mat_ocv, cv::Size{3,3});
}
void check() {
EXPECT_EQ(0, cvtest::norm(out_mat, out_mat_ocv, NORM_INF));
}
};
TEST_F(GFrameTest, Input) {
cv::GFrame in;
auto out = GBlurFrame::on(in);
cv::GComputation c(cv::GIn(in), cv::GOut(out));
auto pkg = cv::gapi::kernels<OCVBlurFrame>();
c.apply(cv::gin(in_mat), cv::gout(out_mat), cv::compile_args(pkg));
check();
}
////////////////////////////////////////////////////////////////////////////////
// cv::MediaFrame tests
namespace {
@ -108,6 +82,7 @@ struct MediaFrame_BGR: public MediaFrame_Test {
M bgr;
MediaFrame_BGR()
: bgr(M::eye(240, 320, CV_8UC3)) {
cv::randn(bgr, cv::Scalar::all(127.0f), cv::Scalar::all(40.f));
frame = MF::Create<TestMediaBGR>(bgr);
}
};
@ -128,6 +103,23 @@ TEST_F(MediaFrame_BGR, Access) {
EXPECT_EQ(bgr.step, view2.stride[0]);
}
TEST_F(MediaFrame_BGR, Input) {
// Run the OpenCV code
cv::Mat out_mat_ocv, out_mat_gapi;
cv::blur(bgr, out_mat_ocv, cv::Size{3,3});
// Run the G-API code
cv::GFrame in;
cv::GMat out = GBlurFrame::on(in);
cv::GComputation(cv::GIn(in), cv::GOut(out))
.apply(cv::gin(frame),
cv::gout(out_mat_gapi),
cv::compile_args(cv::gapi::kernels<OCVBlurFrame>()));
// Compare
EXPECT_EQ(0, cvtest::norm(out_mat_ocv, out_mat_gapi, NORM_INF));
}
struct MediaFrame_NV12: public MediaFrame_Test {
cv::Size sz;
cv::Mat buf, y, uv;