mirror/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2024-11-28 05:06:29 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #18793 from dmatveev:dm/in_graph_metadata

Browse Source 
G-API: Introduce runtime in-graph metadata

* G-API: In-graph metadata -- initial implementation

* G-API: Finish the in-graph metadata implementation for Streaming

* G-API: Fix standalone build & warnings for in-graph metadata

* G-API: In-graph meta -- fixed review comments

* G-API: Fix issues with desync causing failing tests
This commit is contained in:
Dmitry Matveev 2020-11-17 17:04:19 +03:00 committed by GitHub
parent b5c162175b
commit b866d0dc38
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 681 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
modules/gapi/CMakeLists.txt
View File

@ -57,6 +57,7 @@ file(GLOB gapi_ext_hdrs
set(gapi_srcs
# Front-end part
src/api/grunarg.cpp
src/api/gorigin.cpp
src/api/gmat.cpp
src/api/garray.cpp
@ -131,18 +132,19 @@ set(gapi_srcs
src/backends/ie/giebackend.cpp
src/backends/ie/giebackend/giewrapper.cpp
# ONNX Backend.
# ONNX backend
src/backends/onnx/gonnxbackend.cpp
# Render Backend.
# Render backend
src/backends/render/grenderocv.cpp
src/backends/render/ft_render.cpp
#PlaidML Backend
# PlaidML Backend
src/backends/plaidml/gplaidmlcore.cpp
src/backends/plaidml/gplaidmlbackend.cpp
# Compound
# Common backend code
src/backends/common/gmetabackend.cpp
src/backends/common/gcompoundbackend.cpp
src/backends/common/gcompoundkernel.cpp

67
modules/gapi/include/opencv2/gapi/garg.hpp
View File

@ -9,12 +9,14 @@
#define OPENCV_GAPI_GARG_HPP
#include <vector>
#include <unordered_map>
#include <type_traits>
#include <opencv2/gapi/opencv_includes.hpp>
#include <opencv2/gapi/own/mat.hpp>
#include <opencv2/gapi/media.hpp>
#include <opencv2/gapi/util/util.hpp>
#include <opencv2/gapi/util/any.hpp>
#include <opencv2/gapi/util/variant.hpp>
@ -93,7 +95,7 @@ using GArgs = std::vector<GArg>;
// FIXME: Express as M<GProtoArg...>::type
// FIXME: Move to a separate file!
using GRunArg = util::variant<
using GRunArgBase = util::variant<
#if !defined(GAPI_STANDALONE)
cv::UMat,
#endif // !defined(GAPI_STANDALONE)
@ -105,6 +107,61 @@ using GRunArg = util::variant<
cv::detail::OpaqueRef,
cv::MediaFrame
>;
namespace detail {
template<typename,typename>
struct in_variant;
template<typename T, typename... Types>
struct in_variant<T, util::variant<Types...> >
: std::integral_constant<bool, cv::detail::contains<T, Types...>::value > {
};
} // namespace detail
struct GAPI_EXPORTS GRunArg: public GRunArgBase
{
// Metadata information here
using Meta = std::unordered_map<std::string, util::any>;
Meta meta;
// Mimic the old GRunArg semantics here, old of the times when
// GRunArg was an alias to variant<>
GRunArg();
GRunArg(const cv::GRunArg &arg);
GRunArg(cv::GRunArg &&arg);
GRunArg& operator= (const GRunArg &arg);
GRunArg& operator= (GRunArg &&arg);
template <typename T>
GRunArg(const T &t,
const Meta &m = Meta{},
typename std::enable_if< detail::in_variant<T, GRunArgBase>::value, int>::type = 0)
: GRunArgBase(t)
, meta(m)
{
}
template <typename T>
GRunArg(T &&t,
const Meta &m = Meta{},
typename std::enable_if< detail::in_variant<T, GRunArgBase>::value, int>::type = 0)
: GRunArgBase(std::move(t))
, meta(m)
{
}
template <typename T> auto operator= (const T &t)
-> typename std::enable_if< detail::in_variant<T, GRunArgBase>::value, cv::GRunArg>::type&
{
GRunArgBase::operator=(t);
return *this;
}
template <typename T> auto operator= (T&& t)
-> typename std::enable_if< detail::in_variant<T, GRunArgBase>::value, cv::GRunArg>::type&
{
GRunArgBase::operator=(std::move(t));
return *this;
}
};
using GRunArgs = std::vector<GRunArg>;
// TODO: Think about the addition operator
@ -129,11 +186,13 @@ namespace gapi
namespace wip
{
/**
* @brief This aggregate type represents all types which G-API can handle (via variant).
* @brief This aggregate type represents all types which G-API can
* handle (via variant).
*
* It only exists to overcome C++ language limitations (where a `using`-defined class can't be forward-declared).
* It only exists to overcome C++ language limitations (where a
* `using`-defined class can't be forward-declared).
*/
struct Data: public GRunArg
struct GAPI_EXPORTS Data: public GRunArg
{
using GRunArg::GRunArg;
template <typename T>

13
modules/gapi/include/opencv2/gapi/gopaque.hpp
View File

@ -15,6 +15,7 @@
#include <opencv2/gapi/own/exports.hpp>
#include <opencv2/gapi/opencv_includes.hpp>
#include <opencv2/gapi/util/any.hpp>
#include <opencv2/gapi/util/variant.hpp>
#include <opencv2/gapi/util/throw.hpp>
#include <opencv2/gapi/util/type_traits.hpp>
@ -119,6 +120,7 @@ namespace detail
virtual void mov(BasicOpaqueRef &ref) = 0;
virtual const void* ptr() const = 0;
virtual void set(const cv::util::any &a) = 0;
};
template<typename T> class OpaqueRefT final: public BasicOpaqueRef
@ -212,6 +214,10 @@ namespace detail
}
virtual const void* ptr() const override { return &rref(); }
virtual void set(const cv::util::any &a) override {
wref() = util::any_cast<T>(a);
}
};
// This class strips type information from OpaqueRefT<> and makes it usable
@ -285,6 +291,13 @@ namespace detail
// May be used to uniquely identify this object internally
const void *ptr() const { return m_ref->ptr(); }
// Introduced for in-graph meta handling
OpaqueRef& operator= (const cv::util::any &a)
{
m_ref->set(a);
return *this;
}
};
} // namespace detail

30
modules/gapi/include/opencv2/gapi/streaming/cap.hpp
View File

@ -21,9 +21,11 @@
* Note for developers: please don't put videoio dependency in G-API
* because of this file.
*/
#include <chrono>
#include <opencv2/videoio.hpp>
#include <opencv2/gapi/garg.hpp>
#include <opencv2/gapi/streaming/meta.hpp>
namespace cv {
namespace gapi {
@ -55,6 +57,7 @@ protected:
cv::VideoCapture cap;
cv::Mat first;
bool first_pulled = false;
int64_t counter = 0;
void prep()
{
@ -80,19 +83,26 @@ protected:
GAPI_Assert(!first.empty());
first_pulled = true;
data = first; // no need to clone here since it was cloned already
return true;
}
if (!cap.isOpened()) return false;
cv::Mat frame;
if (!cap.read(frame))
else
{
// end-of-stream happened
return false;
if (!cap.isOpened()) return false;
cv::Mat frame;
if (!cap.read(frame))
{
// end-of-stream happened
return false;
}
// Same reason to clone as in prep()
data = frame.clone();
}
// Same reason to clone as in prep()
data = frame.clone();
// Tag data with seq_id/ts
const auto now = std::chrono::system_clock::now();
const auto dur = std::chrono::duration_cast<std::chrono::microseconds>
(now.time_since_epoch());
data.meta[cv::gapi::streaming::meta_tag::timestamp] = int64_t{dur.count()};
data.meta[cv::gapi::streaming::meta_tag::seq_id] = int64_t{counter++};
return true;
}

79
modules/gapi/include/opencv2/gapi/streaming/meta.hpp Normal file
View File

@ -0,0 +1,79 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
//
// Copyright (C) 2020 Intel Corporation
#ifndef OPENCV_GAPI_GSTREAMING_META_HPP
#define OPENCV_GAPI_GSTREAMING_META_HPP
#include <opencv2/gapi/gopaque.hpp>
#include <opencv2/gapi/gcall.hpp>
#include <opencv2/gapi/gkernel.hpp>
#include <opencv2/gapi/gtype_traits.hpp>
namespace cv {
namespace gapi {
namespace streaming {
// FIXME: the name is debatable
namespace meta_tag {
static constexpr const char * timestamp = "org.opencv.gapi.meta.timestamp";
static constexpr const char * seq_id = "org.opencv.gapi.meta.seq_id";
} // namespace meta_tag
namespace detail {
struct GMeta {
static const char *id() {
return "org.opencv.streaming.meta";
}
// A universal yield for meta(), same as in GDesync
template<typename... R, int... IIs>
static std::tuple<R...> yield(cv::GCall &call, cv::detail::Seq<IIs...>) {
return std::make_tuple(cv::detail::Yield<R>::yield(call, IIs)...);
}
// Also a universal outMeta stub here
static GMetaArgs getOutMeta(const GMetaArgs &args, const GArgs &) {
return args;
}
};
} // namespace detail
template<typename T, typename G>
cv::GOpaque<T> meta(G g, const std::string &tag) {
using O = cv::GOpaque<T>;
cv::GKernel k{
detail::GMeta::id() // kernel id
, tag // kernel tag. Use meta tag here
, &detail::GMeta::getOutMeta // outMeta callback
, {cv::detail::GTypeTraits<O>::shape} // output Shape
, {cv::detail::GTypeTraits<G>::op_kind} // input data kinds
, {cv::detail::GObtainCtor<O>::get()} // output template ctors
};
cv::GCall call(std::move(k));
call.pass(g);
return std::get<0>(detail::GMeta::yield<O>(call, cv::detail::MkSeq<1>::type()));
}
template<typename G>
cv::GOpaque<int64_t> timestamp(G g) {
return meta<int64_t>(g, meta_tag::timestamp);
}
template<typename G>
cv::GOpaque<int64_t> seq_id(G g) {
return meta<int64_t>(g, meta_tag::seq_id);
}
template<typename G>
cv::GOpaque<int64_t> seqNo(G g) {
// Old name, compatibility only
return seq_id(g);
}
} // namespace streaming
} // namespace gapi
} // namespace cv
#endif // OPENCV_GAPI_GSTREAMING_META_HPP

38
modules/gapi/src/api/gbackend.cpp
View File

@ -143,6 +143,14 @@ void bindInArg(Mag& mag, const RcDesc &rc, const GRunArg &arg, HandleRMat handle
if (handleRMat == HandleRMat::SKIP) return;
GAPI_Assert(arg.index() == GRunArg::index_of<cv::RMat>());
bindRMat(mag, rc, util::get<cv::RMat>(arg), RMat::Access::R);
// FIXME: Here meta may^WWILL be copied multiple times!
// Replace it is reference-counted object?
mag.meta<cv::RMat>()[rc.id] = arg.meta;
mag.meta<cv::Mat>()[rc.id] = arg.meta;
#if !defined(GAPI_STANDALONE)
mag.meta<cv::UMat>()[rc.id] = arg.meta;
#endif
break;
}
@ -154,19 +162,23 @@ void bindInArg(Mag& mag, const RcDesc &rc, const GRunArg &arg, HandleRMat handle
case GRunArg::index_of<cv::Scalar>() : mag_scalar = util::get<cv::Scalar>(arg); break;
default: util::throw_error(std::logic_error("content type of the runtime argument does not match to resource description ?"));
}
mag.meta<cv::Scalar>()[rc.id] = arg.meta;
break;
}
case GShape::GARRAY:
mag.template slot<cv::detail::VectorRef>()[rc.id] = util::get<cv::detail::VectorRef>(arg);
mag.slot<cv::detail::VectorRef>()[rc.id] = util::get<cv::detail::VectorRef>(arg);
mag.meta<cv::detail::VectorRef>()[rc.id] = arg.meta;
break;
case GShape::GOPAQUE:
mag.template slot<cv::detail::OpaqueRef>()[rc.id] = util::get<cv::detail::OpaqueRef>(arg);
mag.slot<cv::detail::OpaqueRef>()[rc.id] = util::get<cv::detail::OpaqueRef>(arg);
mag.meta<cv::detail::OpaqueRef>()[rc.id] = arg.meta;
break;
case GShape::GFRAME:
mag.template slot<cv::MediaFrame>()[rc.id] = util::get<cv::MediaFrame>(arg);
mag.slot<cv::MediaFrame>()[rc.id] = util::get<cv::MediaFrame>(arg);
mag.meta<cv::MediaFrame>()[rc.id] = arg.meta;
break;
default:
@ -250,13 +262,23 @@ cv::GRunArg getArg(const Mag& mag, const RcDesc &ref)
// Wrap associated CPU object (either host or an internal one)
switch (ref.shape)
{
case GShape::GMAT: return GRunArg(mag.template slot<cv::RMat>().at(ref.id));
case GShape::GSCALAR: return GRunArg(mag.template slot<cv::Scalar>().at(ref.id));
case GShape::GMAT:
return GRunArg(mag.slot<cv::RMat>().at(ref.id),
mag.meta<cv::RMat>().at(ref.id));
case GShape::GSCALAR:
return GRunArg(mag.slot<cv::Scalar>().at(ref.id),
mag.meta<cv::Scalar>().at(ref.id));
// Note: .at() is intentional for GArray and GOpaque as objects MUST be already there
// (and constructed by either bindIn/Out or resetInternal)
case GShape::GARRAY: return GRunArg(mag.template slot<cv::detail::VectorRef>().at(ref.id));
case GShape::GOPAQUE: return GRunArg(mag.template slot<cv::detail::OpaqueRef>().at(ref.id));
case GShape::GFRAME: return GRunArg(mag.template slot<cv::MediaFrame>().at(ref.id));
case GShape::GARRAY:
return GRunArg(mag.slot<cv::detail::VectorRef>().at(ref.id),
mag.meta<cv::detail::VectorRef>().at(ref.id));
case GShape::GOPAQUE:
return GRunArg(mag.slot<cv::detail::OpaqueRef>().at(ref.id),
mag.meta<cv::detail::OpaqueRef>().at(ref.id));
case GShape::GFRAME:
return GRunArg(mag.slot<cv::MediaFrame>().at(ref.id),
mag.meta<cv::MediaFrame>().at(ref.id));
default:
util::throw_error(std::logic_error("Unsupported GShape type"));
break;

33
modules/gapi/src/api/grunarg.cpp Normal file
View File

@ -0,0 +1,33 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
//
// Copyright (C) 2020 Intel Corporation
#include "precomp.hpp"
#include <opencv2/gapi/garg.hpp>
cv::GRunArg::GRunArg() {
}
cv::GRunArg::GRunArg(const cv::GRunArg &arg)
: cv::GRunArgBase(static_cast<const cv::GRunArgBase&>(arg))
, meta(arg.meta) {
}
cv::GRunArg::GRunArg(cv::GRunArg &&arg)
: cv::GRunArgBase(std::move(static_cast<const cv::GRunArgBase&>(arg)))
, meta(std::move(arg.meta)) {
}
cv::GRunArg& cv::GRunArg::operator= (const cv::GRunArg &arg) {
cv::GRunArgBase::operator=(static_cast<const cv::GRunArgBase&>(arg));
meta = arg.meta;
return *this;
}
cv::GRunArg& cv::GRunArg::operator= (cv::GRunArg &&arg) {
cv::GRunArgBase::operator=(std::move(static_cast<const cv::GRunArgBase&>(arg)));
meta = std::move(arg.meta);
return *this;
}

11
modules/gapi/src/backends/common/gbackend.hpp
View File

@ -62,6 +62,8 @@ namespace magazine {
template<typename... Ts> struct Class
{
template<typename T> using MapT = std::unordered_map<int, T>;
using MapM = std::unordered_map<int, GRunArg::Meta>;
template<typename T> MapT<T>& slot()
{
return std::get<ade::util::type_list_index<T, Ts...>::value>(slots);
@ -70,8 +72,17 @@ namespace magazine {
{
return std::get<ade::util::type_list_index<T, Ts...>::value>(slots);
}
template<typename T> MapM& meta()
{
return metas[ade::util::type_list_index<T, Ts...>::value];
}
template<typename T> const MapM& meta() const
{
return metas[ade::util::type_list_index<T, Ts...>::value];
}
private:
std::tuple<MapT<Ts>...> slots;
std::array<MapM, sizeof...(Ts)> metas;
};
} // namespace magazine

105
modules/gapi/src/backends/common/gmetabackend.cpp Normal file
View File

@ -0,0 +1,105 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
//
// Copyright (C) 2020 Intel Corporation
#include "precomp.hpp"
#include <opencv2/gapi/gcommon.hpp> // compile args
#include <opencv2/gapi/util/any.hpp> // any
#include <opencv2/gapi/streaming/meta.hpp> // GMeta
#include "compiler/gobjref.hpp" // RcDesc
#include "compiler/gmodel.hpp" // GModel, Op
#include "backends/common/gbackend.hpp"
#include "api/gbackend_priv.hpp" // FIXME: Make it part of Backend SDK!
#include "backends/common/gmetabackend.hpp"
namespace {
class GraphMetaExecutable final: public cv::gimpl::GIslandExecutable {
std::string m_meta_tag;
public:
GraphMetaExecutable(const ade::Graph& g,
const std::vector<ade::NodeHandle>& nodes);
bool canReshape() const override;
void reshape(ade::Graph&, const cv::GCompileArgs&) override;
void run(std::vector<InObj> &&input_objs,
std::vector<OutObj> &&output_objs) override;
};
bool GraphMetaExecutable::canReshape() const {
return true;
}
void GraphMetaExecutable::reshape(ade::Graph&, const cv::GCompileArgs&) {
// do nothing here
}
GraphMetaExecutable::GraphMetaExecutable(const ade::Graph& g,
const std::vector<ade::NodeHandle>& nodes) {
// There may be only one node in the graph
GAPI_Assert(nodes.size() == 1u);
cv::gimpl::GModel::ConstGraph cg(g);
const auto &op = cg.metadata(nodes[0]).get<cv::gimpl::Op>();
GAPI_Assert(op.k.name == cv::gapi::streaming::detail::GMeta::id());
m_meta_tag = op.k.tag;
}
void GraphMetaExecutable::run(std::vector<InObj> &&input_objs,
std::vector<OutObj> &&output_objs) {
GAPI_Assert(input_objs.size() == 1u);
GAPI_Assert(output_objs.size() == 1u);
const cv::GRunArg in_arg = input_objs[0].second;
cv::GRunArgP out_arg = output_objs[0].second;
auto it = in_arg.meta.find(m_meta_tag);
if (it == in_arg.meta.end()) {
cv::util::throw_error
(std::logic_error("Run-time meta "
+ m_meta_tag
+ " is not found in object "
+ std::to_string(static_cast<int>(input_objs[0].first.shape))
+ "/"
+ std::to_string(input_objs[0].first.id)));
}
cv::util::get<cv::detail::OpaqueRef>(out_arg) = it->second;
}
class GraphMetaBackendImpl final: public cv::gapi::GBackend::Priv {
virtual void unpackKernel(ade::Graph &,
const ade::NodeHandle &,
const cv::GKernelImpl &) override {
// Do nothing here
}
virtual EPtr compile(const ade::Graph& graph,
const cv::GCompileArgs&,
const std::vector<ade::NodeHandle>& nodes,
const std::vector<cv::gimpl::Data>&,
const std::vector<cv::gimpl::Data>&) const override {
return EPtr{new GraphMetaExecutable(graph, nodes)};
}
};
cv::gapi::GBackend graph_meta_backend() {
static cv::gapi::GBackend this_backend(std::make_shared<GraphMetaBackendImpl>());
return this_backend;
}
struct InGraphMetaKernel final: public cv::detail::KernelTag {
using API = cv::gapi::streaming::detail::GMeta;
static cv::gapi::GBackend backend() { return graph_meta_backend(); }
static int kernel() { return 42; }
};
} // anonymous namespace
cv::gapi::GKernelPackage cv::gimpl::meta::kernels() {
return cv::gapi::kernels<InGraphMetaKernel>();
}

16
modules/gapi/src/backends/common/gmetabackend.hpp Normal file
View File

@ -0,0 +1,16 @@
#ifndef OPENCV_GAPI_SRC_COMMON_META_BACKEND_HPP
#define OPENCV_GAPI_SRC_COMMON_META_BACKEND_HPP
#include <opencv2/gapi/gkernel.hpp>
namespace cv {
namespace gimpl {
namespace meta {
cv::gapi::GKernelPackage kernels();
} // namespace meta
} // namespace gimpl
} // namespace cv
#endif // OPENCV_GAPI_SRC_COMMON_META_BACKEND_HPP

4
modules/gapi/src/compiler/gcompiler.cpp
View File

@ -35,6 +35,7 @@
#include "executor/gexecutor.hpp"
#include "executor/gstreamingexecutor.hpp"
#include "backends/common/gbackend.hpp"
#include "backends/common/gmetabackend.hpp"
// <FIXME:>
#if !defined(GAPI_STANDALONE)
@ -58,7 +59,8 @@ namespace
for (const auto &b : pkg.backends()) {
aux_pkg = combine(aux_pkg, b.priv().auxiliaryKernels());
}
return combine(pkg, aux_pkg);
// Always include built-in meta<> implementation
return combine(pkg, aux_pkg, cv::gimpl::meta::kernels());
};
auto has_use_only = cv::gapi::getCompileArg<cv::gapi::use_only>(args);

35
modules/gapi/src/compiler/gislandmodel.cpp
View File

@ -357,26 +357,21 @@ void GIslandExecutable::run(GIslandExecutable::IInput &in, GIslandExecutable::IO
for (auto &&it: ade::util::zip(ade::util::toRange(in_desc),
ade::util::toRange(in_vector)))
{
// FIXME: Not every Island expects a cv::Mat instead of own::Mat on input
// This kludge should go as a result of de-ownification
const cv::GRunArg& in_data_orig = std::get<1>(it);
cv::GRunArg in_data;
#if !defined(GAPI_STANDALONE)
switch (in_data_orig.index())
{
case cv::GRunArg::index_of<cv::Mat>():
in_data = cv::GRunArg{cv::make_rmat<cv::gimpl::RMatAdapter>(cv::util::get<cv::Mat>(in_data_orig))};
break;
case cv::GRunArg::index_of<cv::Scalar>():
in_data = cv::GRunArg{(cv::util::get<cv::Scalar>(in_data_orig))};
// FIXME: This whole construct is ugly, from
// its writing to a need in this in general
in_data = cv::GRunArg{ cv::make_rmat<cv::gimpl::RMatAdapter>(cv::util::get<cv::Mat>(in_data_orig))
, in_data_orig.meta
};
break;
default:
in_data = in_data_orig;
break;
}
#else
in_data = in_data_orig;
#endif // GAPI_STANDALONE
in_objs.emplace_back(std::get<0>(it), std::move(in_data));
}
for (auto &&it: ade::util::indexed(ade::util::toRange(out_desc)))
@ -385,9 +380,27 @@ void GIslandExecutable::run(GIslandExecutable::IInput &in, GIslandExecutable::IO
out.get(ade::util::checked_cast<int>(ade::util::index(it))));
}
run(std::move(in_objs), std::move(out_objs));
// Propagate in-graph meta down to the graph
// Note: this is not a complete implementation! Mainly this is a stub
// and the proper implementation should come later.
//
// Propagating the meta information here has its pros and cons.
// Pros: it works here uniformly for both regular and streaming cases,
// also for the majority of old-fashioned (synchronous) backends
// Cons: backends implementing the asynchronous run(IInput,IOutput)
// won't get it out of the box
cv::GRunArg::Meta stub_meta;
for (auto &&in_arg : in_vector)
{
stub_meta.insert(in_arg.meta.begin(), in_arg.meta.end());
}
// Report output objects as "ready" to the executor, also post
// calculated in-graph meta for the objects
for (auto &&it: out_objs)
{
out.post(std::move(it.second)); // report output objects as "ready" to the executor
out.meta(it.second, stub_meta);
out.post(std::move(it.second));
}
}

4
modules/gapi/src/compiler/gislandmodel.hpp
View File

@ -172,6 +172,10 @@ struct GIslandExecutable::IOutput: public GIslandExecutable::IODesc {
virtual GRunArgP get(int idx) = 0; // Allocate (wrap) a new data object for output idx
virtual void post(GRunArgP&&) = 0; // Release the object back to the framework (mark available)
virtual void post(EndOfStream&&) = 0; // Post end-of-stream marker back to the framework
// Assign accumulated metadata to the given output object.
// This method can only be called after get() and before post().
virtual void meta(const GRunArgP&, const GRunArg::Meta &) = 0;
};
// GIslandEmitter - a backend-specific thing which feeds data into

53
modules/gapi/src/executor/gexecutor.cpp
View File

@ -12,6 +12,8 @@
#include <ade/util/zip_range.hpp>
#include <opencv2/gapi/opencv_includes.hpp>
#include "api/gproto_priv.hpp" // ptr(GRunArgP)
#include "executor/gexecutor.hpp"
#include "compiler/passes/passes.hpp"
@ -105,6 +107,9 @@ void bindInArgExec(Mag& mag, const RcDesc &rc, const GRunArg &arg)
mag_rmat = util::get<cv::RMat>(arg); break;
default: util::throw_error(std::logic_error("content type of the runtime argument does not match to resource description ?"));
}
// FIXME: has to take extra care about meta here for this particuluar
// case, just because this function exists at all
mag.meta<cv::RMat>()[rc.id] = arg.meta;
}
void bindOutArgExec(Mag& mag, const RcDesc &rc, const GRunArgP &arg)
@ -131,7 +136,7 @@ cv::GRunArgP getObjPtrExec(Mag& mag, const RcDesc &rc)
{
return getObjPtr(mag, rc);
}
return GRunArgP(&mag.template slot<cv::RMat>()[rc.id]);
return GRunArgP(&mag.slot<cv::RMat>()[rc.id]);
}
void writeBackExec(const Mag& mag, const RcDesc &rc, GRunArgP &g_arg)
@ -155,6 +160,25 @@ void writeBackExec(const Mag& mag, const RcDesc &rc, GRunArgP &g_arg)
default: util::throw_error(std::logic_error("content type of the runtime argument does not match to resource description ?"));
}
}
void assignMetaStubExec(Mag& mag, const RcDesc &rc, const cv::GRunArg::Meta &meta) {
switch (rc.shape)
{
case GShape::GARRAY: mag.meta<cv::detail::VectorRef>()[rc.id] = meta; break;
case GShape::GOPAQUE: mag.meta<cv::detail::OpaqueRef>()[rc.id] = meta; break;
case GShape::GSCALAR: mag.meta<cv::Scalar>()[rc.id] = meta; break;
case GShape::GFRAME: mag.meta<cv::MediaFrame>()[rc.id] = meta; break;
case GShape::GMAT:
mag.meta<cv::Mat>() [rc.id] = meta;
mag.meta<cv::RMat>()[rc.id] = meta;
#if !defined(GAPI_STANDALONE)
mag.meta<cv::UMat>()[rc.id] = meta;
#endif
break;
default: util::throw_error(std::logic_error("Unsupported GShape type")); break;
}
}
} // anonymous namespace
}}} // namespace cv::gimpl::magazine
@ -231,11 +255,28 @@ public:
class cv::gimpl::GExecutor::Output final: public cv::gimpl::GIslandExecutable::IOutput
{
cv::gimpl::Mag &mag;
virtual GRunArgP get(int idx) override { return magazine::getObjPtrExec(mag, desc()[idx]); }
virtual void post(GRunArgP&&) override { } // Do nothing here
virtual void post(EndOfStream&&) override {} // Do nothing here too
std::unordered_map<const void*, int> out_idx;
GRunArgP get(int idx) override
{
auto r = magazine::getObjPtrExec(mag, desc()[idx]);
// Remember the output port for this output object
out_idx[cv::gimpl::proto::ptr(r)] = idx;
return r;
}
void post(GRunArgP&&) override { } // Do nothing here
void post(EndOfStream&&) override {} // Do nothing here too
void meta(const GRunArgP &out, const GRunArg::Meta &m) override
{
const auto idx = out_idx.at(cv::gimpl::proto::ptr(out));
magazine::assignMetaStubExec(mag, desc()[idx], m);
}
public:
Output(cv::gimpl::Mag &m, const std::vector<RcDesc> &rcs) : mag(m) { set(rcs); }
Output(cv::gimpl::Mag &m, const std::vector<RcDesc> &rcs)
: mag(m)
{
set(rcs);
}
};
void cv::gimpl::GExecutor::run(cv::gimpl::GRuntimeArgs &&args)
@ -330,7 +371,7 @@ void cv::gimpl::GExecutor::run(cv::gimpl::GRuntimeArgs &&args)
// Run the script
for (auto &op : m_ops)
{
// (5)
// (5), (6)
Input i{m_res, op.in_objects};
Output o{m_res, op.out_objects};
op.isl_exec->run(i, o);

43
modules/gapi/src/executor/gstreamingexecutor.cpp
View File

@ -350,16 +350,14 @@ bool QueueReader::getInputVector(std::vector<Q*> &in_queues,
// value-initialized scalar)
// It can also hold a constant value received with
// Stop::Kind::CNST message (see above).
// FIXME: Variant move problem
isl_inputs[id] = const_cast<const cv::GRunArg&>(in_constants[id]);
isl_inputs[id] = in_constants[id];
continue;
}
q->pop(m_cmd[id]);
if (!cv::util::holds_alternative<Stop>(m_cmd[id]))
{
// FIXME: Variant move problem
isl_inputs[id] = const_cast<const cv::GRunArg &>(cv::util::get<cv::GRunArg>(m_cmd[id]));
isl_inputs[id] = cv::util::get<cv::GRunArg>(m_cmd[id]);
}
else // A Stop sign
{
@ -382,7 +380,7 @@ bool QueueReader::getInputVector(std::vector<Q*> &in_queues,
// NEXT time (on a next call to getInputVector()), the
// "q==nullptr" check above will be triggered, but now
// we need to make it manually:
isl_inputs[id] = const_cast<const cv::GRunArg&>(in_constants[id]);
isl_inputs[id] = in_constants[id];
}
else
{
@ -666,8 +664,7 @@ class StreamingOutput final: public cv::gimpl::GIslandExecutable::IOutput
Cmd cmd;
if (cv::util::holds_alternative<cv::GRunArg>(post_iter->data))
{
// FIXME: That ugly VARIANT problem
cmd = Cmd{const_cast<const cv::GRunArg&>(cv::util::get<cv::GRunArg>(post_iter->data))};
cmd = Cmd{cv::util::get<cv::GRunArg>(post_iter->data)};
}
else
{
@ -677,8 +674,7 @@ class StreamingOutput final: public cv::gimpl::GIslandExecutable::IOutput
}
for (auto &&q : m_out_queues[out_idx])
{
// FIXME: This ugly VARIANT problem
q->push(const_cast<const Cmd&>(cmd));
q->push(cmd);
}
post_iter = m_postings[out_idx].erase(post_iter);
}
@ -708,6 +704,15 @@ class StreamingOutput final: public cv::gimpl::GIslandExecutable::IOutput
}
}
}
void meta(const cv::GRunArgP &out, const cv::GRunArg::Meta &m) override
{
const auto it = m_postIdx.find(cv::gimpl::proto::ptr(out));
GAPI_Assert(it != m_postIdx.end());
const auto out_iter = it->second.second;
cv::util::get<cv::GRunArg>(out_iter->data).meta = m;
}
public:
explicit StreamingOutput(const cv::GMetaArgs &metas,
std::vector< std::vector<Q*> > &out_queues,
@ -769,6 +774,7 @@ void islandActorThread(std::vector<cv::gimpl::RcDesc> in_rcs, //
void collectorThread(std::vector<Q*> in_queues,
std::vector<int> in_mapping,
const std::size_t out_size,
const bool handle_stop,
Q& out_queue)
{
// These flags are static now: regardless if the sync or
@ -783,9 +789,14 @@ void collectorThread(std::vector<Q*> in_queues,
while (true)
{
cv::GRunArgs this_result(out_size);
if (!qr.getResultsVector(in_queues, in_mapping, out_size, this_result))
const bool ok = qr.getResultsVector(in_queues, in_mapping, out_size, this_result);
if (!ok)
{
out_queue.push(Cmd{Stop{}});
if (handle_stop)
{
out_queue.push(Cmd{Stop{}});
}
// Terminate the thread anyway
return;
}
out_queue.push(Cmd{Result{std::move(this_result), flags}});
@ -1263,12 +1274,22 @@ void cv::gimpl::GStreamingExecutor::setSource(GRunArgs &&ins)
// If there are desynchronized parts in the graph, there may be
// multiple theads polling every separate (desynchronized)
// branch in the graph individually.
const bool has_main_path = m_sink_sync.end() !=
std::find(m_sink_sync.begin(), m_sink_sync.end(), -1);
for (auto &&info : m_collector_map) {
m_threads.emplace_back(collectorThread,
info.second.queues,
info.second.mapping,
m_sink_queues.size(),
has_main_path ? info.first == -1 : true, // see below (*)
std::ref(m_out_queue));
// (*) - there may be a problem with desynchronized paths when those work
// faster than the main path. In this case, the desync paths get "Stop" message
// earlier and thus broadcast it down to pipeline gets stopped when there is
// some "main path" data to process. This new collectorThread's flag regulates it:
// - desync paths should never post Stop message if there is a main path.
// - if there is no main path, than any desync path can terminate the execution.
}
state = State::READY;
}

195
modules/gapi/test/gapi_graph_meta_tests.cpp Normal file
View File

@ -0,0 +1,195 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
//
// Copyright (C) 2020 Intel Corporation
#include <tuple>
#include <unordered_set>
#include "test_precomp.hpp"
#include "opencv2/gapi/streaming/meta.hpp"
#include "opencv2/gapi/streaming/cap.hpp"
namespace opencv_test {
namespace {
void initTestDataPath() {
#ifndef WINRT
static bool initialized = false;
if (!initialized)
{
// Since G-API has no own test data (yet), it is taken from the common space
const char* testDataPath = getenv("OPENCV_TEST_DATA_PATH");
if (testDataPath != nullptr) {
cvtest::addDataSearchPath(testDataPath);
initialized = true;
}
}
#endif // WINRT
}
} // anonymous namespace
TEST(GraphMeta, Trad_AccessInput) {
cv::GMat in;
cv::GMat out1 = cv::gapi::blur(in, cv::Size(3,3));
cv::GOpaque<int> out2 = cv::gapi::streaming::meta<int>(in, "foo");
cv::GComputation graph(cv::GIn(in), cv::GOut(out1, out2));
cv::Mat in_mat = cv::Mat::eye(cv::Size(64, 64), CV_8UC1);
cv::Mat out_mat;
int out_meta = 0;
// manually set metadata in the input fields
auto inputs = cv::gin(in_mat);
inputs[0].meta["foo"] = 42;
graph.apply(std::move(inputs), cv::gout(out_mat, out_meta));
EXPECT_EQ(42, out_meta);
}
TEST(GraphMeta, Trad_AccessTmp) {
cv::GMat in;
cv::GMat tmp = cv::gapi::blur(in, cv::Size(3,3));
cv::GMat out1 = tmp+1;
cv::GOpaque<float> out2 = cv::gapi::streaming::meta<float>(tmp, "bar");
cv::GComputation graph(cv::GIn(in), cv::GOut(out1, out2));
cv::Mat in_mat = cv::Mat::eye(cv::Size(64, 64), CV_8UC1);
cv::Mat out_mat;
float out_meta = 0.f;
// manually set metadata in the input fields
auto inputs = cv::gin(in_mat);
inputs[0].meta["bar"] = 1.f;
graph.apply(std::move(inputs), cv::gout(out_mat, out_meta));
EXPECT_EQ(1.f, out_meta);
}
TEST(GraphMeta, Trad_AccessOutput) {
cv::GMat in;
cv::GMat out1 = cv::gapi::blur(in, cv::Size(3,3));
cv::GOpaque<std::string> out2 = cv::gapi::streaming::meta<std::string>(out1, "baz");
cv::GComputation graph(cv::GIn(in), cv::GOut(out1, out2));
cv::Mat in_mat = cv::Mat::eye(cv::Size(64, 64), CV_8UC1);
cv::Mat out_mat;
std::string out_meta;
// manually set metadata in the input fields
auto inputs = cv::gin(in_mat);
// NOTE: Assigning explicitly an std::string is important,
// otherwise a "const char*" will be stored and won't be
// translated properly by util::any since std::string is
// used within the graph.
inputs[0].meta["baz"] = std::string("opencv");
graph.apply(std::move(inputs), cv::gout(out_mat, out_meta));
EXPECT_EQ("opencv", out_meta);
}
TEST(GraphMeta, Streaming_AccessInput) {
initTestDataPath();
cv::GMat in;
cv::GMat out1 = cv::gapi::blur(in, cv::Size(3,3));
cv::GOpaque<int64_t> out2 = cv::gapi::streaming::seq_id(in);
cv::GComputation graph(cv::GIn(in), cv::GOut(out1, out2));
auto ccomp = graph.compileStreaming();
ccomp.setSource<cv::gapi::wip::GCaptureSource>(findDataFile("cv/video/768x576.avi", false));
ccomp.start();
cv::Mat out_mat;
int64_t out_meta = 0;
int64_t expected_counter = 0;
while (ccomp.pull(cv::gout(out_mat, out_meta))) {
EXPECT_EQ(expected_counter, out_meta);
++expected_counter;
}
}
TEST(GraphMeta, Streaming_AccessOutput) {
initTestDataPath();
cv::GMat in;
cv::GMat out1 = cv::gapi::blur(in, cv::Size(3,3));
cv::GOpaque<int64_t> out2 = cv::gapi::streaming::seq_id(out1);
cv::GOpaque<int64_t> out3 = cv::gapi::streaming::timestamp(out1);
cv::GComputation graph(cv::GIn(in), cv::GOut(out1, out2, out3));
auto ccomp = graph.compileStreaming();
ccomp.setSource<cv::gapi::wip::GCaptureSource>(findDataFile("cv/video/768x576.avi", false));
ccomp.start();
cv::Mat out_mat;
int64_t out_meta = 0;
int64_t out_timestamp = 0;
int64_t expected_counter = 0;
int64_t prev_timestamp = -1;
while (ccomp.pull(cv::gout(out_mat, out_meta, out_timestamp))) {
EXPECT_EQ(expected_counter, out_meta);
++expected_counter;
EXPECT_NE(prev_timestamp, out_timestamp);
prev_timestamp = out_timestamp;
}
}
TEST(GraphMeta, Streaming_AccessDesync) {
initTestDataPath();
cv::GMat in;
cv::GOpaque<int64_t> out1 = cv::gapi::streaming::seq_id(in);
cv::GOpaque<int64_t> out2 = cv::gapi::streaming::timestamp(in);
cv::GMat out3 = cv::gapi::blur(in, cv::Size(3,3));
cv::GMat tmp = cv::gapi::streaming::desync(in);
cv::GScalar mean = cv::gapi::mean(tmp);
cv::GOpaque<int64_t> out4 = cv::gapi::streaming::seq_id(mean);
cv::GOpaque<int64_t> out5 = cv::gapi::streaming::timestamp(mean);
cv::GComputation graph(cv::GIn(in), cv::GOut(out1, out2, out3, out4, out5));
auto ccomp = graph.compileStreaming();
ccomp.setSource<cv::gapi::wip::GCaptureSource>(findDataFile("cv/video/768x576.avi", false));
ccomp.start();
cv::optional<int64_t> out_sync_id;
cv::optional<int64_t> out_sync_ts;
cv::optional<cv::Mat> out_sync_mat;
cv::optional<int64_t> out_desync_id;
cv::optional<int64_t> out_desync_ts;
std::unordered_set<int64_t> sync_ids;
std::unordered_set<int64_t> desync_ids;
while (ccomp.pull(cv::gout(out_sync_id, out_sync_ts, out_sync_mat,
out_desync_id, out_desync_ts))) {
if (out_sync_id.has_value()) {
CV_Assert(out_sync_ts.has_value());
CV_Assert(out_sync_mat.has_value());
sync_ids.insert(out_sync_id.value());
}
if (out_desync_id.has_value()) {
CV_Assert(out_desync_ts.has_value());
desync_ids.insert(out_desync_id.value());
}
}
// Visually report that everything is really ok
std::cout << sync_ids.size() << " vs " << desync_ids.size() << std::endl;
// Desync path should generate less objects than the synchronized one
EXPECT_GE(sync_ids.size(), desync_ids.size());
// ..but all desynchronized IDs must be present in the synchronized set
for (auto &&d_id : desync_ids) {
EXPECT_TRUE(sync_ids.count(d_id) > 0);
}
}
} // namespace opencv_test