mirror/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2024-11-24 19:20:28 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #20329 from smirnov-alexey:as/mediaframe_serialization

Browse Source 
[G-API]: Add serialization mechanism for cv::MediaFrame

* Stub initial interface

* Fix templates for deserialization

* Fix tests

* Disable a warning on windows

* Address review comments

* Change enable_ifs to other template helpers

* Resolve ambiguous template

* Fix warnings in docs
This commit is contained in:
Alexey Smirnov 2021-07-13 22:31:46 +03:00 committed by GitHub
parent a7b17bfaf0
commit 5179e37bd1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 243 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
modules/gapi/include/opencv2/gapi/media.hpp
View File

@ -15,6 +15,16 @@
#include <opencv2/gapi/gframe.hpp>
#include <opencv2/gapi/util/any.hpp>
// Forward declaration
namespace cv {
namespace gapi {
namespace s11n {
struct IOStream;
struct IIStream;
} // namespace s11n
} // namespace gapi
} // namespace cv
namespace cv {
/** \addtogroup gapi_data_structures
@ -125,6 +135,16 @@ public:
return dynamic_cast<T*>(adapter);
}
/**
* @brief Serialize MediaFrame's data to a byte array.
*
* @note The actual logic is implemented by frame's adapter class.
* Does nothing by default.
*
* @param os Bytestream to store serialized MediaFrame data in.
*/
void serialize(cv::gapi::s11n::IOStream& os) const;
private:
struct Priv;
std::shared_ptr<Priv> m;
@ -221,6 +241,14 @@ public:
// FIXME: design a better solution
// The default implementation does nothing
virtual cv::util::any blobParams() const;
virtual void serialize(cv::gapi::s11n::IOStream&) {
GAPI_Assert(false && "Generic serialize method of MediaFrame::IAdapter does nothing by default. "
"Please, implement it in derived class to properly serialize the object.");
}
virtual void deserialize(cv::gapi::s11n::IIStream&) {
GAPI_Assert(false && "Generic deserialize method of MediaFrame::IAdapter does nothing by default. "
"Please, implement it in derived class to properly deserialize the object.");
}
};
/** @} */

10
modules/gapi/include/opencv2/gapi/rmat.hpp
View File

@ -14,8 +14,8 @@
namespace cv {
namespace gapi {
namespace s11n {
struct IOStream;
struct IIStream;
struct IOStream;
struct IIStream;
} // namespace s11n
} // namespace gapi
} // namespace cv
@ -111,10 +111,12 @@ public:
// is transferred to the device when the view is destroyed
virtual View access(Access) = 0;
virtual void serialize(cv::gapi::s11n::IOStream&) {
GAPI_Assert(false && "Generic serialize method should never be called for RMat adapter");
GAPI_Assert(false && "Generic serialize method of RMat::Adapter does nothing by default. "
"Please, implement it in derived class to properly serialize the object.");
}
virtual void deserialize(cv::gapi::s11n::IIStream&) {
GAPI_Assert(false && "Generic deserialize method should never be called for RMat adapter");
GAPI_Assert(false && "Generic deserialize method of RMat::Adapter does nothing by default. "
"Please, implement it in derived class to properly deserialize the object.");
}
};
using AdapterP = std::shared_ptr<Adapter>;

67
modules/gapi/include/opencv2/gapi/s11n.hpp
View File

@ -13,6 +13,13 @@
#include <opencv2/gapi/s11n/base.hpp>
#include <opencv2/gapi/gcomputation.hpp>
#include <opencv2/gapi/rmat.hpp>
#include <opencv2/gapi/media.hpp>
#include <opencv2/gapi/util/util.hpp>
// FIXME: caused by deserialize_runarg
#if (defined _WIN32 || defined _WIN64) && defined _MSC_VER
#pragma warning(disable: 4702)
#endif
namespace cv {
namespace gapi {
@ -34,8 +41,8 @@ namespace detail {
template<typename... Types>
cv::GCompileArgs getCompileArgs(const std::vector<char> &bytes);
template<typename RMatAdapterType>
cv::GRunArgs getRunArgsWithRMats(const std::vector<char> &bytes);
template<typename... AdapterType>
cv::GRunArgs getRunArgsWithAdapters(const std::vector<char> &bytes);
} // namespace detail
/** @brief Serialize a graph represented by GComputation into an array of bytes.
@ -133,19 +140,18 @@ type deserialize(const std::vector<char> &bytes) {
}
/**
* @brief Deserialize GRunArgs including RMat objects if any from a byte array.
* @brief Deserialize GRunArgs including RMat and MediaFrame objects if any from a byte array.
*
* RMat adapter type is specified in the template.
* @note To be used properly specified adapter type must overload its serialize() and
* deserialize() methods.
* Adapter types are specified in the template.
* @note To be used properly specified adapter types must overload their deserialize() method.
* @param bytes vector of bytes to deserialize GRunArgs object from.
* @return GRunArgs including RMat objects if any.
* @see RMat
* @return GRunArgs including RMat and MediaFrame objects if any.
* @see RMat MediaFrame
*/
template<typename T, typename RMatAdapterType> inline
template<typename T, typename AtLeastOneAdapterT, typename... AdapterTypes> inline
typename std::enable_if<std::is_same<T, GRunArgs>::value, GRunArgs>::
type deserialize(const std::vector<char> &bytes) {
return detail::getRunArgsWithRMats<RMatAdapterType>(bytes);
return detail::getRunArgsWithAdapters<AtLeastOneAdapterT, AdapterTypes...>(bytes);
}
} // namespace gapi
} // namespace cv
@ -399,16 +405,39 @@ static cv::util::optional<GCompileArg> exec(const std::string& tag, cv::gapi::s1
}
};
template<typename T> struct deserialize_runarg;
template<typename ...T>
struct deserialize_arg_with_adapter;
template<typename RMatAdapterType>
template<typename RA, typename TA>
struct deserialize_arg_with_adapter<RA, TA> {
static GRunArg exec(cv::gapi::s11n::IIStream& is) {
std::unique_ptr<TA> ptr(new TA);
ptr->deserialize(is);
return GRunArg { RA(std::move(ptr)) };
}
};
template<typename RA>
struct deserialize_arg_with_adapter<RA, void> {
static GRunArg exec(cv::gapi::s11n::IIStream&) {
GAPI_Assert(false && "No suitable adapter class found during RMat/MediaFrame deserialization. "
"Please, make sure you've passed them in cv::gapi::deserialize() template");
return GRunArg{};
}
};
template<typename... Types>
struct deserialize_runarg {
static GRunArg exec(cv::gapi::s11n::IIStream& is, uint32_t idx) {
if (idx == GRunArg::index_of<RMat>()) {
auto ptr = std::make_shared<RMatAdapterType>();
ptr->deserialize(is);
return GRunArg { RMat(std::move(ptr)) };
} else { // non-RMat arg - use default deserialization
// Type or void (if not found)
using TA = typename cv::util::find_adapter_impl<RMat::Adapter, Types...>::type;
return deserialize_arg_with_adapter<RMat, TA>::exec(is);
} else if (idx == GRunArg::index_of<MediaFrame>()) {
// Type or void (if not found)
using TA = typename cv::util::find_adapter_impl<MediaFrame::IAdapter, Types...>::type;
return deserialize_arg_with_adapter<MediaFrame, TA>::exec(is);
} else { // not an adapter holding type runarg - use default deserialization
GRunArg arg;
getRunArgByIdx(is, arg, idx);
return arg;
@ -451,8 +480,8 @@ cv::GCompileArgs getCompileArgs(const std::vector<char> &sArgs) {
return args;
}
template<typename RMatAdapterType>
cv::GRunArgs getRunArgsWithRMats(const std::vector<char> &bytes) {
template<typename... AdapterTypes>
cv::GRunArgs getRunArgsWithAdapters(const std::vector<char> &bytes) {
std::unique_ptr<cv::gapi::s11n::IIStream> pIs = cv::gapi::s11n::detail::getInStream(bytes);
cv::gapi::s11n::IIStream& is = *pIs;
cv::GRunArgs args;
@ -462,7 +491,7 @@ cv::GRunArgs getRunArgsWithRMats(const std::vector<char> &bytes) {
for (uint32_t i = 0; i < sz; ++i) {
uint32_t idx = 0;
is >> idx;
args.push_back(cv::gapi::detail::deserialize_runarg<RMatAdapterType>::exec(is, idx));
args.push_back(cv::gapi::detail::deserialize_runarg<AdapterTypes...>::exec(is, idx));
}
return args;

24
modules/gapi/include/opencv2/gapi/util/util.hpp
View File

@ -153,7 +153,29 @@ overload_lamba_set<L...> overload_lambdas(L&& ...lambdas)
{
return overload_lamba_set<L...>(std::forward<L>(lambdas)...);
}
}
template<typename ...T>
struct find_adapter_impl;
template<typename AdapterT, typename T>
struct find_adapter_impl<AdapterT, T>
{
using type = typename std::conditional<std::is_base_of<AdapterT, T>::value,
T,
void>::type;
static constexpr bool found = std::is_base_of<AdapterT, T>::value;
};
template<typename AdapterT, typename T, typename... Types>
struct find_adapter_impl<AdapterT, T, Types...>
{
using type = typename std::conditional<std::is_base_of<AdapterT, T>::value,
T,
typename find_adapter_impl<AdapterT, Types...>::type>::type;
static constexpr bool found = std::is_base_of<AdapterT, T>::value ||
find_adapter_impl<AdapterT, Types...>::found;
};
} // namespace util
} // namespace cv
// \endcond

4
modules/gapi/src/api/media.cpp
View File

@ -35,6 +35,10 @@ cv::MediaFrame::IAdapter* cv::MediaFrame::getAdapter() const {
return m->adapter.get();
}
void cv::MediaFrame::serialize(cv::gapi::s11n::IOStream& os) const {
return m->adapter->serialize(os);
}
cv::MediaFrame::View::View(Ptrs&& ptrs, Strides&& strs, Callback &&cb)
: ptr (std::move(ptrs))
, stride(std::move(strs))

14
modules/gapi/src/api/s11n.cpp
View File

@ -76,14 +76,14 @@ cv::GRunArgsP cv::gapi::bind(cv::GRunArgs &out_args)
{
#if !defined(GAPI_STANDALONE)
case T::index_of<cv::UMat>() :
outputs.emplace_back((cv::UMat*)(&(cv::util::get<cv::UMat>(res_obj))));
outputs.emplace_back(&(cv::util::get<cv::UMat>(res_obj)));
break;
#endif
case cv::GRunArg::index_of<cv::Mat>() :
outputs.emplace_back((cv::Mat*)(&(cv::util::get<cv::Mat>(res_obj))));
outputs.emplace_back(&(cv::util::get<cv::Mat>(res_obj)));
break;
case cv::GRunArg::index_of<cv::Scalar>() :
outputs.emplace_back((cv::Scalar*)(&(cv::util::get<cv::Scalar>(res_obj))));
outputs.emplace_back(&(cv::util::get<cv::Scalar>(res_obj)));
break;
case T::index_of<cv::detail::VectorRef>() :
outputs.emplace_back(cv::util::get<cv::detail::VectorRef>(res_obj));
@ -92,7 +92,10 @@ cv::GRunArgsP cv::gapi::bind(cv::GRunArgs &out_args)
outputs.emplace_back(cv::util::get<cv::detail::OpaqueRef>(res_obj));
break;
case cv::GRunArg::index_of<cv::RMat>() :
outputs.emplace_back((cv::RMat*)(&(cv::util::get<cv::RMat>(res_obj))));
outputs.emplace_back(&(cv::util::get<cv::RMat>(res_obj)));
break;
case cv::GRunArg::index_of<cv::MediaFrame>() :
outputs.emplace_back(&(cv::util::get<cv::MediaFrame>(res_obj)));
break;
default:
GAPI_Assert(false && "This value type is not supported!"); // ...maybe because of STANDALONE mode.
@ -130,6 +133,9 @@ cv::GRunArg cv::gapi::bind(cv::GRunArgP &out)
case T::index_of<cv::RMat*>() :
return cv::GRunArg(*cv::util::get<cv::RMat*>(out));
case T::index_of<cv::MediaFrame*>() :
return cv::GRunArg(*cv::util::get<cv::MediaFrame*>(out));
default:
// ...maybe our types were extended
GAPI_Assert(false && "This value type is UNKNOWN!");

14
modules/gapi/src/backends/common/serialization.cpp
View File

@ -201,18 +201,20 @@ IOStream& operator<< (IOStream& os, const cv::RMat& mat) {
return os;
}
IIStream& operator>> (IIStream& is, cv::RMat&) {
util::throw_error(std::logic_error("operator>> for RMat should never be called"));
util::throw_error(std::logic_error("operator>> for RMat should never be called. "
"Instead, cv::gapi::deserialize<cv::GRunArgs, AdapterTypes...>() "
"should be used"));
return is;
}
IOStream& operator<< (IOStream& os, const cv::MediaFrame &) {
// Stub
GAPI_Assert(false && "cv::MediaFrame serialization is not supported!");
IOStream& operator<< (IOStream& os, const cv::MediaFrame &frame) {
frame.serialize(os);
return os;
}
IIStream& operator>> (IIStream& is, cv::MediaFrame &) {
// Stub
GAPI_Assert(false && "cv::MediaFrame serialization is not supported!");
util::throw_error(std::logic_error("operator>> for MediaFrame should never be called. "
"Instead, cv::gapi::deserialize<cv::GRunArgs, AdapterTypes...>() "
"should be used"));
return is;
}

116
modules/gapi/test/s11n/gapi_s11n_tests.cpp
View File

@ -2,6 +2,7 @@
#include "backends/common/serialization.hpp"
#include <opencv2/gapi/rmat.hpp>
#include <opencv2/gapi/media.hpp>
#include <../src/backends/common/gbackend.hpp> // asView
namespace {
@ -148,6 +149,29 @@ public:
int getVal() { return m_value; }
std::string getStr() { return m_str; }
};
class MyMediaFrameAdapter : public cv::MediaFrame::IAdapter {
cv::Mat m_mat;
int m_value;
std::string m_str;
public:
MyMediaFrameAdapter() = default;
MyMediaFrameAdapter(cv::Mat m, int value, const std::string& str)
: m_mat(m), m_value(value), m_str(str)
{}
virtual cv::MediaFrame::View access(cv::MediaFrame::Access) override {
return cv::MediaFrame::View({m_mat.data}, {m_mat.step});
}
virtual cv::GFrameDesc meta() const override { return {cv::MediaFormat::BGR, m_mat.size()}; }
virtual void serialize(cv::gapi::s11n::IOStream& os) override {
os << m_value << m_str;
}
virtual void deserialize(cv::gapi::s11n::IIStream& is) override {
is >> m_value >> m_str;
}
int getVal() { return m_value; }
std::string getStr() { return m_str; }
};
}
namespace opencv_test {
@ -581,6 +605,17 @@ TEST_F(S11N_Basic, Test_Vector_Of_Strings) {
EXPECT_EQ("42", des[2]);
}
TEST_F(S11N_Basic, Test_RunArg) {
cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3);
auto v = cv::GRunArgs{ cv::GRunArg{ mat } };
const std::vector<char> sargsin = cv::gapi::serialize(v);
cv::GRunArgs out = cv::gapi::deserialize<cv::GRunArgs>(sargsin);
cv::Mat out_mat = cv::util::get<cv::Mat>(out[0]);
EXPECT_EQ(0, cv::norm(mat, out_mat));
}
TEST_F(S11N_Basic, Test_RunArg_RMat) {
cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3);
cv::RMat rmat = cv::make_rmat<MyRMatAdapter>(mat, 42, "It actually works");
@ -614,6 +649,87 @@ TEST_F(S11N_Basic, Test_RunArg_RMat_Scalar_Mat) {
EXPECT_EQ(0, cv::norm(mat, out_mat));
}
TEST_F(S11N_Basic, Test_RunArg_MediaFrame) {
cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3);
auto frame = cv::MediaFrame::Create<MyMediaFrameAdapter>(mat, 42, "It actually works");
auto v = cv::GRunArgs{ cv::GRunArg{ frame } };
const std::vector<char> sargsin = cv::gapi::serialize(v);
cv::GRunArgs out = cv::gapi::deserialize<cv::GRunArgs, MyMediaFrameAdapter>(sargsin);
cv::MediaFrame out_mat = cv::util::get<cv::MediaFrame>(out[0]);
auto adapter = out_mat.get<MyMediaFrameAdapter>();
EXPECT_EQ(42, adapter->getVal());
EXPECT_EQ("It actually works", adapter->getStr());
}
TEST_F(S11N_Basic, Test_RunArg_MediaFrame_Scalar_Mat) {
cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3);
auto frame = cv::MediaFrame::Create<MyMediaFrameAdapter>(mat, 42, "It actually works");
cv::Scalar sc(111);
auto v = cv::GRunArgs{ cv::GRunArg{ frame }, cv::GRunArg{ sc }, cv::GRunArg{ mat } };
const std::vector<char> sargsin = cv::gapi::serialize(v);
cv::GRunArgs out = cv::gapi::deserialize<cv::GRunArgs, MyMediaFrameAdapter>(sargsin);
cv::MediaFrame out_frame = cv::util::get<cv::MediaFrame>(out[0]);
auto adapter = out_frame.get<MyMediaFrameAdapter>();
EXPECT_EQ(42, adapter->getVal());
EXPECT_EQ("It actually works", adapter->getStr());
cv::Scalar out_sc = cv::util::get<cv::Scalar>(out[1]);
EXPECT_EQ(sc, out_sc);
cv::Mat out_mat = cv::util::get<cv::Mat>(out[2]);
EXPECT_EQ(0, cv::norm(mat, out_mat));
}
TEST_F(S11N_Basic, Test_RunArg_MediaFrame_RMat) {
cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3);
cv::Mat mat2 = cv::Mat::eye(cv::Size(128, 64), CV_8UC3);
auto frame = cv::MediaFrame::Create<MyMediaFrameAdapter>(mat, 42, "It actually works");
auto rmat = cv::make_rmat<MyRMatAdapter>(mat2, 24, "Hello there");
auto v = cv::GRunArgs{ cv::GRunArg{ frame }, cv::GRunArg{ rmat } };
const std::vector<char> sargsin = cv::gapi::serialize(v);
cv::GRunArgs out = cv::gapi::deserialize<cv::GRunArgs, MyMediaFrameAdapter, MyRMatAdapter>(sargsin);
cv::MediaFrame out_frame = cv::util::get<cv::MediaFrame>(out[0]);
cv::RMat out_rmat = cv::util::get<cv::RMat>(out[1]);
auto adapter = out_frame.get<MyMediaFrameAdapter>();
EXPECT_EQ(42, adapter->getVal());
EXPECT_EQ("It actually works", adapter->getStr());
auto adapter2 = out_rmat.get<MyRMatAdapter>();
EXPECT_EQ(24, adapter2->getVal());
EXPECT_EQ("Hello there", adapter2->getStr());
}
TEST_F(S11N_Basic, Test_RunArg_RMat_MediaFrame) {
cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3);
cv::Mat mat2 = cv::Mat::eye(cv::Size(128, 64), CV_8UC3);
auto frame = cv::MediaFrame::Create<MyMediaFrameAdapter>(mat, 42, "It actually works");
auto rmat = cv::make_rmat<MyRMatAdapter>(mat2, 24, "Hello there");
auto v = cv::GRunArgs{ cv::GRunArg{ rmat }, cv::GRunArg{ frame } };
const std::vector<char> sargsin = cv::gapi::serialize(v);
cv::GRunArgs out = cv::gapi::deserialize<cv::GRunArgs, MyMediaFrameAdapter, MyRMatAdapter>(sargsin);
cv::RMat out_rmat = cv::util::get<cv::RMat>(out[0]);
cv::MediaFrame out_frame = cv::util::get<cv::MediaFrame>(out[1]);
auto adapter = out_frame.get<MyMediaFrameAdapter>();
EXPECT_EQ(42, adapter->getVal());
EXPECT_EQ("It actually works", adapter->getStr());
auto adapter2 = out_rmat.get<MyRMatAdapter>();
EXPECT_EQ(24, adapter2->getVal());
EXPECT_EQ("Hello there", adapter2->getStr());
}
namespace {
template <cv::detail::OpaqueKind K, typename T>
bool verifyOpaqueKind(T&& in) {