From 3673b45437e4c1e0490c0c48ba76439c4f135361 Mon Sep 17 00:00:00 2001 From: Ruslan Garnov Date: Fri, 24 Sep 2021 14:25:57 +0300 Subject: [PATCH] Added desync RMats and MediaFrames support --- .../gapi/include/opencv2/gapi/gstreaming.hpp | 12 +++- .../include/opencv2/gapi/streaming/desync.hpp | 5 +- modules/gapi/src/api/kernels_streaming.cpp | 7 +- .../gapi/src/executor/gstreamingexecutor.cpp | 5 +- .../test/streaming/gapi_streaming_tests.cpp | 68 ++++++++++++++++++- 5 files changed, 90 insertions(+), 7 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/gstreaming.hpp b/modules/gapi/include/opencv2/gapi/gstreaming.hpp index 0e7d2a847f..890eb584fb 100644 --- a/modules/gapi/include/opencv2/gapi/gstreaming.hpp +++ b/modules/gapi/include/opencv2/gapi/gstreaming.hpp @@ -2,7 +2,7 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // -// Copyright (C) 2018 Intel Corporation +// Copyright (C) 2018-2021 Intel Corporation #ifndef OPENCV_GAPI_GSTREAMING_COMPILED_HPP @@ -65,6 +65,7 @@ using OptionalOpaqueRef = OptRef; using GOptRunArgP = util::variant< optional*, optional*, + optional*, optional*, cv::detail::OptionalVectorRef, cv::detail::OptionalOpaqueRef @@ -74,6 +75,7 @@ using GOptRunArgsP = std::vector; using GOptRunArg = util::variant< optional, optional, + optional, optional, optional, optional @@ -95,6 +97,14 @@ template<> inline GOptRunArgP wrap_opt_arg(optional &m) { return GOptRunArgP{&m}; } +template<> inline GOptRunArgP wrap_opt_arg(optional &m) { + return GOptRunArgP{&m}; +} + +template<> inline GOptRunArgP wrap_opt_arg(optional &f) { + return GOptRunArgP{&f}; +} + template<> inline GOptRunArgP wrap_opt_arg(optional &s) { return GOptRunArgP{&s}; } diff --git a/modules/gapi/include/opencv2/gapi/streaming/desync.hpp b/modules/gapi/include/opencv2/gapi/streaming/desync.hpp index 86de279fe9..1ed6e24b49 100644 --- a/modules/gapi/include/opencv2/gapi/streaming/desync.hpp +++ b/modules/gapi/include/opencv2/gapi/streaming/desync.hpp @@ -2,7 +2,7 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // -// Copyright (C) 2020 Intel Corporation +// Copyright (C) 2020-2021 Intel Corporation #ifndef OPENCV_GAPI_GSTREAMING_DESYNC_HPP @@ -73,9 +73,10 @@ G desync(const G &g) { * which produces an array of cv::util::optional<> objects. * * @note This feature is highly experimental now and is currently - * limited to a single GMat argument only. + * limited to a single GMat/GFrame argument only. */ GAPI_EXPORTS GMat desync(const GMat &g); +GAPI_EXPORTS GFrame desync(const GFrame &f); } // namespace streaming } // namespace gapi diff --git a/modules/gapi/src/api/kernels_streaming.cpp b/modules/gapi/src/api/kernels_streaming.cpp index c88fbede5b..2c50551f4e 100644 --- a/modules/gapi/src/api/kernels_streaming.cpp +++ b/modules/gapi/src/api/kernels_streaming.cpp @@ -2,7 +2,7 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // -// Copyright (C) 2020 Intel Corporation +// Copyright (C) 2020-2021 Intel Corporation #include "precomp.hpp" @@ -75,6 +75,11 @@ cv::GMat cv::gapi::streaming::desync(const cv::GMat &g) { // object will feed both branches of the streaming executable. } +// All notes from the above desync(GMat) are also applicable here +cv::GFrame cv::gapi::streaming::desync(const cv::GFrame &f) { + return cv::gapi::copy(detail::desync(f)); +} + cv::GMat cv::gapi::streaming::BGR(const cv::GFrame& in) { return cv::gapi::streaming::GBGR::on(in); } diff --git a/modules/gapi/src/executor/gstreamingexecutor.cpp b/modules/gapi/src/executor/gstreamingexecutor.cpp index 27049aef63..2379e3e164 100644 --- a/modules/gapi/src/executor/gstreamingexecutor.cpp +++ b/modules/gapi/src/executor/gstreamingexecutor.cpp @@ -186,8 +186,9 @@ void sync_data(cv::gimpl::stream::Result &r, cv::GOptRunArgsP &outputs) // FIXME: this conversion should be unified switch (out_obj.index()) { - HANDLE_CASE(cv::Scalar); break; - HANDLE_CASE(cv::RMat); break; + HANDLE_CASE(cv::Scalar); break; + HANDLE_CASE(cv::RMat); break; + HANDLE_CASE(cv::MediaFrame); break; case T::index_of*>(): { // Mat: special handling. diff --git a/modules/gapi/test/streaming/gapi_streaming_tests.cpp b/modules/gapi/test/streaming/gapi_streaming_tests.cpp index 369a0c069f..4f8b2563ac 100644 --- a/modules/gapi/test/streaming/gapi_streaming_tests.cpp +++ b/modules/gapi/test/streaming/gapi_streaming_tests.cpp @@ -2,7 +2,7 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // -// Copyright (C) 2019-2020 Intel Corporation +// Copyright (C) 2019-2021 Intel Corporation #include "../test_precomp.hpp" @@ -2284,4 +2284,70 @@ TEST(OneVPL_Source, Init) EXPECT_TRUE(stream_data_provider->empty()); } #endif + +TEST(GAPI_Streaming, TestDesyncRMat) { + cv::GMat in; + auto blurred = cv::gapi::blur(in, cv::Size{3,3}); + auto desynced = cv::gapi::streaming::desync(blurred); + auto out = in - blurred; + auto pipe = cv::GComputation(cv::GIn(in), cv::GOut(desynced, out)).compileStreaming(); + + cv::Size sz(32,32); + cv::Mat in_mat(sz, CV_8UC3); + cv::randu(in_mat, cv::Scalar::all(0), cv::Scalar(255)); + pipe.setSource(cv::gin(in_mat)); + pipe.start(); + + cv::optional out_desync; + cv::optional out_rmat; + while (true) { + // Initially it throwed "bad variant access" since there was + // no RMat handling in wrap_opt_arg + EXPECT_NO_THROW(pipe.pull(cv::gout(out_desync, out_rmat))); + if (out_rmat) break; + } +} + +G_API_OP(GTestBlur, , "test.blur") { + static GFrameDesc outMeta(GFrameDesc d) { return d; } +}; +GAPI_OCV_KERNEL(GOcvTestBlur, GTestBlur) { + static void run(const cv::MediaFrame& in, cv::MediaFrame& out) { + auto d = in.desc(); + GAPI_Assert(d.fmt == cv::MediaFormat::BGR); + auto view = in.access(cv::MediaFrame::Access::R); + cv::Mat mat(d.size, CV_8UC3, view.ptr[0]); + cv::Mat blurred; + cv::blur(mat, blurred, cv::Size{3,3}); + out = cv::MediaFrame::Create(blurred); + } +}; + +TEST(GAPI_Streaming, TestDesyncMediaFrame) { + initTestDataPath(); + cv::GFrame in; + auto blurred = GTestBlur::on(in); + auto desynced = cv::gapi::streaming::desync(blurred); + auto out = GTestBlur::on(blurred); + auto pipe = cv::GComputation(cv::GIn(in), cv::GOut(desynced, out)) + .compileStreaming(cv::compile_args(cv::gapi::kernels())); + + std::string filepath = findDataFile("cv/video/768x576.avi"); + try { + pipe.setSource(filepath); + } catch(...) { + throw SkipTestException("Video file can not be opened"); + } + pipe.start(); + + cv::optional out_desync; + cv::optional out_frame; + while (true) { + // Initially it throwed "bad variant access" since there was + // no MediaFrame handling in wrap_opt_arg + EXPECT_NO_THROW(pipe.pull(cv::gout(out_desync, out_frame))); + if (out_frame) break; + } +} + } // namespace opencv_test