From b083c20eb2eba7227f66dd10837609db9704fec6 Mon Sep 17 00:00:00 2001 From: AsyaPronina <155jj@mail.ru> Date: Thu, 4 Jun 2020 19:55:49 +0300 Subject: [PATCH] Enable stateful kernels in G-API OCV Backend --- .../include/opencv2/gapi/cpu/gcpukernel.hpp | 127 +++++++-- .../gapi/include/opencv2/gapi/gcompiled.hpp | 13 + modules/gapi/src/backends/cpu/gcpubackend.cpp | 65 ++++- modules/gapi/src/backends/cpu/gcpubackend.hpp | 11 +- modules/gapi/src/backends/cpu/gcpukernel.cpp | 10 +- modules/gapi/src/backends/ocl/goclbackend.cpp | 8 +- modules/gapi/src/backends/ocl/goclbackend.hpp | 2 +- .../src/backends/render/grenderocvbackend.cpp | 2 +- modules/gapi/src/compiler/gcompiled.cpp | 11 + modules/gapi/src/compiler/gcompiled_priv.hpp | 1 + modules/gapi/src/executor/gexecutor.cpp | 8 + modules/gapi/src/executor/gexecutor.hpp | 2 + .../gapi/src/executor/gstreamingexecutor.cpp | 13 +- .../gapi_ocv_stateful_kernel_test_utils.hpp | 47 ++++ .../cpu/gapi_ocv_stateful_kernel_tests.cpp | 251 ++++++++++++++++++ 15 files changed, 527 insertions(+), 44 deletions(-) create mode 100644 modules/gapi/test/cpu/gapi_ocv_stateful_kernel_test_utils.hpp create mode 100644 modules/gapi/test/cpu/gapi_ocv_stateful_kernel_tests.cpp diff --git a/modules/gapi/include/opencv2/gapi/cpu/gcpukernel.hpp b/modules/gapi/include/opencv2/gapi/cpu/gcpukernel.hpp index 6345d84ab2..3c75baa31c 100644 --- a/modules/gapi/include/opencv2/gapi/cpu/gcpukernel.hpp +++ b/modules/gapi/include/opencv2/gapi/cpu/gcpukernel.hpp @@ -17,6 +17,7 @@ #include #include #include +#include #include //suppress_unused_warning #include @@ -109,11 +110,17 @@ public: return outOpaqueRef(output).wref(); } + GArg state() + { + return m_state; + } + protected: detail::VectorRef& outVecRef(int output); detail::OpaqueRef& outOpaqueRef(int output); std::vector m_args; + GArg m_state; //FIXME: avoid conversion of arguments from internal representation to OpenCV one on each call //to OCV kernel. (This can be achieved by a two single time conversions in GCPUExecutable::run, @@ -127,16 +134,18 @@ protected: class GAPI_EXPORTS GCPUKernel { public: - // This function is kernel's execution entry point (does the processing work) - using F = std::function; + // This function is a kernel's execution entry point (does the processing work) + using RunF = std::function; + // This function is a stateful kernel's setup routine (configures state) + using SetupF = std::function; GCPUKernel(); - explicit GCPUKernel(const F& f); + GCPUKernel(const RunF& runF, const SetupF& setupF = nullptr); - void apply(GCPUContext &ctx); + RunF m_runF = nullptr; + SetupF m_setupF = nullptr; -protected: - F m_f; + bool m_isStateful = false; }; // FIXME: This is an ugly ad-hoc implementation. TODO: refactor @@ -269,12 +278,38 @@ template struct get_out> } }; +template +struct OCVSetupHelper; + +template +struct OCVSetupHelper> +{ + template + static void setup_impl(const GMetaArgs &metaArgs, const GArgs &args, GArg &state, + detail::Seq) + { + // TODO: unique_ptr <-> shared_ptr conversion ? + // To check: Conversion is possible only if the state which should be passed to + // 'setup' user callback isn't required to have previous value + std::shared_ptr stPtr; + Impl::setup(detail::get_in_meta(metaArgs, args, IIs)..., stPtr); + state = GArg(stPtr); + } + + static void setup(const GMetaArgs &metaArgs, const GArgs &args, GArg& state) + { + setup_impl(metaArgs, args, state, + typename detail::MkSeq::type()); + } +}; + +// OCVCallHelper is a helper class to call stateless OCV kernels and OCV kernel functors. template struct OCVCallHelper; // FIXME: probably can be simplified with std::apply or analogue. template -struct OCVCallHelper, std::tuple > +struct OCVCallHelper, std::tuple> { template struct call_and_postprocess @@ -302,19 +337,16 @@ struct OCVCallHelper, std::tuple > //by comparing it's state (data ptr) before and after the call. //This is done by converting each output Mat into tracked_cv_mat object, and binding //them to parameters of ad-hoc function - //Convert own::Scalar to cv::Scalar before call kernel and run kernel - //convert cv::Scalar to own::Scalar after call kernel and write back results call_and_postprocess::get(ctx, IIs))...> - ::call(get_in::get(ctx, IIs)..., - get_out::get(ctx, OIs)...); + ::call(get_in::get(ctx, IIs)..., get_out::get(ctx, OIs)...); } template - static void call_impl(cv::GCPUContext &ctx, Impl& impl, detail::Seq, detail::Seq) + static void call_impl(cv::GCPUContext &ctx, Impl& impl, + detail::Seq, detail::Seq) { - call_and_postprocess::get(ctx, IIs))...> - ::call(impl, cv::detail::get_in::get(ctx, IIs)..., - cv::detail::get_out::get(ctx, OIs)...); + call_and_postprocess::get(ctx, IIs))...> + ::call(impl, get_in::get(ctx, IIs)..., get_out::get(ctx, OIs)...); } static void call(GCPUContext &ctx) @@ -335,23 +367,78 @@ struct OCVCallHelper, std::tuple > } }; +// OCVStCallHelper is a helper class to call stateful OCV kernels. +template +struct OCVStCallHelper; + +template +struct OCVStCallHelper, std::tuple> : + OCVCallHelper, std::tuple> +{ + template + struct call_and_postprocess + { + template + static void call(typename Impl::State& st, Inputs&&... ins, Outputs&&... outs) + { + Impl::run(std::forward(ins)..., outs..., st); + postprocess(outs...); + } + }; + + template + static void call_impl(GCPUContext &ctx, detail::Seq, detail::Seq) + { + auto& st = *ctx.state().get>(); + call_and_postprocess::get(ctx, IIs))...> + ::call(st, get_in::get(ctx, IIs)..., get_out::get(ctx, OIs)...); + } + + static void call(GCPUContext &ctx) + { + call_impl(ctx, + typename detail::MkSeq::type(), + typename detail::MkSeq::type()); + } +}; + } // namespace detail template -class GCPUKernelImpl: public cv::detail::OCVCallHelper, - public cv::detail::KernelTag +class GCPUKernelImpl: public cv::detail::KernelTag { - using P = detail::OCVCallHelper; + using CallHelper = detail::OCVCallHelper; public: using API = K; - static cv::gapi::GBackend backend() { return cv::gapi::cpu::backend(); } - static cv::GCPUKernel kernel() { return GCPUKernel(&P::call); } + static cv::gapi::GBackend backend() { return cv::gapi::cpu::backend(); } + static cv::GCPUKernel kernel() { return GCPUKernel(&CallHelper::call); } +}; + +template +class GCPUStKernelImpl: public cv::detail::KernelTag +{ + using StSetupHelper = detail::OCVSetupHelper; + using StCallHelper = detail::OCVStCallHelper; + +public: + using API = K; + using State = S; + + static cv::gapi::GBackend backend() { return cv::gapi::cpu::backend(); } + static cv::GCPUKernel kernel() { return GCPUKernel(&StCallHelper::call, + &StSetupHelper::setup); } }; #define GAPI_OCV_KERNEL(Name, API) struct Name: public cv::GCPUKernelImpl +// TODO: Reuse Anatoliy's logic for support of types with commas in macro. +// Retrieve the common part from Anatoliy's logic to the separate place. +#define GAPI_OCV_KERNEL_ST(Name, API, State) \ + struct Name:public cv::GCPUStKernelImpl \ + + class gapi::cpu::GOCVFunctor : public gapi::GFunctor { public: diff --git a/modules/gapi/include/opencv2/gapi/gcompiled.hpp b/modules/gapi/include/opencv2/gapi/gcompiled.hpp index b08451af26..26e91f5f96 100644 --- a/modules/gapi/include/opencv2/gapi/gcompiled.hpp +++ b/modules/gapi/include/opencv2/gapi/gcompiled.hpp @@ -208,6 +208,19 @@ public: // FIXME: Why it requires compile args? void reshape(const GMetaArgs& inMetas, const GCompileArgs& args); + /** + * @brief Prepare inner kernels states for a new video-stream. + * + * GCompiled objects may be used to process video streams frame by frame. + * In this case, a GCompiled is called on every image frame individually. + * Starting OpenCV 4.4, some kernels in the graph may have their internal + * states (see GAPI_OCV_KERNEL_ST for the OpenCV backend). + * In this case, if user starts processing another video stream with + * this GCompiled, this method needs to be called to let kernels re-initialize + * their internal states to a new video stream. + */ + void prepareForNewStream(); + protected: /// @private std::shared_ptr m_priv; diff --git a/modules/gapi/src/backends/cpu/gcpubackend.cpp b/modules/gapi/src/backends/cpu/gcpubackend.cpp index c9fe3cad59..220434ed69 100644 --- a/modules/gapi/src/backends/cpu/gcpubackend.cpp +++ b/modules/gapi/src/backends/cpu/gcpubackend.cpp @@ -33,13 +33,13 @@ // // If not, we need to introduce that! using GCPUModel = ade::TypedGraph - < cv::gimpl::Unit + < cv::gimpl::CPUUnit , cv::gimpl::Protocol >; // FIXME: Same issue with Typed and ConstTyped using GConstGCPUModel = ade::ConstTypedGraph - < cv::gimpl::Unit + < cv::gimpl::CPUUnit , cv::gimpl::Protocol >; @@ -53,7 +53,7 @@ namespace { GCPUModel gm(graph); auto cpu_impl = cv::util::any_cast(impl.opaque); - gm.metadata(op_node).set(cv::gimpl::Unit{cpu_impl}); + gm.metadata(op_node).set(cv::gimpl::CPUUnit{cpu_impl}); } virtual EPtr compile(const ade::Graph &graph, @@ -78,11 +78,23 @@ cv::gimpl::GCPUExecutable::GCPUExecutable(const ade::Graph &g, { // Convert list of operations (which is topologically sorted already) // into an execution script. + GConstGCPUModel gcm(m_g); for (auto &nh : nodes) { switch (m_gm.metadata(nh).get().t) { - case NodeType::OP: m_script.push_back({nh, GModel::collectOutputMeta(m_gm, nh)}); break; + case NodeType::OP: + { + m_script.push_back({nh, GModel::collectOutputMeta(m_gm, nh)}); + + // If kernel is stateful then prepare storage for its state. + GCPUKernel k = gcm.metadata(nh).get().k; + if (k.m_isStateful) + { + m_nodesToStates[nh] = GArg{ }; + } + break; + } case NodeType::DATA: { m_dataNodes.push_back(nh); @@ -104,6 +116,9 @@ cv::gimpl::GCPUExecutable::GCPUExecutable(const ade::Graph &g, default: util::throw_error(std::logic_error("Unsupported NodeType type")); } } + + // For each stateful kernel call 'setup' user callback to initialize state. + setupKernelStates(); } // FIXME: Document what it does @@ -140,6 +155,26 @@ cv::GArg cv::gimpl::GCPUExecutable::packArg(const GArg &arg) } } +void cv::gimpl::GCPUExecutable::setupKernelStates() +{ + GConstGCPUModel gcm(m_g); + for (auto& nodeToState : m_nodesToStates) + { + auto& kernelNode = nodeToState.first; + auto& kernelState = nodeToState.second; + + const GCPUKernel& kernel = gcm.metadata(kernelNode).get().k; + kernel.m_setupF(GModel::collectInputMeta(m_gm, kernelNode), + m_gm.metadata(kernelNode).get().args, + kernelState); + } +} + +void cv::gimpl::GCPUExecutable::handleNewStream() +{ + m_newStreamStarted = true; +} + void cv::gimpl::GCPUExecutable::run(std::vector &&input_objs, std::vector &&output_objs) { @@ -167,6 +202,14 @@ void cv::gimpl::GCPUExecutable::run(std::vector &&input_objs, } } + // In case if new video-stream happens - for each stateful kernel + // call 'setup' user callback to re-initialize state. + if (m_newStreamStarted) + { + setupKernelStates(); + m_newStreamStarted = false; + } + // OpenCV backend execution is not a rocket science at all. // Simply invoke our kernels in the proper order. GConstGCPUModel gcm(m_g); @@ -176,7 +219,7 @@ void cv::gimpl::GCPUExecutable::run(std::vector &&input_objs, // Obtain our real execution unit // TODO: Should kernels be copyable? - GCPUKernel k = gcm.metadata(op_info.nh).get().k; + GCPUKernel k = gcm.metadata(op_info.nh).get().k; // Initialize kernel's execution context: // - Input parameters @@ -185,8 +228,8 @@ void cv::gimpl::GCPUExecutable::run(std::vector &&input_objs, using namespace std::placeholders; ade::util::transform(op.args, - std::back_inserter(context.m_args), - std::bind(&GCPUExecutable::packArg, this, _1)); + std::back_inserter(context.m_args), + std::bind(&GCPUExecutable::packArg, this, _1)); // - Output parameters. // FIXME: pre-allocate internal Mats, etc, according to the known meta @@ -198,8 +241,14 @@ void cv::gimpl::GCPUExecutable::run(std::vector &&input_objs, context.m_results[out_port] = magazine::getObjPtr(m_res, out_desc); } + // For stateful kernel add state to its execution context + if (k.m_isStateful) + { + context.m_state = m_nodesToStates.at(op_info.nh); + } + // Now trigger the executable unit - k.apply(context); + k.m_runF(context); //As Kernels are forbidden to allocate memory for (Mat) outputs, //this code seems redundant, at least for Mats diff --git a/modules/gapi/src/backends/cpu/gcpubackend.hpp b/modules/gapi/src/backends/cpu/gcpubackend.hpp index 28cffa1ab8..a2bdbbd282 100644 --- a/modules/gapi/src/backends/cpu/gcpubackend.hpp +++ b/modules/gapi/src/backends/cpu/gcpubackend.hpp @@ -23,7 +23,7 @@ namespace cv { namespace gimpl { -struct Unit +struct CPUUnit { static const char *name() { return "HostKernel"; } GCPUKernel k; @@ -48,6 +48,13 @@ class GCPUExecutable final: public GIslandExecutable // Actual data of all resources in graph (both internal and external) Mag m_res; GArg packArg(const GArg &arg); + void setupKernelStates(); + + // TODO: Check that it is thread-safe + std::unordered_map> m_nodesToStates; + + bool m_newStreamStarted = false; public: GCPUExecutable(const ade::Graph &graph, @@ -62,6 +69,8 @@ public: util::throw_error(std::logic_error("GCPUExecutable::reshape() should never be called")); } + virtual void handleNewStream() override; + virtual void run(std::vector &&input_objs, std::vector &&output_objs) override; }; diff --git a/modules/gapi/src/backends/cpu/gcpukernel.cpp b/modules/gapi/src/backends/cpu/gcpukernel.cpp index 423e29fb95..fe1962b4a4 100644 --- a/modules/gapi/src/backends/cpu/gcpukernel.cpp +++ b/modules/gapi/src/backends/cpu/gcpukernel.cpp @@ -45,13 +45,7 @@ cv::GCPUKernel::GCPUKernel() { } -cv::GCPUKernel::GCPUKernel(const GCPUKernel::F &f) - : m_f(f) +cv::GCPUKernel::GCPUKernel(const GCPUKernel::RunF &runF, const GCPUKernel::SetupF &setupF) + : m_runF(runF), m_setupF(setupF), m_isStateful(m_setupF != nullptr) { } - -void cv::GCPUKernel::apply(GCPUContext &ctx) -{ - GAPI_Assert(m_f); - m_f(ctx); -} diff --git a/modules/gapi/src/backends/ocl/goclbackend.cpp b/modules/gapi/src/backends/ocl/goclbackend.cpp index 1eeafdb425..f8df2cb082 100644 --- a/modules/gapi/src/backends/ocl/goclbackend.cpp +++ b/modules/gapi/src/backends/ocl/goclbackend.cpp @@ -33,13 +33,13 @@ // // If not, we need to introduce that! using GOCLModel = ade::TypedGraph - < cv::gimpl::Unit + < cv::gimpl::OCLUnit , cv::gimpl::Protocol >; // FIXME: Same issue with Typed and ConstTyped using GConstGOCLModel = ade::ConstTypedGraph - < cv::gimpl::Unit + < cv::gimpl::OCLUnit , cv::gimpl::Protocol >; @@ -53,7 +53,7 @@ namespace { GOCLModel gm(graph); auto ocl_impl = cv::util::any_cast(impl.opaque); - gm.metadata(op_node).set(cv::gimpl::Unit{ocl_impl}); + gm.metadata(op_node).set(cv::gimpl::OCLUnit{ocl_impl}); } virtual EPtr compile(const ade::Graph &graph, @@ -198,7 +198,7 @@ void cv::gimpl::GOCLExecutable::run(std::vector &&input_objs, // Obtain our real execution unit // TODO: Should kernels be copyable? - GOCLKernel k = gcm.metadata(op_info.nh).get().k; + GOCLKernel k = gcm.metadata(op_info.nh).get().k; // Initialize kernel's execution context: // - Input parameters diff --git a/modules/gapi/src/backends/ocl/goclbackend.hpp b/modules/gapi/src/backends/ocl/goclbackend.hpp index 52cf6d2636..6d43d7c357 100644 --- a/modules/gapi/src/backends/ocl/goclbackend.hpp +++ b/modules/gapi/src/backends/ocl/goclbackend.hpp @@ -23,7 +23,7 @@ namespace cv { namespace gimpl { -struct Unit +struct OCLUnit { static const char *name() { return "OCLKernel"; } GOCLKernel k; diff --git a/modules/gapi/src/backends/render/grenderocvbackend.cpp b/modules/gapi/src/backends/render/grenderocvbackend.cpp index 05b028565f..abfe4cbbcf 100644 --- a/modules/gapi/src/backends/render/grenderocvbackend.cpp +++ b/modules/gapi/src/backends/render/grenderocvbackend.cpp @@ -93,7 +93,7 @@ void cv::gimpl::render::ocv::GRenderExecutable::run(std::vector &&input_ context.m_args.emplace_back(m_ftpr.get()); - k.apply(context); + k.m_runF(context); for (auto &it : output_objs) magazine::writeBack(m_res, it.first, it.second); } diff --git a/modules/gapi/src/compiler/gcompiled.cpp b/modules/gapi/src/compiler/gcompiled.cpp index bc1d76c79f..e44b7fb34c 100644 --- a/modules/gapi/src/compiler/gcompiled.cpp +++ b/modules/gapi/src/compiler/gcompiled.cpp @@ -72,6 +72,12 @@ void cv::GCompiled::Priv::reshape(const GMetaArgs& inMetas, const GCompileArgs& m_metas = inMetas; } +void cv::GCompiled::Priv::prepareForNewStream() +{ + GAPI_Assert(m_exec); + m_exec->prepareForNewStream(); +} + const cv::gimpl::GModel::Graph& cv::GCompiled::Priv::model() const { GAPI_Assert(nullptr != m_exec); @@ -155,3 +161,8 @@ void cv::GCompiled::reshape(const GMetaArgs& inMetas, const GCompileArgs& args) { m_priv->reshape(inMetas, args); } + +void cv::GCompiled::prepareForNewStream() +{ + m_priv->prepareForNewStream(); +} diff --git a/modules/gapi/src/compiler/gcompiled_priv.hpp b/modules/gapi/src/compiler/gcompiled_priv.hpp index b7bf0f690e..1ce4996a78 100644 --- a/modules/gapi/src/compiler/gcompiled_priv.hpp +++ b/modules/gapi/src/compiler/gcompiled_priv.hpp @@ -48,6 +48,7 @@ public: bool canReshape() const; void reshape(const GMetaArgs& inMetas, const GCompileArgs &args); + void prepareForNewStream(); void run(cv::gimpl::GRuntimeArgs &&args); const GMetaArgs& metas() const; diff --git a/modules/gapi/src/executor/gexecutor.cpp b/modules/gapi/src/executor/gexecutor.cpp index a5351c6486..0eaa6104f1 100644 --- a/modules/gapi/src/executor/gexecutor.cpp +++ b/modules/gapi/src/executor/gexecutor.cpp @@ -265,3 +265,11 @@ void cv::gimpl::GExecutor::reshape(const GMetaArgs& inMetas, const GCompileArgs& passes::inferMeta(ctx, true); m_ops[0].isl_exec->reshape(g, args); } + +void cv::gimpl::GExecutor::prepareForNewStream() +{ + for (auto &op : m_ops) + { + op.isl_exec->handleNewStream(); + } +} diff --git a/modules/gapi/src/executor/gexecutor.hpp b/modules/gapi/src/executor/gexecutor.hpp index a462e557a7..837b80e54e 100644 --- a/modules/gapi/src/executor/gexecutor.hpp +++ b/modules/gapi/src/executor/gexecutor.hpp @@ -91,6 +91,8 @@ public: bool canReshape() const; void reshape(const GMetaArgs& inMetas, const GCompileArgs& args); + void prepareForNewStream(); + const GModel::Graph& model() const; // FIXME: make it ConstGraph? }; diff --git a/modules/gapi/src/executor/gstreamingexecutor.cpp b/modules/gapi/src/executor/gstreamingexecutor.cpp index 92733b0882..82e93c918c 100644 --- a/modules/gapi/src/executor/gstreamingexecutor.cpp +++ b/modules/gapi/src/executor/gstreamingexecutor.cpp @@ -800,6 +800,7 @@ void cv::gimpl::GStreamingExecutor::setSource(GRunArgs &&ins) } } }; + bool islandsRecompiled = false; const auto new_meta = cv::descr_of(ins); // 0 if (gm.metadata().contains()) // (1) { @@ -821,6 +822,8 @@ void cv::gimpl::GStreamingExecutor::setSource(GRunArgs &&ins) } update_int_metas(); // (7) m_reshapable = util::make_optional(is_reshapable); + + islandsRecompiled = true; } else // (8) { @@ -929,7 +932,15 @@ void cv::gimpl::GStreamingExecutor::setSource(GRunArgs &&ins) for (auto &&out_eh : op.nh->outNodes()) { out_queues.push_back(reader_queues(*m_island_graph, out_eh)); } - op.isl_exec->handleNewStream(); + + // If Island Executable is recompiled, all its stuff including internal kernel states + // are recreated and re-initialized automatically. + // But if not, we should notify Island Executable about new started stream to let it update + // its internal variables. + if (!islandsRecompiled) + { + op.isl_exec->handleNewStream(); + } m_threads.emplace_back(islandActorThread, op.in_objects, diff --git a/modules/gapi/test/cpu/gapi_ocv_stateful_kernel_test_utils.hpp b/modules/gapi/test/cpu/gapi_ocv_stateful_kernel_test_utils.hpp new file mode 100644 index 0000000000..040e628460 --- /dev/null +++ b/modules/gapi/test/cpu/gapi_ocv_stateful_kernel_test_utils.hpp @@ -0,0 +1,47 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020 Intel Corporation + +#ifndef OPENCV_GAPI_OCV_STATEFUL_KERNEL_TESTS_UTILS_HPP +#define OPENCV_GAPI_OCV_STATEFUL_KERNEL_TESTS_UTILS_HPP + +#include "../test_precomp.hpp" + +// TODO: Reuse Anatoliy's logic for support of types with commas in macro. +// Retrieve the common part from Anatoliy's logic to the separate place. +#define DEFINE_INITIALIZER(Name, StateType, ...) \ +struct Name \ +{ \ + static StateType value() \ + { \ + return __VA_ARGS__; \ + } \ +} \ + +namespace opencv_test +{ +namespace +{ +struct UserStruct +{ + UserStruct() = default; + UserStruct(short myShortVal, float myFloatVal): + _myShortVal(myShortVal), + _myFloatVal(myFloatVal) { } + + bool operator==(const UserStruct& rhs) const + { + return ((_myShortVal == rhs._myShortVal) && + (_myFloatVal == rhs._myFloatVal)); + } + +private: + short _myShortVal; + float _myFloatVal; +}; +} // anonymous namespace +} // opencv_test + +#endif // OPENCV_GAPI_OCV_STATEFUL_KERNEL_TESTS_UTILS_HPP diff --git a/modules/gapi/test/cpu/gapi_ocv_stateful_kernel_tests.cpp b/modules/gapi/test/cpu/gapi_ocv_stateful_kernel_tests.cpp new file mode 100644 index 0000000000..9e3e20c694 --- /dev/null +++ b/modules/gapi/test/cpu/gapi_ocv_stateful_kernel_tests.cpp @@ -0,0 +1,251 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020 Intel Corporation + +#include "gapi_ocv_stateful_kernel_test_utils.hpp" +#include +#include + +namespace opencv_test +{ +//TODO: test OT, Background Subtractor, Kalman with 3rd version of API +//----------------------------------------------- Simple tests ------------------------------------------------ +namespace +{ + inline void initTestDataPath() + { +#ifndef WINRT + static bool initialized = false; + if (!initialized) + { + // Since G-API has no own test data (yet), it is taken from the common space + const char* testDataPath = getenv("OPENCV_TEST_DATA_PATH"); + GAPI_Assert(testDataPath != nullptr); + + cvtest::addDataSearchPath(testDataPath); + initialized = true; + } +#endif // WINRT + } + + G_TYPED_KERNEL(GCountCalls, (GMat)>, "org.opencv.test.count_calls") + { + static GOpaqueDesc outMeta(GMatDesc /* in */) { return empty_gopaque_desc(); } + }; + + GAPI_OCV_KERNEL_ST(GOCVCountCalls, GCountCalls, int) + { + static void setup(const cv::GMatDesc &/* in */, std::shared_ptr &state) + { + state.reset(new int{ }); + } + + static void run(const cv::Mat &/* in */, int &out, int& state) + { + out = ++state; + } + }; + + G_TYPED_KERNEL(GIsStateUpToDate, (GMat)>, + "org.opencv.test.is_state_up-to-date") + { + static GOpaqueDesc outMeta(GMatDesc /* in */) { return empty_gopaque_desc(); } + }; + + GAPI_OCV_KERNEL_ST(GOCVIsStateUpToDate, GIsStateUpToDate, cv::Size) + { + static void setup(const cv::GMatDesc &in, + std::shared_ptr &state) + { + state.reset(new cv::Size(in.size)); + } + + static void run(const cv::Mat &in , bool &out, cv::Size& state) + { + out = in.size() == state; + } + }; + + G_TYPED_KERNEL(GStInvalidResize, , "org.opencv.test.st_invalid_resize") + { + static GMatDesc outMeta(GMatDesc in, Size, double, double, int) { return in; } + }; + + GAPI_OCV_KERNEL_ST(GOCVStInvalidResize, GStInvalidResize, int) + { + static void setup(const cv::GMatDesc, cv::Size, double, double, int, + std::shared_ptr &/* state */) + { } + + static void run(const cv::Mat& in, cv::Size sz, double fx, double fy, int interp, + cv::Mat &out, int& /* state */) + { + cv::resize(in, out, sz, fx, fy, interp); + } + }; +}; + +TEST(StatefulKernel, StateIsMutableInRuntime) +{ + constexpr int expectedCallsCount = 10; + + cv::Mat dummyIn { 1, 1, CV_8UC1 }; + int actualCallsCount = 0; + + // Declaration of G-API expression + GMat in; + GOpaque out = GCountCalls::on(in); + cv::GComputation comp(cv::GIn(in), cv::GOut(out)); + + const auto pkg = cv::gapi::kernels(); + + // Compilation of G-API expression + auto callsCounter = comp.compile(cv::descr_of(dummyIn), cv::compile_args(pkg)); + + // Simulating video stream: call GCompiled multiple times + for (int i = 0; i < expectedCallsCount; i++) + { + callsCounter(cv::gin(dummyIn), cv::gout(actualCallsCount)); + EXPECT_EQ(i + 1, actualCallsCount); + } + + // End of "video stream" + EXPECT_EQ(expectedCallsCount, actualCallsCount); + + // User asks G-API to prepare for a new stream + callsCounter.prepareForNewStream(); + callsCounter(cv::gin(dummyIn), cv::gout(actualCallsCount)); + EXPECT_EQ(1, actualCallsCount); + +} + +TEST(StatefulKernel, StateIsAutoResetForNewStream) +{ + initTestDataPath(); + + cv::GMat in; + GOpaque out = GIsStateUpToDate::on(in); + cv::GComputation c(cv::GIn(in), cv::GOut(out)); + + const auto pkg = cv::gapi::kernels(); + + // Compilation & testing + auto ccomp = c.compileStreaming(cv::compile_args(pkg)); + + ccomp.setSource(gapi::wip::make_src + (findDataFile("cv/video/768x576.avi"))); + ccomp.start(); + EXPECT_TRUE(ccomp.running()); + + // Process the full video + bool isStateUpToDate = false; + while (ccomp.pull(cv::gout(isStateUpToDate))) { + EXPECT_TRUE(isStateUpToDate); + } + EXPECT_FALSE(ccomp.running()); + + ccomp.setSource(gapi::wip::make_src + (findDataFile("cv/video/1920x1080.avi"))); + ccomp.start(); + EXPECT_TRUE(ccomp.running()); + + while (ccomp.pull(cv::gout(isStateUpToDate))) { + EXPECT_TRUE(isStateUpToDate); + } + EXPECT_FALSE(ccomp.running()); +} + +TEST(StatefulKernel, InvalidReallocatingKernel) +{ + cv::GMat in, out; + cv::Mat in_mat(500, 500, CV_8UC1), out_mat; + out = GStInvalidResize::on(in, cv::Size(300, 300), 0.0, 0.0, cv::INTER_LINEAR); + + const auto pkg = cv::gapi::kernels(); + cv::GComputation comp(cv::GIn(in), cv::GOut(out)); + + EXPECT_THROW(comp.apply(in_mat, out_mat, cv::compile_args(pkg)), std::logic_error); + +} +//------------------------------------------------------------------------------------------------------------- + + +//------------------------------------------- Typed tests on setup() ------------------------------------------ +namespace +{ +template +struct SetupStateTypedTest : public ::testing::Test +{ + using StateT = typename std::tuple_element<0, Tuple>::type; + using SetupT = typename std::tuple_element<1, Tuple>::type; + + G_TYPED_KERNEL(GReturnState, (GMat)>, "org.opencv.test.return_state") + { + static GOpaqueDesc outMeta(GMatDesc /* in */) { return empty_gopaque_desc(); } + }; + + GAPI_OCV_KERNEL_ST(GOCVReturnState, GReturnState, StateT) + { + static void setup(const cv::GMatDesc &/* in */, std::shared_ptr &state) + { + // Don't use input cv::GMatDesc intentionally + state.reset(new StateT(SetupT::value())); + } + + static void run(const cv::Mat &/* in */, StateT &out, StateT& state) + { + out = state; + } + }; +}; + +TYPED_TEST_CASE_P(SetupStateTypedTest); +} // namespace + + +TYPED_TEST_P(SetupStateTypedTest, ReturnInitializedState) +{ + using StateType = typename TestFixture::StateT; + using SetupType = typename TestFixture::SetupT; + + cv::Mat dummyIn { 1, 1, CV_8UC1 }; + StateType retState { }; + + GMat in; + auto out = TestFixture::GReturnState::on(in); + cv::GComputation comp(cv::GIn(in), cv::GOut(out)); + + const auto pkg = cv::gapi::kernels(); + comp.apply(cv::gin(dummyIn), cv::gout(retState), cv::compile_args(pkg)); + + EXPECT_EQ(SetupType::value(), retState); +} + +REGISTER_TYPED_TEST_CASE_P(SetupStateTypedTest, + ReturnInitializedState); + + +DEFINE_INITIALIZER(CharValue, char, 'z'); +DEFINE_INITIALIZER(IntValue, int, 7); +DEFINE_INITIALIZER(FloatValue, float, 42.f); +DEFINE_INITIALIZER(UcharPtrValue, uchar*, nullptr); +namespace +{ +using Std3IntArray = std::array; +} +DEFINE_INITIALIZER(StdArrayValue, Std3IntArray, { 1, 2, 3 }); +DEFINE_INITIALIZER(UserValue, UserStruct, { 5, 7.f }); + +using TypesToVerify = ::testing::Types, + std::tuple, + std::tuple, + std::tuple, + std::tuple, StdArrayValue>, + std::tuple>; + +INSTANTIATE_TYPED_TEST_CASE_P(SetupStateTypedInst, SetupStateTypedTest, TypesToVerify); +//------------------------------------------------------------------------------------------------------------- + +} // opencv_test