From cf5db9b94fd04450ebb0e07927b86e45322cf4e0 Mon Sep 17 00:00:00 2001 From: TolyaTalamanov Date: Mon, 3 Oct 2022 09:43:05 +0000 Subject: [PATCH] Add handle to configure async/sync infer mode --- .../gapi/include/opencv2/gapi/infer/ie.hpp | 37 +++++++++++++++++-- modules/gapi/src/backends/ie/giebackend.cpp | 31 +++++++++++++--- 2 files changed, 58 insertions(+), 10 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/infer/ie.hpp b/modules/gapi/include/opencv2/gapi/infer/ie.hpp index 204bd8f266..27c479624a 100644 --- a/modules/gapi/include/opencv2/gapi/infer/ie.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/ie.hpp @@ -88,6 +88,9 @@ struct ParamDesc { cv::optional vpl_preproc_device; cv::optional vpl_preproc_ctx; + + enum InferMode {Sync, Async}; + InferMode mode; }; } // namespace detail @@ -132,7 +135,8 @@ public: , {} , {} , {} - , {}} { + , {} + , detail::ParamDesc::InferMode::Async} { }; /** @overload @@ -156,7 +160,8 @@ public: , {} , {} , {} - , {}} { + , {} + , detail::ParamDesc::InferMode::Async} { }; /** @brief Specifies sequence of network input layers names for inference. @@ -351,6 +356,22 @@ public: return *this; } + /** @brief Specifies which api will be used to run inference. + + The function is used to specify mode for OpenVINO inference. + OpenVINO has two options to run inference: + 1. Asynchronous (using StartAsync: https://docs.openvino.ai/latest/classInferenceEngine_1_1InferRequest.html#doxid-class-inference-engine-1-1-infer-request-1a405293e8423d82a5b45f642a3bef0d24) + 2. Synchronous (using Infer: https://docs.openvino.ai/latest/classInferenceEngine_1_1InferRequest.html#doxid-class-inference-engine-1-1-infer-request-1a3391ce30894abde730523e9ca9371ce8) + By default asynchronous mode is used. + + @param mode Inference mode which will be used. + @return reference to this parameter structure. + */ + Params& cfgInferMode(detail::ParamDesc::InferMode mode) { + desc.mode = mode; + return *this; + } + // BEGIN(G-API's network parametrization API) GBackend backend() const { return cv::gapi::ie::backend(); } std::string tag() const { return Net::tag(); } @@ -385,7 +406,8 @@ public: const std::string &device) : desc{ model, weights, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Load, true, {}, {}, {}, 1u, - {}, {}, {}, {}}, + {}, {}, {}, {}, + detail::ParamDesc::InferMode::Async }, m_tag(tag) { }; @@ -403,7 +425,8 @@ public: const std::string &device) : desc{ model, {}, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Import, true, {}, {}, {}, 1u, - {}, {}, {}, {}}, + {}, {}, {}, {}, + detail::ParamDesc::InferMode::Async }, m_tag(tag) { }; @@ -476,6 +499,12 @@ public: return *this; } + /** @see ie::Params::cfgInferAPI */ + Params& cfgInferMode(detail::ParamDesc::InferMode mode) { + desc.mode = mode; + return *this; + } + // BEGIN(G-API's network parametrization API) GBackend backend() const { return cv::gapi::ie::backend(); } std::string tag() const { return m_tag; } diff --git a/modules/gapi/src/backends/ie/giebackend.cpp b/modules/gapi/src/backends/ie/giebackend.cpp index 10a5dcab36..af0c97108f 100644 --- a/modules/gapi/src/backends/ie/giebackend.cpp +++ b/modules/gapi/src/backends/ie/giebackend.cpp @@ -375,6 +375,12 @@ struct IEUnit { params.vpl_preproc_ctx.value()); GAPI_LOG_INFO(nullptr, "VPP preproc created successfuly"); } + + if (params.mode == cv::gapi::ie::detail::ParamDesc::InferMode::Sync && + params.nireq != 1u) { + throw std::logic_error( + "Failed: ParamDesc::InferMode::Sync works only with nireq equal to 1."); + } } // This method is [supposed to be] called at Island compilation stage @@ -911,7 +917,8 @@ void AsyncInferExecutor::callback(IInferExecutor::Task task, class cv::gimpl::ie::RequestPool { public: - explicit RequestPool(std::vector&& requests); + explicit RequestPool(cv::gapi::ie::detail::ParamDesc::InferMode mode, + std::vector&& requests); IInferExecutor::Ptr getIdleRequest(); void waitAll(); @@ -929,11 +936,23 @@ void cv::gimpl::ie::RequestPool::release(const size_t id) { } // RequestPool implementation ////////////////////////////////////////////// -cv::gimpl::ie::RequestPool::RequestPool(std::vector&& requests) { +cv::gimpl::ie::RequestPool::RequestPool(cv::gapi::ie::detail::ParamDesc::InferMode mode, + std::vector&& requests) { for (size_t i = 0; i < requests.size(); ++i) { - m_requests.emplace_back( - std::make_shared(std::move(requests[i]), - std::bind(&RequestPool::release, this, i))); + IInferExecutor::Ptr iexec = nullptr; + switch (mode) { + case cv::gapi::ie::detail::ParamDesc::InferMode::Async: + iexec = std::make_shared(std::move(requests[i]), + std::bind(&RequestPool::release, this, i)); + break; + case cv::gapi::ie::detail::ParamDesc::InferMode::Sync: + iexec = std::make_shared(std::move(requests[i]), + std::bind(&RequestPool::release, this, i)); + break; + default: + GAPI_Assert(false && "Unsupported ParamDesc::InferMode"); + } + m_requests.emplace_back(std::move(iexec)); } setup(); } @@ -974,7 +993,7 @@ cv::gimpl::ie::GIEExecutable::GIEExecutable(const ade::Graph &g, if (this_nh == nullptr) { this_nh = nh; this_iec = iem.metadata(this_nh).get().compile(); - m_reqPool.reset(new RequestPool(this_iec.createInferRequests())); + m_reqPool.reset(new RequestPool(this_iec.params.mode, this_iec.createInferRequests())); } else util::throw_error(std::logic_error("Multi-node inference is not supported!"));