mirror/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2024-11-27 20:50:25 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #21688 from sivanov-work:vpp_ie_integration

Browse Source 
G-API: VPP preprocessing GIEBackend integration

* Add ROI in VPP prepro

* Apply comments

* Integration to IE

* Removed extra invocations

* Fix no-vpl compilation

* Fix compilations

* Apply comments

* Use thin wrapper for device & ctx

* Implement Device/Context constructor functions

* Apply samples comment

* Fix compilation

* Fix compilation

* Fix build

* Separate Device&Context from selector, apply other comments

* Fix typo

* Intercept OV exception with pretty print out
This commit is contained in:
Sergey Ivanov 2022-04-01 13:06:47 +03:00 committed by GitHub
parent 7b582b71ba
commit f3945fbddb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 715 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
modules/gapi/CMakeLists.txt
View File

@ -204,6 +204,7 @@ set(gapi_srcs
src/streaming/onevpl/engine/preproc/preproc_engine.cpp
src/streaming/onevpl/engine/preproc/preproc_session.cpp
src/streaming/onevpl/engine/preproc/preproc_dispatcher.cpp
src/streaming/onevpl/engine/preproc_engine_interface.cpp
src/streaming/onevpl/demux/async_mfp_demux_data_provider.cpp
src/streaming/onevpl/data_provider_dispatcher.cpp

19
modules/gapi/include/opencv2/gapi/infer/ie.hpp
View File

@ -20,6 +20,7 @@
#include <opencv2/core/cvdef.h> // GAPI_EXPORTS
#include <opencv2/gapi/gkernel.hpp> // GKernelPackage
#include <opencv2/gapi/infer.hpp> // Generic
#include <opencv2/gapi/streaming/onevpl/accel_types.hpp> // Preproc Dev & Ctx
namespace cv {
namespace gapi {
@ -84,6 +85,9 @@ struct ParamDesc {
// have 2D (Layout::NC) input and if the first dimension not equal to 1
// net.setBatchSize(1) will overwrite it.
cv::optional<size_t> batch_size;
cv::optional<cv::gapi::wip::onevpl::Device> vpl_preproc_device;
cv::optional<cv::gapi::wip::onevpl::Context> vpl_preproc_ctx;
};
} // namespace detail
@ -126,6 +130,8 @@ public:
, {}
, 1u
, {}
, {}
, {}
, {}} {
};
@ -148,6 +154,8 @@ public:
, {}
, 1u
, {}
, {}
, {}
, {}} {
};
@ -336,6 +344,13 @@ public:
return *this;
}
Params<Net>& cfgPreprocessingParams(const cv::gapi::wip::onevpl::Device &device,
const cv::gapi::wip::onevpl::Context &ctx) {
desc.vpl_preproc_device = cv::util::make_optional(device);
desc.vpl_preproc_ctx = cv::util::make_optional(ctx);
return *this;
}
// BEGIN(G-API's network parametrization API)
GBackend backend() const { return cv::gapi::ie::backend(); }
std::string tag() const { return Net::tag(); }
@ -370,7 +385,7 @@ public:
const std::string &device)
: desc{ model, weights, device, {}, {}, {}, 0u, 0u,
detail::ParamDesc::Kind::Load, true, {}, {}, {}, 1u,
{}, {}},
{}, {}, {}, {}},
m_tag(tag) {
};
@ -388,7 +403,7 @@ public:
const std::string &device)
: desc{ model, {}, device, {}, {}, {}, 0u, 0u,
detail::ParamDesc::Kind::Import, true, {}, {}, {}, 1u,
{}, {}},
{}, {}, {}, {}},
m_tag(tag) {
};

72
modules/gapi/include/opencv2/gapi/streaming/onevpl/accel_types.hpp Normal file
View File

@ -0,0 +1,72 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
//
// Copyright (C) 2022 Intel Corporation
#ifndef GAPI_STREAMING_ONEVPL_ACCEL_TYPES_HPP
#define GAPI_STREAMING_ONEVPL_ACCEL_TYPES_HPP
#include <limits>
#include <string>
#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS
namespace cv {
namespace gapi {
namespace wip {
namespace onevpl {
enum class AccelType: uint8_t {
HOST,
DX11,
LAST_VALUE = std::numeric_limits<uint8_t>::max()
};
GAPI_EXPORTS const char* to_cstring(AccelType type);
struct IDeviceSelector;
struct GAPI_EXPORTS Device {
friend struct IDeviceSelector;
using Ptr = void*;
~Device();
const std::string& get_name() const;
Ptr get_ptr() const;
AccelType get_type() const;
private:
Device(Ptr device_ptr, const std::string& device_name,
AccelType device_type);
std::string name;
Ptr ptr;
AccelType type;
};
struct GAPI_EXPORTS Context {
friend struct IDeviceSelector;
using Ptr = void*;
~Context();
Ptr get_ptr() const;
AccelType get_type() const;
private:
Context(Ptr ctx_ptr, AccelType ctx_type);
Ptr ptr;
AccelType type;
};
GAPI_EXPORTS Device create_host_device();
GAPI_EXPORTS Context create_host_context();
GAPI_EXPORTS Device create_dx11_device(Device::Ptr device_ptr,
const std::string& device_name);
GAPI_EXPORTS Context create_dx11_context(Context::Ptr ctx_ptr);
} // namespace onevpl
} // namespace wip
} // namespace gapi
} // namespace cv
#endif // GAPI_STREAMING_ONEVPL_ACCEL_TYPES_HPP

43
modules/gapi/include/opencv2/gapi/streaming/onevpl/device_selector_interface.hpp
View File

@ -12,53 +12,12 @@
#include <string>
#include <vector>
#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS
#include <opencv2/gapi/streaming/onevpl/accel_types.hpp>
namespace cv {
namespace gapi {
namespace wip {
namespace onevpl {
enum class AccelType: uint8_t {
HOST,
DX11,
LAST_VALUE = std::numeric_limits<uint8_t>::max()
};
GAPI_EXPORTS const char* to_cstring(AccelType type);
struct IDeviceSelector;
struct GAPI_EXPORTS Device {
friend struct IDeviceSelector;
using Ptr = void*;
~Device();
const std::string& get_name() const;
Ptr get_ptr() const;
AccelType get_type() const;
private:
Device(Ptr device_ptr, const std::string& device_name,
AccelType device_type);
std::string name;
Ptr ptr;
AccelType type;
};
struct GAPI_EXPORTS Context {
friend struct IDeviceSelector;
using Ptr = void*;
~Context();
Ptr get_ptr() const;
AccelType get_type() const;
private:
Context(Ptr ctx_ptr, AccelType ctx_type);
Ptr ptr;
AccelType type;
};
struct GAPI_EXPORTS IDeviceSelector {
using Ptr = std::shared_ptr<IDeviceSelector>;

4
modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp
View File

@ -46,6 +46,10 @@ public:
void* accel_device_ptr,
void* accel_ctx_ptr);
GSource(const std::string& filePath,
const CfgParams& cfg_params,
const Device &device, const Context &ctx);
GSource(const std::string& filePath,
const CfgParams& cfg_params,
std::shared_ptr<IDeviceSelector> selector);

187
modules/gapi/samples/onevpl_infer_single_roi.cpp
View File

@ -11,7 +11,6 @@
#include <opencv2/gapi/infer/ie.hpp>
#include <opencv2/gapi/render.hpp>
#include <opencv2/gapi/streaming/onevpl/source.hpp>
#include <opencv2/gapi/streaming/onevpl/data_provider_interface.hpp>
#include <opencv2/highgui.hpp> // CommandLineParser
#include <opencv2/gapi/infer/parsers.hpp>
@ -46,26 +45,45 @@ const std::string keys =
"{ cfg_params | <prop name>:<value>;<prop name>:<value> | Semicolon separated list of oneVPL mfxVariants which is used for configuring source (see `MFXSetConfigFilterProperty` by https://spec.oneapi.io/versions/latest/elements/oneVPL/source/index.html) }"
"{ streaming_queue_capacity | 1 | Streaming executor queue capacity. Calculated automaticaly if 0 }"
"{ frames_pool_size | 0 | OneVPL source applies this parameter as preallocated frames pool size}"
"{ vpp_frames_pool_size | 0 | OneVPL source applies this parameter as preallocated frames pool size for VPP preprocessing results}";
"{ vpp_frames_pool_size | 0 | OneVPL source applies this parameter as preallocated frames pool size for VPP preprocessing results}"
"{ roi | -1,-1,-1,-1 | Region of interest (ROI) to use for inference. Identified automatically when not set }";
namespace {
bool is_gpu(const std::string &device_name) {
return device_name.find("GPU") != std::string::npos;
}
std::string get_weights_path(const std::string &model_path) {
const auto EXT_LEN = 4u;
const auto sz = model_path.size();
CV_Assert(sz > EXT_LEN);
GAPI_Assert(sz > EXT_LEN);
auto ext = model_path.substr(sz - EXT_LEN);
std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c){
return static_cast<unsigned char>(std::tolower(c));
});
CV_Assert(ext == ".xml");
GAPI_Assert(ext == ".xml");
return model_path.substr(0u, sz - EXT_LEN) + ".bin";
}
// TODO: It duplicates infer_single_roi sample
cv::util::optional<cv::Rect> parse_roi(const std::string &rc) {
cv::Rect rv;
char delim[3];
std::stringstream is(rc);
is >> rv.x >> delim[0] >> rv.y >> delim[1] >> rv.width >> delim[2] >> rv.height;
if (is.bad()) {
return cv::util::optional<cv::Rect>(); // empty value
}
const auto is_delim = [](char c) {
return c == ',';
};
if (!std::all_of(std::begin(delim), std::end(delim), is_delim)) {
return cv::util::optional<cv::Rect>(); // empty value
}
if (rv.x < 0 || rv.y < 0 || rv.width <= 0 || rv.height <= 0) {
return cv::util::optional<cv::Rect>(); // empty value
}
return cv::util::make_optional(std::move(rv));
}
#ifdef HAVE_INF_ENGINE
#ifdef HAVE_DIRECTX
#ifdef HAVE_D3D11
@ -127,9 +145,15 @@ using GRect = cv::GOpaque<cv::Rect>;
using GSize = cv::GOpaque<cv::Size>;
using GPrims = cv::GArray<cv::gapi::wip::draw::Prim>;
G_API_OP(LocateROI, <GRect(GSize, std::reference_wrapper<const std::string>)>, "sample.custom.locate-roi") {
static cv::GOpaqueDesc outMeta(const cv::GOpaqueDesc &,
std::reference_wrapper<const std::string>) {
G_API_OP(ParseSSD, <GDetections(cv::GMat, GRect, GSize)>, "sample.custom.parse-ssd") {
static cv::GArrayDesc outMeta(const cv::GMatDesc &, const cv::GOpaqueDesc &, const cv::GOpaqueDesc &) {
return cv::empty_array_desc();
}
};
// TODO: It duplicates infer_single_roi sample
G_API_OP(LocateROI, <GRect(GSize)>, "sample.custom.locate-roi") {
static cv::GOpaqueDesc outMeta(const cv::GOpaqueDesc &) {
return cv::empty_gopaque_desc();
}
};
@ -151,12 +175,9 @@ GAPI_OCV_KERNEL(OCVLocateROI, LocateROI) {
// the most convenient aspect ratio for detectors to use)
static void run(const cv::Size& in_size,
std::reference_wrapper<const std::string> device_id_ref,
cv::Rect &out_rect) {
// Identify the central point & square size (- some padding)
// NB: GPU plugin in InferenceEngine doesn't support ROI at now
if (!is_gpu(device_id_ref.get())) {
const auto center = cv::Point{in_size.width/2, in_size.height/2};
auto sqside = std::min(in_size.width, in_size.height);
@ -166,14 +187,6 @@ GAPI_OCV_KERNEL(OCVLocateROI, LocateROI) {
, sqside
, sqside
};
} else {
// use whole frame for GPU device
out_rect = cv::Rect{ 0
, 0
, in_size.width
, in_size.height
};
}
}
};
@ -194,6 +207,55 @@ GAPI_OCV_KERNEL(OCVBBoxes, BBoxes) {
}
};
GAPI_OCV_KERNEL(OCVParseSSD, ParseSSD) {
static void run(const cv::Mat &in_ssd_result,
const cv::Rect &in_roi,
const cv::Size &in_parent_size,
std::vector<cv::Rect> &out_objects) {
const auto &in_ssd_dims = in_ssd_result.size;
GAPI_Assert(in_ssd_dims.dims() == 4u);
const int MAX_PROPOSALS = in_ssd_dims[2];
const int OBJECT_SIZE = in_ssd_dims[3];
GAPI_Assert(OBJECT_SIZE == 7); // fixed SSD object size
const cv::Size up_roi = in_roi.size();
const cv::Rect surface({0,0}, in_parent_size);
out_objects.clear();
const float *data = in_ssd_result.ptr<float>();
for (int i = 0; i < MAX_PROPOSALS; i++) {
const float image_id = data[i * OBJECT_SIZE + 0];
const float label = data[i * OBJECT_SIZE + 1];
const float confidence = data[i * OBJECT_SIZE + 2];
const float rc_left = data[i * OBJECT_SIZE + 3];
const float rc_top = data[i * OBJECT_SIZE + 4];
const float rc_right = data[i * OBJECT_SIZE + 5];
const float rc_bottom = data[i * OBJECT_SIZE + 6];
(void) label; // unused
if (image_id < 0.f) {
break; // marks end-of-detections
}
if (confidence < 0.5f) {
continue; // skip objects with low confidence
}
// map relative coordinates to the original image scale
// taking the ROI into account
cv::Rect rc;
rc.x = static_cast<int>(rc_left * up_roi.width);
rc.y = static_cast<int>(rc_top * up_roi.height);
rc.width = static_cast<int>(rc_right * up_roi.width) - rc.x;
rc.height = static_cast<int>(rc_bottom * up_roi.height) - rc.y;
rc.x += in_roi.x;
rc.y += in_roi.y;
out_objects.emplace_back(rc & surface);
}
}
};
} // namespace custom
namespace cfg {
@ -212,6 +274,7 @@ int main(int argc, char *argv[]) {
// get file name
const auto file_path = cmd.get<std::string>("input");
const auto output = cmd.get<std::string>("output");
const auto opt_roi = parse_roi(cmd.get<std::string>("roi"));
const auto face_model_path = cmd.get<std::string>("facem");
const auto streaming_queue_capacity = cmd.get<uint32_t>("streaming_queue_capacity");
const auto source_decode_queue_capacity = cmd.get<uint32_t>("frames_pool_size");
@ -259,8 +322,8 @@ int main(int argc, char *argv[]) {
// GAPI InferenceEngine backend to provide interoperability with onevpl::GSource
// So GAPI InferenceEngine backend and onevpl::GSource MUST share the same
// device and context
void* accel_device_ptr = nullptr;
void* accel_ctx_ptr = nullptr;
cv::util::optional<cv::gapi::wip::onevpl::Device> accel_device;
cv::util::optional<cv::gapi::wip::onevpl::Context> accel_ctx;
#ifdef HAVE_INF_ENGINE
#ifdef HAVE_DIRECTX
@ -268,7 +331,7 @@ int main(int argc, char *argv[]) {
auto dx11_dev = createCOMPtrGuard<ID3D11Device>();
auto dx11_ctx = createCOMPtrGuard<ID3D11DeviceContext>();
if (is_gpu(device_id)) {
if (device_id.find("GPU") != std::string::npos) {
auto adapter_factory = createCOMPtrGuard<IDXGIFactory>();
{
IDXGIFactory* out_factory = nullptr;
@ -302,8 +365,13 @@ int main(int argc, char *argv[]) {
}
std::tie(dx11_dev, dx11_ctx) = create_device_with_ctx(intel_adapter.get());
accel_device_ptr = reinterpret_cast<void*>(dx11_dev.get());
accel_ctx_ptr = reinterpret_cast<void*>(dx11_ctx.get());
accel_device = cv::util::make_optional(
cv::gapi::wip::onevpl::create_dx11_device(
reinterpret_cast<void*>(dx11_dev.get()),
device_id));
accel_ctx = cv::util::make_optional(
cv::gapi::wip::onevpl::create_dx11_context(
reinterpret_cast<void*>(dx11_ctx.get())));
// put accel type description for VPL source
source_cfgs.push_back(cfg::create_from_string(
@ -315,9 +383,10 @@ int main(int argc, char *argv[]) {
#endif // HAVE_D3D11
#endif // HAVE_DIRECTX
// set ctx_config for GPU device only - no need in case of CPU device type
if (is_gpu(device_id)) {
if (accel_device.has_value() &&
accel_device.value().get_name().find("GPU") != std::string::npos) {
InferenceEngine::ParamMap ctx_config({{"CONTEXT_TYPE", "VA_SHARED"},
{"VA_DEVICE", accel_device_ptr} });
{"VA_DEVICE", accel_device.value().get_ptr()} });
face_net.cfgContextParams(ctx_config);
// NB: consider NV12 surface because it's one of native GPU image format
@ -325,8 +394,16 @@ int main(int argc, char *argv[]) {
}
#endif // HAVE_INF_ENGINE
// turn on preproc
if (accel_device.has_value() && accel_ctx.has_value()) {
face_net.cfgPreprocessingParams(accel_device.value(),
accel_ctx.value());
std::cout << "enforce VPP preprocessing on " << device_id << std::endl;
}
auto kernels = cv::gapi::kernels
< custom::OCVLocateROI
, custom::OCVParseSSD
, custom::OCVBBoxes>();
auto networks = cv::gapi::networks(face_net);
auto face_detection_args = cv::compile_args(networks, kernels);
@ -335,13 +412,12 @@ int main(int argc, char *argv[]) {
}
// Create source
cv::Ptr<cv::gapi::wip::IStreamSource> cap;
cv::gapi::wip::IStreamSource::Ptr cap;
try {
if (is_gpu(device_id)) {
if (accel_device.has_value() && accel_ctx.has_value()) {
cap = cv::gapi::wip::make_onevpl_src(file_path, source_cfgs,
device_id,
accel_device_ptr,
accel_ctx_ptr);
accel_device.value(),
accel_ctx.value());
} else {
cap = cv::gapi::wip::make_onevpl_src(file_path, source_cfgs);
}
@ -353,29 +429,35 @@ int main(int argc, char *argv[]) {
cv::GMetaArg descr = cap->descr_of();
auto frame_descr = cv::util::get<cv::GFrameDesc>(descr);
cv::GOpaque<cv::Rect> in_roi;
auto inputs = cv::gin(cap);
// Now build the graph
cv::GFrame in;
auto size = cv::gapi::streaming::size(in);
auto roi = custom::LocateROI::on(size, std::cref(device_id));
auto blob = cv::gapi::infer<custom::FaceDetector>(in);
cv::GArray<cv::Rect> rcs = cv::gapi::parseSSD(blob, size, 0.5f, true, true);
auto out_frame = cv::gapi::wip::draw::renderFrame(in, custom::BBoxes::on(rcs, roi));
auto out = cv::gapi::streaming::BGR(out_frame);
cv::GStreamingCompiled pipeline;
try {
pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out))
.compileStreaming(std::move(face_detection_args));
} catch (const std::exception& ex) {
std::cerr << "Exception occured during pipeline construction: " << ex.what() << std::endl;
return -1;
auto graph_inputs = cv::GIn(in);
if (!opt_roi.has_value()) {
// Automatically detect ROI to infer. Make it output parameter
std::cout << "ROI is not set or invalid. Locating it automatically"
<< std::endl;
in_roi = custom::LocateROI::on(size);
} else {
// Use the value provided by user
std::cout << "Will run inference for static region "
<< opt_roi.value()
<< " only"
<< std::endl;
graph_inputs += cv::GIn(in_roi);
inputs += cv::gin(opt_roi.value());
}
auto blob = cv::gapi::infer<custom::FaceDetector>(in_roi, in);
cv::GArray<cv::Rect> rcs = custom::ParseSSD::on(blob, in_roi, size);
auto out_frame = cv::gapi::wip::draw::renderFrame(in, custom::BBoxes::on(rcs, in_roi));
auto out = cv::gapi::streaming::BGR(out_frame);
cv::GStreamingCompiled pipeline = cv::GComputation(std::move(graph_inputs), cv::GOut(out)) // and move here
.compileStreaming(std::move(face_detection_args));
// The execution part
// TODO USE may set pool size from outside and set queue_capacity size,
// compile arg: cv::gapi::streaming::queue_capacity
pipeline.setSource(std::move(cap));
pipeline.setSource(std::move(inputs));
pipeline.start();
size_t frames = 0u;
@ -384,7 +466,7 @@ int main(int argc, char *argv[]) {
if (!output.empty() && !writer.isOpened()) {
const auto sz = cv::Size{frame_descr.size.width, frame_descr.size.height};
writer.open(output, cv::VideoWriter::fourcc('M','J','P','G'), 25.0, sz);
CV_Assert(writer.isOpened());
GAPI_Assert(writer.isOpened());
}
cv::Mat outMat;
@ -399,6 +481,7 @@ int main(int argc, char *argv[]) {
}
tm.stop();
std::cout << "Processed " << frames << " frames" << " (" << frames / tm.getTimeSec() << " FPS)" << std::endl;
return 0;
}

2
modules/gapi/samples/pipeline_modeling_tool.cpp
View File

@ -10,7 +10,9 @@
#include <opencv2/core/utils/filesystem.hpp>
#if defined(_WIN32)
#define NOMINMAX
#include <windows.h>
#undef NOMINMAX
#endif
#include "pipeline_modeling_tool/dummy_source.hpp"

252
modules/gapi/src/backends/ie/giebackend.cpp
View File

@ -64,6 +64,9 @@ template<typename T> using QueueClass = cv::gapi::own::concurrent_bounded_queue<
#include "utils/itt.hpp"
#include "streaming/onevpl/engine/preproc_engine_interface.hpp"
#include "streaming/onevpl/engine/preproc/preproc_dispatcher.hpp"
namespace IE = InferenceEngine;
namespace {
@ -261,12 +264,36 @@ struct IEUnit {
InferenceEngine::RemoteContext::Ptr rctx = nullptr;
std::shared_ptr<cv::gapi::wip::IPreprocEngine> preproc_engine_impl;
// FIXME: Unlike loadNetwork case, importNetwork requires that preprocessing
// should be passed as ExecutableNetwork::SetBlob method, so need to collect
// and store this information at the graph compilation stage (outMeta) and use in runtime.
using PreProcMap = std::unordered_map<std::string, IE::PreProcessInfo>;
PreProcMap preproc_map;
// NEW FIXME: Need to aggregate getInputInfo & GetInputInfo from network
// into generic wrapper and invoke it at once in single place instead of
// analyzing ParamDesc::Kind::Load/Import every time when we need to get access
// for network info.
// In term of introducing custom VPP/VPL preprocessing functionality
// It was decided to use GFrameDesc as such aggregated network info with limitation
// that VPP/VPL produces cv::MediaFrame only. But it should be not considered as
// final solution
class InputFramesDesc {
using input_name_type = std::string;
using description_type = cv::GFrameDesc;
std::map<input_name_type, description_type> map;
public:
static bool is_applicable(const cv::GMetaArg &mm);
const description_type &get_param(const input_name_type &input) const;
void set_param(const input_name_type &input,
const IE::TensorDesc& desc);
};
InputFramesDesc net_input_params;
explicit IEUnit(const cv::gapi::ie::detail::ParamDesc &pp)
: params(pp) {
InferenceEngine::ParamMap* ctx_params =
@ -336,6 +363,17 @@ struct IEUnit {
} else {
cv::util::throw_error(std::logic_error("Unsupported ParamDesc::Kind"));
}
using namespace cv::gapi::wip::onevpl;
if (params.vpl_preproc_device.has_value() && params.vpl_preproc_ctx.has_value()) {
using namespace cv::gapi::wip;
GAPI_LOG_INFO(nullptr, "VPP preproc creation requested");
preproc_engine_impl =
IPreprocEngine::create_preproc_engine<onevpl::VPPPreprocDispatcher>(
params.vpl_preproc_device.value(),
params.vpl_preproc_ctx.value());
GAPI_LOG_INFO(nullptr, "VPP preproc created successfuly");
}
}
// This method is [supposed to be] called at Island compilation stage
@ -354,6 +392,39 @@ struct IEUnit {
}
};
bool IEUnit::InputFramesDesc::is_applicable(const cv::GMetaArg &mm) {
return cv::util::holds_alternative<cv::GFrameDesc>(mm);
}
const IEUnit::InputFramesDesc::description_type &
IEUnit::InputFramesDesc::get_param(const input_name_type &input) const {
auto it = map.find(input);
GAPI_Assert(it != map.end() && "No appropriate input is found in InputFramesDesc");
return it->second;
}
void IEUnit::InputFramesDesc::set_param(const input_name_type &input,
const IE::TensorDesc& desc) {
description_type ret;
ret.fmt = cv::MediaFormat::NV12;
const InferenceEngine::SizeVector& inDims = desc.getDims();
auto layout = desc.getLayout();
GAPI_LOG_DEBUG(nullptr, "network input: " << input <<
", tensor dims: " << inDims[0] << ", " << inDims[1] <<
", " << inDims[2] << ", " << inDims[3]);
if (layout != InferenceEngine::NHWC && layout != InferenceEngine::NCHW) {
GAPI_LOG_WARNING(nullptr, "Unsupported layout for VPP preproc: " << layout <<
", input name: " << input);
GAPI_Assert(false && "Unsupported layout for VPP preproc");
}
GAPI_Assert(inDims.size() == 4u);
ret.size.width = static_cast<int>(inDims[3]);
ret.size.height = static_cast<int>(inDims[2]);
auto res = map.emplace(input, ret);
GAPI_Assert(res.second && "Duplicated input info in InputFramesDesc are not allowable");
}
class IECallContext
{
public:
@ -396,6 +467,9 @@ public:
// To store exception appeared in callback.
std::exception_ptr eptr;
using req_key_t = void*;
cv::MediaFrame* prepareKeepAliveFrameSlot(req_key_t key);
size_t releaseKeepAliveFrame(req_key_t key);
private:
cv::detail::VectorRef& outVecRef(std::size_t idx);
@ -417,6 +491,10 @@ private:
// Input parameters passed to an inference operation.
cv::GArgs m_args;
cv::GShapes m_in_shapes;
// keep alive preprocessed frames
std::mutex keep_alive_frames_mutex;
std::unordered_map<req_key_t, cv::MediaFrame> keep_alive_pp_frames;
};
IECallContext::IECallContext(const IEUnit & unit,
@ -516,6 +594,35 @@ cv::GArg IECallContext::packArg(const cv::GArg &arg) {
}
}
cv::MediaFrame* IECallContext::prepareKeepAliveFrameSlot(req_key_t key) {
std::lock_guard<std::mutex> lock(keep_alive_frames_mutex);
return &keep_alive_pp_frames[key];
}
size_t IECallContext::releaseKeepAliveFrame(req_key_t key) {
size_t elapsed_count = 0;
void *prev_slot = nullptr;
// NB: release MediaFrame previously captured by prepareKeepAliveFrameSlot
// We must capture it to keep a reference counter on inner media adapter
// to ensure that frame resource would be locked until inference done.
// Otherwise decoder could seized this frame resource as free/unlocked resource
// from resource pool
// Current function just take a unique frame `key` and overwrite stored
// actual frame by empty frame
{
std::lock_guard<std::mutex> lock(keep_alive_frames_mutex);
auto ka_frame_it = keep_alive_pp_frames.find(key);
if (ka_frame_it != keep_alive_pp_frames.end()) {
prev_slot = &ka_frame_it->second;
ka_frame_it->second = cv::MediaFrame();
}
elapsed_count = keep_alive_pp_frames.size();
}
GAPI_LOG_DEBUG(nullptr, "Release keep alive frame, slot: " << prev_slot <<
", reserved frames count: " << elapsed_count);
return elapsed_count;
}
struct IECallable {
static const char *name() { return "IERequestCallable"; }
using Run = std::function<void(std::shared_ptr<IECallContext>, cv::gimpl::ie::RequestPool&)>;
@ -552,11 +659,65 @@ using GConstGIEModel = ade::ConstTypedGraph
, IECallable
>;
inline IE::Blob::Ptr extractRemoteBlob(IECallContext& ctx, std::size_t i) {
cv::MediaFrame preprocess_frame_impl(cv::MediaFrame &&in_frame, const std::string &layer_name,
IECallContext& ctx,
const cv::util::optional<cv::Rect> &opt_roi,
cv::MediaFrame* out_keep_alive_frame,
bool* out_is_preprocessed) {
cv::util::optional<cv::gapi::wip::pp_params> param =
ctx.uu.preproc_engine_impl->is_applicable(in_frame);
if (param.has_value()) {
GAPI_LOG_DEBUG(nullptr, "VPP preprocessing for decoded remote frame will be used");
cv::GFrameDesc expected_net_input_descr =
ctx.uu.net_input_params.get_param(layer_name);
// TODO: Find a better place to configure media format for GPU
// adjust color conversion to NV12 according to OV GPU limitation
if(ctx.uu.params.device_id.find("GPU") != std::string::npos &&
ctx.uu.rctx) {
auto it = ctx.uu.params.config.find(std::string("GPU_NV12_TWO_INPUTS"));
if (it != ctx.uu.params.config.end()) {
if (it->second == "YES") {
GAPI_LOG_DEBUG(nullptr, "Adjust preprocessing GPU media format to NV12");
expected_net_input_descr.fmt = cv::MediaFormat::NV12;
}
}
}
cv::gapi::wip::pp_session pp_sess =
ctx.uu.preproc_engine_impl->initialize_preproc(param.value(),
expected_net_input_descr);
in_frame = ctx.uu.preproc_engine_impl->run_sync(pp_sess, in_frame, opt_roi);
if (out_keep_alive_frame != nullptr) {
GAPI_LOG_DEBUG(nullptr, "remember preprocessed remote frame to keep it busy from reuse, slot: " <<
out_keep_alive_frame);
*out_keep_alive_frame = in_frame;
}
if (out_is_preprocessed) {
*out_is_preprocessed = true;
}
} // otherwise it is not suitable frame, then check on other preproc backend or rely on IE plugin
return std::move(in_frame);
}
inline IE::Blob::Ptr extractRemoteBlob(IECallContext& ctx, std::size_t i,
const std::string &layer_name,
const cv::util::optional<cv::Rect> &opt_roi,
cv::MediaFrame* out_keep_alive_frame,
bool* out_is_preprocessed) {
GAPI_Assert(ctx.inShape(i) == cv::GShape::GFRAME &&
"Remote blob is supported for MediaFrame only");
cv::MediaFrame frame = ctx.inFrame(i);
if (ctx.uu.preproc_engine_impl) {
GAPI_LOG_DEBUG(nullptr, "Try to use preprocessing for decoded remote frame in remote ctx");
frame = preprocess_frame_impl(std::move(frame), layer_name, ctx, opt_roi,
out_keep_alive_frame, out_is_preprocessed);
}
cv::util::any any_blob_params = ctx.inFrame(i).blobParams();
// Request params for result frame whatever it got preprocessed or not
cv::util::any any_blob_params = frame.blobParams();
using ParamType = std::pair<InferenceEngine::TensorDesc, InferenceEngine::ParamMap>;
using NV12ParamType = std::pair<ParamType, ParamType>;
@ -582,14 +743,24 @@ inline IE::Blob::Ptr extractRemoteBlob(IECallContext& ctx, std::size_t i) {
inline IE::Blob::Ptr extractBlob(IECallContext& ctx,
std::size_t i,
cv::gapi::ie::TraitAs hint) {
cv::gapi::ie::TraitAs hint,
const std::string& layer_name,
const cv::util::optional<cv::Rect> &opt_roi,
cv::MediaFrame* out_keep_alive_frame = nullptr,
bool* out_is_preprocessed = nullptr) {
if (ctx.uu.rctx != nullptr) {
return extractRemoteBlob(ctx, i);
return extractRemoteBlob(ctx, i, layer_name, opt_roi,
out_keep_alive_frame, out_is_preprocessed);
}
switch (ctx.inShape(i)) {
case cv::GShape::GFRAME: {
const auto& frame = ctx.inFrame(i);
auto frame = ctx.inFrame(i);
if (ctx.uu.preproc_engine_impl) {
GAPI_LOG_DEBUG(nullptr, "Try to use preprocessing for decoded frame in local ctx");
frame = preprocess_frame_impl(std::move(frame), layer_name, ctx, opt_roi,
out_keep_alive_frame, out_is_preprocessed);
}
ctx.views.emplace_back(new cv::MediaFrame::View(frame.access(cv::MediaFrame::Access::R)));
return wrapIE(*(ctx.views.back()), frame.desc());
}
@ -626,10 +797,20 @@ static void setROIBlob(InferenceEngine::InferRequest& req,
const IECallContext& ctx) {
if (ctx.uu.params.device_id.find("GPU") != std::string::npos &&
ctx.uu.rctx) {
GAPI_LOG_WARNING(nullptr, "ROI blob creation for device_id: " <<
ctx.uu.params.device_id << ", layer: " << layer_name <<
"is not supported yet");
GAPI_Assert(false && "Unsupported ROI blob creation for GPU remote context");
try {
// NB: make_shared_blob() cannot work with GPU NV12 & ROI at the moment.
// OpenVINO produces exception with unsupported status.
// To do not encounter with silent crash situation we should catch OV exception
// and suggest to avoid this problem by using inner preprocessing feature.
// VPP/VPL proprocessing are supported at the moment
setBlob(req, layer_name, IE::make_shared_blob(blob, toIE(roi)), ctx);
} catch (const std::exception &ex) {
GAPI_LOG_WARNING(nullptr, "cannot set ROI blob for layer: " << layer_name <<
", reason:\n" << ex.what() <<
"\nTry using self GAPI preprocessing feature: "
" Check method `cfgPreprocessingParams` in `cv::gapi::ie::Params`");
throw;
}
} else {
setBlob(req, layer_name, IE::make_shared_blob(blob, toIE(roi)), ctx);
}
@ -975,6 +1156,8 @@ static void PostOutputs(InferenceEngine::InferRequest &request,
ctx->out.meta(output, ctx->input(0).meta);
ctx->out.post(std::move(output), ctx->eptr);
}
ctx->releaseKeepAliveFrame(&request);
}
class PostOutputsList {
@ -1088,6 +1271,12 @@ struct Infer: public cv::detail::KernelTag {
if (isApplicableForResize(ii->getTensorDesc())) {
ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR);
}
// NB: configure input param for further preproc
if (uu.net_input_params.is_applicable(mm)) {
const_cast<IEUnit::InputFramesDesc &>(uu.net_input_params)
.set_param(input_name, ii->getTensorDesc());
}
}
// FIXME: This isn't the best place to call reshape function.
@ -1107,6 +1296,12 @@ struct Infer: public cv::detail::KernelTag {
auto ii = inputs.at(input_name);
const auto & mm = std::get<1>(it);
non_const_prepm->emplace(input_name, configurePreProcInfo(ii, mm));
// NB: configure input param for further preproc
if (uu.net_input_params.is_applicable(mm)) {
const_cast<IEUnit::InputFramesDesc &>(uu.net_input_params)
.set_param(input_name, ii->getTensorDesc());
}
}
}
@ -1145,7 +1340,9 @@ struct Infer: public cv::detail::KernelTag {
(layout == IE::Layout::NCHW || layout == IE::Layout::NHWC)
? cv::gapi::ie::TraitAs::IMAGE : cv::gapi::ie::TraitAs::TENSOR;
IE::Blob::Ptr this_blob = extractBlob(*ctx, i, hint);
IE::Blob::Ptr this_blob = extractBlob(*ctx, i, hint,
layer_name,
cv::util::optional<cv::Rect>{});
setBlob(req, layer_name, this_blob, *ctx);
}
// FIXME: Should it be done by kernel ?
@ -1200,6 +1397,12 @@ struct InferROI: public cv::detail::KernelTag {
if (!input_reshape_table.empty()) {
const_cast<IE::CNNNetwork *>(&uu.net)->reshape(input_reshape_table);
}
// NB: configure input param for further preproc
if (uu.net_input_params.is_applicable(mm)) {
const_cast<IEUnit::InputFramesDesc &>(uu.net_input_params)
.set_param(input_name, ii->getTensorDesc());
}
} else {
GAPI_Assert(uu.params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Import);
auto inputs = uu.this_network.GetInputsInfo();
@ -1207,6 +1410,12 @@ struct InferROI: public cv::detail::KernelTag {
auto* non_const_prepm = const_cast<IEUnit::PreProcMap*>(&uu.preproc_map);
auto ii = inputs.at(input_name);
non_const_prepm->emplace(input_name, configurePreProcInfo(ii, mm));
// NB: configure intput param for further preproc
if (uu.net_input_params.is_applicable(mm)) {
const_cast<IEUnit::InputFramesDesc &>(uu.net_input_params)
.set_param(input_name, ii->getTensorDesc());
}
}
// FIXME: It would be nice here to have an exact number of network's
@ -1236,13 +1445,26 @@ struct InferROI: public cv::detail::KernelTag {
GAPI_Assert(ctx->uu.params.num_in == 1);
auto&& this_roi = ctx->inArg<cv::detail::OpaqueRef>(0).rref<cv::Rect>();
// reserve unique slot for keep alive preprocessed frame
cv::MediaFrame* slot_ptr = ctx->prepareKeepAliveFrameSlot(&req);
// NB: This blob will be used to make roi from its, so
// it should be treated as image
bool preprocessed = false;
IE::Blob::Ptr this_blob =
extractBlob(*ctx, 1, cv::gapi::ie::TraitAs::IMAGE);
extractBlob(*ctx, 1, cv::gapi::ie::TraitAs::IMAGE,
*(ctx->uu.params.input_names.begin()),
cv::util::make_optional(this_roi),
slot_ptr, &preprocessed);
if (!preprocessed) {
setROIBlob(req,
*(ctx->uu.params.input_names.begin()),
this_blob, this_roi, *ctx);
} else {
setBlob(req,
*(ctx->uu.params.input_names.begin()),
this_blob, *ctx);
}
// FIXME: Should it be done by kernel ?
// What about to do that in RequestPool ?
req.StartAsync();
@ -1336,7 +1558,9 @@ struct InferList: public cv::detail::KernelTag {
// NB: This blob will be used to make roi from its, so
// it should be treated as image
IE::Blob::Ptr this_blob = extractBlob(*ctx, 1, cv::gapi::ie::TraitAs::IMAGE);
IE::Blob::Ptr this_blob = extractBlob(*ctx, 1, cv::gapi::ie::TraitAs::IMAGE,
ctx->uu.params.input_names[0u],
cv::util::optional<cv::Rect>{});
std::vector<std::vector<int>> cached_dims(ctx->uu.params.num_out);
for (auto i : ade::util::iota(ctx->uu.params.num_out)) {
@ -1483,7 +1707,9 @@ struct InferList2: public cv::detail::KernelTag {
&& "This operation must have at least two arguments");
// NB: This blob will be used to make roi from its, so
// it should be treated as image
IE::Blob::Ptr blob_0 = extractBlob(*ctx, 0, cv::gapi::ie::TraitAs::IMAGE);
IE::Blob::Ptr blob_0 = extractBlob(*ctx, 0, cv::gapi::ie::TraitAs::IMAGE,
ctx->uu.params.input_names[0u],
cv::util::optional<cv::Rect>{});
const auto list_size = ctx->inArg<cv::detail::VectorRef>(1u).size();
if (list_size == 0u) {
for (auto i : ade::util::iota(ctx->uu.params.num_out)) {

60
modules/gapi/src/streaming/onevpl/cfg_param_device_selector.cpp
View File

@ -38,6 +38,20 @@ namespace gapi {
namespace wip {
namespace onevpl {
std::vector<CfgParam> insertCfgparam(std::vector<CfgParam> &&param_array, AccelType type) {
switch (type) {
case AccelType::HOST:
break;
case AccelType::DX11:
param_array.push_back(CfgParam::create_acceleration_mode(MFX_ACCEL_MODE_VIA_D3D11));
break;
default:
GAPI_DbgAssert(false && "Unexpected AccelType");
break;
}
return std::move(param_array);
}
CfgParamDeviceSelector::CfgParamDeviceSelector(const CfgParams& cfg_params) :
suggested_device(IDeviceSelector::create<Device>(nullptr, "CPU", AccelType::HOST)),
suggested_context(IDeviceSelector::create<Context>(nullptr, AccelType::HOST)) {
@ -231,6 +245,52 @@ CfgParamDeviceSelector::CfgParamDeviceSelector(Device::Ptr device_ptr,
}
}
CfgParamDeviceSelector::CfgParamDeviceSelector(const Device &device,
const Context &ctx,
CfgParams) :
suggested_device(device),
suggested_context(ctx) {
switch(device.get_type()) {
case AccelType::DX11: {
#ifdef HAVE_DIRECTX
#ifdef HAVE_D3D11
ID3D11Device* dx_device_ptr =
reinterpret_cast<ID3D11Device*>(suggested_device.get_ptr());
dx_device_ptr->AddRef();
ID3D11DeviceContext* dx_ctx_ptr =
reinterpret_cast<ID3D11DeviceContext*>(suggested_context.get_ptr());
// oneVPL recommendation
{
ID3D11Multithread *pD11Multithread = nullptr;
dx_ctx_ptr->QueryInterface(IID_PPV_ARGS(&pD11Multithread));
pD11Multithread->SetMultithreadProtected(true);
pD11Multithread->Release();
}
dx_ctx_ptr->AddRef();
break;
#else
GAPI_LOG_WARNING(nullptr, "Unavailable \"" << CfgParam::acceleration_mode_name() <<
": MFX_ACCEL_MODE_VIA_D3D11\""
"was chosen for current project configuration");
throw std::logic_error(std::string("Unsupported \"") +
CfgParam::acceleration_mode_name() + ": MFX_ACCEL_MODE_VIA_D3D11\"");
#endif // HAVE_DIRECTX
#endif // HAVE_D3D11
}
case AccelType::HOST:
break;
default:
throw std::logic_error(std::string("Unsupported \"") + CfgParam::acceleration_mode_name() +
"\" requested: " +
to_cstring(device.get_type()));
break;
}
}
CfgParamDeviceSelector::~CfgParamDeviceSelector() {
GAPI_LOG_INFO(nullptr, "release context: " << suggested_context.get_ptr());
AccelType ctype = suggested_context.get_type();

7
modules/gapi/src/streaming/onevpl/cfg_param_device_selector.hpp
View File

@ -7,14 +7,14 @@
#ifndef GAPI_STREAMING_ONEVPL_CFG_PARAM_DEVICE_SELECTOR_HPP
#define GAPI_STREAMING_ONEVPL_CFG_PARAM_DEVICE_SELECTOR_HPP
#ifdef HAVE_ONEVPL
#include <opencv2/gapi/streaming/onevpl/device_selector_interface.hpp>
#include <opencv2/gapi/streaming/onevpl/cfg_params.hpp>
#include <opencv2/gapi/streaming/onevpl/source.hpp>
#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS
#ifdef HAVE_ONEVPL
namespace cv {
namespace gapi {
namespace wip {
@ -26,6 +26,9 @@ struct GAPI_EXPORTS CfgParamDeviceSelector final: public IDeviceSelector {
const std::string& device_id,
Context::Ptr ctx_ptr,
const CfgParams& params);
CfgParamDeviceSelector(const Device &device_ptr,
const Context &ctx_ptr,
CfgParams params);
~CfgParamDeviceSelector();
DeviceScoreTable select_devices() const override;

36
modules/gapi/src/streaming/onevpl/device_selector_interface.cpp
View File

@ -81,6 +81,42 @@ IDeviceSelector::Score::Type IDeviceSelector::Score::get() const {
IDeviceSelector::~IDeviceSelector() {
}
namespace detail
{
struct DeviceContextCreator : public IDeviceSelector {
DeviceScoreTable select_devices() const override { return {};}
DeviceContexts select_context() override { return {};}
template<typename Entity, typename ...Args>
static Entity create_entity(Args &&...args) {
return IDeviceSelector::create<Entity>(std::forward<Args>(args)...);
}
};
}
Device create_host_device() {
return detail::DeviceContextCreator::create_entity<Device>(nullptr,
"CPU",
AccelType::HOST);
}
Context create_host_context() {
return detail::DeviceContextCreator::create_entity<Context>(nullptr,
AccelType::HOST);
}
Device create_dx11_device(Device::Ptr device_ptr,
const std::string& device_name) {
return detail::DeviceContextCreator::create_entity<Device>(device_ptr,
device_name,
AccelType::DX11);
}
Context create_dx11_context(Context::Ptr ctx_ptr) {
return detail::DeviceContextCreator::create_entity<Context>(ctx_ptr,
AccelType::DX11);
}
} // namespace onevpl
} // namespace wip
} // namespace gapi

32
modules/gapi/src/streaming/onevpl/engine/preproc/preproc_dispatcher.cpp
View File

@ -4,30 +4,33 @@
//
// Copyright (C) 2022 Intel Corporation
#ifdef HAVE_ONEVPL
#include <algorithm>
#include <exception>
#include <opencv2/gapi/streaming/onevpl/data_provider_interface.hpp>
#include "streaming/onevpl/engine/preproc/preproc_dispatcher.hpp"
#ifdef HAVE_ONEVPL
#include "streaming/onevpl/onevpl_export.hpp"
#include "streaming/onevpl/engine/preproc/preproc_engine.hpp"
#include "streaming/onevpl/engine/preproc/preproc_session.hpp"
#include "streaming/onevpl/engine/preproc/preproc_dispatcher.hpp"
#include "streaming/onevpl/accelerators/accel_policy_interface.hpp"
#include "streaming/onevpl/accelerators/surface/surface.hpp"
#include "streaming/onevpl/cfg_params_parser.hpp"
#include "logger.hpp"
#endif // HAVE_ONEVPL
#include "logger.hpp"
namespace cv {
namespace gapi {
namespace wip {
namespace onevpl {
#ifdef HAVE_ONEVPL
cv::util::optional<pp_params> VPPPreprocDispatcher::is_applicable(const cv::MediaFrame& in_frame) {
cv::util::optional<pp_params> param;
GAPI_LOG_DEBUG(nullptr, "workers: " << workers.size());
bool worker_found = false;
for (const auto &w : workers) {
param = w->is_applicable(in_frame);
if (param.has_value()) {
@ -42,11 +45,12 @@ cv::util::optional<pp_params> VPPPreprocDispatcher::is_applicable(const cv::Medi
if (worker_accel_type == adapter->accel_type()){
vpp_param.reserved = reinterpret_cast<void *>(w.get());
GAPI_LOG_DEBUG(nullptr, "selected worker: " << vpp_param.reserved);
worker_found = true;
break;
}
}
}
return param;
return worker_found ? param : cv::util::optional<pp_params>{};
}
pp_session VPPPreprocDispatcher::initialize_preproc(const pp_params& initial_frame_param,
@ -78,8 +82,24 @@ cv::MediaFrame VPPPreprocDispatcher::run_sync(const pp_session &session_handle,
}
GAPI_Assert(false && "Cannot invoke VPP preproc in dispatcher, no suitable worker");
}
#else // HAVE_ONEVPL
cv::util::optional<pp_params> VPPPreprocDispatcher::is_applicable(const cv::MediaFrame&) {
return cv::util::optional<pp_params>{};
}
pp_session VPPPreprocDispatcher::initialize_preproc(const pp_params&,
const GFrameDesc&) {
GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`");
}
cv::MediaFrame VPPPreprocDispatcher::run_sync(const pp_session &,
const cv::MediaFrame&,
const cv::util::optional<cv::Rect> &) {
GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`");
}
#endif // HAVE_ONEVPL
} // namespace onevpl
} // namespace wip
} // namespace gapi
} // namespace cv
#endif // HAVE_ONEVPL

5
modules/gapi/src/streaming/onevpl/engine/preproc/preproc_dispatcher.hpp
View File

@ -11,10 +11,6 @@
#include <vector>
#include "streaming/onevpl/engine/preproc_engine_interface.hpp"
#include "streaming/onevpl/engine/preproc_defines.hpp"
#ifdef HAVE_ONEVPL
#include "streaming/onevpl/onevpl_export.hpp"
namespace cv {
namespace gapi {
@ -49,5 +45,4 @@ private:
} // namespace wip
} // namespace gapi
} // namespace cv
#endif // HAVE_ONEVPL
#endif // GAPI_STREAMING_ONEVPL_PREPROC_DISPATCHER_HPP

4
modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.cpp
View File

@ -51,7 +51,7 @@ void apply_roi(mfxFrameSurface1* surface_handle,
VPPPreprocEngine::VPPPreprocEngine(std::unique_ptr<VPLAccelerationPolicy>&& accel) :
ProcessingEngineBase(std::move(accel)) {
GAPI_LOG_INFO(nullptr, "Create VPP preprocessing engine");
GAPI_LOG_DEBUG(nullptr, "Create VPP preprocessing engine");
preprocessed_frames_count = 0;
create_pipeline(
// 0) preproc decoded surface with VPP params
@ -455,7 +455,7 @@ ProcessingEngineBase::ExecutionStatus VPPPreprocEngine::process_error(mfxStatus
"MFX_ERR_REALLOC_SURFACE is not processed");
break;
case MFX_WRN_IN_EXECUTION:
GAPI_LOG_WARNING(nullptr, "[" << sess.session << "] got MFX_WRN_IN_EXECUTION");
GAPI_LOG_DEBUG(nullptr, "[" << sess.session << "] got MFX_WRN_IN_EXECUTION");
return ExecutionStatus::Continue;
default:
GAPI_LOG_WARNING(nullptr, "Unknown status code: " << mfxstatus_to_string(status) <<

4
modules/gapi/src/streaming/onevpl/engine/preproc_defines.hpp
View File

@ -23,8 +23,8 @@ namespace wip {
#else // VPP_PREPROC_ENGINE
struct empty_pp_params {};
struct empty_pp_session {};
#define GAPI_BACKEND_PP_PARAMS cv::gapi::wip::empty_pp_params;
#define GAPI_BACKEND_PP_SESSIONS cv::gapi::wip::empty_pp_session;
#define GAPI_BACKEND_PP_PARAMS cv::gapi::wip::empty_pp_params
#define GAPI_BACKEND_PP_SESSIONS cv::gapi::wip::empty_pp_session
#endif // VPP_PREPROC_ENGINE
struct pp_params {

83
modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.cpp Normal file
View File

@ -0,0 +1,83 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
//
// Copyright (C) 2022 Intel Corporation
#include <opencv2/gapi/streaming/onevpl/device_selector_interface.hpp>
#include "streaming/onevpl/engine/preproc_engine_interface.hpp"
#include "streaming/onevpl/engine/preproc/preproc_dispatcher.hpp"
#ifdef HAVE_ONEVPL
#include "streaming/onevpl/onevpl_export.hpp"
#include "streaming/onevpl/engine/preproc/preproc_engine.hpp"
#include "streaming/onevpl/accelerators/accel_policy_dx11.hpp"
#include "streaming/onevpl/accelerators/accel_policy_cpu.hpp"
#include "streaming/onevpl/accelerators/surface/surface.hpp"
#include "streaming/onevpl/cfg_param_device_selector.hpp"
#include "streaming/onevpl/cfg_params_parser.hpp"
#endif //HAVE_ONEVPL
#include "logger.hpp"
namespace cv {
namespace gapi {
namespace wip {
template<typename SpecificPreprocEngine, typename ...PreprocEngineArgs >
std::unique_ptr<SpecificPreprocEngine>
IPreprocEngine::create_preproc_engine_impl(const PreprocEngineArgs& ...) {
GAPI_Assert(false && "Unsupported ");
}
template <>
std::unique_ptr<onevpl::VPPPreprocDispatcher>
IPreprocEngine::create_preproc_engine_impl(const onevpl::Device &device,
const onevpl::Context &context) {
using namespace onevpl;
cv::util::suppress_unused_warning(device);
cv::util::suppress_unused_warning(context);
std::unique_ptr<VPPPreprocDispatcher> dispatcher(new VPPPreprocDispatcher);
#ifdef HAVE_ONEVPL
if (device.get_type() == onevpl::AccelType::DX11) {
bool gpu_pp_is_created = false;
#ifdef HAVE_DIRECTX
#ifdef HAVE_D3D11
GAPI_LOG_INFO(nullptr, "Creating DX11 VPP preprocessing engine");
// create GPU VPP preproc engine
dispatcher->insert_worker<VPPPreprocEngine>(
std::unique_ptr<VPLAccelerationPolicy>{
new VPLDX11AccelerationPolicy(
std::make_shared<CfgParamDeviceSelector>(
device, context, CfgParams{}))
});
GAPI_LOG_INFO(nullptr, "DX11 VPP preprocessing engine created");
gpu_pp_is_created = true;
#endif
#endif
GAPI_Assert(gpu_pp_is_created && "VPP preproc for GPU is requested, but it is avaiable only for DX11 at now");
} else {
GAPI_LOG_INFO(nullptr, "Creating CPU VPP preprocessing engine");
dispatcher->insert_worker<VPPPreprocEngine>(
std::unique_ptr<VPLAccelerationPolicy>{
new VPLCPUAccelerationPolicy(
std::make_shared<CfgParamDeviceSelector>(CfgParams{}))});
GAPI_LOG_INFO(nullptr, "CPU VPP preprocessing engine created");
}
#endif // HAVE_ONEVPL
return dispatcher;
}
// Force instantiation
template
std::unique_ptr<onevpl::VPPPreprocDispatcher>
IPreprocEngine::create_preproc_engine_impl<onevpl::VPPPreprocDispatcher,
const onevpl::Device &,const onevpl::Context &>
(const onevpl::Device &device,
const onevpl::Context &ctx);
} // namespace wip
} // namespace gapi
} // namespace cv

10
modules/gapi/src/streaming/onevpl/engine/preproc_engine_interface.hpp
View File

@ -29,6 +29,16 @@ struct IPreprocEngine {
virtual cv::MediaFrame
run_sync(const pp_session &sess, const cv::MediaFrame& in_frame,
const cv::util::optional<cv::Rect> &opt_roi = {}) = 0;
template<typename SpecificPreprocEngine, typename ...PreprocEngineArgs >
static std::unique_ptr<IPreprocEngine> create_preproc_engine(const PreprocEngineArgs& ...args) {
static_assert(std::is_base_of<IPreprocEngine, SpecificPreprocEngine>::value,
"SpecificPreprocEngine must have reachable ancessor IPreprocEngine");
return create_preproc_engine_impl<SpecificPreprocEngine, PreprocEngineArgs...>(args...);
}
private:
template<typename SpecificPreprocEngine, typename ...PreprocEngineArgs >
static std::unique_ptr<SpecificPreprocEngine> create_preproc_engine_impl(const PreprocEngineArgs &...args);
};
} // namespace wip
} // namespace gapi

11
modules/gapi/src/streaming/onevpl/source.cpp
View File

@ -33,6 +33,13 @@ GSource::GSource(const std::string& filePath,
accel_ctx_ptr, cfg_params)) {
}
GSource::GSource(const std::string& filePath,
const CfgParams& cfg_params,
const Device &device, const Context &ctx) :
GSource(filePath, cfg_params,
std::make_shared<CfgParamDeviceSelector>(device, ctx, cfg_params)) {
}
GSource::GSource(const std::string& filePath,
const CfgParams& cfg_params,
std::shared_ptr<IDeviceSelector> selector) :
@ -74,6 +81,10 @@ GSource::GSource(const std::string&, const CfgParams&, const std::string&,
GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`");
}
GSource::GSource(const std::string&, const CfgParams&, const Device &, const Context &) {
GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`");
}
GSource::GSource(const std::string&, const CfgParams&, std::shared_ptr<IDeviceSelector>) {
GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`");
}