diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index 5fd108c0ec..917d1e0814 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -168,6 +168,8 @@ set(gapi_srcs src/streaming/onevpl/source_priv.cpp src/streaming/onevpl/file_data_provider.cpp src/streaming/onevpl/cfg_params.cpp + src/streaming/onevpl/cfg_params_parser.cpp + src/streaming/onevpl/utils.cpp src/streaming/onevpl/data_provider_interface_exception.cpp src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp src/streaming/onevpl/accelerators/surface/surface.cpp diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp index b7bf4f2ffb..a8dbefdf50 100644 --- a/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp +++ b/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp @@ -51,6 +51,8 @@ private: }; } // namespace onevpl +using GVPLSource = onevpl::GSource; + template GAPI_EXPORTS_W cv::Ptr inline make_onevpl_src(Args&&... args) { diff --git a/modules/gapi/samples/onevpl_infer_single_roi.cpp b/modules/gapi/samples/onevpl_infer_single_roi.cpp index 9a3abdf392..f3aee09d42 100644 --- a/modules/gapi/samples/onevpl_infer_single_roi.cpp +++ b/modules/gapi/samples/onevpl_infer_single_roi.cpp @@ -19,6 +19,7 @@ const std::string keys = "{ input | | Path to the input demultiplexed video file }" "{ output | | Path to the output RAW video file. Use .avi extension }" "{ facem | face-detection-adas-0001.xml | Path to OpenVINO IE face detection model (.xml) }" + "{ faced | CPU | Target device for face detection model (e.g. CPU, GPU, VPU, ...) }" "{ cfg_params | :;: | Semicolon separated list of oneVPL mfxVariants which is used for configuring source (see `MFXSetConfigFilterProperty` by https://spec.oneapi.io/versions/latest/elements/oneVPL/source/index.html) }"; @@ -198,7 +199,8 @@ int main(int argc, char *argv[]) { auto face_net = cv::gapi::ie::Params { face_model_path, // path to topology IR - get_weights_path(face_model_path) // path to weights + get_weights_path(face_model_path), // path to weights + cmd.get("faced"), // device specifier }; auto kernels = cv::gapi::kernels < custom::OCVLocateROI diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp index cb27df8661..8365dd20e3 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp @@ -30,7 +30,8 @@ namespace gapi { namespace wip { namespace onevpl { -VPLDX11AccelerationPolicy::VPLDX11AccelerationPolicy() +VPLDX11AccelerationPolicy::VPLDX11AccelerationPolicy() : + hw_handle(nullptr) { #ifdef CPU_ACCEL_ADAPTER adapter.reset(new VPLCPUAccelerationPolicy); diff --git a/modules/gapi/src/streaming/onevpl/cfg_params_parser.cpp b/modules/gapi/src/streaming/onevpl/cfg_params_parser.cpp new file mode 100644 index 0000000000..a683c7478e --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/cfg_params_parser.cpp @@ -0,0 +1,123 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include + +#include +#include + +#include "streaming/onevpl/cfg_params_parser.hpp" +#include "streaming/onevpl/utils.hpp" +#include "logger.hpp" + +#ifdef HAVE_ONEVPL +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +template <> +struct ParamCreator { + template + CfgParam create (const std::string& name, ValueType&& value) { + return CfgParam::create(name, std::forward(value), is_major_flag); + } + bool is_major_flag = false; +}; + +template <> +struct ParamCreator { + template + mfxVariant create (const std::string& name, ValueType&& value) { + static_assert(std::is_same::type, mfxU32>::value, + "ParamCreator supports mfxU32 at the moment. " + "Feel free to extend for more types"); + return create_impl(name, value); + } +private: + mfxVariant create_impl(const std::string&, mfxU32 value) { + mfxVariant ret; + ret.Type = MFX_VARIANT_TYPE_U32; + ret.Data.U32 = value; + return ret; + } +}; + +template +std::vector get_params_from_string(const std::string& str) { + std::vector ret; + std::string::size_type pos = 0; + std::string::size_type endline_pos = std::string::npos; + do + { + endline_pos = str.find_first_of("\r\n", pos); + std::string line = str.substr(pos, endline_pos == std::string::npos ? std::string::npos : endline_pos - pos); + if (line.empty()) break; + + std::string::size_type name_endline_pos = line.find(':'); + if (name_endline_pos == std::string::npos) { + throw std::runtime_error("Cannot parse param from string: " + line + + ". Name and value should be separated by \":\"" ); + } + + std::string name = line.substr(0, name_endline_pos); + std::string value = line.substr(name_endline_pos + 2); + + ParamCreator creator; + if (name == "mfxImplDescription.Impl") { + ret.push_back(creator.create(name, cstr_to_mfx_impl(value.c_str()))); + } else if (name == "mfxImplDescription.mfxDecoderDescription.decoder.CodecID") { + ret.push_back(creator.create(name, cstr_to_mfx_codec_id(value.c_str()))); + } else if (name == "mfxImplDescription.AccelerationMode") { + ret.push_back(creator.create(name, cstr_to_mfx_accel_mode(value.c_str()))); + } else if (name == "mfxImplDescription.ApiVersion.Version") { + ret.push_back(creator.create(name, cstr_to_mfx_version(value.c_str()))); + } else { + GAPI_LOG_DEBUG(nullptr, "Cannot parse configuration param, name: " << name << + ", value: " << value); + } + + pos = endline_pos + 1; + } + while (endline_pos != std::string::npos); + + return ret; +} + +template +std::vector get_params_from_string(const std::string& str); +template +std::vector get_params_from_string(const std::string& str); + +mfxVariant cfg_param_to_mfx_variant(const CfgParam& cfg_val) { + const CfgParam::name_t& name = cfg_val.get_name(); + mfxVariant ret; + cv::util::visit(cv::util::overload_lambdas( + [&ret](uint8_t value) { ret.Type = MFX_VARIANT_TYPE_U8; ret.Data.U8 = value; }, + [&ret](int8_t value) { ret.Type = MFX_VARIANT_TYPE_I8; ret.Data.I8 = value; }, + [&ret](uint16_t value) { ret.Type = MFX_VARIANT_TYPE_U16; ret.Data.U16 = value; }, + [&ret](int16_t value) { ret.Type = MFX_VARIANT_TYPE_I16; ret.Data.I16 = value; }, + [&ret](uint32_t value) { ret.Type = MFX_VARIANT_TYPE_U32; ret.Data.U32 = value; }, + [&ret](int32_t value) { ret.Type = MFX_VARIANT_TYPE_I32; ret.Data.I32 = value; }, + [&ret](uint64_t value) { ret.Type = MFX_VARIANT_TYPE_U64; ret.Data.U64 = value; }, + [&ret](int64_t value) { ret.Type = MFX_VARIANT_TYPE_I64; ret.Data.I64 = value; }, + [&ret](float_t value) { ret.Type = MFX_VARIANT_TYPE_F32; ret.Data.F32 = value; }, + [&ret](double_t value) { ret.Type = MFX_VARIANT_TYPE_F64; ret.Data.F64 = value; }, + [&ret](void* value) { ret.Type = MFX_VARIANT_TYPE_PTR; ret.Data.Ptr = value; }, + [&ret, &name] (const std::string& value) { + auto parsed = get_params_from_string(name + ": " + value + "\n"); + if (parsed.empty()) { + throw std::logic_error("Unsupported parameter, name: " + name + ", value: " + value); + } + ret = *parsed.begin(); + }), cfg_val.get_value()); + return ret; +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/cfg_params_parser.hpp b/modules/gapi/src/streaming/onevpl/cfg_params_parser.hpp new file mode 100644 index 0000000000..6247eef916 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/cfg_params_parser.hpp @@ -0,0 +1,43 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_CFG_PARAM_PARSER_HPP +#define GAPI_STREAMING_ONEVPL_CFG_PARAM_PARSER_HPP + +#ifdef HAVE_ONEVPL +#if (MFX_VERSION >= 2000) +#include +#endif // MFX_VERSION + +#include +#include + +#include +#include + +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +template +std::vector get_params_from_string(const std::string& str); + +template +struct ParamCreator { + template + ReturnType create(const std::string& name, ValueType&& value); +}; + +mfxVariant cfg_param_to_mfx_variant(const CfgParam& value); +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_CFG_PARAM_PARSER_HPP diff --git a/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.cpp b/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.cpp index 3161b1627e..382c3ae88b 100644 --- a/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.cpp +++ b/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.cpp @@ -117,7 +117,7 @@ mfxStatus ReadEncodedStream(mfxBitstream &bs, std::shared_ptr& da return MFX_ERR_NOT_ENOUGH_BUFFER; } - std::copy_n(p0, bs.DataLength, p1); + std::copy_n(p1, bs.DataLength, p0); bs.DataOffset = 0; bs.DataLength += static_cast(data_provider->fetch_data(bs.MaxLength - bs.DataLength, diff --git a/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.hpp b/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.hpp index 7d4869ac66..b307c18112 100644 --- a/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.hpp +++ b/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.hpp @@ -11,6 +11,8 @@ #include "streaming/onevpl/engine/engine_session.hpp" #include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS +#ifdef HAVE_ONEVPL + namespace cv { namespace gapi { namespace wip { @@ -93,4 +95,5 @@ mfxStatus ReadEncodedStream(mfxBitstream &bs, std::shared_ptr& da } // namespace gapi } // namespace cv +#endif // HAVE_ONEVPL #endif // GAPI_STREAMING_ONEVPL_ENGINE_PROCESSING_ENGINE_BASE_HPP diff --git a/modules/gapi/src/streaming/onevpl/source_priv.cpp b/modules/gapi/src/streaming/onevpl/source_priv.cpp index ed838d6adc..28d438a947 100644 --- a/modules/gapi/src/streaming/onevpl/source_priv.cpp +++ b/modules/gapi/src/streaming/onevpl/source_priv.cpp @@ -7,6 +7,12 @@ #include #include +#include "streaming/onevpl/engine/decode/decode_engine_legacy.hpp" +#include "streaming/onevpl/accelerators/accel_policy_dx11.hpp" +#include "streaming/onevpl/accelerators/accel_policy_cpu.hpp" +#include "streaming/onevpl/utils.hpp" +#include "streaming/onevpl/cfg_params_parser.hpp" + #include "streaming/onevpl/source_priv.hpp" #include "logger.hpp" @@ -32,32 +38,346 @@ namespace cv { namespace gapi { namespace wip { namespace onevpl { + +enum { + VPL_NEW_API_MAJOR_VERSION = 2, + VPL_NEW_API_MINOR_VERSION = 2 +}; + + GSource::Priv::Priv() : - mfx_handle(MFXLoad()) + mfx_handle(MFXLoad()), + mfx_impl_description(), + mfx_handle_configs(), + cfg_params(), + mfx_session(), + description(), + description_is_valid(false), + engine() { GAPI_LOG_INFO(nullptr, "Initialized MFX handle: " << mfx_handle); - description_is_valid = false; } -GSource::Priv::Priv(std::shared_ptr, const std::vector&) : +GSource::Priv::Priv(std::shared_ptr provider, const std::vector& params) : GSource::Priv() { + // Enable Config + if (params.empty()) + { + GAPI_LOG_INFO(nullptr, "No special cfg params requested - use default"); + this->cfg_params = getDefaultCfgParams(); + } + else + { + this->cfg_params = params; + } + + GAPI_LOG_DEBUG(nullptr, "Requested cfg params count: " << cfg_params.size()); + this->mfx_handle_configs.resize(cfg_params.size()); + + // Build VPL handle config from major input params + // VPL dispatcher then uses this config handle to look up for all existing VPL impl + // satisfying major input params and available in the system + GAPI_LOG_INFO(nullptr, "Creating VPL config from input params"); + auto cfg_param_it = cfg_params.begin(); + for (mfxConfig& cfg_inst : mfx_handle_configs) { + cfg_inst = MFXCreateConfig(mfx_handle); + GAPI_Assert(cfg_inst && "MFXCreateConfig failed"); + + if (!cfg_param_it->is_major()) { + GAPI_LOG_DEBUG(nullptr, "Skip not major param: " << cfg_param_it->get_name()); + ++cfg_param_it; + continue; + } + + GAPI_LOG_DEBUG(nullptr, "Apply major param: " << cfg_param_it->get_name()); + mfxVariant mfx_param = cfg_param_to_mfx_variant(*cfg_param_it); + mfxStatus sts = MFXSetConfigFilterProperty(cfg_inst, + (mfxU8 *)cfg_param_it->get_name().c_str(), + mfx_param); + if (sts != MFX_ERR_NONE ) + { + GAPI_LOG_WARNING(nullptr, "MFXSetConfigFilterProperty failed, error: " << + mfxstatus_to_string(sts) << + " - for \"" << cfg_param_it->get_name() << "\""); + GAPI_Assert(false && "MFXSetConfigFilterProperty failed"); + } + + ++cfg_param_it; + } + + // collect optional-preferred input parameters from input params + // which may (optionally) or may not be used to choose the most preferrable + // VPL implementation (for example, specific API version or Debug/Release VPL build) + std::vector preferred_params; + std::copy_if(cfg_params.begin(), cfg_params.end(), std::back_inserter(preferred_params), + [] (const CfgParam& param) { return !param.is_major(); }); + std::sort(preferred_params.begin(), preferred_params.end()); + + GAPI_LOG_DEBUG(nullptr, "Find MFX better implementation from handle: " << mfx_handle << + " is satisfying preferrable params count: " << preferred_params.size()); + int i = 0; + mfxImplDescription *idesc = nullptr; + std::vector available_impl_descriptions; + std::map matches_count; + while (MFX_ERR_NONE == MFXEnumImplementations(mfx_handle, + i, + MFX_IMPLCAPS_IMPLDESCSTRUCTURE, + reinterpret_cast(&idesc))) { + + available_impl_descriptions.push_back(idesc); + + std::stringstream ss; + mfxHDL hImplPath = nullptr; + if (MFX_ERR_NONE == MFXEnumImplementations(mfx_handle, i, MFX_IMPLCAPS_IMPLPATH, &hImplPath)) { + if (hImplPath) { + ss << "Implementation path: " << reinterpret_cast(hImplPath) << std::endl; + MFXDispReleaseImplDescription(mfx_handle, hImplPath); + } + } + ss << *idesc << std::endl; + + GAPI_LOG_INFO(nullptr, "Implementation index: " << i << "\n" << ss.str()); + + // Only one VPL implementation is required for GSource here. + // Let's find intersection params from available impl with preferrable input params + // to find best match. + // An available VPL implementation with max matching count + std::vector impl_params = get_params_from_string(ss.str()); + std::sort(impl_params.begin(), impl_params.end()); + GAPI_LOG_DEBUG(nullptr, "Find implementation cfg params count" << impl_params.size()); + + std::vector matched_params; + std::set_intersection(impl_params.begin(), impl_params.end(), + preferred_params.begin(), preferred_params.end(), + std::back_inserter(matched_params)); + + if (preferred_params.empty()) { + // in case of no input preferrance we consider all params are matched + // for the first available VPL implementation. It will be a chosen one + matches_count.emplace(impl_params.size(), i++); + GAPI_LOG_DEBUG(nullptr, "No preferrable params, use the first one implementation"); + break; + } else { + GAPI_LOG_DEBUG(nullptr, "Equal param intersection count: " << matched_params.size()); + matches_count.emplace(matches_count.size(), i++); + } + } + + // Extract the most suitable VPL implementation by max score + auto max_match_it = matches_count.rbegin(); + GAPI_Assert(max_match_it != matches_count.rend() && + "Cannot find matched MFX implementation for requested configuration"); + + int impl_number = max_match_it->second; + GAPI_LOG_INFO(nullptr, "Chosen implementation index: " << impl_number); + + // release unusable impl available_impl_descriptions + std::swap(mfx_impl_description, available_impl_descriptions[impl_number]); + for (mfxImplDescription* unusable_impl_descr : available_impl_descriptions) { + if (unusable_impl_descr) { + MFXDispReleaseImplDescription(mfx_handle, unusable_impl_descr); + } + } + available_impl_descriptions.clear(); + + // create session for implementation + mfxStatus sts = MFXCreateSession(mfx_handle, impl_number, &mfx_session); + if (MFX_ERR_NONE != sts) { + GAPI_LOG_WARNING(nullptr, "Cannot create MFX Session for implementation index:" << + std::to_string(impl_number) << + ", error: " << mfxstatus_to_string(sts)); + } + + GAPI_LOG_INFO(nullptr, "Initialized MFX session: " << mfx_session); + + // initialize decoder + // Find codec ID from config + auto dec_it = std::find_if(cfg_params.begin(), cfg_params.end(), [] (const CfgParam& value) { + return value.get_name() == "mfxImplDescription.mfxDecoderDescription.decoder.CodecID"; + }); + GAPI_Assert (dec_it != cfg_params.end() && "Cannot determine DecoderID from oneVPL config. Abort"); + + // create session driving engine if required + if (!engine) { + std::unique_ptr acceleration = initializeHWAccel(); + + // TODO Add factory static method in ProcessingEngineBase + if (mfx_impl_description->ApiVersion.Major >= VPL_NEW_API_MAJOR_VERSION) { + GAPI_Assert(false && + "GSource mfx_impl_description->ApiVersion.Major >= VPL_NEW_API_MAJOR_VERSION" + " - is not implemented"); + } else { + engine.reset(new VPLLegacyDecodeEngine(std::move(acceleration))); + } + } + + //create decoder for session accoring to header recovered from source file + DecoderParams decoder_param = create_decoder_from_file(*dec_it, provider); + + // create engine session for processing mfx session pipeline + engine->initialize_session(mfx_session, std::move(decoder_param), + provider); + + //prepare session for processing + engine->process(mfx_session); } GSource::Priv::~Priv() { + GAPI_LOG_INFO(nullptr, "Unload MFX implementation description: " << mfx_impl_description); + MFXDispReleaseImplDescription(mfx_handle, mfx_impl_description); GAPI_LOG_INFO(nullptr, "Unload MFX handle: " << mfx_handle); MFXUnload(mfx_handle); } -bool GSource::Priv::pull(cv::gapi::wip::Data&) +DecoderParams GSource::Priv::create_decoder_from_file(const CfgParam& decoder_cfg, + std::shared_ptr provider) { - return false; + GAPI_DbgAssert(provider && "Cannot create decoder, data provider is nullptr"); + + mfxBitstream bitstream{}; + const int BITSTREAM_BUFFER_SIZE = 2000000; + bitstream.MaxLength = BITSTREAM_BUFFER_SIZE; + bitstream.Data = (mfxU8 *)calloc(bitstream.MaxLength, sizeof(mfxU8)); + if(!bitstream.Data) { + throw std::runtime_error("Cannot allocate bitstream.Data bytes: " + + std::to_string(bitstream.MaxLength * sizeof(mfxU8))); + } + + mfxVariant decoder = cfg_param_to_mfx_variant(decoder_cfg); + // according to oneVPL documentation references + // https://spec.oneapi.io/versions/latest/elements/oneVPL/source/API_ref/VPL_disp_api_struct.html + // mfxVariant is an `union` type and considered different meaning for different param ids + // So CodecId has U32 data type + bitstream.CodecId = decoder.Data.U32; + + mfxStatus sts = ReadEncodedStream(bitstream, provider); + if(MFX_ERR_NONE != sts) { + throw std::runtime_error("Error reading bitstream, error: " + + mfxstatus_to_string(sts)); + } + + // Retrieve the frame information from input stream + mfxVideoParam mfxDecParams {}; + mfxDecParams.mfx.CodecId = decoder.Data.U32; + mfxDecParams.IOPattern = MFX_IOPATTERN_OUT_SYSTEM_MEMORY;//MFX_IOPATTERN_OUT_VIDEO_MEMORY; + sts = MFXVideoDECODE_DecodeHeader(mfx_session, &bitstream, &mfxDecParams); + if(MFX_ERR_NONE != sts) { + throw std::runtime_error("Error decoding header, error: " + + mfxstatus_to_string(sts)); + } + + // Input parameters finished, now initialize decode + sts = MFXVideoDECODE_Init(mfx_session, &mfxDecParams); + if (MFX_ERR_NONE != sts) { + throw std::runtime_error("Error initializing Decode, error: " + + mfxstatus_to_string(sts)); + } + + // set valid description + description.size = cv::Size { + mfxDecParams.mfx.FrameInfo.Width, + mfxDecParams.mfx.FrameInfo.Height}; + switch(mfxDecParams.mfx.FrameInfo.FourCC) { + case MFX_FOURCC_I420: + GAPI_Assert(false && "Cannot create GMetaArg description: " + "MediaFrame doesn't support I420 type"); + case MFX_FOURCC_NV12: + description.fmt = cv::MediaFormat::NV12; + break; + default: + { + GAPI_LOG_WARNING(nullptr, "Cannot create GMetaArg description: " + "MediaFrame unknown 'fmt' type: " << + std::to_string(mfxDecParams.mfx.FrameInfo.FourCC)); + GAPI_Assert(false && "Cannot create GMetaArg description: invalid value"); + } + } + description_is_valid = true; + + return {bitstream, mfxDecParams}; +} + +std::unique_ptr GSource::Priv::initializeHWAccel() +{ + std::unique_ptr ret; + + auto accel_mode_it = std::find_if(cfg_params.begin(), cfg_params.end(), [] (const CfgParam& value) { + return value.get_name() == "mfxImplDescription.AccelerationMode"; + }); + if (accel_mode_it == cfg_params.end()) + { + GAPI_LOG_DEBUG(nullptr, "No HW Accel requested. Use CPU"); + + ret.reset(new VPLCPUAccelerationPolicy); + return ret; + } + + GAPI_LOG_DEBUG(nullptr, "Add HW acceleration support"); + mfxVariant accel_mode = cfg_param_to_mfx_variant(*accel_mode_it); + + switch(accel_mode.Data.U32) { + case MFX_ACCEL_MODE_VIA_D3D11: + { + std::unique_ptr cand(new VPLDX11AccelerationPolicy); + ret = std::move(cand); + break; + } + case MFX_ACCEL_MODE_NA: + { + std::unique_ptr cand(new VPLCPUAccelerationPolicy); + ret = std::move(cand); + break; + } + default: + { + GAPI_LOG_WARNING(nullptr, "Cannot initialize HW Accel: " + "invalid accelerator type: " << + std::to_string(accel_mode.Data.U32)); + GAPI_Assert(false && "Cannot initialize HW Accel"); + } + } + + return ret; +} + +const std::vector& GSource::Priv::getDefaultCfgParams() +{ + static const std::vector def_params = + get_params_from_string( + "mfxImplDescription.Impl: MFX_IMPL_TYPE_HARDWARE\n" + "mfxImplDescription.AccelerationMode: MFX_ACCEL_MODE_VIA_D3D11\n"); + + return def_params; +} + +const std::vector& GSource::Priv::getCfgParams() const +{ + return cfg_params; +} + +bool GSource::Priv::pull(cv::gapi::wip::Data& data) +{ + ProcessingEngineBase::ExecutionStatus status = ProcessingEngineBase::ExecutionStatus::Continue; + while (0 == engine->get_ready_frames_count() && + status == ProcessingEngineBase::ExecutionStatus::Continue) { + status = engine->process(mfx_session); + } + + if (engine->get_ready_frames_count()) { + engine->get_frame(data); + return true; + } else { + return false; + } } GMetaArg GSource::Priv::descr_of() const { - return {}; + GAPI_Assert(description_is_valid); + GMetaArg arg(description); + return arg; } } // namespace onevpl } // namespace wip diff --git a/modules/gapi/src/streaming/onevpl/source_priv.hpp b/modules/gapi/src/streaming/onevpl/source_priv.hpp index c278934253..cdaab4eb6a 100644 --- a/modules/gapi/src/streaming/onevpl/source_priv.hpp +++ b/modules/gapi/src/streaming/onevpl/source_priv.hpp @@ -25,23 +25,44 @@ #include +#include "streaming/onevpl/engine/processing_engine_base.hpp" + namespace cv { namespace gapi { namespace wip { namespace onevpl { +struct VPLAccelerationPolicy; +class ProcessingEngineBase; + struct GSource::Priv { explicit Priv(std::shared_ptr provider, const std::vector& params); ~Priv(); + static const std::vector& getDefaultCfgParams(); + const std::vector& getCfgParams() const; + bool pull(cv::gapi::wip::Data& data); GMetaArg descr_of() const; private: Priv(); + DecoderParams create_decoder_from_file(const CfgParam& decoder, + std::shared_ptr provider); + std::unique_ptr initializeHWAccel(); + mfxLoader mfx_handle; + mfxImplDescription *mfx_impl_description; + std::vector mfx_handle_configs; + std::vector cfg_params; + + mfxSession mfx_session; + + cv::GFrameDesc description; bool description_is_valid; + + std::unique_ptr engine; }; } // namespace onevpl } // namespace wip diff --git a/modules/gapi/src/streaming/onevpl/utils.cpp b/modules/gapi/src/streaming/onevpl/utils.cpp new file mode 100644 index 0000000000..72b5c75303 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/utils.cpp @@ -0,0 +1,454 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include + +#include +#include + +#ifdef HAVE_ONEVPL + +#include "streaming/onevpl/utils.hpp" +#include "logger.hpp" + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +const char* mfx_impl_to_cstr(const mfxIMPL impl) { + switch (impl) { + case MFX_IMPL_TYPE_SOFTWARE: + return "MFX_IMPL_TYPE_SOFTWARE"; + case MFX_IMPL_TYPE_HARDWARE: + return "MFX_IMPL_TYPE_HARDWARE"; + default: + return "unknown mfxIMPL"; + } +} + +mfxIMPL cstr_to_mfx_impl(const char* cstr) { + if (!strcmp(cstr, "MFX_IMPL_TYPE_SOFTWARE")) { + return MFX_IMPL_TYPE_SOFTWARE; + } else if (!strcmp(cstr, "MFX_IMPL_TYPE_HARDWARE")) { + return MFX_IMPL_TYPE_HARDWARE; + } + + throw std::logic_error(std::string("Invalid \"mfxImplDescription.Impl\":") + cstr); +} + +const char* mfx_accel_mode_to_cstr (const mfxAccelerationMode mode) { + switch (mode) { + case MFX_ACCEL_MODE_NA: + return "MFX_ACCEL_MODE_NA"; + case MFX_ACCEL_MODE_VIA_D3D9: + return "MFX_ACCEL_MODE_VIA_D3D9"; + case MFX_ACCEL_MODE_VIA_D3D11: + return "MFX_ACCEL_MODE_VIA_D3D11"; + case MFX_ACCEL_MODE_VIA_VAAPI: + return "MFX_ACCEL_MODE_VIA_VAAPI"; + case MFX_ACCEL_MODE_VIA_VAAPI_DRM_MODESET: + return "MFX_ACCEL_MODE_VIA_VAAPI_DRM_MODESET"; + case MFX_ACCEL_MODE_VIA_VAAPI_GLX: + return "MFX_ACCEL_MODE_VIA_VAAPI_GLX"; + case MFX_ACCEL_MODE_VIA_VAAPI_X11: + return "MFX_ACCEL_MODE_VIA_VAAPI_X11"; + case MFX_ACCEL_MODE_VIA_VAAPI_WAYLAND: + return "MFX_ACCEL_MODE_VIA_VAAPI_WAYLAND"; + case MFX_ACCEL_MODE_VIA_HDDLUNITE: + return "MFX_ACCEL_MODE_VIA_HDDLUNITE"; + default: + return "unknown mfxAccelerationMode"; + } + return "unknown mfxAccelerationMode"; +} + +mfxAccelerationMode cstr_to_mfx_accel_mode(const char* cstr) { + if (!strcmp(cstr, "MFX_ACCEL_MODE_NA")) { + return MFX_ACCEL_MODE_NA; + } else if (!strcmp(cstr, "MFX_ACCEL_MODE_VIA_D3D9")) { + return MFX_ACCEL_MODE_VIA_D3D9; + } else if (!strcmp(cstr, "MFX_ACCEL_MODE_VIA_D3D11")) { + return MFX_ACCEL_MODE_VIA_D3D11; + } else if (!strcmp(cstr, "MFX_ACCEL_MODE_VIA_VAAPI")) { + return MFX_ACCEL_MODE_VIA_VAAPI; + } else if (!strcmp(cstr, "MFX_ACCEL_MODE_VIA_VAAPI_DRM_MODESET")) { + return MFX_ACCEL_MODE_VIA_VAAPI_DRM_MODESET; + } else if (!strcmp(cstr, "MFX_ACCEL_MODE_VIA_VAAPI_GLX")) { + return MFX_ACCEL_MODE_VIA_VAAPI_GLX; + } else if (!strcmp(cstr, "MFX_ACCEL_MODE_VIA_VAAPI_X11")) { + return MFX_ACCEL_MODE_VIA_VAAPI_X11; + } else if (!strcmp(cstr, "MFX_ACCEL_MODE_VIA_VAAPI_WAYLAND")) { + return MFX_ACCEL_MODE_VIA_VAAPI_WAYLAND; + } else if (!strcmp(cstr, "MFX_ACCEL_MODE_VIA_HDDLUNITE")) { + return MFX_ACCEL_MODE_VIA_HDDLUNITE; + } + throw std::logic_error(std::string("Invalid \"mfxImplDescription.AccelerationMode\":") + cstr); +} + +const char* mfx_resource_type_to_cstr (const mfxResourceType type) { + switch (type) { + case MFX_RESOURCE_SYSTEM_SURFACE: + return "MFX_RESOURCE_SYSTEM_SURFACE"; + case MFX_RESOURCE_VA_SURFACE: + return "MFX_RESOURCE_VA_SURFACE"; + case MFX_RESOURCE_VA_BUFFER: + return "MFX_RESOURCE_VA_BUFFER"; + case MFX_RESOURCE_DX9_SURFACE: + return "MFX_RESOURCE_DX9_SURFACE"; + case MFX_RESOURCE_DX11_TEXTURE: + return "MFX_RESOURCE_DX11_TEXTURE"; + case MFX_RESOURCE_DX12_RESOURCE: + return "MFX_RESOURCE_DX12_RESOURCE"; + case MFX_RESOURCE_DMA_RESOURCE: + return "MFX_RESOURCE_DMA_RESOURCE"; + case MFX_RESOURCE_HDDLUNITE_REMOTE_MEMORY: + return "MFX_RESOURCE_HDDLUNITE_REMOTE_MEMORY"; + default: + return "unknown mfxResourceType"; + } +} + +mfxResourceType cstr_to_mfx_resource_type(const char* cstr) { + if (!strcmp(cstr, "MFX_RESOURCE_SYSTEM_SURFACE")) { + return MFX_RESOURCE_SYSTEM_SURFACE; + } else if (!strcmp(cstr, "MFX_RESOURCE_VA_SURFACE")) { + return MFX_RESOURCE_VA_SURFACE; + } else if (!strcmp(cstr, "MFX_RESOURCE_VA_BUFFER")) { + return MFX_RESOURCE_VA_BUFFER; + } else if (!strcmp(cstr, "MFX_RESOURCE_DX9_SURFACE")) { + return MFX_RESOURCE_DX9_SURFACE; + } else if (!strcmp(cstr, "MFX_RESOURCE_DX11_TEXTURE")) { + return MFX_RESOURCE_DX11_TEXTURE; + } else if (!strcmp(cstr, "MFX_RESOURCE_DX12_RESOURCE")) { + return MFX_RESOURCE_DX12_RESOURCE; + } else if (!strcmp(cstr, "MFX_RESOURCE_DMA_RESOURCE")) { + return MFX_RESOURCE_DMA_RESOURCE; + } else if (!strcmp(cstr, "MFX_RESOURCE_HDDLUNITE_REMOTE_MEMORY")) { + return MFX_RESOURCE_HDDLUNITE_REMOTE_MEMORY; + } + throw std::logic_error(std::string("Invalid \"decoder.Profiles.MemDesc.MemHandleType\":") + cstr); +} + +mfxU32 cstr_to_mfx_codec_id(const char* cstr) { + if (!strcmp(cstr, "MFX_CODEC_AVC")) { + return MFX_CODEC_AVC; + } else if (!strcmp(cstr, "MFX_CODEC_HEVC")) { + return MFX_CODEC_HEVC; + } else if (!strcmp(cstr, "MFX_CODEC_MPEG2")) { + return MFX_CODEC_MPEG2; + } else if (!strcmp(cstr, "MFX_CODEC_VC1")) { + return MFX_CODEC_VC1; + } else if (!strcmp(cstr, "MFX_CODEC_CAPTURE")) { + return MFX_CODEC_CAPTURE; + } else if (!strcmp(cstr, "MFX_CODEC_VP9")) { + return MFX_CODEC_VP9; + } else if (!strcmp(cstr, "MFX_CODEC_AV1")) { + return MFX_CODEC_AV1; + } + throw std::logic_error(std::string("Cannot parse \"mfxImplDescription.mfxDecoderDescription.decoder.CodecID\" value: ") + cstr); +} + +const char* mfx_codec_type_to_cstr(const mfxU32 fourcc, const mfxU32 type) { + switch (fourcc) { + case MFX_CODEC_JPEG: { + switch (type) { + case MFX_PROFILE_UNKNOWN: + return "MFX_PROFILE_UNKNOWN"; + case MFX_PROFILE_JPEG_BASELINE: + return "MFX_PROFILE_JPEG_BASELINE"; + default: + return "::max(); + } + + const char* delim = strchr(cstr, '.'); + if (!delim) { + // in digital form - return as is + return std::stoul(cstr, nullptr, 10); + } + std::string major (cstr, delim - cstr); + std::string minor (delim + 1); + mfxU32 major_val = std::stoul(major, nullptr, 10); + mfxU32 minor_val = std::stoul(minor, nullptr, 10); + + // pack to digital form + return {major_val << 16 | minor_val}; +} + +std::ostream& operator<< (std::ostream& out, const mfxImplDescription& idesc) +{ + out << "mfxImplDescription.Version: " << static_cast(idesc.Version.Major) + << "." << static_cast(idesc.Version.Minor) << std::endl; + out << "mfxImplDescription.Impl: " << mfx_impl_to_cstr(idesc.Impl) << std::endl; + out << "(*)mfxImplDescription.AccelerationMode: " << mfx_accel_mode_to_cstr(idesc.AccelerationMode) << std::endl; + out << "mfxImplDescription.ApiVersion: " << idesc.ApiVersion.Major << "." << idesc.ApiVersion.Minor << std::endl; + out << "(*)mfxImplDescription.ApiVersion.Version: " << idesc.ApiVersion.Version << std::endl; + out << "mfxImplDescription.ImplName: " << idesc.ImplName << std::endl; + out << "mfxImplDescription.License: " << idesc.License << std::endl; + out << "mfxImplDescription.Keywords: " << idesc.Keywords << std::endl; + out << "mfxImplDescription.VendorID: " << idesc.VendorID << std::endl; + out << "mfxImplDescription.VendorImplID: " << idesc.VendorImplID << std::endl; + + const mfxAccelerationModeDescription &accel = idesc.AccelerationModeDescription; + out << "mfxImplDescription.mfxAccelerationMode.Version: " << static_cast(accel.Version.Major) + << "." << static_cast(accel.Version.Minor) << std::endl; + for (int mode = 0; mode < accel.NumAccelerationModes; mode++) { + out << "mfxImplDescription.mfxAccelerationMode.Mode: " << mfx_accel_mode_to_cstr(accel.Mode[mode]) << std::endl; + } + + const mfxDeviceDescription &dev = idesc.Dev; + out << "mfxImplDescription.mfxDeviceDescription.Version: " << static_cast(dev.Version.Major) + << "." << static_cast(dev.Version.Minor) << std::endl; + out << "mfxImplDescription.mfxDeviceDescription.DeviceID: " << dev.DeviceID << std::endl; + for (int subdevice = 0; subdevice < dev.NumSubDevices; subdevice++) { + out << "mfxImplDescription.mfxDeviceDescription.subdevices.Index: " << dev.SubDevices[subdevice].Index << std::endl; + out << "mfxImplDescription.mfxDeviceDescription.subdevices.SubDeviceID: " << dev.SubDevices[subdevice].SubDeviceID << std::endl; + } + + /* mfxDecoderDescription */ + const mfxDecoderDescription &dec = idesc.Dec; + out << "mfxImplDescription.mfxDecoderDescription.Version: " << static_cast(dec.Version.Major) + << "." << static_cast(dec.Version.Minor) << std::endl; + for (int codec = 0; codec < dec.NumCodecs; codec++) { + auto cid = dec.Codecs[codec].CodecID; + out << "(*)mfxImplDescription.mfxDecoderDescription.decoder.CodecID: " << cid;//(cid & 0xff) << "." << (cid >> 8 & 0xff) << "." << (cid >> 16 & 0xff) << "." << (cid >> 24 & 0xff) << std::endl; + out << "mfxImplDescription.mfxDecoderDescription.decoder.MaxcodecLevel: " << dec.Codecs[codec].MaxcodecLevel << std::endl; + for (int profile = 0; profile < dec.Codecs[codec].NumProfiles; profile++) { + out << "mfxImplDescription.mfxDecoderDescription.decoder.Profiles: " + << mfx_codec_type_to_cstr(dec.Codecs[codec].CodecID, + dec.Codecs[codec].Profiles[profile].Profile) << std::endl; + for (int memtype = 0; memtype < dec.Codecs[codec].Profiles[profile].NumMemTypes; memtype++) { + out << "mfxImplDescription.mfxDecoderDescription.decoder.Profiles.MemDesc.MemHandleType: " + << mfx_resource_type_to_cstr(dec.Codecs[codec].Profiles[profile].MemDesc[memtype].MemHandleType) << std::endl; + out << "mfxImplDescription.mfxDecoderDescription.decoder.Profiles.MemDesc.Width.Min: " + << dec.Codecs[codec].Profiles[profile].MemDesc[memtype].Width.Min << std::endl; + out << "mfxImplDescription.mfxDecoderDescription.decoder.Profiles.MemDesc.Width.Max: " + << dec.Codecs[codec].Profiles[profile].MemDesc[memtype].Width.Max << std::endl; + out << "mfxImplDescription.mfxDecoderDescription.decoder.Profiles.MemDesc.Width.Step: " + << dec.Codecs[codec].Profiles[profile].MemDesc[memtype].Width.Step << std::endl; + out << "mfxImplDescription.mfxDecoderDescription.decoder.Profiles.MemDesc.Height.Min: " + << dec.Codecs[codec].Profiles[profile].MemDesc[memtype].Height.Min << std::endl; + out << "mfxImplDescription.mfxDecoderDescription.decoder.Profiles.MemDesc.Height.Max: " + << dec.Codecs[codec].Profiles[profile].MemDesc[memtype].Height.Max << std::endl; + out << "mfxImplDescription.mfxDecoderDescription.decoder.Profiles.MemDesc.Height.Step: " + << dec.Codecs[codec].Profiles[profile].MemDesc[memtype].Height.Step << std::endl; + } + } + } + + out << "mfxImplDescription.NumExtParam: " << idesc.NumExtParam << std::endl; + + out << "\n(*) - configurable params" << std::endl; + return out; +} + +std::string mfxstatus_to_string(mfxStatus err) { + switch(err) + { + case MFX_ERR_NONE: + return "MFX_ERR_NONE"; + case MFX_ERR_UNKNOWN: + return "MFX_ERR_UNKNOWN"; + case MFX_ERR_NULL_PTR: + return "MFX_ERR_NULL_PTR"; + case MFX_ERR_UNSUPPORTED: + return "MFX_ERR_UNSUPPORTED"; + case MFX_ERR_MEMORY_ALLOC: + return "MFX_ERR_MEMORY_ALLOC"; + case MFX_ERR_NOT_ENOUGH_BUFFER: + return "MFX_ERR_NOT_ENOUGH_BUFFER"; + case MFX_ERR_INVALID_HANDLE: + return "MFX_ERR_INVALID_HANDLE"; + case MFX_ERR_LOCK_MEMORY: + return "MFX_ERR_LOCK_MEMORY"; + case MFX_ERR_NOT_INITIALIZED: + return "MFX_ERR_NOT_INITIALIZED"; + case MFX_ERR_NOT_FOUND: + return "MFX_ERR_NOT_FOUND"; + case MFX_ERR_MORE_DATA: + return "MFX_ERR_MORE_DATA"; + case MFX_ERR_MORE_SURFACE: + return "MFX_ERR_MORE_SURFACE"; + case MFX_ERR_ABORTED: + return "MFX_ERR_ABORTED"; + case MFX_ERR_DEVICE_LOST: + return "MFX_ERR_DEVICE_LOST"; + case MFX_ERR_INCOMPATIBLE_VIDEO_PARAM: + return "MFX_ERR_INCOMPATIBLE_VIDEO_PARAM"; + case MFX_ERR_INVALID_VIDEO_PARAM: + return "MFX_ERR_INVALID_VIDEO_PARAM"; + case MFX_ERR_UNDEFINED_BEHAVIOR: + return "MFX_ERR_UNDEFINED_BEHAVIOR"; + case MFX_ERR_DEVICE_FAILED: + return "MFX_ERR_DEVICE_FAILED"; + case MFX_ERR_MORE_BITSTREAM: + return "MFX_ERR_MORE_BITSTREAM"; + case MFX_ERR_GPU_HANG: + return "MFX_ERR_GPU_HANG"; + case MFX_ERR_REALLOC_SURFACE: + return "MFX_ERR_REALLOC_SURFACE"; + case MFX_ERR_RESOURCE_MAPPED: + return "MFX_ERR_RESOURCE_MAPPED"; + case MFX_ERR_NOT_IMPLEMENTED: + return "MFX_ERR_NOT_IMPLEMENTED"; + case MFX_WRN_DEVICE_BUSY: + return "MFX_WRN_DEVICE_BUSY"; + case MFX_WRN_VIDEO_PARAM_CHANGED: + return "MFX_WRN_VIDEO_PARAM_CHANGED"; + + + default: + break; + } + + std::string ret(""; + return ret; +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/utils.hpp b/modules/gapi/src/streaming/onevpl/utils.hpp index 0512c4f687..94f5a249e8 100644 --- a/modules/gapi/src/streaming/onevpl/utils.hpp +++ b/modules/gapi/src/streaming/onevpl/utils.hpp @@ -26,9 +26,27 @@ namespace gapi { namespace wip { namespace onevpl { -inline std::string mfxstatus_to_string(mfxStatus) { - return "UNKNOWN"; -} +const char* mfx_impl_to_cstr(const mfxIMPL impl); + +mfxIMPL cstr_to_mfx_impl(const char* cstr); + +const char* mfx_accel_mode_to_cstr (const mfxAccelerationMode mode); + +mfxAccelerationMode cstr_to_mfx_accel_mode(const char* cstr); + +const char* mfx_resource_type_to_cstr (const mfxResourceType type); + +mfxResourceType cstr_to_mfx_resource_type(const char* cstr); + +mfxU32 cstr_to_mfx_codec_id(const char* cstr); + +const char* mfx_codec_type_to_cstr(const mfxU32 fourcc, const mfxU32 type); + +mfxU32 cstr_to_mfx_version(const char* cstr); + +std::string mfxstatus_to_string(mfxStatus err); + +std::ostream& operator<< (std::ostream& out, const mfxImplDescription& idesc); } // namespace onevpl } // namespace wip