diff --git a/modules/gapi/include/opencv2/gapi/infer/bindings_onnx.hpp b/modules/gapi/include/opencv2/gapi/infer/bindings_onnx.hpp index 4ba829df09..c418c0d496 100644 --- a/modules/gapi/include/opencv2/gapi/infer/bindings_onnx.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/bindings_onnx.hpp @@ -39,6 +39,12 @@ public: GAPI_WRAP PyParams& cfgAddExecutionProvider(ep::DirectML ep); + GAPI_WRAP + PyParams& cfgAddExecutionProvider(ep::CUDA ep); + + GAPI_WRAP + PyParams& cfgAddExecutionProvider(ep::TensorRT ep); + GAPI_WRAP PyParams& cfgDisableMemPattern(); diff --git a/modules/gapi/include/opencv2/gapi/infer/onnx.hpp b/modules/gapi/include/opencv2/gapi/infer/onnx.hpp index 64b855acd7..ff5febcf90 100644 --- a/modules/gapi/include/opencv2/gapi/infer/onnx.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/onnx.hpp @@ -32,6 +32,56 @@ namespace onnx { */ namespace ep { +/** + * @brief This structure provides functions + * that fill inference options for CUDA Execution Provider. + * Please follow https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html#cuda-execution-provider + */ +struct GAPI_EXPORTS_W_SIMPLE CUDA { + // NB: Used from python. + /// @private -- Exclude this constructor from OpenCV documentation + GAPI_WRAP + CUDA() = default; + + /** @brief Class constructor. + + Constructs CUDA parameters based on device type information. + + @param dev_id Target device id to use. + */ + GAPI_WRAP + explicit CUDA(const int dev_id) + : device_id(dev_id) { + } + + int device_id; +}; + +/** + * @brief This structure provides functions + * that fill inference options for TensorRT Execution Provider. + * Please follow https://onnxruntime.ai/docs/execution-providers/TensorRT-ExecutionProvider.html#tensorrt-execution-provider + */ +struct GAPI_EXPORTS_W_SIMPLE TensorRT { + // NB: Used from python. + /// @private -- Exclude this constructor from OpenCV documentation + GAPI_WRAP + TensorRT() = default; + + /** @brief Class constructor. + + Constructs TensorRT parameters based on device type information. + + @param dev_id Target device id to use. + */ + GAPI_WRAP + explicit TensorRT(const int dev_id) + : device_id(dev_id) { + } + + int device_id; +}; + /** * @brief This structure provides functions * that fill inference options for ONNX OpenVINO Execution Provider. @@ -143,7 +193,11 @@ public: DeviceDesc ddesc; }; -using EP = cv::util::variant; +using EP = cv::util::variant< cv::util::monostate + , OpenVINO + , DirectML + , CUDA + , TensorRT>; } // namespace ep @@ -431,6 +485,34 @@ public: return *this; } + /** @brief Adds execution provider for runtime. + + The function is used to add ONNX Runtime CUDA Execution Provider options. + + @param ep CUDA Execution Provider options. + @see cv::gapi::onnx::ep::CUDA. + + @return the reference on modified object. + */ + Params& cfgAddExecutionProvider(ep::CUDA&& ep) { + desc.execution_providers.emplace_back(std::move(ep)); + return *this; + } + + /** @brief Adds execution provider for runtime. + + The function is used to add ONNX Runtime TensorRT Execution Provider options. + + @param ep TensorRT Execution Provider options. + @see cv::gapi::onnx::ep::TensorRT. + + @return the reference on modified object. + */ + Params& cfgAddExecutionProvider(ep::TensorRT&& ep) { + desc.execution_providers.emplace_back(std::move(ep)); + return *this; + } + /** @brief Disables the memory pattern optimization. @return the reference on modified object. @@ -491,6 +573,16 @@ public: desc.execution_providers.emplace_back(std::move(ep)); } + /** @see onnx::Params::cfgAddExecutionProvider. */ + void cfgAddExecutionProvider(ep::CUDA&& ep) { + desc.execution_providers.emplace_back(std::move(ep)); + } + + /** @see onnx::Params::cfgAddExecutionProvider. */ + void cfgAddExecutionProvider(ep::TensorRT&& ep) { + desc.execution_providers.emplace_back(std::move(ep)); + } + /** @see onnx::Params::cfgDisableMemPattern. */ void cfgDisableMemPattern() { desc.disable_mem_pattern = true; diff --git a/modules/gapi/misc/python/pyopencv_gapi.hpp b/modules/gapi/misc/python/pyopencv_gapi.hpp index 60d5f85479..3269a7d470 100644 --- a/modules/gapi/misc/python/pyopencv_gapi.hpp +++ b/modules/gapi/misc/python/pyopencv_gapi.hpp @@ -31,6 +31,8 @@ using map_string_and_vector_float = std::map>; using map_int_and_double = std::map; using ep_OpenVINO = cv::gapi::onnx::ep::OpenVINO; using ep_DirectML = cv::gapi::onnx::ep::DirectML; +using ep_CUDA = cv::gapi::onnx::ep::CUDA; +using ep_TensorRT = cv::gapi::onnx::ep::TensorRT; // NB: Python wrapper generate T_U for T // This behavior is only observed for inputs diff --git a/modules/gapi/src/backends/onnx/bindings_onnx.cpp b/modules/gapi/src/backends/onnx/bindings_onnx.cpp index 6051c6bb4d..b41ec7b1b1 100644 --- a/modules/gapi/src/backends/onnx/bindings_onnx.cpp +++ b/modules/gapi/src/backends/onnx/bindings_onnx.cpp @@ -33,6 +33,18 @@ cv::gapi::onnx::PyParams::cfgAddExecutionProvider(cv::gapi::onnx::ep::DirectML e return *this; } +cv::gapi::onnx::PyParams& +cv::gapi::onnx::PyParams::cfgAddExecutionProvider(cv::gapi::onnx::ep::CUDA ep) { + m_priv->cfgAddExecutionProvider(std::move(ep)); + return *this; +} + +cv::gapi::onnx::PyParams& +cv::gapi::onnx::PyParams::cfgAddExecutionProvider(cv::gapi::onnx::ep::TensorRT ep) { + m_priv->cfgAddExecutionProvider(std::move(ep)); + return *this; +} + cv::gapi::onnx::PyParams& cv::gapi::onnx::PyParams::cfgDisableMemPattern() { m_priv->cfgDisableMemPattern(); diff --git a/modules/gapi/src/backends/onnx/gonnxbackend.cpp b/modules/gapi/src/backends/onnx/gonnxbackend.cpp index b90d4d6974..c552b8b0e6 100644 --- a/modules/gapi/src/backends/onnx/gonnxbackend.cpp +++ b/modules/gapi/src/backends/onnx/gonnxbackend.cpp @@ -145,9 +145,39 @@ public: void run(); }; +static void addCUDAExecutionProvider(Ort::SessionOptions *session_options, + const cv::gapi::onnx::ep::CUDA &cuda_ep) { + OrtCUDAProviderOptions options{}; + options.device_id = cuda_ep.device_id; + + try { + session_options->AppendExecutionProvider_CUDA(options); + } catch (const std::exception &e) { + std::stringstream ss; + ss << "ONNX Backend: Failed to enable CUDA" + << " Execution Provider: " << e.what(); + cv::util::throw_error(std::runtime_error(ss.str())); + } +} + +static void addTensorRTExecutionProvider(Ort::SessionOptions *session_options, + const cv::gapi::onnx::ep::TensorRT &trt_ep) { + OrtTensorRTProviderOptions options{}; + options.device_id = trt_ep.device_id; + + try { + session_options->AppendExecutionProvider_TensorRT(options); + } catch (const std::exception &e) { + std::stringstream ss; + ss << "ONNX Backend: Failed to enable TensorRT" + << " Execution Provider: " << e.what(); + cv::util::throw_error(std::runtime_error(ss.str())); + } +} + static void addOpenVINOExecutionProvider(Ort::SessionOptions *session_options, const cv::gapi::onnx::ep::OpenVINO &ov_ep) { - OrtOpenVINOProviderOptions options; + OrtOpenVINOProviderOptions options{}; options.device_type = ov_ep.device_type.c_str(); options.cache_dir = ov_ep.cache_dir.c_str(); options.num_of_threads = ov_ep.num_of_threads; @@ -181,6 +211,18 @@ static void addExecutionProvider(Ort::SessionOptions *session_options, addDMLExecutionProvider(session_options, dml_ep); break; } + case ep::EP::index_of(): { + GAPI_LOG_INFO(NULL, "CUDA Execution Provider is added."); + const auto &cuda_ep = cv::util::get(execution_provider); + addCUDAExecutionProvider(session_options, cuda_ep); + break; + } + case ep::EP::index_of(): { + GAPI_LOG_INFO(NULL, "TensorRT Execution Provider is added."); + const auto &trt_ep = cv::util::get(execution_provider); + addTensorRTExecutionProvider(session_options, trt_ep); + break; + } default: GAPI_LOG_INFO(NULL, "CPU Execution Provider is added."); break;