mirror/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2024-11-27 20:50:25 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #24059 from TolyaTalamanov:at/add-onnx-cuda-execution-provider

Browse Source 
G-API: Support CUDA & TensoRT Execution Providers for ONNXRT Backend #24059

### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [ ] I agree to contribute to the project under Apache 2 License.
- [ ] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [ ] The PR is proposed to the proper branch
- [ ] There is a reference to the original bug report and related work
- [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [ ] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
Anatoliy Talamanov 2023-08-02 12:13:07 +01:00 committed by GitHub
parent 0883c6a913
commit f46f7eff0c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 156 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
modules/gapi/include/opencv2/gapi/infer/bindings_onnx.hpp
View File

@ -39,6 +39,12 @@ public:
GAPI_WRAP
PyParams& cfgAddExecutionProvider(ep::DirectML ep);
GAPI_WRAP
PyParams& cfgAddExecutionProvider(ep::CUDA ep);
GAPI_WRAP
PyParams& cfgAddExecutionProvider(ep::TensorRT ep);
GAPI_WRAP
PyParams& cfgDisableMemPattern();

94
modules/gapi/include/opencv2/gapi/infer/onnx.hpp
View File

@ -32,6 +32,56 @@ namespace onnx {
*/
namespace ep {
/**
* @brief This structure provides functions
* that fill inference options for CUDA Execution Provider.
* Please follow https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html#cuda-execution-provider
*/
struct GAPI_EXPORTS_W_SIMPLE CUDA {
// NB: Used from python.
/// @private -- Exclude this constructor from OpenCV documentation
GAPI_WRAP
CUDA() = default;
/** @brief Class constructor.
Constructs CUDA parameters based on device type information.
@param dev_id Target device id to use.
*/
GAPI_WRAP
explicit CUDA(const int dev_id)
: device_id(dev_id) {
}
int device_id;
};
/**
* @brief This structure provides functions
* that fill inference options for TensorRT Execution Provider.
* Please follow https://onnxruntime.ai/docs/execution-providers/TensorRT-ExecutionProvider.html#tensorrt-execution-provider
*/
struct GAPI_EXPORTS_W_SIMPLE TensorRT {
// NB: Used from python.
/// @private -- Exclude this constructor from OpenCV documentation
GAPI_WRAP
TensorRT() = default;
/** @brief Class constructor.
Constructs TensorRT parameters based on device type information.
@param dev_id Target device id to use.
*/
GAPI_WRAP
explicit TensorRT(const int dev_id)
: device_id(dev_id) {
}
int device_id;
};
/**
* @brief This structure provides functions
* that fill inference options for ONNX OpenVINO Execution Provider.
@ -143,7 +193,11 @@ public:
DeviceDesc ddesc;
};
using EP = cv::util::variant<cv::util::monostate, OpenVINO, DirectML>;
using EP = cv::util::variant< cv::util::monostate
, OpenVINO
, DirectML
, CUDA
, TensorRT>;
} // namespace ep
@ -431,6 +485,34 @@ public:
return *this;
}
/** @brief Adds execution provider for runtime.
The function is used to add ONNX Runtime CUDA Execution Provider options.
@param ep CUDA Execution Provider options.
@see cv::gapi::onnx::ep::CUDA.
@return the reference on modified object.
*/
Params<Net>& cfgAddExecutionProvider(ep::CUDA&& ep) {
desc.execution_providers.emplace_back(std::move(ep));
return *this;
}
/** @brief Adds execution provider for runtime.
The function is used to add ONNX Runtime TensorRT Execution Provider options.
@param ep TensorRT Execution Provider options.
@see cv::gapi::onnx::ep::TensorRT.
@return the reference on modified object.
*/
Params<Net>& cfgAddExecutionProvider(ep::TensorRT&& ep) {
desc.execution_providers.emplace_back(std::move(ep));
return *this;
}
/** @brief Disables the memory pattern optimization.
@return the reference on modified object.
@ -491,6 +573,16 @@ public:
desc.execution_providers.emplace_back(std::move(ep));
}
/** @see onnx::Params::cfgAddExecutionProvider. */
void cfgAddExecutionProvider(ep::CUDA&& ep) {
desc.execution_providers.emplace_back(std::move(ep));
}
/** @see onnx::Params::cfgAddExecutionProvider. */
void cfgAddExecutionProvider(ep::TensorRT&& ep) {
desc.execution_providers.emplace_back(std::move(ep));
}
/** @see onnx::Params::cfgDisableMemPattern. */
void cfgDisableMemPattern() {
desc.disable_mem_pattern = true;

2
modules/gapi/misc/python/pyopencv_gapi.hpp
View File

@ -31,6 +31,8 @@ using map_string_and_vector_float = std::map<std::string, std::vector<float>>;
using map_int_and_double = std::map<int, double>;
using ep_OpenVINO = cv::gapi::onnx::ep::OpenVINO;
using ep_DirectML = cv::gapi::onnx::ep::DirectML;
using ep_CUDA = cv::gapi::onnx::ep::CUDA;
using ep_TensorRT = cv::gapi::onnx::ep::TensorRT;
// NB: Python wrapper generate T_U for T<U>
// This behavior is only observed for inputs

12
modules/gapi/src/backends/onnx/bindings_onnx.cpp
View File

@ -33,6 +33,18 @@ cv::gapi::onnx::PyParams::cfgAddExecutionProvider(cv::gapi::onnx::ep::DirectML e
return *this;
}
cv::gapi::onnx::PyParams&
cv::gapi::onnx::PyParams::cfgAddExecutionProvider(cv::gapi::onnx::ep::CUDA ep) {
m_priv->cfgAddExecutionProvider(std::move(ep));
return *this;
}
cv::gapi::onnx::PyParams&
cv::gapi::onnx::PyParams::cfgAddExecutionProvider(cv::gapi::onnx::ep::TensorRT ep) {
m_priv->cfgAddExecutionProvider(std::move(ep));
return *this;
}
cv::gapi::onnx::PyParams&
cv::gapi::onnx::PyParams::cfgDisableMemPattern() {
m_priv->cfgDisableMemPattern();

44
modules/gapi/src/backends/onnx/gonnxbackend.cpp
View File

@ -145,9 +145,39 @@ public:
void run();
};
static void addCUDAExecutionProvider(Ort::SessionOptions *session_options,
const cv::gapi::onnx::ep::CUDA &cuda_ep) {
OrtCUDAProviderOptions options{};
options.device_id = cuda_ep.device_id;
try {
session_options->AppendExecutionProvider_CUDA(options);
} catch (const std::exception &e) {
std::stringstream ss;
ss << "ONNX Backend: Failed to enable CUDA"
<< " Execution Provider: " << e.what();
cv::util::throw_error(std::runtime_error(ss.str()));
}
}
static void addTensorRTExecutionProvider(Ort::SessionOptions *session_options,
const cv::gapi::onnx::ep::TensorRT &trt_ep) {
OrtTensorRTProviderOptions options{};
options.device_id = trt_ep.device_id;
try {
session_options->AppendExecutionProvider_TensorRT(options);
} catch (const std::exception &e) {
std::stringstream ss;
ss << "ONNX Backend: Failed to enable TensorRT"
<< " Execution Provider: " << e.what();
cv::util::throw_error(std::runtime_error(ss.str()));
}
}
static void addOpenVINOExecutionProvider(Ort::SessionOptions *session_options,
const cv::gapi::onnx::ep::OpenVINO &ov_ep) {
OrtOpenVINOProviderOptions options;
OrtOpenVINOProviderOptions options{};
options.device_type = ov_ep.device_type.c_str();
options.cache_dir = ov_ep.cache_dir.c_str();
options.num_of_threads = ov_ep.num_of_threads;
@ -181,6 +211,18 @@ static void addExecutionProvider(Ort::SessionOptions *session_options,
addDMLExecutionProvider(session_options, dml_ep);
break;
}
case ep::EP::index_of<ep::CUDA>(): {
GAPI_LOG_INFO(NULL, "CUDA Execution Provider is added.");
const auto &cuda_ep = cv::util::get<ep::CUDA>(execution_provider);
addCUDAExecutionProvider(session_options, cuda_ep);
break;
}
case ep::EP::index_of<ep::TensorRT>(): {
GAPI_LOG_INFO(NULL, "TensorRT Execution Provider is added.");
const auto &trt_ep = cv::util::get<ep::TensorRT>(execution_provider);
addTensorRTExecutionProvider(session_options, trt_ep);
break;
}
default:
GAPI_LOG_INFO(NULL, "CPU Execution Provider is added.");
break;