mirror/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2025-08-05 22:19:14 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #21564 from alalek:dnn_fix_openvino_outputs

Browse Source
This commit is contained in:
Alexander Alekhin 2022-02-06 16:06:23 +00:00
commit a00a0dbfcd
4 changed files with 74 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
modules/dnn/include/opencv2/dnn/shape_utils.hpp
View File

@ -184,7 +184,8 @@ static inline MatShape concat(const MatShape& a, const MatShape& b)
return c;
}
static inline std::string toString(const MatShape& shape, const String& name = "")
template<typename _Tp>
static inline std::string toString(const std::vector<_Tp>& shape, const String& name = "")
{
std::ostringstream ss;
if (!name.empty())
@ -195,11 +196,14 @@ static inline std::string toString(const MatShape& shape, const String& name = "
ss << " ]";
return ss.str();
}
static inline void print(const MatShape& shape, const String& name = "")
template<typename _Tp>
static inline void print(const std::vector<_Tp>& shape, const String& name = "")
{
std::cout << toString(shape, name) << std::endl;
}
static inline std::ostream& operator<<(std::ostream &out, const MatShape& shape)
template<typename _Tp>
static inline std::ostream& operator<<(std::ostream &out, const std::vector<_Tp>& shape)
{
out << toString(shape);
return out;

49
modules/dnn/src/dnn.cpp
View File

@ -1937,10 +1937,15 @@ struct Net::Impl : public detail::NetImplBase
#ifdef HAVE_DNN_NGRAPH
/** mark input pins as outputs from other subnetworks
* FIXIT must be done by DNN engine not ngraph.
*/
void addNgraphOutputs(LayerData &ld)
{
CV_TRACE_FUNCTION();
CV_LOG_DEBUG(NULL, "DNN/IE: layer of new subnet: " << ld.name << "@" << ld.type);
Ptr<InfEngineNgraphNet> layerNet;
auto it = ld.backendNodes.find(preferableBackend);
if (it != ld.backendNodes.end())
@ -1964,8 +1969,8 @@ struct Net::Impl : public detail::NetImplBase
CV_Assert(!ieInpNode.empty()); CV_Assert(!ieInpNode->net.empty());
if (layerNet != ieInpNode->net)
{
ieInpNode->net->addOutput(ieInpNode->node->get_friendly_name());
ieInpNode->net->setUnconnectedNodes(ieInpNode);
CV_LOG_DEBUG(NULL, "DNN/IE: pin output between subnets: " << ieInpNode->node->get_friendly_name());
ieInpNode->net->addOutput(ieInpNode);
}
}
}
@ -2064,13 +2069,19 @@ struct Net::Impl : public detail::NetImplBase
{
LayerData& ld = it->second;
CV_LOG_DEBUG(NULL, "DNN/IE: processing layer " << ld.name << "@" << ld.type << " (" << ld.id << ") ...");
if (ld.id == 0 && ld.skip)
{
CV_LOG_DEBUG(NULL, "DNN/IE: SKIP!");
continue;
}
bool fused = ld.skip;
Ptr<Layer> layer = ld.layerInstance;
if (!fused && !layer->supportBackend(preferableBackend))
{
CV_LOG_DEBUG(NULL, "DNN/IE: NOT supported!");
bool customizable = ld.id != 0 && supportsCPUFallback;
// TODO: there is a bug in Myriad plugin with custom layers shape infer.
@ -2097,6 +2108,7 @@ struct Net::Impl : public detail::NetImplBase
if (!customizable)
{
CV_LOG_DEBUG(NULL, "DNN/IE: NOT customizable!");
addNgraphOutputs(ld);
net = Ptr<InfEngineNgraphNet>();
layer->preferableTarget = DNN_TARGET_CPU;
@ -2108,7 +2120,7 @@ struct Net::Impl : public detail::NetImplBase
if (!inpNode.empty()) {
Ptr<InfEngineNgraphNode> ieNode = inpNode.dynamicCast<InfEngineNgraphNode>();
CV_Assert(!ieNode.empty());
ieNode->net->setUnconnectedNodes(ieNode);
ieNode->net->addOutput(ieNode);
}
}
continue;
@ -2221,21 +2233,30 @@ struct Net::Impl : public detail::NetImplBase
if (layer->supportBackend(preferableBackend))
{
CV_LOG_DEBUG(NULL, "DNN/IE: wrap layer " << ld.name << "@" << ld.type << " - outputs: " << ld.outputBlobsWrappers.size());
node = layer->initNgraph(ld.inputBlobsWrappers, inputNodes);
#if 0 // FIXIT doesn't work with multiple outputs (set name is applied to the same node)
for (int i = 0; i < ld.outputBlobsWrappers.size(); ++i)
{
InferenceEngine::DataPtr dataPtr = ngraphDataNode(ld.outputBlobsWrappers[i]);
node.dynamicCast<InfEngineNgraphNode>()->setName(dataPtr->getName());
}
#else
node.dynamicCast<InfEngineNgraphNode>()->setName(layer->name);
#endif
}
else
{
CV_LOG_DEBUG(NULL, "DNN/IE: layer is not supported: " << ld.name << "@" << ld.type);
node = Ptr<BackendNode>(new InfEngineNgraphNode(inputNodes,
ld.layerInstance, ld.inputBlobs, ld.outputBlobs, ld.internals));
}
}
else if (node.empty())
{
CV_LOG_DEBUG(NULL, "DNN/IE: node.empty() bypass...");
continue;
}
ld.backendNodes[preferableBackend] = node;
@ -2243,15 +2264,11 @@ struct Net::Impl : public detail::NetImplBase
CV_Assert(!ieNode.empty());
ieNode->net = net;
if (ld.consumers.empty()) {
// TF EAST_text_detection
ieNode->net->setUnconnectedNodes(ieNode);
}
for (const auto& pin : blobsToKeep_)
{
if (pin.lid == ld.id)
{
ieNode->net->addOutput(ieNode->node->get_friendly_name());
ieNode->net->addOutput(ieNode);
break;
}
}
@ -2282,7 +2299,7 @@ struct Net::Impl : public detail::NetImplBase
if (!ieNode->net->isInitialized())
{
ieNode->net->setUnconnectedNodes(ieNode);
ieNode->net->addOutput(ieNode);
ieNode->net->createNet((Target)preferableTarget);
ld.skip = false;
}
@ -2412,8 +2429,15 @@ struct Net::Impl : public detail::NetImplBase
preferableBackend != DNN_BACKEND_INFERENCE_ENGINE_NGRAPH))
return;
#if 0 // FIXIT mode without fusion is broken due to unsupported layers and handling of "custom" nodes
if (preferableBackend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH)
return;
#endif
// scan through all the layers. If there is convolution layer followed by the activation layer,
// we try to embed this activation into the convolution and disable separate execution of the activation
// FIXIT replace by layersToKeep to avoid hacks like "LayerPin(lid, 0)"
std::set<LayerPin> pinsToKeep(blobsToKeep_.begin(),
blobsToKeep_.end());
for (MapIdToLayerData::const_iterator it = layers.begin(); it != layers.end(); it++)
@ -2438,6 +2462,13 @@ struct Net::Impl : public detail::NetImplBase
LayerPin lpNext(ld.consumers[0].lid, 0);
while (nextData)
{
#ifdef HAVE_INF_ENGINE
if (preferableBackend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && pinsToKeep.count(lpNext) != 0)
{
CV_LOG_DEBUG(NULL, "DNN/IE: skip fusing with 'output' node: " << nextData->name << "@" << nextData->type);
break;
}
#endif
Ptr<Layer> nextLayer = nextData->layerInstance;
if (currLayer->tryFuse(nextLayer))
{

45
modules/dnn/src/ie_ngraph.cpp
View File

@ -379,16 +379,21 @@ InfEngineNgraphNet::InfEngineNgraphNet(detail::NetImplBase& netImpl, InferenceEn
device_name = "CPU";
}
void InfEngineNgraphNet::addOutput(const std::string& name)
void InfEngineNgraphNet::addOutput(const Ptr<InfEngineNgraphNode>& node)
{
requestedOutputs.push_back(name);
CV_Assert(node);
CV_Assert(node->node);
const std::string& name = node->node->get_friendly_name();
requestedOutputs.insert({name, node});
}
void InfEngineNgraphNet::setNodePtr(std::shared_ptr<ngraph::Node>* ptr) {
all_nodes.emplace((*ptr)->get_friendly_name(), ptr);
}
void InfEngineNgraphNet::release() {
void InfEngineNgraphNet::release()
{
// FIXIT release should not be conditional, release ALL
for (auto& node : components.back()) {
#if INF_ENGINE_VER_MAJOR_GT(INF_ENGINE_RELEASE_2020_4)
if (!(ngraph::op::is_parameter(node) || ngraph::op::is_output(node) || ngraph::op::is_constant(node)) ) {
@ -397,7 +402,6 @@ void InfEngineNgraphNet::setNodePtr(std::shared_ptr<ngraph::Node>* ptr) {
#endif
auto it = all_nodes.find(node->get_friendly_name());
if (it != all_nodes.end()) {
unconnectedNodes.erase(*(it->second));
it->second->reset();
all_nodes.erase(it);
}
@ -422,7 +426,8 @@ void InfEngineNgraphNet::dfs(std::shared_ptr<ngraph::Node>& node,
}
}
int InfEngineNgraphNet::getNumComponents() {
int InfEngineNgraphNet::getNumComponents()
{
if (!components.empty()) {
return components.size();
}
@ -445,17 +450,21 @@ int InfEngineNgraphNet::getNumComponents() {
void InfEngineNgraphNet::createNet(Target targetId) {
if (!hasNetOwner)
{
CV_Assert(!unconnectedNodes.empty());
CV_Assert(!requestedOutputs.empty());
ngraph::ResultVector outs;
for (auto& node : unconnectedNodes)
for (auto output_node_it = requestedOutputs.begin(); output_node_it != requestedOutputs.end(); ++output_node_it)
{
auto out = std::make_shared<ngraph::op::Result>(node);
CV_LOG_DEBUG(NULL, "DNN/NGRAPH: Add 'Result' output: " << output_node_it->first);
CV_Assert(output_node_it->second);
auto out = std::make_shared<ngraph::op::Result>(output_node_it->second->node);
outs.push_back(out);
}
CV_Assert_N(!inputs_vec.empty(), !outs.empty());
ngraph_function = std::make_shared<ngraph::Function>(outs, inputs_vec);
int num_comp = getNumComponents();
CV_LOG_DEBUG(NULL, "DNN/IE: number of subgraphs: " << num_comp);
if (num_comp > 1) {
for (int i = num_comp - 1; i >= 0; --i) {
ngraph::ResultVector outputs;
@ -466,6 +475,7 @@ void InfEngineNgraphNet::createNet(Target targetId) {
#else
if (node->is_parameter()) {
#endif
CV_LOG_DEBUG(NULL, "DNN/IE: subgraph[" << i << "]: +input[" << inps.size() << "] = '" << node->get_friendly_name() << "'");
auto parameter = std::dynamic_pointer_cast<ngraph::op::Parameter>(node);
inps.push_back(parameter);
}
@ -474,10 +484,12 @@ void InfEngineNgraphNet::createNet(Target targetId) {
#else
else if (node->is_output()) {
#endif
CV_LOG_DEBUG(NULL, "DNN/IE: subgraph[" << i << "]: +output[" << outputs.size() << "] = '" << node->get_friendly_name() << "'");
auto result = std::dynamic_pointer_cast<ngraph::op::Result>(node);
outputs.push_back(result);
}
}
CV_LOG_DEBUG(NULL, "DNN/IE: subgraph[" << i << ": nodes=" << components.back().size() << " inputs=" << inps.size() << " outputs=" << outputs.size());
isInit = false;
CV_Assert_N(!inps.empty(), !outputs.empty());
ngraph_function = std::make_shared<ngraph::Function>(outputs, inps);
@ -571,7 +583,7 @@ void InfEngineNgraphNet::init(Target targetId)
auto node = ngraph_function->output(i).get_node();
for (size_t j = 0; j < node->get_input_size(); ++j) {
std::string name = node->input_value(j).get_node()->get_friendly_name();
auto iter = std::find(requestedOutputs.begin(), requestedOutputs.end(), name);
auto iter = requestedOutputs.find(name);
if (iter != requestedOutputs.end()) {
requestedOutputs.erase(iter);
cnn.addOutput(name);
@ -579,10 +591,6 @@ void InfEngineNgraphNet::init(Target targetId)
}
}
}
for (const auto& name : requestedOutputs)
{
cnn.addOutput(name);
}
for (const auto& it : cnn.getInputsInfo())
{
@ -627,9 +635,6 @@ ngraph::ParameterVector InfEngineNgraphNet::setInputs(const std::vector<cv::Mat>
return current_inp;
}
void InfEngineNgraphNet::setUnconnectedNodes(Ptr<InfEngineNgraphNode>& node) {
unconnectedNodes.insert(node->node);
}
void InfEngineNgraphNet::initPlugin(InferenceEngine::CNNNetwork& net)
{
@ -729,10 +734,10 @@ void InfEngineNgraphNet::initPlugin(InferenceEngine::CNNNetwork& net)
}
}
}
if (isHetero)
netExec = ie.LoadNetwork(net, "HETERO:" + device_name + ",CPU", config);
else
netExec = ie.LoadNetwork(net, device_name, config);
std::string ieDevice = isHetero ? ("HETERO:" + device_name + ",CPU") : device_name;
CV_LOG_INFO(NULL, "DNN/IE: Calling LoadNetwork(device=" << ieDevice << ")...");
netExec = ie.LoadNetwork(net, ieDevice, config);
}
catch (const std::exception& ex)
{

6
modules/dnn/src/ie_ngraph.hpp
View File

@ -37,7 +37,7 @@ public:
InfEngineNgraphNet(detail::NetImplBase& netImpl);
InfEngineNgraphNet(detail::NetImplBase& netImpl, InferenceEngine::CNNNetwork& net);
void addOutput(const std::string& name);
void addOutput(const Ptr<InfEngineNgraphNode>& node);
bool isInitialized();
void init(Target targetId);
@ -47,7 +47,6 @@ public:
void initPlugin(InferenceEngine::CNNNetwork& net);
ngraph::ParameterVector setInputs(const std::vector<cv::Mat>& inputs, const std::vector<std::string>& names);
void setUnconnectedNodes(Ptr<InfEngineNgraphNode>& node);
void addBlobs(const std::vector<cv::Ptr<BackendWrapper> >& ptrs);
void createNet(Target targetId);
@ -88,8 +87,7 @@ public:
InferenceEngine::CNNNetwork cnn;
bool hasNetOwner;
std::vector<std::string> requestedOutputs;
std::unordered_set<std::shared_ptr<ngraph::Node>> unconnectedNodes;
std::unordered_map<std::string, Ptr<InfEngineNgraphNode> > requestedOutputs;
std::map<std::string, InferenceEngine::TensorDesc> outputsDesc;
};