mirror/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2025-07-25 22:57:53 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #26208 from alexlyulkov:al/new-engine-caffe-parser

Browse Source 
Modified Caffe parser to support the new dnn engine #26208

Now the Caffe parser supports both the old and the new engine. It can be selected using newEngine argument in PopulateNet.

All cpu Caffe tests work fine except:

- Test_Caffe_nets.Colorization
- Test_Caffe_layers.FasterRCNN_Proposal

Both these tests doesn't work because of the bug in the new net.forward function. The function takes the name of the desired target last layer, but uses this name as the name of the desired output tensor.
Also Colorization test contains a strange model with a Silence layer in the end, so it doesn't have outputs. The old parser just ignored it. I think, the proper solution is to run this model until the (number_of_layers - 2) layer using proper net.forward arguments in the test.

### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [ ] The PR is proposed to the proper branch
- [ ] There is a reference to the original bug report and related work
- [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [x] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
alexlyulkov 2024-10-28 11:32:07 +03:00 committed by GitHub
parent 7b0a082dd4
commit a2fa1d49a4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 366 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
modules/dnn/include/opencv2/dnn/dnn.hpp
View File

@ -1051,17 +1051,24 @@ CV__DNN_INLINE_NS_BEGIN
/** @brief Reads a network model stored in <a href="http://caffe.berkeleyvision.org">Caffe</a> framework's format. /** @brief Reads a network model stored in <a href="http://caffe.berkeleyvision.org">Caffe</a> framework's format.
* @param prototxt path to the .prototxt file with text description of the network architecture. * @param prototxt path to the .prototxt file with text description of the network architecture.
* @param caffeModel path to the .caffemodel file with learned network. * @param caffeModel path to the .caffemodel file with learned network.
* @param engine select DNN engine to be used. With auto selection the new engine is used.
* Please pay attention that the new DNN does not support non-CPU back-ends for now.
* @returns Net object. * @returns Net object.
*/ */
CV_EXPORTS_W Net readNetFromCaffe(CV_WRAP_FILE_PATH const String &prototxt, CV_WRAP_FILE_PATH const String &caffeModel = String()); CV_EXPORTS_W Net readNetFromCaffe(CV_WRAP_FILE_PATH const String &prototxt,
CV_WRAP_FILE_PATH const String &caffeModel = String(),
int engine = ENGINE_AUTO);
/** @brief Reads a network model stored in Caffe model in memory. /** @brief Reads a network model stored in Caffe model in memory.
* @param bufferProto buffer containing the content of the .prototxt file * @param bufferProto buffer containing the content of the .prototxt file
* @param bufferModel buffer containing the content of the .caffemodel file * @param bufferModel buffer containing the content of the .caffemodel file
* @param engine select DNN engine to be used. With auto selection the new engine is used.
* Please pay attention that the new DNN does not support non-CPU back-ends for now.
* @returns Net object. * @returns Net object.
*/ */
CV_EXPORTS_W Net readNetFromCaffe(const std::vector<uchar>& bufferProto, CV_EXPORTS_W Net readNetFromCaffe(const std::vector<uchar>& bufferProto,
const std::vector<uchar>& bufferModel = std::vector<uchar>()); const std::vector<uchar>& bufferModel = std::vector<uchar>(),
int engine = ENGINE_AUTO);
/** @brief Reads a network model stored in Caffe model in memory. /** @brief Reads a network model stored in Caffe model in memory.
* @details This is an overloaded member function, provided for convenience. * @details This is an overloaded member function, provided for convenience.
@ -1070,10 +1077,13 @@ CV__DNN_INLINE_NS_BEGIN
* @param lenProto length of bufferProto * @param lenProto length of bufferProto
* @param bufferModel buffer containing the content of the .caffemodel file * @param bufferModel buffer containing the content of the .caffemodel file
* @param lenModel length of bufferModel * @param lenModel length of bufferModel
* @param engine select DNN engine to be used. With auto selection the new engine is used.
* Please pay attention that the new DNN does not support non-CPU back-ends for now.
* @returns Net object. * @returns Net object.
*/ */
CV_EXPORTS Net readNetFromCaffe(const char *bufferProto, size_t lenProto, CV_EXPORTS Net readNetFromCaffe(const char *bufferProto, size_t lenProto,
const char *bufferModel = NULL, size_t lenModel = 0); const char *bufferModel = NULL, size_t lenModel = 0,
int engine = ENGINE_AUTO);
/** @brief Reads a network model stored in <a href="https://www.tensorflow.org/">TensorFlow</a> framework's format. /** @brief Reads a network model stored in <a href="https://www.tensorflow.org/">TensorFlow</a> framework's format.
* @param model path to the .pb file with binary protobuf description of the network architecture * @param model path to the .pb file with binary protobuf description of the network architecture

4
modules/dnn/misc/objc/gen_dict.json
View File

@ -1,8 +1,8 @@
{ {
"func_arg_fix" : { "func_arg_fix" : {
"Dnn": { "Dnn": {
"(Net*)readNetFromCaffe:(NSString*)prototxt caffeModel:(NSString*)caffeModel" : { "readNetFromCaffe" : {"name" : "readNetFromCaffeFile"} }, "(Net*)readNetFromCaffe:(NSString*)prototxt caffeModel:(NSString*)caffeModel engine:(int)engine" : { "readNetFromCaffe" : {"name" : "readNetFromCaffeFile"} },
"(Net*)readNetFromCaffe:(ByteVector*)bufferProto bufferModel:(ByteVector*)bufferModel" : { "readNetFromCaffe" : {"name" : "readNetFromCaffeBuffer"} }, "(Net*)readNetFromCaffe:(ByteVector*)bufferProto bufferModel:(ByteVector*)bufferModel engine:(int)engine" : { "readNetFromCaffe" : {"name" : "readNetFromCaffeBuffer"} },
"(Net*)readNetFromDarknet:(NSString*)cfgFile darknetModel:(NSString*)darknetModel" : { "readNetFromDarknet" : {"name" : "readNetFromDarknetFile"} }, "(Net*)readNetFromDarknet:(NSString*)cfgFile darknetModel:(NSString*)darknetModel" : { "readNetFromDarknet" : {"name" : "readNetFromDarknetFile"} },
"(Net*)readNetFromDarknet:(ByteVector*)bufferCfg bufferModel:(ByteVector*)bufferModel" : { "readNetFromDarknet" : {"name" : "readNetFromDarknetBuffer"} }, "(Net*)readNetFromDarknet:(ByteVector*)bufferCfg bufferModel:(ByteVector*)bufferModel" : { "readNetFromDarknet" : {"name" : "readNetFromDarknetBuffer"} },
"(Net*)readNetFromONNX:(NSString*)onnxFile engine:(int)engine" : { "readNetFromONNX" : {"name" : "readNetFromONNXFile"} }, "(Net*)readNetFromONNX:(NSString*)onnxFile engine:(int)engine" : { "readNetFromONNX" : {"name" : "readNetFromONNXFile"} },

9
modules/dnn/misc/python/test/test_dnn.py
View File

@ -112,7 +112,7 @@ class dnn_test(NewOpenCVTests):
def checkIETarget(self, backend, target): def checkIETarget(self, backend, target):
proto = self.find_dnn_file('dnn/layers/layer_convolution.prototxt') proto = self.find_dnn_file('dnn/layers/layer_convolution.prototxt')
model = self.find_dnn_file('dnn/layers/layer_convolution.caffemodel') model = self.find_dnn_file('dnn/layers/layer_convolution.caffemodel')
net = cv.dnn.readNet(proto, model) net = cv.dnn.readNet(proto, model, engine=cv.dnn.ENGINE_CLASSIC)
try: try:
net.setPreferableBackend(backend) net.setPreferableBackend(backend)
net.setPreferableTarget(target) net.setPreferableTarget(target)
@ -324,6 +324,9 @@ class dnn_test(NewOpenCVTests):
testScores, testBoxes, 0.5) testScores, testBoxes, 0.5)
def test_async(self): def test_async(self):
# bug: https://github.com/opencv/opencv/issues/26376
raise unittest.SkipTest("The new dnn engine does not support async inference")
timeout = 10*1000*10**6 # in nanoseconds (10 sec) timeout = 10*1000*10**6 # in nanoseconds (10 sec)
proto = self.find_dnn_file('dnn/layers/layer_convolution.prototxt') proto = self.find_dnn_file('dnn/layers/layer_convolution.prototxt')
model = self.find_dnn_file('dnn/layers/layer_convolution.caffemodel') model = self.find_dnn_file('dnn/layers/layer_convolution.caffemodel')
@ -337,7 +340,7 @@ class dnn_test(NewOpenCVTests):
printParams(backend, target) printParams(backend, target)
netSync = cv.dnn.readNet(proto, model) netSync = cv.dnn.readNet(proto, model, engine=cv.dnn.ENGINE_CLASSIC)
netSync.setPreferableBackend(backend) netSync.setPreferableBackend(backend)
netSync.setPreferableTarget(target) netSync.setPreferableTarget(target)
@ -463,7 +466,7 @@ class dnn_test(NewOpenCVTests):
for backend, target in self.dnnBackendsAndTargets: for backend, target in self.dnnBackendsAndTargets:
printParams(backend, target) printParams(backend, target)
net = cv.dnn.readNet(model) net = cv.dnn.readNet(model, engine=cv.dnn.ENGINE_CLASSIC)
net.setPreferableBackend(backend) net.setPreferableBackend(backend)
net.setPreferableTarget(target) net.setPreferableTarget(target)

292
modules/dnn/src/caffe/caffe_importer.cpp
View File

@ -40,6 +40,7 @@
//M*/ //M*/
#include "../precomp.hpp" #include "../precomp.hpp"
#include "../net_impl.hpp"
#ifdef HAVE_PROTOBUF #ifdef HAVE_PROTOBUF
#include <iostream> #include <iostream>
@ -53,8 +54,10 @@
#include "caffe_io.hpp" #include "caffe_io.hpp"
#endif #endif
#include <opencv2/core/utils/configuration.private.hpp>
#include <opencv2/core/utils/fp_control_utils.hpp> #include <opencv2/core/utils/fp_control_utils.hpp>
namespace cv { namespace cv {
namespace dnn { namespace dnn {
CV__DNN_INLINE_NS_BEGIN CV__DNN_INLINE_NS_BEGIN
@ -320,6 +323,30 @@ public:
} }
} }
Ptr<Layer> addLayer(Net& dstNet,
const String& type,
const String& name,
LayerParams& layerParams,
const std::vector<String>& inputs,
const std::vector<String>& outputs)
{
layerParams.type = type;
layerParams.name = name;
Ptr<Layer> layer = LayerFactory::createLayerInstance(type, layerParams);
if (!layer) {
CV_Error(Error::StsError, "Can't create layer " + name + " with type " + type);
return nullptr;
}
for (const String& inputName : inputs)
layer->inputs.push_back(dstNet.getArg(inputName));
for (const String& outputName : outputs)
layer->outputs.push_back(dstNet.getArg(outputName));
layer->netimpl = dstNet.getImpl();
CV_Assert(dstNet.getImpl()->dump_indent == 3);
return layer;
}
struct BlobNote struct BlobNote
{ {
BlobNote(const std::string &_name, int _layerId, int _outNum) : BlobNote(const std::string &_name, int _layerId, int _outNum) :
@ -332,24 +359,41 @@ public:
std::vector<BlobNote> addedBlobs; std::vector<BlobNote> addedBlobs;
std::map<String, int> layerCounter; std::map<String, int> layerCounter;
void populateNet(Net dstNet) void populateNet(Net dstNet, bool newEngine)
{ {
CV_TRACE_FUNCTION(); CV_TRACE_FUNCTION();
int layersSize = net.layer_size(); int layersSize = net.layer_size();
layerCounter.clear(); layerCounter.clear();
addedBlobs.clear();
addedBlobs.reserve(layersSize + 1);
//setup input layer names // OLD ENGINE
if(!newEngine)
{
addedBlobs.clear();
addedBlobs.reserve(layersSize + 1);
}
std::vector<String> netInputs(net.input_size()); std::vector<String> netInputs(net.input_size());
std::vector<MatShape> inp_shapes; std::vector<MatShape> inp_shapes;
// NEW ENGINE
Net::Impl* netImpl = dstNet.getImpl();
std::vector<Ptr<Layer>> curr_prog;
std::vector<Arg> modelInputs, modelOutputs;
{ {
int net_input_size = net.input_size(); int net_input_size = net.input_size();
for (int inNum = 0; inNum < net_input_size; inNum++) for (int inNum = 0; inNum < net_input_size; inNum++)
{ {
addedBlobs.push_back(BlobNote(net.input(inNum), 0, inNum)); if (newEngine)
netInputs[inNum] = net.input(inNum); {
modelInputs.push_back(netImpl->newArg(net.input(inNum), DNN_ARG_INPUT));
netImpl->args.at(modelInputs.back().idx).type = CV_32F;
}
else
{
addedBlobs.push_back(BlobNote(net.input(inNum), 0, inNum));
netInputs[inNum] = net.input(inNum);
}
} }
if (net.input_dim_size() > 0) // deprecated in Caffe proto if (net.input_dim_size() > 0) // deprecated in Caffe proto
@ -365,7 +409,10 @@ public:
shape[1] = net.input_dim(dim+1); shape[1] = net.input_dim(dim+1);
shape[2] = net.input_dim(dim+2); shape[2] = net.input_dim(dim+2);
shape[3] = net.input_dim(dim+3); shape[3] = net.input_dim(dim+3);
inp_shapes.push_back(shape); if (newEngine)
netImpl->args.at(modelInputs[inp_id].idx).shape = shape;
else
inp_shapes.push_back(shape);
} }
} }
else if (net.input_shape_size() > 0) // deprecated in Caffe proto else if (net.input_shape_size() > 0) // deprecated in Caffe proto
@ -375,7 +422,10 @@ public:
for (int inp_id = 0; inp_id < net_input_shape_size; inp_id++) for (int inp_id = 0; inp_id < net_input_shape_size; inp_id++)
{ {
MatShape shape = parseBlobShape(net.input_shape(inp_id)); MatShape shape = parseBlobShape(net.input_shape(inp_id));
inp_shapes.push_back(shape); if (newEngine)
netImpl->args.at(modelInputs[inp_id].idx).shape = shape;
else
inp_shapes.push_back(shape);
} }
} }
else else
@ -383,11 +433,20 @@ public:
for (int inp_id = 0; inp_id < net_input_size; inp_id++) for (int inp_id = 0; inp_id < net_input_size; inp_id++)
{ {
MatShape shape; // empty MatShape shape; // empty
inp_shapes.push_back(shape); if (newEngine)
netImpl->args.at(modelInputs[inp_id].idx).shape = shape;
else
inp_shapes.push_back(shape);
} }
} }
} }
if (newEngine && net.layer(layersSize - 1).type() == "Silence")
{
CV_LOG_WARNING(NULL, "Caffe parser: Silence layer was ignored");
layersSize--;
}
for (int li = 0; li < layersSize; li++) for (int li = 0; li < layersSize; li++)
{ {
const caffe::LayerParameter &layer = net.layer(li); const caffe::LayerParameter &layer = net.layer(li);
@ -398,6 +457,12 @@ public:
extractLayerParams(layer, layerParams); extractLayerParams(layer, layerParams);
extractBinaryLayerParams(layer, layerParams); extractBinaryLayerParams(layer, layerParams);
if (newEngine && li == layersSize - 1)
{
for (int outNum = 0; outNum < layer.top_size(); outNum++)
modelOutputs.push_back(netImpl->newArg(layer.top(outNum), DNN_ARG_OUTPUT));
}
int repetitions = layerCounter[name]++; int repetitions = layerCounter[name]++;
if (repetitions) if (repetitions)
name += String("_") + toString(repetitions); name += String("_") + toString(repetitions);
@ -406,9 +471,17 @@ public:
{ {
for (int outNum = 0; outNum < layer.top_size(); outNum++) for (int outNum = 0; outNum < layer.top_size(); outNum++)
{ {
addOutput(layer, 0, outNum); if (newEngine)
addedBlobs.back().outNum = netInputs.size(); {
netInputs.push_back(addedBlobs.back().name); modelInputs.push_back(netImpl->newArg(layer.top(outNum), DNN_ARG_INPUT));
netImpl->args.at(modelInputs.back().idx).type = CV_32F;
}
else
{
addOutput(layer, 0, outNum);
addedBlobs.back().outNum = netInputs.size();
netInputs.push_back(addedBlobs.back().name);
}
} }
if (layer.has_input_param()) if (layer.has_input_param())
{ {
@ -418,7 +491,15 @@ public:
for (int inp_id = 0; inp_id < input_shape_size; inp_id++) for (int inp_id = 0; inp_id < input_shape_size; inp_id++)
{ {
MatShape shape = parseBlobShape(inputParameter.shape(inp_id)); MatShape shape = parseBlobShape(inputParameter.shape(inp_id));
inp_shapes.push_back(shape); if (newEngine)
{
int inputIdx = modelInputs.size() - input_shape_size + inp_id;
netImpl->args.at(modelInputs[inputIdx].idx).shape = shape;
}
else
{
inp_shapes.push_back(shape);
}
} }
} }
continue; continue;
@ -437,12 +518,24 @@ public:
if (repetitions) if (repetitions)
mvnName += String("_") + toString(repetitions); mvnName += String("_") + toString(repetitions);
int mvnId = dstNet.addLayer(mvnName, "MVN", mvnParams); if (newEngine)
addInput(layer.bottom(0), mvnId, 0, dstNet); {
addOutput(layer, mvnId, 0); Ptr<Layer> netLayer = addLayer(
net.mutable_layer(li)->set_bottom(0, layer.top(0)); dstNet, "MVN", mvnName, mvnParams,
layerParams.blobs[0].setTo(0); // mean {layer.bottom(0)},
layerParams.blobs[1].setTo(1); // std {layer.top(0)});
curr_prog.push_back(netLayer);
continue;
}
else
{
int mvnId = dstNet.addLayer(mvnName, "MVN", mvnParams);
addInput(layer.bottom(0), mvnId, 0, dstNet);
addOutput(layer, mvnId, 0);
net.mutable_layer(li)->set_bottom(0, layer.top(0));
layerParams.blobs[0].setTo(0); // mean
layerParams.blobs[1].setTo(1); // std
}
} }
} }
else if (type == "Axpy") else if (type == "Axpy")
@ -458,13 +551,34 @@ public:
LayerParams scaleParams; LayerParams scaleParams;
scaleParams.set("axis", 1); scaleParams.set("axis", 1);
scaleParams.set("has_bias", false); scaleParams.set("has_bias", false);
int scaleId = dstNet.addLayer(scaleName, "Scale", scaleParams);
addInput(layer.bottom(2), scaleId, 0, dstNet); if (newEngine)
addInput(layer.bottom(0), scaleId, 1, dstNet); {
addOutput(layer, scaleId, 0); std::string intermediateTensor = scaleName + "_intermediate_output";
net.mutable_layer(li)->set_bottom(0, layer.top(0)); Ptr<Layer> netLayerScale= addLayer(
net.mutable_layer(li)->mutable_bottom()->RemoveLast(); dstNet, "Scale", scaleName, scaleParams,
type = "Eltwise"; {layer.bottom(2), layer.bottom(0)},
{intermediateTensor});
curr_prog.push_back(netLayerScale);
LayerParams eltwiseParams;
Ptr<Layer> netLayerEltwise = addLayer(
dstNet, "Eltwise", name, eltwiseParams,
{intermediateTensor, layer.bottom(1)},
{layer.top(0)});
curr_prog.push_back(netLayerEltwise);
continue;
}
else
{
int scaleId = dstNet.addLayer(scaleName, "Scale", scaleParams);
addInput(layer.bottom(2), scaleId, 0, dstNet);
addInput(layer.bottom(0), scaleId, 1, dstNet);
addOutput(layer, scaleId, 0);
net.mutable_layer(li)->set_bottom(0, layer.top(0));
net.mutable_layer(li)->mutable_bottom()->RemoveLast();
type = "Eltwise";
}
} }
else if (type == "Resample") else if (type == "Resample")
{ {
@ -489,9 +603,19 @@ public:
CV_Assert(layer.bottom_size() == layer.top_size()); CV_Assert(layer.bottom_size() == layer.top_size());
for (int i = 0; i < layer.bottom_size(); i++) for (int i = 0; i < layer.bottom_size(); i++)
{ {
int conv_id = dstNet.addLayer(layer.top(i), type, layerParams); if (newEngine)
addInput(layer.bottom(i), conv_id, 0, dstNet); {
addedBlobs.push_back(BlobNote(layer.top(i), conv_id, 0)); Ptr<Layer> netLayer = addLayer(
dstNet, type, layer.top(i), layerParams,
{layer.bottom(i)}, {layer.top(i)});
curr_prog.push_back(netLayer);
}
else
{
int conv_id = dstNet.addLayer(layer.top(i), type, layerParams);
addInput(layer.bottom(i), conv_id, 0, dstNet);
addedBlobs.push_back(BlobNote(layer.top(i), conv_id, 0));
}
} }
continue; continue;
} }
@ -504,25 +628,77 @@ public:
if(!layerParams.has("axis")) if(!layerParams.has("axis"))
layerParams.set("axis", 1); layerParams.set("axis", 1);
} }
else if ("Proposal" == type)
{
if (newEngine && layer.top_size() == 1)
{
// Add unused optional second output and create the Proposal layer
std::vector<string> layerInputs;
for (int inNum = 0; inNum < layer.bottom_size(); inNum++)
layerInputs.push_back(layer.bottom(inNum));
Ptr<Layer> netLayer = addLayer(
dstNet, type, name, layerParams,
layerInputs, {layer.top(0), name + "___output_scores"});
curr_prog.push_back(netLayer);
continue;
}
}
else if ("Silence" == type)
{
if (newEngine)
{
CV_LOG_WARNING(NULL, "Caffe parser: Silence layer was ignored");
continue;
}
}
int id = dstNet.addLayer(name, type, layerParams); if (newEngine)
{
std::vector<string> layerInputs, layerOutputs;
for (int inNum = 0; inNum < layer.bottom_size(); inNum++)
layerInputs.push_back(layer.bottom(inNum));
for (int outNum = 0; outNum < layer.top_size(); outNum++)
layerOutputs.push_back(layer.top(outNum));
for (int inNum = 0; inNum < layer.bottom_size(); inNum++) Ptr<Layer> netLayer = addLayer(
addInput(layer.bottom(inNum), id, inNum, dstNet); dstNet, type, name, layerParams,
layerInputs, layerOutputs);
for (int outNum = 0; outNum < layer.top_size(); outNum++) curr_prog.push_back(netLayer);
addOutput(layer, id, outNum); }
else
{
int id = dstNet.addLayer(name, type, layerParams);
for (int inNum = 0; inNum < layer.bottom_size(); inNum++)
addInput(layer.bottom(inNum), id, inNum, dstNet);
for (int outNum = 0; outNum < layer.top_size(); outNum++)
addOutput(layer, id, outNum);
}
} }
dstNet.setInputsNames(netInputs);
if (inp_shapes.size() > 0) if (newEngine)
{ {
CV_CheckEQ(inp_shapes.size(), netInputs.size(), ""); Ptr<Graph> curr_graph = netImpl->newGraph(net.name(), modelInputs, true);
for (int inp_id = 0; inp_id < inp_shapes.size(); inp_id++) curr_graph->setOutputs(modelOutputs);
dstNet.setInputShape(netInputs[inp_id], inp_shapes[inp_id]); curr_graph->setProg(curr_prog);
}
addedBlobs.clear(); netImpl->mainGraph = curr_graph;
netImpl->modelFormat = DNN_MODEL_CAFFE;
netImpl->originalLayout = DATA_LAYOUT_NCHW;
netImpl->prepareForInference();
}
else
{
dstNet.setInputsNames(netInputs);
if (inp_shapes.size() > 0)
{
CV_CheckEQ(inp_shapes.size(), netInputs.size(), "");
for (int inp_id = 0; inp_id < inp_shapes.size(); inp_id++)
dstNet.setInputShape(netInputs[inp_id], inp_shapes[inp_id]);
}
addedBlobs.clear();
}
} }
void addOutput(const caffe::LayerParameter &layer, int layerId, int outNum) void addOutput(const caffe::LayerParameter &layer, int layerId, int outNum)
@ -569,45 +745,59 @@ public:
} }
Net readNetFromCaffe(const String &prototxt, const String &caffeModel /*= String()*/) Net readNetFromCaffe(const String &prototxt,
const String &caffeModel, /*= String()*/
int engine)
{ {
static const int engine_forced = (int)utils::getConfigurationParameterSizeT("OPENCV_FORCE_DNN_ENGINE", ENGINE_AUTO);
if(engine_forced != ENGINE_AUTO)
engine = engine_forced;
CaffeImporter caffeImporter(prototxt.c_str(), caffeModel.c_str()); CaffeImporter caffeImporter(prototxt.c_str(), caffeModel.c_str());
Net net; Net net;
caffeImporter.populateNet(net); caffeImporter.populateNet(net, engine == ENGINE_NEW || engine == ENGINE_AUTO);
return net; return net;
} }
Net readNetFromCaffe(const char *bufferProto, size_t lenProto, Net readNetFromCaffe(const char *bufferProto, size_t lenProto,
const char *bufferModel, size_t lenModel) const char *bufferModel, size_t lenModel,
int engine)
{ {
static const int engine_forced = (int)utils::getConfigurationParameterSizeT("OPENCV_FORCE_DNN_ENGINE", ENGINE_AUTO);
if(engine_forced != ENGINE_AUTO)
engine = engine_forced;
CaffeImporter caffeImporter(bufferProto, lenProto, bufferModel, lenModel); CaffeImporter caffeImporter(bufferProto, lenProto, bufferModel, lenModel);
Net net; Net net;
caffeImporter.populateNet(net); caffeImporter.populateNet(net, engine == ENGINE_NEW || engine == ENGINE_AUTO);
return net; return net;
} }
Net readNetFromCaffe(const std::vector<uchar>& bufferProto, const std::vector<uchar>& bufferModel) Net readNetFromCaffe(const std::vector<uchar>& bufferProto,
const std::vector<uchar>& bufferModel,
int engine)
{ {
const char* bufferProtoPtr = reinterpret_cast<const char*>(&bufferProto[0]); const char* bufferProtoPtr = reinterpret_cast<const char*>(&bufferProto[0]);
const char* bufferModelPtr = bufferModel.empty() ? NULL : const char* bufferModelPtr = bufferModel.empty() ? NULL :
reinterpret_cast<const char*>(&bufferModel[0]); reinterpret_cast<const char*>(&bufferModel[0]);
return readNetFromCaffe(bufferProtoPtr, bufferProto.size(), return readNetFromCaffe(bufferProtoPtr, bufferProto.size(),
bufferModelPtr, bufferModel.size()); bufferModelPtr, bufferModel.size(),
engine);
} }
#else // HAVE_PROTOBUF #else // HAVE_PROTOBUF
#define DNN_PROTOBUF_UNSUPPORTED() CV_Error(Error::StsError, "DNN/Caffe: Build OpenCV with Protobuf to import Caffe models") #define DNN_PROTOBUF_UNSUPPORTED() CV_Error(Error::StsError, "DNN/Caffe: Build OpenCV with Protobuf to import Caffe models")
Net readNetFromCaffe(const String &, const String &) { Net readNetFromCaffe(const String &, const String &, int) {
DNN_PROTOBUF_UNSUPPORTED(); DNN_PROTOBUF_UNSUPPORTED();
} }
Net readNetFromCaffe(const char *, size_t, const char *, size_t) { Net readNetFromCaffe(const char *, size_t, const char *, size_t, int) {
DNN_PROTOBUF_UNSUPPORTED(); DNN_PROTOBUF_UNSUPPORTED();
} }
Net readNetFromCaffe(const std::vector<uchar>&, const std::vector<uchar>&) { Net readNetFromCaffe(const std::vector<uchar>&, const std::vector<uchar>&, int) {
DNN_PROTOBUF_UNSUPPORTED(); DNN_PROTOBUF_UNSUPPORTED();
} }

4
modules/dnn/src/dnn_read.cpp
View File

@ -21,7 +21,7 @@ Net readNet(const String& _model, const String& _config, const String& _framewor
{ {
if (modelExt == "prototxt" || configExt == "caffemodel") if (modelExt == "prototxt" || configExt == "caffemodel")
std::swap(model, config); std::swap(model, config);
return readNetFromCaffe(config, model); return readNetFromCaffe(config, model, engine);
} }
if (framework == "tensorflow" || modelExt == "pb" || configExt == "pb" || modelExt == "pbtxt" || configExt == "pbtxt") if (framework == "tensorflow" || modelExt == "pb" || configExt == "pb" || modelExt == "pbtxt" || configExt == "pbtxt")
{ {
@ -61,7 +61,7 @@ Net readNet(const String& _framework, const std::vector<uchar>& bufferModel,
if (framework == "onnx") if (framework == "onnx")
return readNetFromONNX(bufferModel, engine); return readNetFromONNX(bufferModel, engine);
else if (framework == "caffe") else if (framework == "caffe")
return readNetFromCaffe(bufferConfig, bufferModel); return readNetFromCaffe(bufferConfig, bufferModel, engine);
else if (framework == "tensorflow") else if (framework == "tensorflow")
return readNetFromTensorflow(bufferModel, bufferConfig); return readNetFromTensorflow(bufferModel, bufferConfig);
else if (framework == "darknet") else if (framework == "darknet")

17
modules/dnn/src/model.cpp
View File

@ -114,14 +114,18 @@ public:
} }
Mat blob = dnn::blobFromImageWithParams(frame, param); // [1, 10, 10, 4] Mat blob = dnn::blobFromImageWithParams(frame, param); // [1, 10, 10, 4]
net.setInput(blob);
// Faster-RCNN or R-FCN // Faster-RCNN or R-FCN
if (!net.getMainGraph() && net.getLayer(0)->outputNameToIndex("im_info") != -1) if ((net.getMainGraph() && net.haveArg("im_info") && net.argKind(net.getArg("im_info")) == DNN_ARG_INPUT) ||
(!net.getMainGraph() && net.getLayer(0)->outputNameToIndex("im_info") != -1))
{ {
net.setInput(blob, "data");
Mat imInfo(Matx13f(size.height, size.width, 1.6f)); Mat imInfo(Matx13f(size.height, size.width, 1.6f));
net.setInput(imInfo, "im_info"); net.setInput(imInfo, "im_info");
} }
else
{
net.setInput(blob);
}
net.forward(outs, outNames); net.forward(outs, outNames);
} }
@ -507,7 +511,12 @@ void DetectionModel::detect(InputArray frame, CV_OUT std::vector<int>& classIds,
int frameWidth = frame.cols(); int frameWidth = frame.cols();
int frameHeight = frame.rows(); int frameHeight = frame.rows();
if (getNetwork_().getLayer(0)->outputNameToIndex("im_info") != -1) if ((getNetwork_().getMainGraph() &&
getNetwork_().haveArg("im_info") &&
getNetwork_().argKind(getNetwork_().getArg("im_info")) == DNN_ARG_INPUT)
||
(!getNetwork_().getMainGraph() &&
getNetwork_().getLayer(0)->outputNameToIndex("im_info") != -1))
{ {
frameWidth = impl->size.width; frameWidth = impl->size.width;
frameHeight = impl->size.height; frameHeight = impl->size.height;

33
modules/dnn/test/test_backends.cpp
View File

@ -41,7 +41,13 @@ public:
Net netDefault = readNet(weights, proto); Net netDefault = readNet(weights, proto);
netDefault.setPreferableBackend(DNN_BACKEND_OPENCV); netDefault.setPreferableBackend(DNN_BACKEND_OPENCV);
netDefault.setInput(inp); netDefault.setInput(inp);
Mat outDefault = netDefault.forward(outputLayer).clone();
// BUG: https://github.com/opencv/opencv/issues/26349
Mat outDefault;
if(netDefault.getMainGraph())
outDefault = netDefault.forward().clone();
else
outDefault = netDefault.forward(outputLayer).clone();
net = readNet(weights, proto); net = readNet(weights, proto);
net.setInput(inp); net.setInput(inp);
@ -51,7 +57,12 @@ public:
if (target == DNN_TARGET_CPU_FP16) if (target == DNN_TARGET_CPU_FP16)
net.enableWinograd(false); net.enableWinograd(false);
Mat out = net.forward(outputLayer).clone(); // BUG: https://github.com/opencv/opencv/issues/26349
Mat out;
if(net.getMainGraph())
out = net.forward().clone();
else
out = net.forward(outputLayer).clone();
check(outDefault, out, outputLayer, l1, lInf, detectionConfThresh, "First run"); check(outDefault, out, outputLayer, l1, lInf, detectionConfThresh, "First run");
@ -65,8 +76,17 @@ public:
} }
netDefault.setInput(inp); netDefault.setInput(inp);
net.setInput(inp); net.setInput(inp);
outDefault = netDefault.forward(outputLayer).clone();
out = net.forward(outputLayer).clone(); if(netDefault.getMainGraph())
outDefault = netDefault.forward().clone();
else
outDefault = netDefault.forward(outputLayer).clone();
if(net.getMainGraph())
out = net.forward().clone();
else
out = net.forward(outputLayer).clone();
check(outDefault, out, outputLayer, l1, lInf, detectionConfThresh, "Second run"); check(outDefault, out, outputLayer, l1, lInf, detectionConfThresh, "Second run");
} }
@ -514,9 +534,8 @@ TEST_P(DNNTestNetwork, FastNeuralStyle_eccv16)
#if defined(HAVE_INF_ENGINE) && INF_ENGINE_VER_MAJOR_GE(2019010000) #if defined(HAVE_INF_ENGINE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
expectNoFallbacksFromIE(net); expectNoFallbacksFromIE(net);
#endif #endif
// BUG: https://github.com/opencv/opencv/issues/26306
// Temporarily disabled check for no "fallbacks", since the new engine does not support CUDA yet expectNoFallbacksFromCUDA(net);
//expectNoFallbacksFromCUDA(net);
} }
INSTANTIATE_TEST_CASE_P(/*nothing*/, DNNTestNetwork, dnnBackendsAndTargets(/* withInferenceEngine = */ true, INSTANTIATE_TEST_CASE_P(/*nothing*/, DNNTestNetwork, dnnBackendsAndTargets(/* withInferenceEngine = */ true,

33
modules/dnn/test/test_caffe_importer.cpp
View File

@ -230,7 +230,14 @@ TEST_P(Reproducibility_AlexNet, Accuracy)
ASSERT_TRUE(!sample.empty()); ASSERT_TRUE(!sample.empty());
net.setInput(blobFromImage(sample, 1.0f, Size(227, 227), Scalar(), false), "data"); net.setInput(blobFromImage(sample, 1.0f, Size(227, 227), Scalar(), false), "data");
Mat out = net.forward("prob");
Mat out;
// BUG: https://github.com/opencv/opencv/issues/26349
if (net.getMainGraph())
out = net.forward();
else
out = net.forward("prob");
Mat ref = blobFromNPY(_tf("caffe_alexnet_prob.npy")); Mat ref = blobFromNPY(_tf("caffe_alexnet_prob.npy"));
normAssert(ref, out, "", l1, lInf); normAssert(ref, out, "", l1, lInf);
} }
@ -259,7 +266,13 @@ TEST(Reproducibility_FCN, Accuracy)
net.getMemoryConsumption(shape(1,3,227,227), CV_32F, layerIds, weights, blobs); net.getMemoryConsumption(shape(1,3,227,227), CV_32F, layerIds, weights, blobs);
net.setInput(blobFromImage(sample, 1.0f, Size(500, 500), Scalar(), false), "data"); net.setInput(blobFromImage(sample, 1.0f, Size(500, 500), Scalar(), false), "data");
Mat out = net.forward("score");
Mat out;
// BUG: https://github.com/opencv/opencv/issues/26349
if (net.getMainGraph())
out = net.forward();
else
out = net.forward("score");
Mat refData = imread(_tf("caffe_fcn8s_prob.png"), IMREAD_ANYDEPTH); Mat refData = imread(_tf("caffe_fcn8s_prob.png"), IMREAD_ANYDEPTH);
int shape[] = {1, 21, 500, 500}; int shape[] = {1, 21, 500, 500};
@ -292,7 +305,13 @@ TEST(Reproducibility_SSD, Accuracy)
Mat in_blob = blobFromImage(sample, 1.0f, Size(300, 300), Scalar(), false); Mat in_blob = blobFromImage(sample, 1.0f, Size(300, 300), Scalar(), false);
net.setInput(in_blob, "data"); net.setInput(in_blob, "data");
Mat out = net.forward("detection_out");
// BUG: https://github.com/opencv/opencv/issues/26349
Mat out;
if(net.getMainGraph())
out = net.forward();
else
out = net.forward("detection_out");
Mat ref = blobFromNPY(_tf("ssd_out.npy")); Mat ref = blobFromNPY(_tf("ssd_out.npy"));
normAssertDetections(ref, out, "", 0.06); normAssertDetections(ref, out, "", 0.06);
@ -495,7 +514,13 @@ TEST(Reproducibility_GoogLeNet_fp16, Accuracy)
ASSERT_TRUE(!inpMats[0].empty() && !inpMats[1].empty()); ASSERT_TRUE(!inpMats[0].empty() && !inpMats[1].empty());
net.setInput(blobFromImages(inpMats, 1.0f, Size(), Scalar(), false), "data"); net.setInput(blobFromImages(inpMats, 1.0f, Size(), Scalar(), false), "data");
Mat out = net.forward("prob");
// BUG: https://github.com/opencv/opencv/issues/26349
Mat out;
if(net.getMainGraph())
out = net.forward();
else
out = net.forward("prob");
Mat ref = blobFromNPY(_tf("googlenet_prob.npy")); Mat ref = blobFromNPY(_tf("googlenet_prob.npy"));
normAssert(out, ref, "", l1, lInf); normAssert(out, ref, "", l1, lInf);

5
modules/dnn/test/test_common.hpp
View File

@ -195,6 +195,11 @@ public:
void expectNoFallbacks(Net& net, bool raiseError = true) void expectNoFallbacks(Net& net, bool raiseError = true)
{ {
// The new DNN engine does not support back-ends for now
// bug: https://github.com/opencv/opencv/issues/26198
if (net.getMainGraph())
return;
// Check if all the layers are supported with current backend and target. // Check if all the layers are supported with current backend and target.
// Some layers might be fused so their timings equal to zero. // Some layers might be fused so their timings equal to zero.
std::vector<double> timings; std::vector<double> timings;

14
modules/dnn/test/test_googlenet.cpp
View File

@ -80,7 +80,13 @@ TEST_P(Reproducibility_GoogLeNet, Batching)
ASSERT_TRUE(!inpMats[0].empty() && !inpMats[1].empty()); ASSERT_TRUE(!inpMats[0].empty() && !inpMats[1].empty());
net.setInput(blobFromImages(inpMats, 1.0f, Size(), Scalar(), false), "data"); net.setInput(blobFromImages(inpMats, 1.0f, Size(), Scalar(), false), "data");
Mat out = net.forward("prob");
// BUG: https://github.com/opencv/opencv/issues/26349
Mat out;
if(net.getMainGraph())
out = net.forward();
else
out = net.forward("prob");
Mat ref = blobFromNPY(_tf("googlenet_prob.npy")); Mat ref = blobFromNPY(_tf("googlenet_prob.npy"));
normAssert(out, ref); normAssert(out, ref);
@ -93,8 +99,9 @@ TEST_P(Reproducibility_GoogLeNet, IntermediateBlobs)
applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
if (targetId == DNN_TARGET_CPU_FP16) if (targetId == DNN_TARGET_CPU_FP16)
applyTestTag(CV_TEST_TAG_DNN_SKIP_CPU_FP16); applyTestTag(CV_TEST_TAG_DNN_SKIP_CPU_FP16);
// BUG: https://github.com/opencv/opencv/issues/26349
Net net = readNetFromCaffe(findDataFile("dnn/bvlc_googlenet.prototxt"), Net net = readNetFromCaffe(findDataFile("dnn/bvlc_googlenet.prototxt"),
findDataFile("dnn/bvlc_googlenet.caffemodel", false)); findDataFile("dnn/bvlc_googlenet.caffemodel", false), ENGINE_CLASSIC);
net.setPreferableBackend(DNN_BACKEND_OPENCV); net.setPreferableBackend(DNN_BACKEND_OPENCV);
net.setPreferableTarget(targetId); net.setPreferableTarget(targetId);
@ -126,8 +133,9 @@ TEST_P(Reproducibility_GoogLeNet, SeveralCalls)
applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
if (targetId == DNN_TARGET_CPU_FP16) if (targetId == DNN_TARGET_CPU_FP16)
applyTestTag(CV_TEST_TAG_DNN_SKIP_CPU_FP16); applyTestTag(CV_TEST_TAG_DNN_SKIP_CPU_FP16);
// BUG: https://github.com/opencv/opencv/issues/26349
Net net = readNetFromCaffe(findDataFile("dnn/bvlc_googlenet.prototxt"), Net net = readNetFromCaffe(findDataFile("dnn/bvlc_googlenet.prototxt"),
findDataFile("dnn/bvlc_googlenet.caffemodel", false)); findDataFile("dnn/bvlc_googlenet.caffemodel", false), ENGINE_CLASSIC);
net.setPreferableBackend(DNN_BACKEND_OPENCV); net.setPreferableBackend(DNN_BACKEND_OPENCV);
net.setPreferableTarget(targetId); net.setPreferableTarget(targetId);

8
modules/dnn/test/test_layers.cpp
View File

@ -408,7 +408,11 @@ TEST_P(Test_Caffe_layers, Reshape_Split_Slice)
rng.fill(input, RNG::UNIFORM, -1, 1); rng.fill(input, RNG::UNIFORM, -1, 1);
net.setInput(input, "input"); net.setInput(input, "input");
Mat output = net.forward("output"); Mat output;
if (net.getMainGraph())
output = net.forward();
else
output = net.forward("output");
normAssert(input, output, "", default_l1, default_lInf); normAssert(input, output, "", default_l1, default_lInf);
} }
@ -864,7 +868,7 @@ TEST_P(Test_Caffe_layers, FasterRCNN_Proposal)
std::vector<Mat> outs; std::vector<Mat> outs;
net.setPreferableBackend(backend); net.setPreferableBackend(backend);
net.setPreferableTarget(target); net.setPreferableTarget(target);
net.forward(outs, "output"); net.forward(outs);
for (int i = 0; i < 2; ++i) for (int i = 0; i < 2; ++i)
{ {

14
modules/dnn/test/test_misc.cpp
View File

@ -330,7 +330,10 @@ TEST_P(dump, Regression)
Net net = readNet(findDataFile("dnn/squeezenet_v1.1.prototxt"), Net net = readNet(findDataFile("dnn/squeezenet_v1.1.prototxt"),
findDataFile("dnn/squeezenet_v1.1.caffemodel", false)); findDataFile("dnn/squeezenet_v1.1.caffemodel", false));
ASSERT_EQ(net.getLayerInputs(net.getLayerId("fire2/concat")).size(), 2); if (net.getMainGraph())
ASSERT_EQ(net.getLayer(net.getLayerId("fire2/concat"))->inputs.size(), 2);
else
ASSERT_EQ(net.getLayerInputs(net.getLayerId("fire2/concat")).size(), 2);
int size[] = {1, 3, 227, 227}; int size[] = {1, 3, 227, 227};
Mat input = cv::Mat::ones(4, size, CV_32F); Mat input = cv::Mat::ones(4, size, CV_32F);
@ -602,7 +605,14 @@ TEST(Net, forwardAndRetrieve)
outNames.push_back("testLayer"); outNames.push_back("testLayer");
std::vector<std::vector<Mat> > outBlobs; std::vector<std::vector<Mat> > outBlobs;
net.forward(outBlobs, outNames); if (net.getMainGraph())
{
// Issue: https://github.com/opencv/opencv/issues/26349
outBlobs.push_back({});
net.forward(outBlobs[0]);
}
else
net.forward(outBlobs, outNames);
EXPECT_EQ(outBlobs.size(), 1); EXPECT_EQ(outBlobs.size(), 1);
EXPECT_EQ(outBlobs[0].size(), 2); EXPECT_EQ(outBlobs[0].size(), 2);