From 01048e56030d768a471c33f7b307451d8941e232 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 21 Feb 2020 22:39:54 +0300 Subject: [PATCH] Merge pull request #16616 from alalek:dnn_fix_input_shape * dnn: fix processing of input shapes - importer: avoid using of .setInput() => .setInputShape() - setInput: shape limitation check (partial) * dnn(test): test .setInput() in readNet() --- modules/core/include/opencv2/core/check.hpp | 1 + modules/core/src/check.cpp | 4 + modules/core/src/matrix_wrap.cpp | 1 + modules/dnn/include/opencv2/dnn/dnn.hpp | 4 + .../dnn/include/opencv2/dnn/shape_utils.hpp | 10 +++ modules/dnn/src/caffe/caffe_importer.cpp | 5 +- modules/dnn/src/dnn.cpp | 86 ++++++++++++++++--- modules/dnn/test/test_misc.cpp | 59 +++++++++++++ 8 files changed, 152 insertions(+), 18 deletions(-) diff --git a/modules/core/include/opencv2/core/check.hpp b/modules/core/include/opencv2/core/check.hpp index 604447e8d7..0e0c7cbf31 100644 --- a/modules/core/include/opencv2/core/check.hpp +++ b/modules/core/include/opencv2/core/check.hpp @@ -79,6 +79,7 @@ CV_EXPORTS void CV_NORETURN check_failed_auto(const size_t v, const CheckContext CV_EXPORTS void CV_NORETURN check_failed_auto(const float v, const CheckContext& ctx); CV_EXPORTS void CV_NORETURN check_failed_auto(const double v, const CheckContext& ctx); CV_EXPORTS void CV_NORETURN check_failed_auto(const Size_ v, const CheckContext& ctx); +CV_EXPORTS void CV_NORETURN check_failed_auto(const std::string& v1, const CheckContext& ctx); CV_EXPORTS void CV_NORETURN check_failed_MatDepth(const int v, const CheckContext& ctx); CV_EXPORTS void CV_NORETURN check_failed_MatType(const int v, const CheckContext& ctx); CV_EXPORTS void CV_NORETURN check_failed_MatChannels(const int v, const CheckContext& ctx); diff --git a/modules/core/src/check.cpp b/modules/core/src/check.cpp index 59c2bbe5de..4988b87c34 100644 --- a/modules/core/src/check.cpp +++ b/modules/core/src/check.cpp @@ -171,6 +171,10 @@ void check_failed_auto(const Size_ v, const CheckContext& ctx) { check_failed_auto_< Size_ >(v, ctx); } +void check_failed_auto(const std::string& v, const CheckContext& ctx) +{ + check_failed_auto_< std::string >(v, ctx); +} }} // namespace diff --git a/modules/core/src/matrix_wrap.cpp b/modules/core/src/matrix_wrap.cpp index e16e2f3f83..4c5efd6ba5 100644 --- a/modules/core/src/matrix_wrap.cpp +++ b/modules/core/src/matrix_wrap.cpp @@ -569,6 +569,7 @@ int _InputArray::sizend(int* arrsz, int i) const } else { + CV_CheckLE(dims(i), 2, "Not supported"); // TODO Support EXPR with 3+ dims Size sz2d = size(i); d = 2; if(arrsz) diff --git a/modules/dnn/include/opencv2/dnn/dnn.hpp b/modules/dnn/include/opencv2/dnn/dnn.hpp index 1b763bac70..113893813c 100644 --- a/modules/dnn/include/opencv2/dnn/dnn.hpp +++ b/modules/dnn/include/opencv2/dnn/dnn.hpp @@ -484,6 +484,10 @@ CV__DNN_EXPERIMENTAL_NS_BEGIN */ CV_WRAP void setInputsNames(const std::vector &inputBlobNames); + /** @brief Specify shape of network input. + */ + CV_WRAP void setInputShape(const String &inputName, const MatShape& shape); + /** @brief Runs forward pass to compute output of layer with name @p outputName. * @param outputName name for layer which output is needed to get * @return blob for first output of specified layer. diff --git a/modules/dnn/include/opencv2/dnn/shape_utils.hpp b/modules/dnn/include/opencv2/dnn/shape_utils.hpp index b0ed3afc54..c975fcff04 100644 --- a/modules/dnn/include/opencv2/dnn/shape_utils.hpp +++ b/modules/dnn/include/opencv2/dnn/shape_utils.hpp @@ -138,6 +138,16 @@ static inline MatShape shape(const UMat& mat) return shape(mat.size.p, mat.dims); } +#if 0 // issues with MatExpr wrapped into InputArray +static inline +MatShape shape(InputArray input) +{ + int sz[CV_MAX_DIM]; + int ndims = input.sizend(sz); + return shape(sz, ndims); +} +#endif + namespace {inline bool is_neg(int i) { return i < 0; }} static inline MatShape shape(int a0, int a1=-1, int a2=-1, int a3=-1) diff --git a/modules/dnn/src/caffe/caffe_importer.cpp b/modules/dnn/src/caffe/caffe_importer.cpp index 7d3d15b1b8..16860a9256 100644 --- a/modules/dnn/src/caffe/caffe_importer.cpp +++ b/modules/dnn/src/caffe/caffe_importer.cpp @@ -484,10 +484,7 @@ public: { CV_CheckEQ(inp_shapes.size(), netInputs.size(), ""); for (int inp_id = 0; inp_id < inp_shapes.size(); inp_id++) - { - if (!inp_shapes[inp_id].empty()) - dstNet.setInput(Mat(inp_shapes[inp_id], CV_32F), netInputs[inp_id]); - } + dstNet.setInputShape(netInputs[inp_id], inp_shapes[inp_id]); } addedBlobs.clear(); diff --git a/modules/dnn/src/dnn.cpp b/modules/dnn/src/dnn.cpp index 9e2d255bb0..b0c52b101a 100644 --- a/modules/dnn/src/dnn.cpp +++ b/modules/dnn/src/dnn.cpp @@ -722,6 +722,18 @@ struct DataLayer : public Layer void setNames(const std::vector &names) { outNames.assign(names.begin(), names.end()); + shapes.clear(); shapes.resize(outNames.size()); + } + + void setInputShape(const String& tgtName, const MatShape& shape) + { + std::vector::const_iterator it = std::find(outNames.begin(), outNames.end(), tgtName); + CV_Check(tgtName, it != outNames.end(), "Unknown input"); + int idx = (int)(it - outNames.begin()); + + CV_Assert(idx < (int)shapes.size()); + CV_Check(tgtName, shapes[idx].empty(), "Input shape redefinition is not allowed"); + shapes[idx] = shape; } bool getMemoryShapes(const std::vector &inputs, @@ -784,6 +796,7 @@ struct DataLayer : public Layer #endif // HAVE_INF_ENGINE std::vector outNames; + std::vector shapes; // Preprocessing parameters for each network's input. std::vector scaleFactors; std::vector means; @@ -2842,8 +2855,25 @@ struct Net::Impl } else { - inOutShapes[0].out.clear(); - return; + const std::vector& inputShapes = netInputLayer->shapes; + bool none = true; + for (size_t i = 0; i < inputShapes.size(); i++) + { + if (!inputShapes[i].empty()) + { + none = false; + break; + } + } + if (none) + { + inOutShapes[0].out.clear(); + return; + } + else + { + inOutShapes[0].in = inputShapes; + } } } @@ -3069,7 +3099,7 @@ Net Net::Impl::createNetworkFromModelOptimizer(InferenceEngine::CNNNetwork& ieNe // set empty input to determine input shapes for (int inp_id = 0; inp_id < inputsNames.size(); ++inp_id) { - cvNet.setInput(Mat(inp_shapes[inp_id], CV_32F), inputsNames[inp_id]); + cvNet.setInputShape(inputsNames[inp_id], inp_shapes[inp_id]); } Ptr backendNode; @@ -3494,6 +3524,13 @@ void Net::setInputsNames(const std::vector &inputBlobNames) impl->netInputLayer->setNames(inputBlobNames); } +void Net::setInputShape(const String &inputName, const MatShape& shape) +{ + CV_TRACE_FUNCTION(); + + impl->netInputLayer->setInputShape(inputName, shape); +} + void Net::setInput(InputArray blob, const String& name, double scalefactor, const Scalar& mean) { CV_TRACE_FUNCTION(); @@ -3506,6 +3543,33 @@ void Net::setInput(InputArray blob, const String& name, double scalefactor, cons if (!pin.valid()) CV_Error(Error::StsObjectNotFound, "Requested blob \"" + name + "\" not found"); + Mat blob_ = blob.getMat(); // can't use InputArray directly due MatExpr stuff + MatShape blobShape = shape(blob_); + + if (pin.lid == 0) + { + CV_Assert(!impl->netInputLayer.empty()); + const DataLayer& netInputLayer = *impl->netInputLayer.get(); + if (!netInputLayer.shapes.empty()) + { + CV_CheckLT(pin.oid, (int)netInputLayer.shapes.size(), ""); + const MatShape& inputShapeLimitation = netInputLayer.shapes[pin.oid]; + if (!inputShapeLimitation.empty()) + { + CV_CheckEQ(inputShapeLimitation.size(), blobShape.size(), ""); +#if 0 // TODO: DNNTestNetwork.MobileNet_SSD_Caffe_Different_Width_Height/0 + const size_t dims = inputShapeLimitation.size(); + for (size_t dim = 0; dim < dims; dim++) + { + if (dims >= 3 && dim == 0 && inputShapeLimitation[0] == 1) + continue; // don't limit batch + CV_CheckEQ(inputShapeLimitation[dim], blobShape[dim], ""); + } +#endif + } + } + } + LayerData &ld = impl->layers[pin.lid]; const int numInputs = std::max(pin.oid+1, (int)ld.requiredOutputs.size()); ld.outputBlobs.resize(numInputs); @@ -3515,17 +3579,11 @@ void Net::setInput(InputArray blob, const String& name, double scalefactor, cons impl->netInputLayer->means.resize(numInputs); MatShape prevShape = shape(impl->netInputLayer->inputsData[pin.oid]); - Mat blob_ = blob.getMat(); - bool oldShape = prevShape == shape(blob_); - if (oldShape) - { - blob_.copyTo(impl->netInputLayer->inputsData[pin.oid]); - } - else - { - ld.outputBlobs[pin.oid] = blob_.clone(); - impl->netInputLayer->inputsData[pin.oid] = ld.outputBlobs[pin.oid]; - } + bool oldShape = prevShape == blobShape; + + blob_.copyTo(impl->netInputLayer->inputsData[pin.oid]); + if (!oldShape) + ld.outputBlobs[pin.oid] = impl->netInputLayer->inputsData[pin.oid]; if (!ld.outputBlobsWrappers[pin.oid].empty()) { diff --git a/modules/dnn/test/test_misc.cpp b/modules/dnn/test/test_misc.cpp index ca60b9111a..eccf171539 100644 --- a/modules/dnn/test/test_misc.cpp +++ b/modules/dnn/test/test_misc.cpp @@ -78,6 +78,65 @@ TEST(readNet, Regression) EXPECT_FALSE(net.empty()); } +TEST(readNet, do_not_call_setInput) // https://github.com/opencv/opencv/issues/16618 +{ + // 1. load network + const string proto = findDataFile("dnn/squeezenet_v1.1.prototxt"); + const string model = findDataFile("dnn/squeezenet_v1.1.caffemodel", false); + Net net = readNetFromCaffe(proto, model); + + // 2. mistake: no inputs are specified through .setInput() + + // 3. try inference + Mat res; + EXPECT_THROW( + { + res = net.forward(); // no inputs after loading => should fail + }, cv::Exception); + EXPECT_TRUE(res.empty()) << res.size; +} + +#ifdef HAVE_INF_ENGINE +static +void test_readNet_IE_do_not_call_setInput(Backend backendId) +{ + const Target targetId = DNN_TARGET_CPU; + + const std::string& model = findDataFile("dnn/layers/layer_convolution.bin"); + const std::string& proto = findDataFile("dnn/layers/layer_convolution.xml"); + + if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + setInferenceEngineBackendType(CV_DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_API); + else if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + setInferenceEngineBackendType(CV_DNN_BACKEND_INFERENCE_ENGINE_NGRAPH); + else + FAIL() << "Unknown backendId"; + + Net net = readNet(model, proto); + net.setPreferableBackend(backendId); + net.setPreferableTarget(targetId); + + // 2. mistake: no inputs are specified through .setInput() + + // 3. try inference + Mat res; + EXPECT_THROW( + { + res = net.forward(); // no inputs after loading => should fail + }, cv::Exception); + EXPECT_TRUE(res.empty()) << res.size; +} + +TEST(readNet, do_not_call_setInput_IE_NN_BUILDER_2019) +{ + test_readNet_IE_do_not_call_setInput(DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019); +} +TEST(readNet, do_not_call_setInput_IE_NGRAPH) +{ + test_readNet_IE_do_not_call_setInput(DNN_BACKEND_INFERENCE_ENGINE_NGRAPH); +} +#endif // HAVE_INF_ENGINE + typedef testing::TestWithParam > dump; TEST_P(dump, Regression) {