Merge pull request #16616 from alalek:dnn_fix_input_shape

* dnn: fix processing of input shapes

- importer: avoid using of .setInput() => .setInputShape()
- setInput: shape limitation check (partial)

* dnn(test): test .setInput() in readNet()

modules/core/include/opencv2/core/check.hpp

@@ -79,6 +79,7 @@ CV_EXPORTS void CV_NORETURN check_failed_auto(const size_t v, const CheckContext
 CV_EXPORTS void CV_NORETURN check_failed_auto(const float v, const CheckContext& ctx);
 CV_EXPORTS void CV_NORETURN check_failed_auto(const double v, const CheckContext& ctx);
 CV_EXPORTS void CV_NORETURN check_failed_auto(const Size_<int> v, const CheckContext& ctx);
+CV_EXPORTS void CV_NORETURN check_failed_auto(const std::string& v1, const CheckContext& ctx);
 CV_EXPORTS void CV_NORETURN check_failed_MatDepth(const int v, const CheckContext& ctx);
 CV_EXPORTS void CV_NORETURN check_failed_MatType(const int v, const CheckContext& ctx);
 CV_EXPORTS void CV_NORETURN check_failed_MatChannels(const int v, const CheckContext& ctx);

modules/core/src/check.cpp

@@ -171,6 +171,10 @@ void check_failed_auto(const Size_<int> v, const CheckContext& ctx)
 {
     check_failed_auto_< Size_<int> >(v, ctx);
 }
+void check_failed_auto(const std::string& v, const CheckContext& ctx)
+{
+    check_failed_auto_< std::string >(v, ctx);
+}
 
 
 }} // namespace

modules/core/src/matrix_wrap.cpp

@@ -569,6 +569,7 @@ int _InputArray::sizend(int* arrsz, int i) const
     }
     else
     {
+        CV_CheckLE(dims(i), 2, "Not supported");  // TODO Support EXPR with 3+ dims
         Size sz2d = size(i);
         d = 2;
         if(arrsz)

modules/dnn/include/opencv2/dnn/dnn.hpp

@@ -484,6 +484,10 @@ CV__DNN_EXPERIMENTAL_NS_BEGIN
          */
         CV_WRAP void setInputsNames(const std::vector<String> &inputBlobNames);
 
+        /** @brief Specify shape of network input.
+         */
+        CV_WRAP void setInputShape(const String &inputName, const MatShape& shape);
+
         /** @brief Runs forward pass to compute output of layer with name @p outputName.
          *  @param outputName name for layer which output is needed to get
          *  @return blob for first output of specified layer.

modules/dnn/include/opencv2/dnn/shape_utils.hpp

@@ -138,6 +138,16 @@ static inline MatShape shape(const UMat& mat)
     return shape(mat.size.p, mat.dims);
 }
 
+#if 0  // issues with MatExpr wrapped into InputArray
+static inline
+MatShape shape(InputArray input)
+{
+    int sz[CV_MAX_DIM];
+    int ndims = input.sizend(sz);
+    return shape(sz, ndims);
+}
+#endif
+
 namespace {inline bool is_neg(int i) { return i < 0; }}
 
 static inline MatShape shape(int a0, int a1=-1, int a2=-1, int a3=-1)

modules/dnn/src/caffe/caffe_importer.cpp

@@ -484,10 +484,7 @@ public:
         {
             CV_CheckEQ(inp_shapes.size(), netInputs.size(), "");
             for (int inp_id = 0; inp_id < inp_shapes.size(); inp_id++)
-            {
+                dstNet.setInputShape(netInputs[inp_id], inp_shapes[inp_id]);
-                if (!inp_shapes[inp_id].empty())
-                    dstNet.setInput(Mat(inp_shapes[inp_id], CV_32F), netInputs[inp_id]);
-            }
         }
 
         addedBlobs.clear();

modules/dnn/src/dnn.cpp

@@ -722,6 +722,18 @@ struct DataLayer : public Layer
     void setNames(const std::vector<String> &names)
     {
         outNames.assign(names.begin(), names.end());
+        shapes.clear(); shapes.resize(outNames.size());
+    }
+
+    void setInputShape(const String& tgtName, const MatShape& shape)
+    {
+        std::vector<String>::const_iterator it = std::find(outNames.begin(), outNames.end(), tgtName);
+        CV_Check(tgtName, it != outNames.end(), "Unknown input");
+        int idx = (int)(it - outNames.begin());
+
+        CV_Assert(idx < (int)shapes.size());
+        CV_Check(tgtName, shapes[idx].empty(), "Input shape redefinition is not allowed");
+        shapes[idx] = shape;
     }
 
     bool getMemoryShapes(const std::vector<MatShape> &inputs,
@@ -784,6 +796,7 @@ struct DataLayer : public Layer
 #endif  // HAVE_INF_ENGINE
 
     std::vector<String> outNames;
+    std::vector<MatShape> shapes;
     // Preprocessing parameters for each network's input.
     std::vector<double> scaleFactors;
     std::vector<Scalar> means;
@@ -2842,8 +2855,25 @@ struct Net::Impl
             }
             else
             {
-                inOutShapes[0].out.clear();
+                const std::vector<MatShape>& inputShapes = netInputLayer->shapes;
-                return;
+                bool none = true;
+                for (size_t i = 0; i < inputShapes.size(); i++)
+                {
+                    if (!inputShapes[i].empty())
+                    {
+                        none = false;
+                        break;
+                    }
+                }
+                if (none)
+                {
+                    inOutShapes[0].out.clear();
+                    return;
+                }
+                else
+                {
+                    inOutShapes[0].in = inputShapes;
+                }
             }
         }
 
@@ -3069,7 +3099,7 @@ Net Net::Impl::createNetworkFromModelOptimizer(InferenceEngine::CNNNetwork& ieNe
     // set empty input to determine input shapes
     for (int inp_id = 0; inp_id < inputsNames.size(); ++inp_id)
     {
-        cvNet.setInput(Mat(inp_shapes[inp_id], CV_32F), inputsNames[inp_id]);
+        cvNet.setInputShape(inputsNames[inp_id], inp_shapes[inp_id]);
     }
 
     Ptr<BackendNode> backendNode;
@@ -3494,6 +3524,13 @@ void Net::setInputsNames(const std::vector<String> &inputBlobNames)
     impl->netInputLayer->setNames(inputBlobNames);
 }
 
+void Net::setInputShape(const String &inputName, const MatShape& shape)
+{
+    CV_TRACE_FUNCTION();
+
+    impl->netInputLayer->setInputShape(inputName, shape);
+}
+
 void Net::setInput(InputArray blob, const String& name, double scalefactor, const Scalar& mean)
 {
     CV_TRACE_FUNCTION();
@@ -3506,6 +3543,33 @@ void Net::setInput(InputArray blob, const String& name, double scalefactor, cons
     if (!pin.valid())
         CV_Error(Error::StsObjectNotFound, "Requested blob \"" + name + "\" not found");
 
+    Mat blob_ = blob.getMat();  // can't use InputArray directly due MatExpr stuff
+    MatShape blobShape = shape(blob_);
+
+    if (pin.lid == 0)
+    {
+        CV_Assert(!impl->netInputLayer.empty());
+        const DataLayer& netInputLayer = *impl->netInputLayer.get();
+        if (!netInputLayer.shapes.empty())
+        {
+            CV_CheckLT(pin.oid, (int)netInputLayer.shapes.size(), "");
+            const MatShape& inputShapeLimitation = netInputLayer.shapes[pin.oid];
+            if (!inputShapeLimitation.empty())
+            {
+                CV_CheckEQ(inputShapeLimitation.size(), blobShape.size(), "");
+#if 0  // TODO: DNNTestNetwork.MobileNet_SSD_Caffe_Different_Width_Height/0
+                const size_t dims = inputShapeLimitation.size();
+                for (size_t dim = 0; dim < dims; dim++)
+                {
+                    if (dims >= 3 && dim == 0 && inputShapeLimitation[0] == 1)
+                        continue;  // don't limit batch
+                    CV_CheckEQ(inputShapeLimitation[dim], blobShape[dim], "");
+                }
+#endif
+            }
+        }
+    }
+
     LayerData &ld = impl->layers[pin.lid];
     const int numInputs = std::max(pin.oid+1, (int)ld.requiredOutputs.size());
     ld.outputBlobs.resize(numInputs);
@@ -3515,17 +3579,11 @@ void Net::setInput(InputArray blob, const String& name, double scalefactor, cons
     impl->netInputLayer->means.resize(numInputs);
 
     MatShape prevShape = shape(impl->netInputLayer->inputsData[pin.oid]);
-    Mat blob_ = blob.getMat();
+    bool oldShape = prevShape == blobShape;
-    bool oldShape = prevShape == shape(blob_);
+
-    if (oldShape)
+    blob_.copyTo(impl->netInputLayer->inputsData[pin.oid]);
-    {
+    if (!oldShape)
-        blob_.copyTo(impl->netInputLayer->inputsData[pin.oid]);
+        ld.outputBlobs[pin.oid] = impl->netInputLayer->inputsData[pin.oid];
-    }
-    else
-    {
-        ld.outputBlobs[pin.oid] = blob_.clone();
-        impl->netInputLayer->inputsData[pin.oid] = ld.outputBlobs[pin.oid];
-    }
 
     if (!ld.outputBlobsWrappers[pin.oid].empty())
     {

modules/dnn/test/test_misc.cpp

@@ -78,6 +78,65 @@ TEST(readNet, Regression)
     EXPECT_FALSE(net.empty());
 }
 
+TEST(readNet, do_not_call_setInput)  // https://github.com/opencv/opencv/issues/16618
+{
+    // 1. load network
+    const string proto = findDataFile("dnn/squeezenet_v1.1.prototxt");
+    const string model = findDataFile("dnn/squeezenet_v1.1.caffemodel", false);
+    Net net = readNetFromCaffe(proto, model);
+
+    // 2. mistake: no inputs are specified through .setInput()
+
+    // 3. try inference
+    Mat res;
+    EXPECT_THROW(
+    {
+        res = net.forward();  // no inputs after loading => should fail
+    }, cv::Exception);
+    EXPECT_TRUE(res.empty()) << res.size;
+}
+
+#ifdef HAVE_INF_ENGINE
+static
+void test_readNet_IE_do_not_call_setInput(Backend backendId)
+{
+    const Target targetId = DNN_TARGET_CPU;
+
+    const std::string& model = findDataFile("dnn/layers/layer_convolution.bin");
+    const std::string& proto = findDataFile("dnn/layers/layer_convolution.xml");
+
+    if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019)
+        setInferenceEngineBackendType(CV_DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_API);
+    else if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH)
+        setInferenceEngineBackendType(CV_DNN_BACKEND_INFERENCE_ENGINE_NGRAPH);
+    else
+        FAIL() << "Unknown backendId";
+
+    Net net = readNet(model, proto);
+    net.setPreferableBackend(backendId);
+    net.setPreferableTarget(targetId);
+
+    // 2. mistake: no inputs are specified through .setInput()
+
+    // 3. try inference
+    Mat res;
+    EXPECT_THROW(
+    {
+        res = net.forward();  // no inputs after loading => should fail
+    }, cv::Exception);
+    EXPECT_TRUE(res.empty()) << res.size;
+}
+
+TEST(readNet, do_not_call_setInput_IE_NN_BUILDER_2019)
+{
+    test_readNet_IE_do_not_call_setInput(DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019);
+}
+TEST(readNet, do_not_call_setInput_IE_NGRAPH)
+{
+    test_readNet_IE_do_not_call_setInput(DNN_BACKEND_INFERENCE_ENGINE_NGRAPH);
+}
+#endif  // HAVE_INF_ENGINE
+
 typedef testing::TestWithParam<tuple<Backend, Target> > dump;
 TEST_P(dump, Regression)
 {