Broadcasting from ONNX

modules/dnn/src/layers/scale_layer.cpp

@@ -46,14 +46,14 @@ public:
     {
         std::vector<Mat> inputs;
         inputs_arr.getMatVector(inputs);
-        hasWeights = blobs.size() == 2 || (blobs.size() == 1 && !hasBias);
+        hasWeights = blobs.size() == 2 || (blobs.size() <= 1 && !hasBias);
         CV_Assert((inputs.size() == 2 && blobs.empty()) || blobs.size() == (int)hasWeights + (int)hasBias);
     }
 
     virtual bool supportBackend(int backendId) CV_OVERRIDE
     {
         return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_HALIDE ||
-               (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && axis == 1) ||
+               (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && axis == 1 && !blobs.empty()) ||
                (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && axis > 0);
     }
 
@@ -78,10 +78,9 @@ public:
         Mat &outBlob = outputs[0];
         // There is a mode when we multiply a first blob by a second one
         // instead of trainable weights.
-        Mat weights = blobs.empty() ? inputs[1] : (hasWeights ? blobs[0] : Mat());
+        Mat weights = hasWeights ? (blobs.empty() ? inputs[1] : blobs[0]).reshape(1, 1) : Mat();;
-        Mat bias = hasBias ? blobs.back().reshape(1, 1) : Mat();
+        Mat bias = hasBias ? (blobs.empty() ? inputs[1] : blobs.back()).reshape(1, 1) : Mat();
-        if (!weights.empty())
+
-            weights = weights.reshape(1, 1);
         MatShape inpShape = shape(inpBlob);
         const int numWeights = !weights.empty() ? weights.total() : bias.total();
         CV_Assert(numWeights != 0);
@@ -229,28 +228,40 @@ public:
 #ifdef HAVE_DNN_NGRAPH
     virtual Ptr<BackendNode> initNgraph(const std::vector<Ptr<BackendWrapper> >& inputs, const std::vector<Ptr<BackendNode> >& nodes) CV_OVERRIDE
     {
-        CV_Assert(!blobs.empty());
+        auto ieInpNode0 = nodes[0].dynamicCast<InfEngineNgraphNode>()->node;
-        const size_t numChannels = blobs[0].total();
+        auto ieInpNode1 = nodes.size() > 1 ? nodes[1].dynamicCast<InfEngineNgraphNode>()->node : nullptr;
-        auto ieInpNode = nodes[0].dynamicCast<InfEngineNgraphNode>()->node;
 
-        std::vector<size_t> shape(ieInpNode->get_shape().size(), 1);
+        size_t numChannels = 1;
+        if (blobs.empty())
+            for (const size_t& dim : ieInpNode1->get_shape())
+                numChannels *= dim;
+        else
+            numChannels = blobs[0].total();
+
+        std::vector<size_t> shape(ieInpNode0->get_shape().size(), 1);
         int cAxis = clamp(axis, shape.size());
         shape[cAxis] = numChannels;
 
-        auto node = ieInpNode;
+        auto node = ieInpNode0;
         if (hasWeights)
         {
-            auto weight = std::make_shared<ngraph::op::Constant>(ngraph::element::f32,
+            auto weight = blobs.empty() ? ieInpNode1 :
-                                                                 ngraph::Shape(shape), blobs[0].data);
+                          std::make_shared<ngraph::op::Constant>(ngraph::element::f32, ngraph::Shape(shape), blobs[0].data);
+
             node = std::make_shared<ngraph::op::v1::Multiply>(node, weight, ngraph::op::AutoBroadcastType::NUMPY);
         }
         if (hasBias || !hasWeights)
         {
-            auto bias = hasBias ?
+            std::shared_ptr<ngraph::Node> bias;
-                        std::make_shared<ngraph::op::Constant>(ngraph::element::f32,
+            if (hasBias)
-                                                               ngraph::Shape(shape), blobs.back().data) :
+            {
-                        std::make_shared<ngraph::op::Constant>(ngraph::element::f32,
+                bias = blobs.empty() ? ieInpNode1 :
-                                                               ngraph::Shape(shape), std::vector<float>(numChannels, 0).data());
+                       std::make_shared<ngraph::op::Constant>(ngraph::element::f32,
+                                                              ngraph::Shape(shape), blobs.back().data);
+            }
+            else
+                bias = std::make_shared<ngraph::op::Constant>(ngraph::element::f32,
+                                                              ngraph::Shape(shape), std::vector<float>(numChannels, 0).data());
             node = std::make_shared<ngraph::op::v1::Add>(node, bias, ngraph::op::AutoBroadcastType::NUMPY);
         }
         return Ptr<BackendNode>(new InfEngineNgraphNode(node));
@@ -259,8 +270,8 @@ public:
 
     void getScaleShift(Mat& scale, Mat& shift) const CV_OVERRIDE
     {
-        scale = hasWeights ? blobs[0] : Mat();
+        scale = (hasWeights && !blobs.empty()) ? blobs[0] : Mat();
-        shift = hasBias ? blobs.back() : Mat();
+        shift = (hasBias && !blobs.empty()) ? blobs.back() : Mat();
     }
 
     virtual int64 getFLOPS(const std::vector<MatShape> &inputs,

modules/dnn/src/onnx/onnx_importer.cpp

@@ -427,24 +427,57 @@ void ONNXImporter::populateNet(Net dstNet)
             }
             layerParams.type = "Slice";
         }
-        else if (layer_type == "Add" || layer_type == "Sum")
+        else if (layer_type == "Add" || layer_type == "Sum" || layer_type == "Sub")
         {
+            bool isSub = layer_type == "Sub";
+            CV_CheckEQ(node_proto.input_size(), 2, "");
             if (layer_id.find(node_proto.input(1)) == layer_id.end())
             {
                 Mat blob = getBlob(node_proto, constBlobs, 1);
                 blob = blob.reshape(1, 1);
                 if (blob.total() == 1) {
                     layerParams.type = "Power";
-                    layerParams.set("shift", blob.at<float>(0));
+                    layerParams.set("shift", (isSub ? -1 : 1) * blob.at<float>(0));
                 }
                 else {
                     layerParams.type = "Scale";
                     layerParams.set("bias_term", true);
-                    layerParams.blobs.push_back(blob);
+                    layerParams.blobs.push_back((isSub ? -1 : 1) * blob);
                 }
             }
-            else {
+            else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(1)])
+            {
                 layerParams.type = "Eltwise";
+                if (isSub)
+                {
+                    static float subCoeffs[] = {1.f, -1.f};
+                    layerParams.set("coeff", DictValue::arrayReal<float*>(subCoeffs, 2));
+                }
+            }
+            else
+            {
+                if (isSub)
+                {
+                    LayerParams powerParams;
+                    powerParams.name = layerParams.name + "/neg";
+                    powerParams.type = "Power";
+                    powerParams.set("scale", -1);
+
+                    //Create Power layer
+                    int id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams);
+                    //Connect to input
+                    layerId = layer_id.find(node_proto.input(1));
+                    CV_Assert(layerId != layer_id.end());
+                    dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0);
+                    //Add shape
+                    layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0)));
+                    outShapes[powerParams.name] = outShapes[node_proto.input(1)];
+
+                    //Replace input to Power
+                    node_proto.set_input(1, powerParams.name);
+                }
+                layerParams.type = "Scale";
+                layerParams.set("bias_term", true);
             }
         }
         else if (layer_type == "Max")
@@ -452,19 +485,6 @@ void ONNXImporter::populateNet(Net dstNet)
             layerParams.type = "Eltwise";
             layerParams.set("operation", "max");
         }
-        else if (layer_type == "Sub")
-        {
-            Mat blob = getBlob(node_proto, constBlobs, 1);
-            if (blob.total() == 1) {
-                layerParams.type = "Power";
-                layerParams.set("shift", -blob.at<float>(0));
-            }
-            else {
-                layerParams.type = "Scale";
-                layerParams.set("has_bias", true);
-                layerParams.blobs.push_back(-1.0f * blob.reshape(1, 1));
-            }
-        }
         else if (layer_type == "Neg")
         {
             layerParams.type = "Power";
@@ -643,10 +663,35 @@ void ONNXImporter::populateNet(Net dstNet)
                     layerParams.type = "Scale";
                 }
             }
-            else {
+            else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(1)])
+            {
                 layerParams.type = "Eltwise";
                 layerParams.set("operation", isDiv ? "div" : "prod");
             }
+            else
+            {
+                if (isDiv)
+                {
+                    LayerParams powerParams;
+                    powerParams.name = layerParams.name + "/inv";
+                    powerParams.type = "Power";
+                    powerParams.set("power", -1);
+
+                    //Create Power layer
+                    int id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams);
+                    //Connect to input
+                    layerId = layer_id.find(node_proto.input(1));
+                    CV_Assert(layerId != layer_id.end());
+                    dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0);
+                    //Add shape
+                    layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0)));
+                    outShapes[powerParams.name] = outShapes[node_proto.input(1)];
+
+                    //Replace input to Power
+                    node_proto.set_input(1, powerParams.name);
+                }
+                layerParams.type = "Scale";
+            }
 
             if (!haveVariables)
             {

modules/dnn/test/test_onnx_importer.cpp

@@ -32,29 +32,33 @@ public:
 
     void testONNXModels(const String& basename, const Extension ext = npy,
                         const double l1 = 0, const float lInf = 0, const bool useSoftmax = false,
-                        bool checkNoFallbacks = true)
+                        bool checkNoFallbacks = true, int numInps = 1)
     {
         String onnxmodel = _tf("models/" + basename + ".onnx", required);
-        Mat inp, ref;
+        std::vector<Mat> inps(numInps);
+        Mat ref;
         if (ext == npy) {
-            inp = blobFromNPY(_tf("data/input_" + basename + ".npy"));
+            for (int i = 0; i < numInps; ++i)
+                inps[i] = blobFromNPY(_tf("data/input_" + basename + (numInps > 1 ? format("_%d", i) : "") + ".npy"));
             ref = blobFromNPY(_tf("data/output_" + basename + ".npy"));
         }
         else if (ext == pb) {
-            inp = readTensorFromONNX(_tf("data/input_" + basename + ".pb"));
+            for (int i = 0; i < numInps; ++i)
+                inps[i] = readTensorFromONNX(_tf("data/input_" + basename + (numInps > 1 ? format("_%d", i) : "") + ".pb"));
             ref = readTensorFromONNX(_tf("data/output_" + basename + ".pb"));
         }
         else
             CV_Error(Error::StsUnsupportedFormat, "Unsupported extension");
 
-        checkBackend(&inp, &ref);
+        checkBackend(&inps[0], &ref);
         Net net = readNetFromONNX(onnxmodel);
         ASSERT_FALSE(net.empty());
 
         net.setPreferableBackend(backend);
         net.setPreferableTarget(target);
 
-        net.setInput(inp);
+        for (int i = 0; i < numInps; ++i)
+            net.setInput(inps[i], numInps > 1 ? format("%d", i) : "");
         Mat out = net.forward("");
 
         if (useSoftmax)
@@ -328,25 +332,14 @@ TEST_P(Test_ONNX_layers, ResizeUnfused)
 
 TEST_P(Test_ONNX_layers, MultyInputs)
 {
-    const String model =  _tf("models/multy_inputs.onnx");
+    testONNXModels("multy_inputs", npy, 0, 0, false, true, 2);
+}
 
-    Net net = readNetFromONNX(model);
+TEST_P(Test_ONNX_layers, Broadcast)
-    ASSERT_FALSE(net.empty());
+{
-
+    if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019)
-    net.setPreferableBackend(backend);
+        applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER);
-    net.setPreferableTarget(target);
+    testONNXModels("channel_broadcast", npy, 0, 0, false, true, 2);
-
-    Mat inp1 = blobFromNPY(_tf("data/input_multy_inputs_0.npy"));
-    Mat inp2 = blobFromNPY(_tf("data/input_multy_inputs_1.npy"));
-    Mat ref  = blobFromNPY(_tf("data/output_multy_inputs.npy"));
-    checkBackend(&inp1, &ref);
-
-    net.setInput(inp1, "0");
-    net.setInput(inp2, "1");
-    Mat out = net.forward();
-
-    normAssert(ref, out, "", default_l1,  default_lInf);
-    expectNoFallbacksFromIE(net);
 }
 
 TEST_P(Test_ONNX_layers, Div)