Merge pull request #20402 from rogday:tf_diag_dummy

modules/dnn/src/dnn.cpp

@@ -99,6 +99,15 @@ bool DNN_DIAGNOSTICS_RUN = false;
 void enableModelDiagnostics(bool isDiagnosticsMode)
 {
     DNN_DIAGNOSTICS_RUN = isDiagnosticsMode;
+
+    if (DNN_DIAGNOSTICS_RUN)
+    {
+        detail::NotImplemented::Register();
+    }
+    else
+    {
+        detail::NotImplemented::unRegister();
+    }
 }
 
 using std::vector;
@@ -4014,13 +4023,24 @@ int Net::addLayer(const String &name, const String &type, LayerParams &params)
 {
     CV_TRACE_FUNCTION();
 
-    if (impl->getLayerId(name) >= 0)
+    int id = impl->getLayerId(name);
+    if (id >= 0)
     {
-        CV_Error(Error::StsBadArg, "Layer \"" + name + "\" already into net");
-        return -1;
+        if (!DNN_DIAGNOSTICS_RUN || type != "NotImplemented")
+        {
+            CV_Error(Error::StsBadArg, "Layer \"" + name + "\" already into net");
+            return -1;
+        }
+        else
+        {
+            LayerData& ld = impl->layers.find(id)->second;
+            ld.type = type;
+            ld.params = params;
+            return -1;
+        }
     }
 
-    int id = ++impl->lastLayerId;
+    id = ++impl->lastLayerId;
     impl->layerNameToId.insert(std::make_pair(name, id));
     impl->layers.insert(std::make_pair(id, LayerData(id, name, type, params)));
     if (params.get<bool>("has_dynamic_shapes", false))

modules/dnn/src/dnn_common.hpp

@@ -15,6 +15,15 @@ void initializeLayerFactory();
 
 namespace detail {
 
+class NotImplemented : public Layer
+{
+public:
+    static Ptr<Layer> create(const LayerParams &params);
+
+    static void Register();
+    static void unRegister();
+};
+
 struct NetImplBase
 {
     const int networkId;  // network global identifier

modules/dnn/src/layers/not_implemented_layer.cpp

@@ -0,0 +1,194 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+
+#include "../precomp.hpp"
+#include "../dnn_common.hpp"
+
+namespace cv { namespace dnn {
+CV__DNN_INLINE_NS_BEGIN
+
+namespace detail {
+
+class NotImplementedImpl CV_FINAL : public NotImplemented
+{
+public:
+    NotImplementedImpl(const LayerParams& params)
+    {
+        setParamsFrom(params);
+        CV_Assert(params.has("type"));
+        std::stringstream ss;
+        ss << "Node for layer '" << params.name << "' of type '" << params.get("type") << "' wasn't initialized.";
+        msg = ss.str();
+    }
+
+    CV_DEPRECATED_EXTERNAL
+    virtual void finalize(const std::vector<Mat*> &input, std::vector<Mat> &output) CV_OVERRIDE
+    {
+        CV_Error(Error::StsNotImplemented, msg);
+    }
+
+    virtual void finalize(InputArrayOfArrays inputs, OutputArrayOfArrays outputs) CV_OVERRIDE
+    {
+        CV_Error(Error::StsNotImplemented, msg);
+    }
+
+    CV_DEPRECATED_EXTERNAL
+    virtual void forward(std::vector<Mat*> &input, std::vector<Mat> &output, std::vector<Mat> &internals) CV_OVERRIDE
+    {
+        CV_Error(Error::StsNotImplemented, msg);
+    }
+
+    virtual void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE
+    {
+        CV_Error(Error::StsNotImplemented, msg);
+    }
+
+    void forward_fallback(InputArrayOfArrays inputs, OutputArrayOfArrays outputs, OutputArrayOfArrays internals)
+    {
+        CV_Error(Error::StsNotImplemented, msg);
+    }
+
+    CV_DEPRECATED_EXTERNAL
+    void finalize(const std::vector<Mat> &inputs, CV_OUT std::vector<Mat> &outputs)
+    {
+        CV_Error(Error::StsNotImplemented, msg);
+    }
+
+    CV_DEPRECATED std::vector<Mat> finalize(const std::vector<Mat> &inputs)
+    {
+        CV_Error(Error::StsNotImplemented, msg);
+    }
+
+    CV_DEPRECATED void run(const std::vector<Mat> &inputs,
+                           CV_OUT std::vector<Mat> &outputs,
+                           CV_IN_OUT std::vector<Mat> &internals)
+    {
+        CV_Error(Error::StsNotImplemented, msg);
+    }
+
+    virtual int inputNameToIndex(String inputName) CV_OVERRIDE
+    {
+        CV_Error(Error::StsNotImplemented, msg);
+    }
+
+    virtual int outputNameToIndex(const String& outputName) CV_OVERRIDE
+    {
+        CV_Error(Error::StsNotImplemented, msg);
+    }
+
+    virtual bool supportBackend(int backendId) CV_OVERRIDE
+    {
+        CV_Error(Error::StsNotImplemented, msg);
+    }
+
+    virtual Ptr<BackendNode> initHalide(const std::vector<Ptr<BackendWrapper> > &inputs) CV_OVERRIDE
+    {
+        CV_Error(Error::StsNotImplemented, msg);
+    }
+
+    virtual Ptr<BackendNode> initInfEngine(const std::vector<Ptr<BackendWrapper> > &inputs) CV_OVERRIDE
+    {
+        CV_Error(Error::StsNotImplemented, msg);
+    }
+
+    virtual Ptr<BackendNode> initNgraph(const std::vector<Ptr<BackendWrapper> > &inputs,
+                                        const std::vector<Ptr<BackendNode> >& nodes) CV_OVERRIDE
+    {
+        CV_Error(Error::StsNotImplemented, msg);
+    }
+
+    virtual Ptr<BackendNode> initVkCom(const std::vector<Ptr<BackendWrapper> > &inputs) CV_OVERRIDE
+    {
+        CV_Error(Error::StsNotImplemented, msg);
+    }
+
+    virtual Ptr<BackendNode> initCUDA(
+            void *context,
+            const std::vector<Ptr<BackendWrapper>>& inputs,
+            const std::vector<Ptr<BackendWrapper>>& outputs
+    ) CV_OVERRIDE
+    {
+        CV_Error(Error::StsNotImplemented, msg);
+    }
+
+    virtual void applyHalideScheduler(Ptr<BackendNode>& node,
+                                      const std::vector<Mat*> &inputs,
+                                      const std::vector<Mat> &outputs,
+                                      int targetId) const CV_OVERRIDE
+    {
+        CV_Error(Error::StsNotImplemented, msg);
+    }
+
+    virtual Ptr<BackendNode> tryAttach(const Ptr<BackendNode>& node) CV_OVERRIDE
+    {
+        CV_Error(Error::StsNotImplemented, msg);
+    }
+
+    virtual bool setActivation(const Ptr<ActivationLayer>& layer) CV_OVERRIDE
+    {
+        CV_Error(Error::StsNotImplemented, msg);
+    }
+
+    virtual bool tryFuse(Ptr<Layer>& top) CV_OVERRIDE
+    {
+        CV_Error(Error::StsNotImplemented, msg);
+    }
+
+    virtual void getScaleShift(Mat& scale, Mat& shift) const CV_OVERRIDE
+    {
+        CV_Error(Error::StsNotImplemented, msg);
+    }
+
+    virtual void unsetAttached() CV_OVERRIDE
+    {
+        CV_Error(Error::StsNotImplemented, msg);
+    }
+
+    virtual bool getMemoryShapes(const std::vector<MatShape> &inputs,
+                                 const int requiredOutputs,
+                                 std::vector<MatShape> &outputs,
+                                 std::vector<MatShape> &internals) const CV_OVERRIDE
+    {
+        CV_Error(Error::StsNotImplemented, msg);
+    }
+
+    virtual int64 getFLOPS(const std::vector<MatShape> &inputs,
+                           const std::vector<MatShape> &outputs) const CV_OVERRIDE
+    {
+        CV_Error(Error::StsNotImplemented, msg);
+    }
+
+    virtual bool updateMemoryShapes(const std::vector<MatShape> &inputs) CV_OVERRIDE
+    {
+        CV_Error(Error::StsNotImplemented, msg);
+    }
+
+private:
+    std::string msg;
+};
+
+Ptr<Layer> NotImplemented::create(const LayerParams& params)
+{
+    return makePtr<NotImplementedImpl>(params);
+}
+
+Ptr<Layer> notImplementedRegisterer(LayerParams &params)
+{
+    return detail::NotImplemented::create(params);
+}
+
+void NotImplemented::Register()
+{
+    LayerFactory::registerLayer("NotImplemented", detail::notImplementedRegisterer);
+}
+
+void NotImplemented::unRegister()
+{
+    LayerFactory::unregisterLayer("NotImplemented");
+}
+
+} // namespace detail
+
+CV__DNN_INLINE_NS_END
+}}  // namespace cv::dnn

modules/dnn/src/tensorflow/tf_importer.cpp

@@ -466,6 +466,8 @@ void ExcludeLayer(tensorflow::GraphDef& net, const int layer_index, const int in
         net.mutable_node()->DeleteSubrange(layer_index, 1);
 }
 
+class LayerHandler;
+
 class TFImporter
 {
 public:
@@ -473,6 +475,7 @@ public:
     TFImporter(Net& net, const char *dataModel, size_t lenModel,
                const char *dataConfig = NULL, size_t lenConfig = 0);
 protected:
+    std::unique_ptr<LayerHandler> layerHandler;
     std::unique_ptr<Net> utilNet;
     Net& dstNet;
     void populateNet();
@@ -514,6 +517,7 @@ protected:
 private:
     void addPermuteLayer(const int* order, const std::string& permName, Pin& inpId);
 
+    friend class LayerHandler;
     typedef void (TFImporter::*TFImporterNodeParser)(tensorflow::GraphDef&, const tensorflow::NodeDef&, LayerParams&);
     typedef std::map<std::string, TFImporterNodeParser> DispatchMap;
 
@@ -554,6 +558,20 @@ private:
     void parseCustomLayer        (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams);
 };
 
+class LayerHandler
+{
+public:
+    LayerHandler(TFImporter* importer_);
+    ~LayerHandler() = default;
+
+    bool handleMissing(const opencv_tensorflow::NodeDef& layer);
+    void handleFailed(const opencv_tensorflow::NodeDef& layer);
+
+private:
+    TFImporter* importer;
+    std::set<std::string> layers;
+};
+
 const TFImporter::DispatchMap TFImporter::buildDispatchMap()
 {
     static DispatchMap dispatch;
@@ -2340,7 +2358,8 @@ void TFImporter::parseCustomLayer(tensorflow::GraphDef& net, const tensorflow::N
 }
 
 TFImporter::TFImporter(Net& net, const char *model, const char *config)
-    :  utilNet(DNN_DIAGNOSTICS_RUN ?  new Net : nullptr),
+    :  layerHandler(DNN_DIAGNOSTICS_RUN ?  new LayerHandler(this) : nullptr),
+        utilNet(DNN_DIAGNOSTICS_RUN ?  new Net : nullptr),
         dstNet(DNN_DIAGNOSTICS_RUN ? *utilNet : net), dispatch(buildDispatchMap())
 {
     if (model && model[0])
@@ -2362,7 +2381,8 @@ TFImporter::TFImporter(
         const char *dataModel, size_t lenModel,
         const char *dataConfig, size_t lenConfig
 )
-    : utilNet(DNN_DIAGNOSTICS_RUN ?  new Net : nullptr),
+    :  layerHandler(DNN_DIAGNOSTICS_RUN ?  new LayerHandler(this) : nullptr),
+       utilNet(DNN_DIAGNOSTICS_RUN ?  new Net : nullptr),
        dstNet(DNN_DIAGNOSTICS_RUN ? *utilNet : net), dispatch(buildDispatchMap())
 {
     if (dataModel != NULL && lenModel > 0)
@@ -2620,11 +2640,6 @@ DataLayout TFImporter::predictOutputDataLayout(const tensorflow::NodeDef& layer)
     return it->second;
 }
 
-Ptr<Layer> dummy_constructor(LayerParams & params)
-{
-    return new Layer(params);
-}
-
 void TFImporter::populateNet()
 {
     CV_Assert(netBin.ByteSize() || netTxt.ByteSize());
@@ -2727,7 +2742,6 @@ void TFImporter::populateNet()
     addConstNodes(netBin, value_id, layers_to_ignore);
     addConstNodes(netTxt, value_id, layers_to_ignore);
 
-
     for (int li = 0; li < layersSize; li++)
     {
         const tensorflow::NodeDef& layer = net.node(li);
@@ -2785,41 +2799,64 @@ void TFImporter::parseNode(const tensorflow::NodeDef& layer)
         {
             ((*this).*(iter->second))(net, layer, layerParams);
         }
-        else
+        else if (!DNN_DIAGNOSTICS_RUN || !layerHandler->handleMissing(layer))
         {
-            if (DNN_DIAGNOSTICS_RUN && !LayerFactory::createLayerInstance(type, layerParams))
-            {
-                CV_LOG_ERROR(NULL, "DNN/TF: Node='" << name << "' of type='"<< type
-                                                    << "' is not supported. This error won't be displayed again.");
-                LayerFactory::registerLayer(type, dummy_constructor);
-            }
-
             parseCustomLayer(net, layer, layerParams);
         }
     }
     catch (const std::exception& e)
     {
-        if (!DNN_DIAGNOSTICS_RUN)
+        CV_LOG_ERROR(NULL, "DNN/TF: Can't parse layer for node='" << name << "' of type='" << type
+                                                                  << "'. Exception: " << e.what());
+
+        if (DNN_DIAGNOSTICS_RUN)
         {
-            CV_LOG_ERROR(NULL, "DNN/TF: Can't parse layer for node='" << name << "' of type='" << type
-                                << "'. Exception: " << e.what());
-            throw;
+            layerHandler->handleFailed(layer);
         }
         else
         {
-            CV_LOG_ERROR(NULL, "DNN/TF: Can't parse layer for node='" << name << "' of type='" << type
-                                << "'. Exception: " << e.what());
-
-            // internal layer failure (didnt call addLayer)
-            if (dstNet.getLayerId(name) == -1)
-            {
-                int id = dstNet.addLayer(name, type, layerParams);
-                layer_id[name] = id;
-            }
+            throw;
         }
     }
 }
 
+LayerHandler::LayerHandler(TFImporter* importer_) : importer(importer_) {}
+
+void LayerHandler::handleFailed(const opencv_tensorflow::NodeDef& layer)
+{
+    LayerParams lp;
+    lp.name = layer.name();
+    lp.type = "NotImplemented";
+    lp.set("type", layer.op());
+
+    // the layer will be created or its params and type will be replaced
+    int id = importer->dstNet.addLayer(lp.name, "NotImplemented", lp);
+    if (id != -1) // internal layer failure before the call to addLayer()
+    {
+        importer->layer_id[lp.name] = id;
+    }
+}
+
+bool LayerHandler::handleMissing(const opencv_tensorflow::NodeDef& layer)
+{
+    LayerParams lp;
+    // If we didn't add it, but can create it, it's custom and not missing.
+    if (layers.find(layer.op()) == layers.end() && LayerFactory::createLayerInstance(layer.op(), lp))
+    {
+        return false;
+    }
+
+    if (layers.insert(layer.op()).second)
+    {
+        CV_LOG_ERROR(NULL, "DNN/TF: Node='" << layer.name() << "' of type='"<< layer.op()
+                                            << "' is not supported. This error won't be displayed again.");
+    }
+
+    handleFailed(layer);
+
+    return true;
+}
+
 } // namespace
 
 #endif //HAVE_PROTOBUF

modules/dnn/test/test_tf_importer.cpp

@@ -568,6 +568,39 @@ TEST_P(Test_TensorFlow_layers, l2_normalize_3d)
     runTensorFlowNet("l2_normalize_3d");
 }
 
+class Test_TensorFlow_diagnostics : public DNNTestLayer {
+public:
+    Test_TensorFlow_diagnostics()
+    {
+        enableModelDiagnostics(true);
+    }
+
+    ~Test_TensorFlow_diagnostics()
+    {
+        enableModelDiagnostics(false);
+    }
+
+    void runFailingTensorFlowNet(const std::string& prefix, bool hasText = false)
+    {
+        std::string netPath = path(prefix + "_net.pb");
+        std::string netConfig = (hasText ? path(prefix + "_net.pbtxt") : "");
+
+        Net net = readNetFromTensorflow(netPath, netConfig);
+    }
+};
+
+TEST_P(Test_TensorFlow_diagnostics, not_implemented_layer)
+{
+    runFailingTensorFlowNet("not_implemented_layer");
+}
+
+TEST_P(Test_TensorFlow_diagnostics, broken_parameters)
+{
+    runFailingTensorFlowNet("broken_layer");
+}
+
+INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_diagnostics, dnnBackendsAndTargets());
+
 class Test_TensorFlow_nets : public DNNTestLayer {};
 
 TEST_P(Test_TensorFlow_nets, MobileNet_SSD)