diff --git a/modules/dnn/src/dnn.cpp b/modules/dnn/src/dnn.cpp index b1123e2195..52a5fcba28 100644 --- a/modules/dnn/src/dnn.cpp +++ b/modules/dnn/src/dnn.cpp @@ -99,6 +99,15 @@ bool DNN_DIAGNOSTICS_RUN = false; void enableModelDiagnostics(bool isDiagnosticsMode) { DNN_DIAGNOSTICS_RUN = isDiagnosticsMode; + + if (DNN_DIAGNOSTICS_RUN) + { + detail::NotImplemented::Register(); + } + else + { + detail::NotImplemented::unRegister(); + } } using std::vector; @@ -4014,13 +4023,24 @@ int Net::addLayer(const String &name, const String &type, LayerParams ¶ms) { CV_TRACE_FUNCTION(); - if (impl->getLayerId(name) >= 0) + int id = impl->getLayerId(name); + if (id >= 0) { - CV_Error(Error::StsBadArg, "Layer \"" + name + "\" already into net"); - return -1; + if (!DNN_DIAGNOSTICS_RUN || type != "NotImplemented") + { + CV_Error(Error::StsBadArg, "Layer \"" + name + "\" already into net"); + return -1; + } + else + { + LayerData& ld = impl->layers.find(id)->second; + ld.type = type; + ld.params = params; + return -1; + } } - int id = ++impl->lastLayerId; + id = ++impl->lastLayerId; impl->layerNameToId.insert(std::make_pair(name, id)); impl->layers.insert(std::make_pair(id, LayerData(id, name, type, params))); if (params.get("has_dynamic_shapes", false)) diff --git a/modules/dnn/src/dnn_common.hpp b/modules/dnn/src/dnn_common.hpp index ff8f5e8467..46fae41cc2 100644 --- a/modules/dnn/src/dnn_common.hpp +++ b/modules/dnn/src/dnn_common.hpp @@ -15,6 +15,15 @@ void initializeLayerFactory(); namespace detail { +class NotImplemented : public Layer +{ +public: + static Ptr create(const LayerParams ¶ms); + + static void Register(); + static void unRegister(); +}; + struct NetImplBase { const int networkId; // network global identifier diff --git a/modules/dnn/src/layers/not_implemented_layer.cpp b/modules/dnn/src/layers/not_implemented_layer.cpp new file mode 100644 index 0000000000..c4b1343902 --- /dev/null +++ b/modules/dnn/src/layers/not_implemented_layer.cpp @@ -0,0 +1,194 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "../dnn_common.hpp" + +namespace cv { namespace dnn { +CV__DNN_INLINE_NS_BEGIN + +namespace detail { + +class NotImplementedImpl CV_FINAL : public NotImplemented +{ +public: + NotImplementedImpl(const LayerParams& params) + { + setParamsFrom(params); + CV_Assert(params.has("type")); + std::stringstream ss; + ss << "Node for layer '" << params.name << "' of type '" << params.get("type") << "' wasn't initialized."; + msg = ss.str(); + } + + CV_DEPRECATED_EXTERNAL + virtual void finalize(const std::vector &input, std::vector &output) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual void finalize(InputArrayOfArrays inputs, OutputArrayOfArrays outputs) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + CV_DEPRECATED_EXTERNAL + virtual void forward(std::vector &input, std::vector &output, std::vector &internals) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + void forward_fallback(InputArrayOfArrays inputs, OutputArrayOfArrays outputs, OutputArrayOfArrays internals) + { + CV_Error(Error::StsNotImplemented, msg); + } + + CV_DEPRECATED_EXTERNAL + void finalize(const std::vector &inputs, CV_OUT std::vector &outputs) + { + CV_Error(Error::StsNotImplemented, msg); + } + + CV_DEPRECATED std::vector finalize(const std::vector &inputs) + { + CV_Error(Error::StsNotImplemented, msg); + } + + CV_DEPRECATED void run(const std::vector &inputs, + CV_OUT std::vector &outputs, + CV_IN_OUT std::vector &internals) + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual int inputNameToIndex(String inputName) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual int outputNameToIndex(const String& outputName) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual Ptr initHalide(const std::vector > &inputs) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual Ptr initInfEngine(const std::vector > &inputs) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual Ptr initNgraph(const std::vector > &inputs, + const std::vector >& nodes) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual Ptr initVkCom(const std::vector > &inputs) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual Ptr initCUDA( + void *context, + const std::vector>& inputs, + const std::vector>& outputs + ) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual void applyHalideScheduler(Ptr& node, + const std::vector &inputs, + const std::vector &outputs, + int targetId) const CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual Ptr tryAttach(const Ptr& node) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual bool setActivation(const Ptr& layer) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual bool tryFuse(Ptr& top) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual void getScaleShift(Mat& scale, Mat& shift) const CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual void unsetAttached() CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual int64 getFLOPS(const std::vector &inputs, + const std::vector &outputs) const CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual bool updateMemoryShapes(const std::vector &inputs) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + +private: + std::string msg; +}; + +Ptr NotImplemented::create(const LayerParams& params) +{ + return makePtr(params); +} + +Ptr notImplementedRegisterer(LayerParams ¶ms) +{ + return detail::NotImplemented::create(params); +} + +void NotImplemented::Register() +{ + LayerFactory::registerLayer("NotImplemented", detail::notImplementedRegisterer); +} + +void NotImplemented::unRegister() +{ + LayerFactory::unregisterLayer("NotImplemented"); +} + +} // namespace detail + +CV__DNN_INLINE_NS_END +}} // namespace cv::dnn diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index 15f88007b4..10670bfef9 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -466,6 +466,8 @@ void ExcludeLayer(tensorflow::GraphDef& net, const int layer_index, const int in net.mutable_node()->DeleteSubrange(layer_index, 1); } +class LayerHandler; + class TFImporter { public: @@ -473,6 +475,7 @@ public: TFImporter(Net& net, const char *dataModel, size_t lenModel, const char *dataConfig = NULL, size_t lenConfig = 0); protected: + std::unique_ptr layerHandler; std::unique_ptr utilNet; Net& dstNet; void populateNet(); @@ -514,6 +517,7 @@ protected: private: void addPermuteLayer(const int* order, const std::string& permName, Pin& inpId); + friend class LayerHandler; typedef void (TFImporter::*TFImporterNodeParser)(tensorflow::GraphDef&, const tensorflow::NodeDef&, LayerParams&); typedef std::map DispatchMap; @@ -554,6 +558,20 @@ private: void parseCustomLayer (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); }; +class LayerHandler +{ +public: + LayerHandler(TFImporter* importer_); + ~LayerHandler() = default; + + bool handleMissing(const opencv_tensorflow::NodeDef& layer); + void handleFailed(const opencv_tensorflow::NodeDef& layer); + +private: + TFImporter* importer; + std::set layers; +}; + const TFImporter::DispatchMap TFImporter::buildDispatchMap() { static DispatchMap dispatch; @@ -2340,7 +2358,8 @@ void TFImporter::parseCustomLayer(tensorflow::GraphDef& net, const tensorflow::N } TFImporter::TFImporter(Net& net, const char *model, const char *config) - : utilNet(DNN_DIAGNOSTICS_RUN ? new Net : nullptr), + : layerHandler(DNN_DIAGNOSTICS_RUN ? new LayerHandler(this) : nullptr), + utilNet(DNN_DIAGNOSTICS_RUN ? new Net : nullptr), dstNet(DNN_DIAGNOSTICS_RUN ? *utilNet : net), dispatch(buildDispatchMap()) { if (model && model[0]) @@ -2362,7 +2381,8 @@ TFImporter::TFImporter( const char *dataModel, size_t lenModel, const char *dataConfig, size_t lenConfig ) - : utilNet(DNN_DIAGNOSTICS_RUN ? new Net : nullptr), + : layerHandler(DNN_DIAGNOSTICS_RUN ? new LayerHandler(this) : nullptr), + utilNet(DNN_DIAGNOSTICS_RUN ? new Net : nullptr), dstNet(DNN_DIAGNOSTICS_RUN ? *utilNet : net), dispatch(buildDispatchMap()) { if (dataModel != NULL && lenModel > 0) @@ -2620,11 +2640,6 @@ DataLayout TFImporter::predictOutputDataLayout(const tensorflow::NodeDef& layer) return it->second; } -Ptr dummy_constructor(LayerParams & params) -{ - return new Layer(params); -} - void TFImporter::populateNet() { CV_Assert(netBin.ByteSize() || netTxt.ByteSize()); @@ -2727,7 +2742,6 @@ void TFImporter::populateNet() addConstNodes(netBin, value_id, layers_to_ignore); addConstNodes(netTxt, value_id, layers_to_ignore); - for (int li = 0; li < layersSize; li++) { const tensorflow::NodeDef& layer = net.node(li); @@ -2785,41 +2799,64 @@ void TFImporter::parseNode(const tensorflow::NodeDef& layer) { ((*this).*(iter->second))(net, layer, layerParams); } - else + else if (!DNN_DIAGNOSTICS_RUN || !layerHandler->handleMissing(layer)) { - if (DNN_DIAGNOSTICS_RUN && !LayerFactory::createLayerInstance(type, layerParams)) - { - CV_LOG_ERROR(NULL, "DNN/TF: Node='" << name << "' of type='"<< type - << "' is not supported. This error won't be displayed again."); - LayerFactory::registerLayer(type, dummy_constructor); - } - parseCustomLayer(net, layer, layerParams); } } catch (const std::exception& e) { - if (!DNN_DIAGNOSTICS_RUN) + CV_LOG_ERROR(NULL, "DNN/TF: Can't parse layer for node='" << name << "' of type='" << type + << "'. Exception: " << e.what()); + + if (DNN_DIAGNOSTICS_RUN) { - CV_LOG_ERROR(NULL, "DNN/TF: Can't parse layer for node='" << name << "' of type='" << type - << "'. Exception: " << e.what()); - throw; + layerHandler->handleFailed(layer); } else { - CV_LOG_ERROR(NULL, "DNN/TF: Can't parse layer for node='" << name << "' of type='" << type - << "'. Exception: " << e.what()); - - // internal layer failure (didnt call addLayer) - if (dstNet.getLayerId(name) == -1) - { - int id = dstNet.addLayer(name, type, layerParams); - layer_id[name] = id; - } + throw; } } } +LayerHandler::LayerHandler(TFImporter* importer_) : importer(importer_) {} + +void LayerHandler::handleFailed(const opencv_tensorflow::NodeDef& layer) +{ + LayerParams lp; + lp.name = layer.name(); + lp.type = "NotImplemented"; + lp.set("type", layer.op()); + + // the layer will be created or its params and type will be replaced + int id = importer->dstNet.addLayer(lp.name, "NotImplemented", lp); + if (id != -1) // internal layer failure before the call to addLayer() + { + importer->layer_id[lp.name] = id; + } +} + +bool LayerHandler::handleMissing(const opencv_tensorflow::NodeDef& layer) +{ + LayerParams lp; + // If we didn't add it, but can create it, it's custom and not missing. + if (layers.find(layer.op()) == layers.end() && LayerFactory::createLayerInstance(layer.op(), lp)) + { + return false; + } + + if (layers.insert(layer.op()).second) + { + CV_LOG_ERROR(NULL, "DNN/TF: Node='" << layer.name() << "' of type='"<< layer.op() + << "' is not supported. This error won't be displayed again."); + } + + handleFailed(layer); + + return true; +} + } // namespace #endif //HAVE_PROTOBUF diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index 2c36134724..35751b4824 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -568,6 +568,39 @@ TEST_P(Test_TensorFlow_layers, l2_normalize_3d) runTensorFlowNet("l2_normalize_3d"); } +class Test_TensorFlow_diagnostics : public DNNTestLayer { +public: + Test_TensorFlow_diagnostics() + { + enableModelDiagnostics(true); + } + + ~Test_TensorFlow_diagnostics() + { + enableModelDiagnostics(false); + } + + void runFailingTensorFlowNet(const std::string& prefix, bool hasText = false) + { + std::string netPath = path(prefix + "_net.pb"); + std::string netConfig = (hasText ? path(prefix + "_net.pbtxt") : ""); + + Net net = readNetFromTensorflow(netPath, netConfig); + } +}; + +TEST_P(Test_TensorFlow_diagnostics, not_implemented_layer) +{ + runFailingTensorFlowNet("not_implemented_layer"); +} + +TEST_P(Test_TensorFlow_diagnostics, broken_parameters) +{ + runFailingTensorFlowNet("broken_layer"); +} + +INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_diagnostics, dnnBackendsAndTargets()); + class Test_TensorFlow_nets : public DNNTestLayer {}; TEST_P(Test_TensorFlow_nets, MobileNet_SSD)