mirror of
https://github.com/opencv/opencv.git
synced 2025-01-19 15:04:01 +08:00
Merge pull request #26330 from alexlyulkov:al/new-engine-tflite-parser2
Modified TFLite parser for the new dnn engine #26330 The new dnn graph is creating just by defining input and output names of each layer. Some TFLite layers has fused activation, which doesn't have layer name and input and output names. Also some layers require additional preprocessing layers (e.g. NHWC -> NCHW). All these layers should be added to the graph with some unique layer and input and output names. I solve this problem by adding additionalPreLayer and additionalPostLayer layers. If a layer has a fused activation, I add additionalPostLayer and change input and output names this way: **original**: conv_relu(conv123, conv123_input, conv123_output) **new**: conv(conv123, conv123_input, conv123_output_additional_post_layer) + relu(conv123_relu, conv1_output_additional_post_layer, conv123_output) If a layer has additional preprocessing layer, I change input and output names this way: **original**: permute_reshape(reshape345, reshape345_input, reshape345_output) **new**: permute(reshape345_permute, reshape345_input, reshape345_input_additional_pre_layer) + reshape(reshape345, reshape345_input_additional_pre_layer, reshape345_output) ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
alexlyulkov
GitHub
parent
6648482b69
commit
a40ceff215
@ -1105,23 +1105,29 @@ CV__DNN_INLINE_NS_BEGIN
|
||||
|
||||
/** @brief Reads a network model stored in <a href="https://www.tensorflow.org/lite">TFLite</a> framework's format.
|
||||
* @param model path to the .tflite file with binary flatbuffers description of the network architecture
|
||||
* @param engine select DNN engine to be used. With auto selection the new engine is used first and falls back to classic.
|
||||
* Please pay attention that the new DNN does not support non-CPU back-ends for now.
|
||||
* @returns Net object.
|
||||
*/
|
||||
CV_EXPORTS_W Net readNetFromTFLite(CV_WRAP_FILE_PATH const String &model);
|
||||
CV_EXPORTS_W Net readNetFromTFLite(CV_WRAP_FILE_PATH const String &model, int engine=ENGINE_AUTO);
|
||||
|
||||
/** @brief Reads a network model stored in <a href="https://www.tensorflow.org/lite">TFLite</a> framework's format.
|
||||
* @param bufferModel buffer containing the content of the tflite file
|
||||
* @param engine select DNN engine to be used. With auto selection the new engine is used first and falls back to classic.
|
||||
* Please pay attention that the new DNN does not support non-CPU back-ends for now.
|
||||
* @returns Net object.
|
||||
*/
|
||||
CV_EXPORTS_W Net readNetFromTFLite(const std::vector<uchar>& bufferModel);
|
||||
CV_EXPORTS_W Net readNetFromTFLite(const std::vector<uchar>& bufferModel, int engine=ENGINE_AUTO);
|
||||
|
||||
/** @brief Reads a network model stored in <a href="https://www.tensorflow.org/lite">TFLite</a> framework's format.
|
||||
* @details This is an overloaded member function, provided for convenience.
|
||||
* It differs from the above function only in what argument(s) it accepts.
|
||||
* @param bufferModel buffer containing the content of the tflite file
|
||||
* @param lenModel length of bufferModel
|
||||
* @param engine select DNN engine to be used. With auto selection the new engine is used first and falls back to classic.
|
||||
* Please pay attention that the new DNN does not support non-CPU back-ends for now.
|
||||
*/
|
||||
CV_EXPORTS Net readNetFromTFLite(const char *bufferModel, size_t lenModel);
|
||||
CV_EXPORTS Net readNetFromTFLite(const char *bufferModel, size_t lenModel, int engine=ENGINE_AUTO);
|
||||
|
||||
/**
|
||||
* @brief Read deep learning network represented in one of the supported formats.
|
||||
|
||||
@ -9,8 +9,8 @@
|
||||
"(Net*)readNetFromONNX:(ByteVector*)buffer engine:(int)engine" : { "readNetFromONNX" : {"name" : "readNetFromONNXBuffer"} },
|
||||
"(Net*)readNetFromTensorflow:(NSString*)model config:(NSString*)config" : { "readNetFromTensorflow" : {"name" : "readNetFromTensorflowFile"} },
|
||||
"(Net*)readNetFromTensorflow:(ByteVector*)bufferModel bufferConfig:(ByteVector*)bufferConfig" : { "readNetFromTensorflow" : {"name" : "readNetFromTensorflowBuffer"} },
|
||||
"(Net*)readNetFromTFLite:(NSString*)model" : { "readNetFromTFLite" : {"name" : "readNetFromTFLiteFile"} },
|
||||
"(Net*)readNetFromTFLite:(ByteVector*)buffer" : { "readNetFromTFLite" : {"name" : "readNetFromTFLiteBuffer"} }
|
||||
"(Net*)readNetFromTFLite:(NSString*)model engine:(int)engine" : { "readNetFromTFLite" : {"name" : "readNetFromTFLiteFile"} },
|
||||
"(Net*)readNetFromTFLite:(ByteVector*)buffer engine:(int)engine" : { "readNetFromTFLite" : {"name" : "readNetFromTFLiteBuffer"} }
|
||||
},
|
||||
"Net": {
|
||||
"(void)forward:(NSMutableArray<Mat*>*)outputBlobs outputName:(NSString*)outputName" : { "forward" : {"name" : "forwardOutputBlobs"} },
|
||||
|
||||
@ -3,12 +3,14 @@
|
||||
// of this distribution and at http://opencv.org/license.html.
|
||||
|
||||
#include "../precomp.hpp"
|
||||
#include "../net_impl.hpp"
|
||||
|
||||
#ifdef HAVE_FLATBUFFERS
|
||||
#include "schema_generated.h"
|
||||
#include "builtin_op_data.h"
|
||||
#endif
|
||||
|
||||
#include <opencv2/core/utils/configuration.private.hpp>
|
||||
#include <opencv2/core/utils/logger.defines.hpp>
|
||||
#undef CV_LOG_STRIP_LEVEL
|
||||
#define CV_LOG_STRIP_LEVEL CV_LOG_LEVEL_VERBOSE + 1
|
||||
@ -24,13 +26,15 @@ using namespace opencv_tflite;
|
||||
|
||||
class TFLiteImporter {
|
||||
public:
|
||||
TFLiteImporter(Net& net, const char* modelBuffer, size_t bufSize);
|
||||
TFLiteImporter(Net& net, const char* modelBuffer, size_t bufSize, bool newEngine);
|
||||
|
||||
private:
|
||||
bool newEngine;
|
||||
const opencv_tflite::Model* model;
|
||||
const flatbuffers::Vector<flatbuffers::Offset<opencv_tflite::Tensor> >* modelTensors;
|
||||
std::map<int, Mat> allTensors;
|
||||
Net& dstNet;
|
||||
std::vector<Ptr<Layer>> curProg;
|
||||
|
||||
// This is a vector of pairs (layerId, outputId) where we iterate over
|
||||
// indices from TFLite notation and get created OpenCV layers.
|
||||
@ -75,11 +79,12 @@ private:
|
||||
|
||||
void parseFusedActivation(const Operator& op, ActivationFunctionType activ);
|
||||
void parseActivation(const Operator& op, const std::string& opcode, LayerParams& layerParams, bool isFused);
|
||||
void addLayer(LayerParams& layerParams, const Operator& op);
|
||||
int addPermuteLayer(const std::vector<int>& order, const std::string& permName, const std::pair<int, int>& inpId, int dtype);
|
||||
void addLayer(LayerParams& layerParams, const Operator& op, bool additionalPreLayer = false, bool additionalPostLayer = false);
|
||||
void addLayer(LayerParams& layerParams, const std::vector<std::string>& inputTensors, const std::vector<std::string>& outputTensors);
|
||||
int addPermuteLayer(const std::vector<int>& order, const std::string& permName, const std::pair<int, int>& inpId, int dtype, int inpTensorId);
|
||||
int addReshapeLayer(const std::vector<int>& shape, int axis, int num_axes,
|
||||
const std::string& name, const std::pair<int, int>& inpId, int dtype);
|
||||
int addFlattenLayer(int axis, int end_axis, const std::string& name, const std::pair<int, int>& inpId, int dtype);
|
||||
const std::string& name, const std::pair<int, int>& inpId, int dtype, int inpTensorId);
|
||||
int addFlattenLayer(int axis, int end_axis, const std::string& name, const std::pair<int, int>& inpId, int dtype, int outTensorId);
|
||||
|
||||
inline bool isInt8(const Operator& op);
|
||||
inline void getQuantParams(const Operator& op, float& inpScale, int& inpZero, float& outScale, int& outZero);
|
||||
@ -120,8 +125,8 @@ Mat TFLiteImporter::parseTensor(const Tensor& tensor)
|
||||
return shape.empty() ? Mat() : Mat(shape, dtype, const_cast<void*>(data));
|
||||
}
|
||||
|
||||
TFLiteImporter::TFLiteImporter(Net& dstNet, const char* modelBuffer, size_t bufSize)
|
||||
: dstNet(dstNet), dispatch(buildDispatchMap())
|
||||
TFLiteImporter::TFLiteImporter(Net& dstNet, const char* modelBuffer, size_t bufSize, bool newEngine)
|
||||
: newEngine(newEngine), dstNet(dstNet), dispatch(buildDispatchMap())
|
||||
{
|
||||
flatbuffers::Verifier verifier((const uint8_t*)modelBuffer, bufSize);
|
||||
if (!VerifyModelBuffer(verifier)) {
|
||||
@ -178,6 +183,12 @@ void TFLiteImporter::populateNet()
|
||||
size_t subgraph_inputs_size = subgraph_inputs->size();
|
||||
std::vector<std::string> inputsNames(subgraph_inputs_size);
|
||||
std::vector<MatShape> inputsShapes(subgraph_inputs_size);
|
||||
|
||||
// NEW ENGINE
|
||||
Net::Impl* netImpl = dstNet.getImpl();
|
||||
std::vector<Arg> modelInputs, modelOutputs;
|
||||
Ptr<Graph> curr_graph;
|
||||
|
||||
for (size_t i = 0; i < subgraph_inputs_size; ++i)
|
||||
{
|
||||
size_t idx = subgraph_inputs->Get(i);
|
||||
@ -196,16 +207,46 @@ void TFLiteImporter::populateNet()
|
||||
std::swap(shape[1], shape[2]);
|
||||
}
|
||||
inputsShapes[i] = shape;
|
||||
|
||||
if (newEngine)
|
||||
{
|
||||
modelInputs.push_back(netImpl->newArg(tensor->name()->str(), DNN_ARG_INPUT));
|
||||
netImpl->args.at(modelInputs.back().idx).type = CV_32F;
|
||||
netImpl->args.at(modelInputs.back().idx).shape = shape;
|
||||
}
|
||||
}
|
||||
|
||||
dstNet.setInputsNames(inputsNames);
|
||||
for (size_t i = 0; i < subgraph_inputs_size; ++i)
|
||||
if (!newEngine)
|
||||
{
|
||||
dstNet.setInputShape(inputsNames[i], inputsShapes[i]);
|
||||
dstNet.setInputsNames(inputsNames);
|
||||
for (size_t i = 0; i < subgraph_inputs_size; ++i)
|
||||
{
|
||||
dstNet.setInputShape(inputsNames[i], inputsShapes[i]);
|
||||
}
|
||||
}
|
||||
|
||||
const auto& all_operators = *subgraph_operators;
|
||||
const size_t all_operators_size = all_operators.size();
|
||||
|
||||
if (newEngine)
|
||||
{
|
||||
const auto last_op = all_operators[all_operators_size - 1];
|
||||
const auto op_outputs = last_op->outputs();
|
||||
std::string type = EnumNameBuiltinOperator(
|
||||
BuiltinOperator(opCodes->Get(last_op->opcode_index())->deprecated_builtin_code()));
|
||||
for (int idx : *op_outputs)
|
||||
{
|
||||
std::string tensorName = modelTensors->Get(idx)->name()->str();
|
||||
modelOutputs.push_back(netImpl->newArg(tensorName, DNN_ARG_OUTPUT));
|
||||
|
||||
// TFLite_Detection_PostProcess layer returns 4 outputs
|
||||
// (num_bboxes, bboxes, classes, conf)
|
||||
// but our detection_output layer returns one output with all the data
|
||||
if (type == "TFLite_Detection_PostProcess")
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
for (size_t op_idx = 0; op_idx < all_operators_size; ++op_idx)
|
||||
{
|
||||
const auto op = all_operators[op_idx];
|
||||
@ -259,6 +300,17 @@ void TFLiteImporter::populateNet()
|
||||
throw;
|
||||
}
|
||||
}
|
||||
if (newEngine)
|
||||
{
|
||||
Ptr<Graph> curr_graph = netImpl->newGraph(subgraph->name()->str(), modelInputs, true);
|
||||
curr_graph->setOutputs(modelOutputs);
|
||||
curr_graph->setProg(curProg);
|
||||
|
||||
netImpl->mainGraph = curr_graph;
|
||||
netImpl->modelFormat = DNN_MODEL_TFLITE;
|
||||
netImpl->originalLayout = DATA_LAYOUT_NCHW; // TODO Should we set NHWC?
|
||||
netImpl->prepareForInference();
|
||||
}
|
||||
}
|
||||
|
||||
TFLiteImporter::DispatchMap TFLiteImporter::buildDispatchMap()
|
||||
@ -293,7 +345,7 @@ TFLiteImporter::DispatchMap TFLiteImporter::buildDispatchMap()
|
||||
return dispatch;
|
||||
}
|
||||
|
||||
void TFLiteImporter::addLayer(LayerParams& layerParams, const Operator& op) {
|
||||
void TFLiteImporter::addLayer(LayerParams& layerParams, const Operator& op, bool additionalPreLayer, bool additionalPostLayer) {
|
||||
const auto op_inputs = op.inputs();
|
||||
const auto op_outputs = op.outputs();
|
||||
|
||||
@ -329,10 +381,14 @@ void TFLiteImporter::addLayer(LayerParams& layerParams, const Operator& op) {
|
||||
layerParams.set("zeropoints", outZero);
|
||||
}
|
||||
}
|
||||
int layerId = dstNet.addLayer(layerParams.name, layerParams.type, dtype, layerParams);
|
||||
|
||||
int layerId = -1;
|
||||
if (!newEngine)
|
||||
layerId = dstNet.addLayer(layerParams.name, layerParams.type, dtype, layerParams);
|
||||
|
||||
// Connect layer to inputs
|
||||
int i = 0;
|
||||
std::vector<string> inputTensors;
|
||||
std::vector<DataLayout> inpLayouts;
|
||||
for (int idx : *op_inputs) {
|
||||
if (layerIds.find(idx) == layerIds.end()) {
|
||||
@ -340,9 +396,19 @@ void TFLiteImporter::addLayer(LayerParams& layerParams, const Operator& op) {
|
||||
}
|
||||
inpLayouts.push_back(layouts[idx]);
|
||||
|
||||
auto it = layerIds.find(idx);
|
||||
CV_Assert(it != layerIds.end());
|
||||
dstNet.connect(it->second.first, it->second.second, layerId, i++);
|
||||
if (newEngine)
|
||||
{
|
||||
std::string tensorName = modelTensors->Get(idx)->name()->str();
|
||||
if (additionalPreLayer)
|
||||
tensorName += "_additional_pre_layer";
|
||||
inputTensors.push_back(tensorName);
|
||||
}
|
||||
else
|
||||
{
|
||||
auto it = layerIds.find(idx);
|
||||
CV_Assert(it != layerIds.end());
|
||||
dstNet.connect(it->second.first, it->second.second, layerId, i++);
|
||||
}
|
||||
}
|
||||
|
||||
// Predict output layout. Some layer-specific parsers may set them explicitly.
|
||||
@ -364,9 +430,44 @@ void TFLiteImporter::addLayer(LayerParams& layerParams, const Operator& op) {
|
||||
|
||||
// Register outputs
|
||||
i = 0;
|
||||
std::vector<string> outputTensors;
|
||||
for (int idx : *op_outputs) {
|
||||
layerIds[idx] = std::make_pair(layerId, i++);
|
||||
if (newEngine)
|
||||
{
|
||||
std::string tensorName = modelTensors->Get(idx)->name()->str();
|
||||
if (additionalPostLayer)
|
||||
tensorName = tensorName + "_additional_post_layer";
|
||||
outputTensors.push_back(tensorName);
|
||||
layerIds[idx] = std::make_pair(-1, -1);
|
||||
}
|
||||
else
|
||||
layerIds[idx] = std::make_pair(layerId, i++);
|
||||
|
||||
// TFLite_Detection_PostProcess layer returns 4 outputs
|
||||
// (num_bboxes, bboxes, classes, conf)
|
||||
// but our detection_output layer returns one output with all the data
|
||||
if (layerParams.type == "DetectionOutput")
|
||||
break;
|
||||
}
|
||||
|
||||
if (newEngine)
|
||||
{
|
||||
if (additionalPostLayer)
|
||||
layerParams.name += "_pre";
|
||||
addLayer(layerParams, inputTensors, outputTensors);
|
||||
}
|
||||
}
|
||||
|
||||
void TFLiteImporter::addLayer(LayerParams& layerParams, const std::vector<std::string>& inputTensors, const std::vector<std::string>& outputTensors) {
|
||||
Ptr<Layer> layer = LayerFactory::createLayerInstance(layerParams.type, layerParams);
|
||||
if (!layer)
|
||||
CV_Error(Error::StsError, "Can't create layer " + layerParams.name + " with type " + layerParams.type);
|
||||
layer->netimpl = dstNet.getImpl();
|
||||
for (const std::string& inputName : inputTensors)
|
||||
layer->inputs.push_back(dstNet.getArg(inputName));
|
||||
for (const std::string& outputName : outputTensors)
|
||||
layer->outputs.push_back(dstNet.getArg(outputName));
|
||||
curProg.push_back(layer);
|
||||
}
|
||||
|
||||
void TFLiteImporter::parseConvolution(const Operator& op, const std::string& opcode, LayerParams& layerParams) {
|
||||
@ -428,7 +529,10 @@ void TFLiteImporter::parseConvolution(const Operator& op, const std::string& opc
|
||||
}
|
||||
}
|
||||
}
|
||||
addLayer(layerParams, op);
|
||||
|
||||
std::string fusedActivationType = EnumNameActivationFunctionType(options->fused_activation_function());
|
||||
bool haveFusedActivation = fusedActivationType != "NONE";
|
||||
addLayer(layerParams, op, false, haveFusedActivation);
|
||||
parseFusedActivation(op, options->fused_activation_function());
|
||||
}
|
||||
|
||||
@ -490,7 +594,10 @@ void TFLiteImporter::parseDWConvolution(const Operator& op, const std::string& o
|
||||
}
|
||||
}
|
||||
}
|
||||
addLayer(layerParams, op);
|
||||
|
||||
std::string fusedActivationType = EnumNameActivationFunctionType(options->fused_activation_function());
|
||||
bool haveFusedActivation = fusedActivationType != "NONE";
|
||||
addLayer(layerParams, op, false, haveFusedActivation);
|
||||
parseFusedActivation(op, options->fused_activation_function());
|
||||
}
|
||||
|
||||
@ -557,7 +664,10 @@ void TFLiteImporter::parseEltwise(const Operator& op, const std::string& opcode,
|
||||
layerParams.set("scales", outScale);
|
||||
layerParams.set("zeropoints", outZero);
|
||||
}
|
||||
addLayer(layerParams, op);
|
||||
|
||||
std::string fusedActivationType = EnumNameActivationFunctionType(activ);
|
||||
bool haveFusedActivation = fusedActivationType != "NONE";
|
||||
addLayer(layerParams, op, false, haveFusedActivation);
|
||||
parseFusedActivation(op, activ);
|
||||
}
|
||||
|
||||
@ -576,7 +686,10 @@ void TFLiteImporter::parsePooling(const Operator& op, const std::string& opcode,
|
||||
layerParams.set("pool", "ave");
|
||||
else
|
||||
CV_Error(Error::StsNotImplemented, "Pool type selection for " + opcode);
|
||||
addLayer(layerParams, op);
|
||||
|
||||
std::string fusedActivationType = EnumNameActivationFunctionType(options->fused_activation_function());
|
||||
bool haveFusedActivation = fusedActivationType != "NONE";
|
||||
addLayer(layerParams, op, false, haveFusedActivation);
|
||||
parseFusedActivation(op, options->fused_activation_function());
|
||||
}
|
||||
|
||||
@ -628,6 +741,7 @@ void TFLiteImporter::parseReshape(const Operator& op, const std::string& opcode,
|
||||
shape.assign(options->new_shape()->begin(), options->new_shape()->end());
|
||||
}
|
||||
|
||||
bool additionalPreLayer = false;
|
||||
if (inpLayout == DNN_LAYOUT_NHWC) {
|
||||
if (shape.size() == 4) {
|
||||
// Keep data but change a shape to OpenCV's NCHW order
|
||||
@ -638,13 +752,14 @@ void TFLiteImporter::parseReshape(const Operator& op, const std::string& opcode,
|
||||
std::vector<int> order = {0, 2, 3, 1};
|
||||
const std::string name = layerParams.name + "/permute";
|
||||
auto inpId = layerIds[op.inputs()->Get(0)];
|
||||
int permId = addPermuteLayer(order, name, inpId, isInt8(op) ? CV_8S : CV_32F); // NCHW -> NHWC
|
||||
int permId = addPermuteLayer(order, name, inpId, isInt8(op) ? CV_8S : CV_32F, op.inputs()->Get(0)); // NCHW -> NHWC
|
||||
layerIds[op.inputs()->Get(0)] = std::make_pair(permId, 0);
|
||||
layouts[op.outputs()->Get(0)] = DNN_LAYOUT_NCHW;
|
||||
additionalPreLayer = true;
|
||||
}
|
||||
}
|
||||
layerParams.set("dim", DictValue::arrayInt<int*>(shape.data(), shape.size()));
|
||||
addLayer(layerParams, op);
|
||||
addLayer(layerParams, op, additionalPreLayer);
|
||||
}
|
||||
|
||||
void TFLiteImporter::parseConcat(const Operator& op, const std::string& opcode, LayerParams& layerParams) {
|
||||
@ -660,7 +775,10 @@ void TFLiteImporter::parseConcat(const Operator& op, const std::string& opcode,
|
||||
axis = remap[axis];
|
||||
}
|
||||
layerParams.set("axis", axis);
|
||||
addLayer(layerParams, op);
|
||||
|
||||
std::string fusedActivationType = EnumNameActivationFunctionType(options->fused_activation_function());
|
||||
bool haveFusedActivation = fusedActivationType != "NONE";
|
||||
addLayer(layerParams, op, false, haveFusedActivation);
|
||||
parseFusedActivation(op, options->fused_activation_function());
|
||||
}
|
||||
|
||||
@ -690,14 +808,14 @@ void TFLiteImporter::parsePack(const Operator& op, const std::string& opcode, La
|
||||
}
|
||||
const auto name = modelTensors->Get(inp)->name()->str() + "/reshape";
|
||||
int reshapeId = addReshapeLayer(shape, axis == dims ? dims - 1 : axis, 1,
|
||||
name, inpId, isInt8(op) ? CV_8S : CV_32F);
|
||||
name, inpId, isInt8(op) ? CV_8S : CV_32F, inp);
|
||||
|
||||
originLayerIds[inp] = layerIds[inp];
|
||||
layerIds[inp] = std::make_pair(reshapeId, 0);
|
||||
}
|
||||
layerParams.type = "Concat";
|
||||
layerParams.set("axis", axis);
|
||||
addLayer(layerParams, op);
|
||||
addLayer(layerParams, op, true, false);
|
||||
|
||||
// Restore origin layer inputs
|
||||
for (const auto& ids : originLayerIds) {
|
||||
@ -787,11 +905,11 @@ void TFLiteImporter::parseGlobalPooling(const Operator& op, const std::string& o
|
||||
if (!keep_dims) {
|
||||
const auto name = layerParams.name;
|
||||
layerParams.name += "/global_pooling";
|
||||
addLayer(layerParams, op);
|
||||
addLayer(layerParams, op, false, true);
|
||||
|
||||
int out = op.outputs()->Get(0);
|
||||
auto outId = layerIds[out];
|
||||
int flattenId = addFlattenLayer(1, -1, name, outId, isInt8(op) ? CV_8S : CV_32F);
|
||||
int flattenId = addFlattenLayer(1, -1, name, outId, isInt8(op) ? CV_8S : CV_32F, out);
|
||||
layerIds[out] = std::make_pair(flattenId, 0);
|
||||
}
|
||||
else {
|
||||
@ -800,35 +918,68 @@ void TFLiteImporter::parseGlobalPooling(const Operator& op, const std::string& o
|
||||
}
|
||||
|
||||
int TFLiteImporter::addPermuteLayer(const std::vector<int>& order, const std::string& permName,
|
||||
const std::pair<int, int>& inpId, int dtype)
|
||||
const std::pair<int, int>& inpId, int dtype, int inpTensorId)
|
||||
{
|
||||
LayerParams permLP;
|
||||
permLP.set("order", DictValue::arrayInt<const int*>(order.data(), order.size()));
|
||||
int permId = dstNet.addLayer(permName, "Permute", dtype, permLP);
|
||||
dstNet.connect(inpId.first, inpId.second, permId, 0);
|
||||
return permId;
|
||||
if (newEngine)
|
||||
{
|
||||
permLP.type = "Permute";
|
||||
permLP.name = permName;
|
||||
std::string tensorName = modelTensors->Get(inpTensorId)->name()->str();
|
||||
addLayer(permLP, {tensorName}, {tensorName + "_additional_pre_layer"});
|
||||
return -1;
|
||||
}
|
||||
else
|
||||
{
|
||||
int permId = dstNet.addLayer(permName, "Permute", dtype, permLP);
|
||||
dstNet.connect(inpId.first, inpId.second, permId, 0);
|
||||
return permId;
|
||||
}
|
||||
}
|
||||
|
||||
int TFLiteImporter::addReshapeLayer(const std::vector<int>& shape, int axis, int num_axes,
|
||||
const std::string& name, const std::pair<int, int>& inpId, int dtype)
|
||||
const std::string& name, const std::pair<int, int>& inpId, int dtype, int inpTensorId)
|
||||
{
|
||||
LayerParams lp;
|
||||
lp.set("axis", axis);
|
||||
lp.set("dim", DictValue::arrayInt<const int*>(shape.data(), shape.size()));
|
||||
lp.set("num_axes", num_axes);
|
||||
int id = dstNet.addLayer(name, "Reshape", dtype, lp);
|
||||
dstNet.connect(inpId.first, inpId.second, id, 0);
|
||||
return id;
|
||||
if (newEngine)
|
||||
{
|
||||
lp.type = "Reshape";
|
||||
lp.name = name;
|
||||
std::string tensorName = modelTensors->Get(inpTensorId)->name()->str();
|
||||
addLayer(lp, {tensorName}, {tensorName + "_additional_pre_layer"});
|
||||
return -1;
|
||||
}
|
||||
else
|
||||
{
|
||||
int id = dstNet.addLayer(name, "Reshape", dtype, lp);
|
||||
dstNet.connect(inpId.first, inpId.second, id, 0);
|
||||
return id;
|
||||
}
|
||||
}
|
||||
|
||||
int TFLiteImporter::addFlattenLayer(int axis, int end_axis, const std::string& name, const std::pair<int, int>& inpId, int dtype)
|
||||
int TFLiteImporter::addFlattenLayer(int axis, int end_axis, const std::string& name, const std::pair<int, int>& inpId, int dtype, int outTensorId)
|
||||
{
|
||||
LayerParams lp;
|
||||
lp.set("axis", axis);
|
||||
lp.set("end_axis", end_axis);
|
||||
int id = dstNet.addLayer(name, "Flatten", dtype, lp);
|
||||
dstNet.connect(inpId.first, inpId.second, id, 0);
|
||||
return id;
|
||||
if (newEngine)
|
||||
{
|
||||
lp.type = "Flatten";
|
||||
lp.name = name;
|
||||
std::string tensorName = modelTensors->Get(outTensorId)->name()->str();
|
||||
addLayer(lp, {tensorName + "_additional_post_layer"}, {tensorName});
|
||||
return -1;
|
||||
}
|
||||
else
|
||||
{
|
||||
int id = dstNet.addLayer(name, "Flatten", dtype, lp);
|
||||
dstNet.connect(inpId.first, inpId.second, id, 0);
|
||||
return id;
|
||||
}
|
||||
}
|
||||
|
||||
void TFLiteImporter::parseDeconvolution(const Operator& op, const std::string& opcode, LayerParams& layerParams) {
|
||||
@ -926,7 +1077,10 @@ void TFLiteImporter::parseFullyConnected(const Operator& op, const std::string&
|
||||
|
||||
layerParams.set("transB", true);
|
||||
layerParams.set("constB", true);
|
||||
addLayer(layerParams, op);
|
||||
|
||||
std::string fusedActivationType = EnumNameActivationFunctionType(options->fused_activation_function());
|
||||
bool haveFusedActivation = fusedActivationType != "NONE";
|
||||
addLayer(layerParams, op, false, haveFusedActivation);
|
||||
parseFusedActivation(op, options->fused_activation_function());
|
||||
}
|
||||
|
||||
@ -1013,14 +1167,25 @@ void TFLiteImporter::parseDetectionPostProcess(const Operator& op, const std::st
|
||||
priorsLP.type = "Const";
|
||||
priorsLP.blobs.resize(1, priors);
|
||||
|
||||
int priorsId = dstNet.addLayer(priorsLP.name, priorsLP.type, priorsLP);
|
||||
layerIds[op.inputs()->Get(2)] = std::make_pair(priorsId, 0);
|
||||
if (newEngine)
|
||||
{
|
||||
std::string outTensorName = modelTensors->Get(op.inputs()->Get(2))->name()->str();
|
||||
addLayer(priorsLP, {}, {outTensorName});
|
||||
layerIds[op.inputs()->Get(2)] = std::make_pair(-1, -1);
|
||||
}
|
||||
else
|
||||
{
|
||||
int priorsId = dstNet.addLayer(priorsLP.name, priorsLP.type, priorsLP);
|
||||
layerIds[op.inputs()->Get(2)] = std::make_pair(priorsId, 0);
|
||||
}
|
||||
addLayer(layerParams, op);
|
||||
}
|
||||
|
||||
void TFLiteImporter::parseFusedActivation(const Operator& op, ActivationFunctionType activ) {
|
||||
LayerParams activParams;
|
||||
activParams.name = modelTensors->Get(op.outputs()->Get(0))->name()->str() + "/activ";
|
||||
activParams.name = modelTensors->Get(op.outputs()->Get(0))->name()->str();
|
||||
if (!newEngine)
|
||||
activParams.name += "/activ";
|
||||
parseActivation(op, EnumNameActivationFunctionType(activ), activParams, true);
|
||||
}
|
||||
|
||||
@ -1092,13 +1257,22 @@ void TFLiteImporter::parseActivation(const Operator& op, const std::string& opco
|
||||
}
|
||||
|
||||
if (isFused) {
|
||||
int dtype = isInt8(op) ? CV_8S : CV_32F;
|
||||
int layerId = dstNet.addLayerToPrev(activParams.name, activParams.type, dtype, activParams);
|
||||
if (newEngine)
|
||||
{
|
||||
std::string tensorName = modelTensors->Get(op.outputs()->Get(0))->name()->str();
|
||||
addLayer(activParams, {tensorName + "_additional_post_layer"}, {tensorName});
|
||||
layerIds[op.outputs()->Get(0)] = std::make_pair(-1, -1);
|
||||
}
|
||||
else
|
||||
{
|
||||
int dtype = isInt8(op) ? CV_8S : CV_32F;
|
||||
int layerId = dstNet.addLayerToPrev(activParams.name, activParams.type, dtype, activParams);
|
||||
|
||||
// Override layer ids mapping
|
||||
int i = 0;
|
||||
for (int idx : *op.outputs()) {
|
||||
layerIds[idx] = std::make_pair(layerId, i++);
|
||||
// Override layer ids mapping
|
||||
int i = 0;
|
||||
for (int idx : *op.outputs()) {
|
||||
layerIds[idx] = std::make_pair(layerId, i++);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
addLayer(activParams, op);
|
||||
@ -1136,7 +1310,11 @@ void TFLiteImporter::getQuantParams(const Operator& op, float& inpScale, int& in
|
||||
}
|
||||
}
|
||||
|
||||
Net readNetFromTFLite(const String &modelPath) {
|
||||
Net readNetFromTFLite(const String &modelPath, int engine) {
|
||||
static const int engine_forced = utils::getConfigurationParameterSizeT("OPENCV_FORCE_DNN_ENGINE", ENGINE_AUTO);
|
||||
if(engine_forced != ENGINE_AUTO)
|
||||
engine = engine_forced;
|
||||
|
||||
Net net;
|
||||
|
||||
std::vector<char> content;
|
||||
@ -1155,17 +1333,21 @@ Net readNetFromTFLite(const String &modelPath) {
|
||||
ifs.read(content.data(), sz);
|
||||
CV_Assert(!ifs.bad());
|
||||
|
||||
TFLiteImporter(net, content.data(), content.size());
|
||||
TFLiteImporter(net, content.data(), content.size(), engine == ENGINE_NEW || engine == ENGINE_AUTO);
|
||||
return net;
|
||||
}
|
||||
|
||||
Net readNetFromTFLite(const std::vector<uchar>& bufferModel) {
|
||||
Net readNetFromTFLite(const std::vector<uchar>& bufferModel, int engine) {
|
||||
return readNetFromTFLite((const char*)bufferModel.data(), bufferModel.size());
|
||||
}
|
||||
|
||||
Net readNetFromTFLite(const char *bufferModel, size_t bufSize) {
|
||||
Net readNetFromTFLite(const char *bufferModel, size_t bufSize, int engine) {
|
||||
static const int engine_forced = utils::getConfigurationParameterSizeT("OPENCV_FORCE_DNN_ENGINE", ENGINE_AUTO);
|
||||
if(engine_forced != ENGINE_AUTO)
|
||||
engine = engine_forced;
|
||||
|
||||
Net net;
|
||||
TFLiteImporter(net, bufferModel, bufSize);
|
||||
TFLiteImporter(net, bufferModel, bufSize, engine == ENGINE_NEW || engine == ENGINE_AUTO);
|
||||
return net;
|
||||
}
|
||||
|
||||
@ -1173,15 +1355,15 @@ Net readNetFromTFLite(const char *bufferModel, size_t bufSize) {
|
||||
|
||||
#define DNN_TFLITE_UNSUPPORTED() CV_Error(Error::StsError, "DNN/TFLite: Build OpenCV with FlatBuffers to import TFLite models: https://github.com/opencv/opencv/pull/23161")
|
||||
|
||||
Net readNetFromTFLite(const String &) {
|
||||
Net readNetFromTFLite(const String &, int) {
|
||||
DNN_TFLITE_UNSUPPORTED();
|
||||
}
|
||||
|
||||
Net readNetFromTFLite(const std::vector<uchar>&) {
|
||||
Net readNetFromTFLite(const std::vector<uchar>&, int) {
|
||||
DNN_TFLITE_UNSUPPORTED();
|
||||
}
|
||||
|
||||
Net readNetFromTFLite(const char *, size_t) {
|
||||
Net readNetFromTFLite(const char *, size_t, int) {
|
||||
DNN_TFLITE_UNSUPPORTED();
|
||||
}
|
||||
|
||||
|
||||
@ -155,6 +155,9 @@ TEST_P(Test_TFLite, max_unpooling)
|
||||
net.setPreferableBackend(backend);
|
||||
net.setPreferableTarget(target);
|
||||
|
||||
if (net.getMainGraph())
|
||||
throw SkipTestException("The new dnn engine doesn't support forward to specified layers"); // https://github.com/opencv/opencv/issues/26349
|
||||
|
||||
Mat input = imread(findDataFile("cv/shared/lena.png"));
|
||||
cvtColor(input, input, COLOR_BGR2RGBA);
|
||||
input = input.mul(Scalar(1, 1, 1, 0));
|
||||
|
||||
Loading…
Reference in New Issue
Block a user