Merge pull request #11104 from asciian:reading_from_stream

modules/dnn/include/opencv2/dnn/dnn.hpp

@@ -644,6 +644,24 @@ CV__DNN_EXPERIMENTAL_NS_BEGIN
     */
     CV_EXPORTS_W Net readNetFromDarknet(const String &cfgFile, const String &darknetModel = String());
 
+    /** @brief Reads a network model stored in <a href="https://pjreddie.com/darknet/">Darknet</a> model files.
+     *  @param bufferCfg   A buffer contains a content of .cfg file with text description of the network architecture.
+     *  @param bufferModel A buffer contains a content of .weights file with learned network.
+     *  @returns Net object.
+     */
+    CV_EXPORTS_W Net readNetFromDarknet(const std::vector<uchar>& bufferCfg,
+                                        const std::vector<uchar>& bufferModel = std::vector<uchar>());
+
+    /** @brief Reads a network model stored in <a href="https://pjreddie.com/darknet/">Darknet</a> model files.
+     *  @param bufferCfg   A buffer contains a content of .cfg file with text description of the network architecture.
+     *  @param lenCfg      Number of bytes to read from bufferCfg
+     *  @param bufferModel A buffer contains a content of .weights file with learned network.
+     *  @param lenModel    Number of bytes to read from bufferModel
+     *  @returns Net object.
+     */
+    CV_EXPORTS Net readNetFromDarknet(const char *bufferCfg, size_t lenCfg,
+                                      const char *bufferModel = NULL, size_t lenModel = 0);
+
     /** @brief Reads a network model stored in <a href="http://caffe.berkeleyvision.org">Caffe</a> framework's format.
       * @param prototxt   path to the .prototxt file with text description of the network architecture.
       * @param caffeModel path to the .caffemodel file with learned network.
@@ -651,6 +669,14 @@ CV__DNN_EXPERIMENTAL_NS_BEGIN
       */
     CV_EXPORTS_W Net readNetFromCaffe(const String &prototxt, const String &caffeModel = String());
 
+    /** @brief Reads a network model stored in Caffe model in memory.
+      * @param bufferProto buffer containing the content of the .prototxt file
+      * @param bufferModel buffer containing the content of the .caffemodel file
+      * @returns Net object.
+      */
+    CV_EXPORTS_W Net readNetFromCaffe(const std::vector<uchar>& bufferProto,
+                                      const std::vector<uchar>& bufferModel = std::vector<uchar>());
+
     /** @brief Reads a network model stored in Caffe model in memory.
       * @details This is an overloaded member function, provided for convenience.
       * It differs from the above function only in what argument(s) it accepts.
@@ -672,6 +698,14 @@ CV__DNN_EXPERIMENTAL_NS_BEGIN
       */
     CV_EXPORTS_W Net readNetFromTensorflow(const String &model, const String &config = String());
 
+    /** @brief Reads a network model stored in <a href="https://www.tensorflow.org/">TensorFlow</a> framework's format.
+      * @param bufferModel buffer containing the content of the pb file
+      * @param bufferConfig buffer containing the content of the pbtxt file
+      * @returns Net object.
+      */
+    CV_EXPORTS_W Net readNetFromTensorflow(const std::vector<uchar>& bufferModel,
+                                           const std::vector<uchar>& bufferConfig = std::vector<uchar>());
+
     /** @brief Reads a network model stored in <a href="https://www.tensorflow.org/">TensorFlow</a> framework's format.
       * @details This is an overloaded member function, provided for convenience.
       * It differs from the above function only in what argument(s) it accepts.
@@ -735,6 +769,18 @@ CV__DNN_EXPERIMENTAL_NS_BEGIN
       */
      CV_EXPORTS_W Net readNet(const String& model, const String& config = "", const String& framework = "");
 
+     /**
+      * @brief Read deep learning network represented in one of the supported formats.
+      * @details This is an overloaded member function, provided for convenience.
+      *          It differs from the above function only in what argument(s) it accepts.
+      * @param[in] framework    Name of origin framework.
+      * @param[in] bufferModel  A buffer with a content of binary file with weights
+      * @param[in] bufferConfig A buffer with a content of text file contains network configuration.
+      * @returns Net object.
+      */
+     CV_EXPORTS_W Net readNet(const String& framework, const std::vector<uchar>& bufferModel,
+                              const std::vector<uchar>& bufferConfig = std::vector<uchar>());
+
     /** @brief Loads blob which was serialized as torch.Tensor object of Torch7 framework.
      *  @warning This function has the same limitations as readNetFromTorch().
      */

modules/dnn/misc/java/test/DnnTensorFlowTest.java

@@ -1,10 +1,14 @@
 package org.opencv.test.dnn;
 
 import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import org.opencv.core.Core;
 import org.opencv.core.Mat;
+import org.opencv.core.MatOfFloat;
+import org.opencv.core.MatOfByte;
 import org.opencv.core.Scalar;
 import org.opencv.core.Size;
 import org.opencv.dnn.DictValue;
@@ -26,6 +30,15 @@ public class DnnTensorFlowTest extends OpenCVTestCase {
 
     Net net;
 
+    private static void normAssert(Mat ref, Mat test) {
+        final double l1 = 1e-5;
+        final double lInf = 1e-4;
+        double normL1 = Core.norm(ref, test, Core.NORM_L1) / ref.total();
+        double normLInf = Core.norm(ref, test, Core.NORM_INF) / ref.total();
+        assertTrue(normL1 < l1);
+        assertTrue(normLInf < lInf);
+    }
+
     @Override
     protected void setUp() throws Exception {
         super.setUp();
@@ -46,7 +59,7 @@ public class DnnTensorFlowTest extends OpenCVTestCase {
 
         File testDataPath = new File(envTestDataPath);
 
-        File f = new File(testDataPath, "dnn/space_shuttle.jpg");
+        File f = new File(testDataPath, "dnn/grace_hopper_227.png");
         sourceImageFile = f.toString();
         if(!f.exists()) throw new Exception("Test image is missing: " + sourceImageFile);
 
@@ -77,31 +90,55 @@ public class DnnTensorFlowTest extends OpenCVTestCase {
 
     }
 
-    public void testTestNetForward() {
+    public void checkInceptionNet(Net net)
-        Mat rawImage = Imgcodecs.imread(sourceImageFile);
+    {
+        Mat image = Imgcodecs.imread(sourceImageFile);
+        assertNotNull("Loading image from file failed!", image);
 
-        assertNotNull("Loading image from file failed!", rawImage);
+        Mat inputBlob = Dnn.blobFromImage(image, 1.0, new Size(224, 224), new Scalar(0), true, true);
-
-        Mat image = new Mat();
-        Imgproc.resize(rawImage, image, new Size(224,224));
-
-        Mat inputBlob = Dnn.blobFromImage(image);
         assertNotNull("Converting image to blob failed!", inputBlob);
 
-        Mat inputBlobP = new Mat();
+        net.setInput(inputBlob, "input");
-        Core.subtract(inputBlob, new Scalar(117.0), inputBlobP);
-
-        net.setInput(inputBlobP, "input" );
-
-        Mat result = net.forward();
 
+        Mat result = new Mat();
+        try {
+            net.setPreferableBackend(Dnn.DNN_BACKEND_OPENCV);
+            result = net.forward("softmax2");
+        }
+        catch (Exception e) {
+            fail("DNN forward failed: " + e.getMessage());
+        }
         assertNotNull("Net returned no result!", result);
 
-        Core.MinMaxLocResult minmax = Core.minMaxLoc(result.reshape(1, 1));
+        result = result.reshape(1, 1);
+        Core.MinMaxLocResult minmax = Core.minMaxLoc(result);
+        assertEquals("Wrong prediction", (int)minmax.maxLoc.x, 866);
 
-        assertTrue("No image recognized!", minmax.maxVal > 0.9);
+        Mat top5RefScores = new MatOfFloat(new float[] {
+            0.63032645f, 0.2561979f, 0.032181446f, 0.015721032f, 0.014785315f
+        }).reshape(1, 1);
 
+        Core.sort(result, result, Core.SORT_DESCENDING);
 
+        normAssert(result.colRange(0, 5), top5RefScores);
     }
 
+    public void testTestNetForward() {
+        checkInceptionNet(net);
+    }
+
+    public void testReadFromBuffer() {
+        File modelFile = new File(modelFileName);
+        byte[] modelBuffer = new byte[ (int)modelFile.length() ];
+
+        try {
+            FileInputStream fis = new FileInputStream(modelFile);
+            fis.read(modelBuffer);
+            fis.close();
+        } catch (IOException e) {
+            fail("Failed to read a model: " + e.getMessage());
+        }
+        net = Dnn.readNetFromTensorflow(new MatOfByte(modelBuffer));
+        checkInceptionNet(net);
+    }
 }

modules/dnn/src/caffe/caffe_importer.cpp

@@ -453,6 +453,15 @@ Net readNetFromCaffe(const char *bufferProto, size_t lenProto,
     return net;
 }
 
+Net readNetFromCaffe(const std::vector<uchar>& bufferProto, const std::vector<uchar>& bufferModel)
+{
+    const char* bufferProtoPtr = reinterpret_cast<const char*>(&bufferProto[0]);
+    const char* bufferModelPtr = bufferModel.empty() ? NULL :
+                                 reinterpret_cast<const char*>(&bufferModel[0]);
+    return readNetFromCaffe(bufferProtoPtr, bufferProto.size(),
+                            bufferModelPtr, bufferModel.size());
+}
+
 #endif //HAVE_PROTOBUF
 
 CV__DNN_EXPERIMENTAL_NS_END

modules/dnn/src/darknet/darknet_importer.cpp

@@ -44,6 +44,7 @@
 #include "../precomp.hpp"
 
 #include <iostream>
+#include <fstream>
 #include <algorithm>
 #include <vector>
 #include <map>
@@ -66,14 +67,19 @@ public:
 
     DarknetImporter() {}
 
-    DarknetImporter(const char *cfgFile, const char *darknetModel)
+    DarknetImporter(std::istream &cfgStream, std::istream &darknetModelStream)
     {
         CV_TRACE_FUNCTION();
 
-        ReadNetParamsFromCfgFileOrDie(cfgFile, &net);
+        ReadNetParamsFromCfgStreamOrDie(cfgStream, &net);
+        ReadNetParamsFromBinaryStreamOrDie(darknetModelStream, &net);
+    }
 
-        if (darknetModel && darknetModel[0])
+    DarknetImporter(std::istream &cfgStream)
-            ReadNetParamsFromBinaryFileOrDie(darknetModel, &net);
+    {
+        CV_TRACE_FUNCTION();
+
+        ReadNetParamsFromCfgStreamOrDie(cfgStream, &net);
     }
 
     struct BlobNote
@@ -175,15 +181,75 @@ public:
     }
 };
 
-}
+static Net readNetFromDarknet(std::istream &cfgFile, std::istream &darknetModel)
-
-Net readNetFromDarknet(const String &cfgFile, const String &darknetModel /*= String()*/)
 {
-    DarknetImporter darknetImporter(cfgFile.c_str(), darknetModel.c_str());
     Net net;
+    DarknetImporter darknetImporter(cfgFile, darknetModel);
     darknetImporter.populateNet(net);
     return net;
 }
 
+static Net readNetFromDarknet(std::istream &cfgFile)
+{
+    Net net;
+    DarknetImporter darknetImporter(cfgFile);
+    darknetImporter.populateNet(net);
+    return net;
+}
+
+}
+
+Net readNetFromDarknet(const String &cfgFile, const String &darknetModel /*= String()*/)
+{
+    std::ifstream cfgStream(cfgFile.c_str());
+    if (!cfgStream.is_open())
+    {
+        CV_Error(cv::Error::StsParseError, "Failed to parse NetParameter file: " + std::string(cfgFile));
+    }
+    if (darknetModel != String())
+    {
+        std::ifstream darknetModelStream(darknetModel.c_str(), std::ios::binary);
+        if (!darknetModelStream.is_open())
+        {
+            CV_Error(cv::Error::StsParseError, "Failed to parse NetParameter file: " + std::string(darknetModel));
+        }
+        return readNetFromDarknet(cfgStream, darknetModelStream);
+    }
+    else
+        return readNetFromDarknet(cfgStream);
+}
+
+struct BufferStream : public std::streambuf
+{
+    BufferStream(const char* s, std::size_t n)
+    {
+        char* ptr = const_cast<char*>(s);
+        setg(ptr, ptr, ptr + n);
+    }
+};
+
+Net readNetFromDarknet(const char *bufferCfg, size_t lenCfg, const char *bufferModel, size_t lenModel)
+{
+    BufferStream cfgBufferStream(bufferCfg, lenCfg);
+    std::istream cfgStream(&cfgBufferStream);
+    if (lenModel)
+    {
+        BufferStream weightsBufferStream(bufferModel, lenModel);
+        std::istream weightsStream(&weightsBufferStream);
+        return readNetFromDarknet(cfgStream, weightsStream);
+    }
+    else
+        return readNetFromDarknet(cfgStream);
+}
+
+Net readNetFromDarknet(const std::vector<uchar>& bufferCfg, const std::vector<uchar>& bufferModel)
+{
+    const char* bufferCfgPtr = reinterpret_cast<const char*>(&bufferCfg[0]);
+    const char* bufferModelPtr = bufferModel.empty() ? NULL :
+                                 reinterpret_cast<const char*>(&bufferModel[0]);
+    return readNetFromDarknet(bufferCfgPtr, bufferCfg.size(),
+                              bufferModelPtr, bufferModel.size());
+}
+
 CV__DNN_EXPERIMENTAL_NS_END
 }} // namespace

modules/dnn/src/darknet/darknet_io.cpp

@@ -476,68 +476,61 @@ namespace cv {
                 return dst;
             }
 
-            bool ReadDarknetFromCfgFile(const char *cfgFile, NetParameter *net)
+            bool ReadDarknetFromCfgStream(std::istream &ifile, NetParameter *net)
             {
-                std::ifstream ifile;
+                bool read_net = false;
-                ifile.open(cfgFile);
+                int layers_counter = -1;
-                if (ifile.is_open())
+                for (std::string line; std::getline(ifile, line);) {
-                {
+                    line = escapeString(line);
-                    bool read_net = false;
+                    if (line.empty()) continue;
-                    int layers_counter = -1;
+                    switch (line[0]) {
-                    for (std::string line; std::getline(ifile, line);) {
+                    case '\0': break;
-                        line = escapeString(line);
+                    case '#': break;
-                        if (line.empty()) continue;
+                    case ';': break;
-                        switch (line[0]) {
+                    case '[':
-                        case '\0': break;
+                        if (line == "[net]") {
-                        case '#': break;
+                            read_net = true;
-                        case ';': break;
+                        }
-                        case '[':
+                        else {
-                            if (line == "[net]") {
+                            // read section
-                                read_net = true;
+                            read_net = false;
-                            }
+                            ++layers_counter;
-                            else {
+                            const size_t layer_type_size = line.find("]") - 1;
-                                // read section
+                            CV_Assert(layer_type_size < line.size());
-                                read_net = false;
+                            std::string layer_type = line.substr(1, layer_type_size);
-                                ++layers_counter;
+                            net->layers_cfg[layers_counter]["type"] = layer_type;
-                                const size_t layer_type_size = line.find("]") - 1;
+                        }
-                                CV_Assert(layer_type_size < line.size());
+                        break;
-                                std::string layer_type = line.substr(1, layer_type_size);
+                    default:
-                                net->layers_cfg[layers_counter]["type"] = layer_type;
+                        // read entry
-                            }
+                        const size_t separator_index = line.find('=');
-                            break;
+                        CV_Assert(separator_index < line.size());
-                        default:
+                        if (separator_index != std::string::npos) {
-                            // read entry
+                            std::string name = line.substr(0, separator_index);
-                            const size_t separator_index = line.find('=');
+                            std::string value = line.substr(separator_index + 1, line.size() - (separator_index + 1));
-                            CV_Assert(separator_index < line.size());
+                            name = escapeString(name);
-                            if (separator_index != std::string::npos) {
+                            value = escapeString(value);
-                                std::string name = line.substr(0, separator_index);
+                            if (name.empty() || value.empty()) continue;
-                                std::string value = line.substr(separator_index + 1, line.size() - (separator_index + 1));
+                            if (read_net)
-                                name = escapeString(name);
+                                net->net_cfg[name] = value;
-                                value = escapeString(value);
+                            else
-                                if (name.empty() || value.empty()) continue;
+                                net->layers_cfg[layers_counter][name] = value;
-                                if (read_net)
-                                    net->net_cfg[name] = value;
-                                else
-                                    net->layers_cfg[layers_counter][name] = value;
-                            }
                         }
                     }
-
-                    std::string anchors = net->layers_cfg[net->layers_cfg.size() - 1]["anchors"];
-                    std::vector<float> vec = getNumbers<float>(anchors);
-                    std::map<std::string, std::string> &net_params = net->net_cfg;
-                    net->width = getParam(net_params, "width", 416);
-                    net->height = getParam(net_params, "height", 416);
-                    net->channels = getParam(net_params, "channels", 3);
-                    CV_Assert(net->width > 0 && net->height > 0 && net->channels > 0);
                 }
-                else
+
-                    return false;
+                std::string anchors = net->layers_cfg[net->layers_cfg.size() - 1]["anchors"];
+                std::vector<float> vec = getNumbers<float>(anchors);
+                std::map<std::string, std::string> &net_params = net->net_cfg;
+                net->width = getParam(net_params, "width", 416);
+                net->height = getParam(net_params, "height", 416);
+                net->channels = getParam(net_params, "channels", 3);
+                CV_Assert(net->width > 0 && net->height > 0 && net->channels > 0);
 
                 int current_channels = net->channels;
                 net->out_channels_vec.resize(net->layers_cfg.size());
 
-                int layers_counter = -1;
+                layers_counter = -1;
 
                 setLayersParams setParams(net);
 
@@ -676,13 +669,8 @@ namespace cv {
                 return true;
             }
 
-
+            bool ReadDarknetFromWeightsStream(std::istream &ifile, NetParameter *net)
-            bool ReadDarknetFromWeightsFile(const char *darknetModel, NetParameter *net)
             {
-                std::ifstream ifile;
-                ifile.open(darknetModel, std::ios::binary);
-                CV_Assert(ifile.is_open());
-
                 int32_t major_ver, minor_ver, revision;
                 ifile.read(reinterpret_cast<char *>(&major_ver), sizeof(int32_t));
                 ifile.read(reinterpret_cast<char *>(&minor_ver), sizeof(int32_t));
@@ -778,19 +766,18 @@ namespace cv {
         }
 
 
-        void ReadNetParamsFromCfgFileOrDie(const char *cfgFile, darknet::NetParameter *net)
+        void ReadNetParamsFromCfgStreamOrDie(std::istream &ifile, darknet::NetParameter *net)
         {
-            if (!darknet::ReadDarknetFromCfgFile(cfgFile, net)) {
+            if (!darknet::ReadDarknetFromCfgStream(ifile, net)) {
-                CV_Error(cv::Error::StsParseError, "Failed to parse NetParameter file: " + std::string(cfgFile));
+                CV_Error(cv::Error::StsParseError, "Failed to parse NetParameter stream");
             }
         }
 
-        void ReadNetParamsFromBinaryFileOrDie(const char *darknetModel, darknet::NetParameter *net)
+        void ReadNetParamsFromBinaryStreamOrDie(std::istream &ifile, darknet::NetParameter *net)
         {
-            if (!darknet::ReadDarknetFromWeightsFile(darknetModel, net)) {
+            if (!darknet::ReadDarknetFromWeightsStream(ifile, net)) {
-                CV_Error(cv::Error::StsParseError, "Failed to parse NetParameter file: " + std::string(darknetModel));
+                CV_Error(cv::Error::StsParseError, "Failed to parse NetParameter stream");
             }
         }
-
     }
 }

modules/dnn/src/darknet/darknet_io.hpp

@@ -109,10 +109,9 @@ namespace cv {
             };
         }
 
-        // Read parameters from a file into a NetParameter message.
+        // Read parameters from a stream into a NetParameter message.
-        void ReadNetParamsFromCfgFileOrDie(const char *cfgFile, darknet::NetParameter *net);
+        void ReadNetParamsFromCfgStreamOrDie(std::istream &ifile, darknet::NetParameter *net);
-        void ReadNetParamsFromBinaryFileOrDie(const char *darknetModel, darknet::NetParameter *net);
+        void ReadNetParamsFromBinaryStreamOrDie(std::istream &ifile, darknet::NetParameter *net);
-
     }
 }
 #endif

modules/dnn/src/dnn.cpp

@@ -3126,6 +3126,23 @@ Net readNet(const String& _model, const String& _config, const String& _framewor
                                       model + (config.empty() ? "" : ", " + config));
 }
 
+Net readNet(const String& _framework, const std::vector<uchar>& bufferModel,
+            const std::vector<uchar>& bufferConfig)
+{
+    String framework = _framework.toLowerCase();
+    if (framework == "caffe")
+        return readNetFromCaffe(bufferConfig, bufferModel);
+    else if (framework == "tensorflow")
+        return readNetFromTensorflow(bufferModel, bufferConfig);
+    else if (framework == "darknet")
+        return readNetFromDarknet(bufferConfig, bufferModel);
+    else if (framework == "torch")
+        CV_Error(Error::StsNotImplemented, "Reading Torch models from buffers");
+    else if (framework == "dldt")
+        CV_Error(Error::StsNotImplemented, "Reading Intel's Model Optimizer models from buffers");
+    CV_Error(Error::StsError, "Cannot determine an origin framework with a name " + framework);
+}
+
 Net readNetFromModelOptimizer(const String &xml, const String &bin)
 {
     return Net::readFromModelOptimizer(xml, bin);

modules/dnn/src/tensorflow/tf_importer.cpp

@@ -1856,5 +1856,14 @@ Net readNetFromTensorflow(const char* bufferModel, size_t lenModel,
     return net;
 }
 
+Net readNetFromTensorflow(const std::vector<uchar>& bufferModel, const std::vector<uchar>& bufferConfig)
+{
+    const char* bufferModelPtr = reinterpret_cast<const char*>(&bufferModel[0]);
+    const char* bufferConfigPtr = bufferConfig.empty() ? NULL :
+                                  reinterpret_cast<const char*>(&bufferConfig[0]);
+    return readNetFromTensorflow(bufferModelPtr, bufferModel.size(),
+                                 bufferConfigPtr, bufferConfig.size());
+}
+
 CV__DNN_EXPERIMENTAL_NS_END
 }} // namespace

modules/dnn/test/test_darknet_importer.cpp

@@ -65,6 +65,34 @@ TEST(Test_Darknet, read_yolo_voc)
     ASSERT_FALSE(net.empty());
 }
 
+TEST(Test_Darknet, read_yolo_voc_stream)
+{
+    Mat ref;
+    Mat sample = imread(_tf("dog416.png"));
+    Mat inp = blobFromImage(sample, 1.0/255, Size(416, 416), Scalar(), true, false);
+    const std::string cfgFile = findDataFile("dnn/yolo-voc.cfg", false);
+    const std::string weightsFile = findDataFile("dnn/yolo-voc.weights", false);
+    // Import by paths.
+    {
+        Net net = readNetFromDarknet(cfgFile, weightsFile);
+        net.setInput(inp);
+        net.setPreferableBackend(DNN_BACKEND_OPENCV);
+        ref = net.forward();
+    }
+    // Import from bytes array.
+    {
+        std::string cfg, weights;
+        readFileInMemory(cfgFile, cfg);
+        readFileInMemory(weightsFile, weights);
+
+        Net net = readNetFromDarknet(&cfg[0], cfg.size(), &weights[0], weights.size());
+        net.setInput(inp);
+        net.setPreferableBackend(DNN_BACKEND_OPENCV);
+        Mat out = net.forward();
+        normAssert(ref, out);
+    }
+}
+
 class Test_Darknet_layers : public DNNTestLayer
 {
 public: