Merge pull request #17436 from vpisarev:fix_python_io

* fixed #17044
1. fixed Python part of the tutorial about using OpenCV XML-YAML-JSON I/O functionality from C++ and Python.
2. added startWriteStruct() and endWriteStruct() methods to FileStorage
3. modifed FileStorage::write() methods to make them work well inside sequences, not only mappings.

* try to fix the doc builder

* added Python regression test for FileStorage I/O API ([TODO] iterating through long sequences can be very slow)

* fixed yaml testing

doc/tutorials/core/file_input_output_with_xml_yml/file_input_output_with_xml_yml.markdown

@@ -107,8 +107,9 @@ you may access it. For sequences you need to go through them to query a specific
     then we have to specify if our output is either a sequence or map.
 
     For sequence before the first element print the "[" character and after the last one the "]"
-    character. With Python, the "]" character could be written with the name of the sequence or
-    the last element of the sequence depending on the number of elements:
+    character. With Python, call `FileStorage.startWriteStruct(structure_name, struct_type)`,
+    where `struct_type` is `cv2.FileNode_MAP` or `cv2.FileNode_SEQ` to start writing the structure.
+    Call `FileStorage.endWriteStruct()` to finish the structure:
     @add_toggle_cpp
     @snippet cpp/tutorial_code/core/file_input_output/file_input_output.cpp writeStr
     @end_toggle

modules/core/include/opencv2/core/persistence.hpp

@@ -429,7 +429,7 @@ public:
     /** @brief Writes the registered C structure (CvMat, CvMatND, CvSeq).
     @param name Name of the written object.
     @param obj Pointer to the object.
-    @see ocvWrite for details.
+    @see cvWrite for details.
      */
     void writeObj( const String& name, const void* obj );
 
@@ -456,6 +456,17 @@ public:
      */
     CV_WRAP void writeComment(const String& comment, bool append = false);
 
+    /** @brief Starts to write a nested structure (sequence or a mapping).
+    @param name name of the structure (if it's a member of parent mapping, otherwise it should be empty
+    @param flags type of the structure (FileNode::MAP or FileNode::SEQ (both with optional FileNode::FLOW)).
+    @param typeName usually an empty string
+    */
+    CV_WRAP void startWriteStruct(const String& name, int flags, const String& typeName=String());
+
+    /** @brief Finishes writing nested structure (should pair startWriteStruct())
+    */
+    CV_WRAP void endWriteStruct();
+
     /** @brief Returns the normalized object name for the specified name of a file.
     @param filename Name of a file
     @returns The normalized object name.

modules/core/src/persistence_cpp.cpp

@@ -166,22 +166,24 @@ void FileStorage::writeObj( const String& name, const void* obj )
 
 void FileStorage::write( const String& name, int val )
 {
-    *this << name << val;
+    cvWriteInt(fs, name.c_str(), val);
 }
 
 void FileStorage::write( const String& name, double val )
 {
-    *this << name << val;
+    cvWriteReal(fs, name.c_str(), val);
 }
 
 void FileStorage::write( const String& name, const String& val )
 {
-    *this << name << val;
+    cvWriteString(fs, name.c_str(), val.c_str());
 }
 
 void FileStorage::write( const String& name, InputArray val )
 {
-    *this << name << val.getMat();
+    if(state & INSIDE_MAP)
+        *this << name;
+    *this << val.getMat();
 }
 
 void FileStorage::writeComment( const String& comment, bool append )
@@ -189,6 +191,26 @@ void FileStorage::writeComment( const String& comment, bool append )
     cvWriteComment(fs, comment.c_str(), append ? 1 : 0);
 }
 
+void FileStorage::startWriteStruct(const String& name, int flags, const String& typeName)
+{
+    int struct_type = flags & FileNode::TYPE_MASK;
+    bool isflow = (flags & FileNode::FLOW) != 0;
+    CV_Assert(struct_type == FileNode::SEQ || struct_type == FileNode::MAP);
+    char strbegin_[] = { (struct_type == FileNode::SEQ ? '[' : '{'), (isflow ? ':' : '\0'), '\0' };
+    String strbegin = strbegin_;
+    if (!typeName.empty())
+        strbegin += typeName;
+    *this << name << strbegin;
+}
+
+void FileStorage::endWriteStruct()
+{
+    if( structs.empty() )
+        CV_Error( CV_StsError, "Extra endWriteStruct()" );
+    char openparen = structs.back();
+    *this << (openparen == '[' ? "]" : "}");
+}
+
 String FileStorage::getDefaultObjectName(const String& _filename)
 {
     static const char* stubname = "unnamed";

modules/core/src/persistence_xml.cpp

@@ -853,7 +853,7 @@ void icvXMLWriteScalar( CvFileStorage* fs, const char* key, const char* data, in
         char* ptr = fs->buffer;
         int new_offset = (int)(ptr - fs->buffer_start) + len;
 
-        if( key )
+        if( key && key[0] != '\0' )
             CV_Error( CV_StsBadArg, "elements with keys can not be written to sequence" );
 
         fs->struct_flags = CV_NODE_SEQ;

modules/python/test/test_filestorage_io.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python
+"""Algorithm serialization test."""
+from __future__ import print_function
+import tempfile
+import os
+import cv2 as cv
+import numpy as np
+from tests_common import NewOpenCVTests
+
+class MyData:
+    def __init__(self):
+        self.A = 97
+        self.X = np.pi
+        self.name = 'mydata1234'
+
+    def write(self, fs, name):
+        fs.startWriteStruct(name, cv.FileNode_MAP|cv.FileNode_FLOW)
+        fs.write('A', self.A)
+        fs.write('X', self.X)
+        fs.write('name', self.name)
+        fs.endWriteStruct()
+
+    def read(self, node):
+        if (not node.empty()):
+            self.A = int(node.getNode('A').real())
+            self.X = node.getNode('X').real()
+            self.name = node.getNode('name').string()
+        else:
+            self.A = self.X = 0
+            self.name = ''
+
+class filestorage_io_test(NewOpenCVTests):
+    strings_data = ['image1.jpg', 'Awesomeness', '../data/baboon.jpg']
+    R0 = np.eye(3,3)
+    T0 = np.zeros((3,1))
+
+    def write_data(self, fname):
+        fs = cv.FileStorage(fname, cv.FileStorage_WRITE)
+        R = self.R0
+        T = self.T0
+        m = MyData()
+
+        fs.write('iterationNr', 100)
+
+        fs.startWriteStruct('strings', cv.FileNode_SEQ)
+        for elem in self.strings_data:
+            fs.write('', elem)
+        fs.endWriteStruct()
+
+        fs.startWriteStruct('Mapping', cv.FileNode_MAP)
+        fs.write('One', 1)
+        fs.write('Two', 2)
+        fs.endWriteStruct()
+
+        fs.write('R_MAT', R)
+        fs.write('T_MAT', T)
+
+        m.write(fs, 'MyData')
+        fs.release()
+
+    def read_data_and_check(self, fname):
+        fs = cv.FileStorage(fname, cv.FileStorage_READ)
+
+        n = fs.getNode('iterationNr')
+        itNr = int(n.real())
+        self.assertEqual(itNr, 100)
+
+        n = fs.getNode('strings')
+        self.assertTrue(n.isSeq())
+        self.assertEqual(n.size(), len(self.strings_data))
+
+        for i in range(n.size()):
+            self.assertEqual(n.at(i).string(), self.strings_data[i])
+
+        n = fs.getNode('Mapping')
+        self.assertEqual(int(n.getNode('Two').real()), 2)
+        self.assertEqual(int(n.getNode('One').real()), 1)
+
+        R = fs.getNode('R_MAT').mat()
+        T = fs.getNode('T_MAT').mat()
+
+        self.assertEqual(cv.norm(R, self.R0, cv.NORM_INF), 0)
+        self.assertEqual(cv.norm(T, self.T0, cv.NORM_INF), 0)
+
+        m0 = MyData()
+        m = MyData()
+        m.read(fs.getNode('MyData'))
+        self.assertEqual(m.A, m0.A)
+        self.assertEqual(m.X, m0.X)
+        self.assertEqual(m.name, m0.name)
+
+        n = fs.getNode('NonExisting')
+        self.assertTrue(n.isNone())
+        fs.release()
+
+    def run_fs_test(self, ext):
+        fd, fname = tempfile.mkstemp(prefix="opencv_python_sample_filestorage", suffix=ext)
+        os.close(fd)
+        self.write_data(fname)
+        self.read_data_and_check(fname)
+        os.remove(fname)
+
+    def test_xml(self):
+        self.run_fs_test(".xml")
+
+    def test_yml(self):
+        self.run_fs_test(".yml")
+
+    def test_json(self):
+        self.run_fs_test(".json")
+
+if __name__ == '__main__':
+    NewOpenCVTests.bootstrap()

samples/python/tutorial_code/core/file_input_output/file_input_output.py

@@ -29,12 +29,12 @@ class MyData:
         return s
 
     ## [inside]
-    def write(self, fs):
-        fs.write('MyData','{')
+    def write(self, fs, name):
+        fs.startWriteStruct(name, cv.FileNode_MAP|cv.FileNode_FLOW)
         fs.write('A', self.A)
         fs.write('X', self.X)
         fs.write('name', self.name)
-        fs.write('MyData','}')
+        fs.endWriteStruct()
 
     def read(self, node):
         if (not node.empty()):
@@ -74,25 +74,26 @@ def main(argv):
     ## [writeNum]
 
     ## [writeStr]
-    s.write('strings', '[')
-    s.write('image1.jpg','Awesomeness')
-    s.write('../data/baboon.jpg',']')
+    s.startWriteStruct('strings', cv.FileNode_SEQ)
+    for elem in ['image1.jpg', 'Awesomeness', '../data/baboon.jpg']:
+        s.write('', elem)
+    s.endWriteStruct()
     ## [writeStr]
 
     ## [writeMap]
-    s.write ('Mapping', '{')
-    s.write ('One', 1)
-    s.write ('Two', 2)
-    s.write ('Mapping', '}')
+    s.startWriteStruct('Mapping', cv.FileNode_MAP)
+    s.write('One', 1)
+    s.write('Two', 2)
+    s.endWriteStruct()
     ## [writeMap]
 
     ## [iomatw]
-    s.write ('R_MAT', R)
-    s.write ('T_MAT', T)
+    s.write('R_MAT', R)
+    s.write('T_MAT', T)
     ## [iomatw]
 
     ## [customIOw]
-    m.write(s)
+    m.write(s, 'MyData')
     ## [customIOw]
     ## [close]
     s.release()