From 5489735258db761b4eccd5bce25375e0b8a735ef Mon Sep 17 00:00:00 2001 From: Vadim Pisarevsky Date: Mon, 1 Jun 2020 14:33:09 +0300 Subject: [PATCH] Merge pull request #17436 from vpisarev:fix_python_io * fixed #17044 1. fixed Python part of the tutorial about using OpenCV XML-YAML-JSON I/O functionality from C++ and Python. 2. added startWriteStruct() and endWriteStruct() methods to FileStorage 3. modifed FileStorage::write() methods to make them work well inside sequences, not only mappings. * try to fix the doc builder * added Python regression test for FileStorage I/O API ([TODO] iterating through long sequences can be very slow) * fixed yaml testing --- .../file_input_output_with_xml_yml.markdown | 5 +- .../core/include/opencv2/core/persistence.hpp | 13 +- modules/core/src/persistence_cpp.cpp | 30 ++++- modules/core/src/persistence_xml.cpp | 2 +- modules/python/test/test_filestorage_io.py | 113 ++++++++++++++++++ .../file_input_output/file_input_output.py | 27 +++-- 6 files changed, 169 insertions(+), 21 deletions(-) create mode 100755 modules/python/test/test_filestorage_io.py diff --git a/doc/tutorials/core/file_input_output_with_xml_yml/file_input_output_with_xml_yml.markdown b/doc/tutorials/core/file_input_output_with_xml_yml/file_input_output_with_xml_yml.markdown index 4e426c62a9..402804e13c 100644 --- a/doc/tutorials/core/file_input_output_with_xml_yml/file_input_output_with_xml_yml.markdown +++ b/doc/tutorials/core/file_input_output_with_xml_yml/file_input_output_with_xml_yml.markdown @@ -107,8 +107,9 @@ you may access it. For sequences you need to go through them to query a specific then we have to specify if our output is either a sequence or map. For sequence before the first element print the "[" character and after the last one the "]" - character. With Python, the "]" character could be written with the name of the sequence or - the last element of the sequence depending on the number of elements: + character. With Python, call `FileStorage.startWriteStruct(structure_name, struct_type)`, + where `struct_type` is `cv2.FileNode_MAP` or `cv2.FileNode_SEQ` to start writing the structure. + Call `FileStorage.endWriteStruct()` to finish the structure: @add_toggle_cpp @snippet cpp/tutorial_code/core/file_input_output/file_input_output.cpp writeStr @end_toggle diff --git a/modules/core/include/opencv2/core/persistence.hpp b/modules/core/include/opencv2/core/persistence.hpp index 1194885b11..e3026e3c9f 100644 --- a/modules/core/include/opencv2/core/persistence.hpp +++ b/modules/core/include/opencv2/core/persistence.hpp @@ -429,7 +429,7 @@ public: /** @brief Writes the registered C structure (CvMat, CvMatND, CvSeq). @param name Name of the written object. @param obj Pointer to the object. - @see ocvWrite for details. + @see cvWrite for details. */ void writeObj( const String& name, const void* obj ); @@ -456,6 +456,17 @@ public: */ CV_WRAP void writeComment(const String& comment, bool append = false); + /** @brief Starts to write a nested structure (sequence or a mapping). + @param name name of the structure (if it's a member of parent mapping, otherwise it should be empty + @param flags type of the structure (FileNode::MAP or FileNode::SEQ (both with optional FileNode::FLOW)). + @param typeName usually an empty string + */ + CV_WRAP void startWriteStruct(const String& name, int flags, const String& typeName=String()); + + /** @brief Finishes writing nested structure (should pair startWriteStruct()) + */ + CV_WRAP void endWriteStruct(); + /** @brief Returns the normalized object name for the specified name of a file. @param filename Name of a file @returns The normalized object name. diff --git a/modules/core/src/persistence_cpp.cpp b/modules/core/src/persistence_cpp.cpp index 32c72f75be..8072eff017 100644 --- a/modules/core/src/persistence_cpp.cpp +++ b/modules/core/src/persistence_cpp.cpp @@ -166,22 +166,24 @@ void FileStorage::writeObj( const String& name, const void* obj ) void FileStorage::write( const String& name, int val ) { - *this << name << val; + cvWriteInt(fs, name.c_str(), val); } void FileStorage::write( const String& name, double val ) { - *this << name << val; + cvWriteReal(fs, name.c_str(), val); } void FileStorage::write( const String& name, const String& val ) { - *this << name << val; + cvWriteString(fs, name.c_str(), val.c_str()); } void FileStorage::write( const String& name, InputArray val ) { - *this << name << val.getMat(); + if(state & INSIDE_MAP) + *this << name; + *this << val.getMat(); } void FileStorage::writeComment( const String& comment, bool append ) @@ -189,6 +191,26 @@ void FileStorage::writeComment( const String& comment, bool append ) cvWriteComment(fs, comment.c_str(), append ? 1 : 0); } +void FileStorage::startWriteStruct(const String& name, int flags, const String& typeName) +{ + int struct_type = flags & FileNode::TYPE_MASK; + bool isflow = (flags & FileNode::FLOW) != 0; + CV_Assert(struct_type == FileNode::SEQ || struct_type == FileNode::MAP); + char strbegin_[] = { (struct_type == FileNode::SEQ ? '[' : '{'), (isflow ? ':' : '\0'), '\0' }; + String strbegin = strbegin_; + if (!typeName.empty()) + strbegin += typeName; + *this << name << strbegin; +} + +void FileStorage::endWriteStruct() +{ + if( structs.empty() ) + CV_Error( CV_StsError, "Extra endWriteStruct()" ); + char openparen = structs.back(); + *this << (openparen == '[' ? "]" : "}"); +} + String FileStorage::getDefaultObjectName(const String& _filename) { static const char* stubname = "unnamed"; diff --git a/modules/core/src/persistence_xml.cpp b/modules/core/src/persistence_xml.cpp index 276eb78db6..aaaa613bc2 100644 --- a/modules/core/src/persistence_xml.cpp +++ b/modules/core/src/persistence_xml.cpp @@ -853,7 +853,7 @@ void icvXMLWriteScalar( CvFileStorage* fs, const char* key, const char* data, in char* ptr = fs->buffer; int new_offset = (int)(ptr - fs->buffer_start) + len; - if( key ) + if( key && key[0] != '\0' ) CV_Error( CV_StsBadArg, "elements with keys can not be written to sequence" ); fs->struct_flags = CV_NODE_SEQ; diff --git a/modules/python/test/test_filestorage_io.py b/modules/python/test/test_filestorage_io.py new file mode 100755 index 0000000000..62b540d79c --- /dev/null +++ b/modules/python/test/test_filestorage_io.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python +"""Algorithm serialization test.""" +from __future__ import print_function +import tempfile +import os +import cv2 as cv +import numpy as np +from tests_common import NewOpenCVTests + +class MyData: + def __init__(self): + self.A = 97 + self.X = np.pi + self.name = 'mydata1234' + + def write(self, fs, name): + fs.startWriteStruct(name, cv.FileNode_MAP|cv.FileNode_FLOW) + fs.write('A', self.A) + fs.write('X', self.X) + fs.write('name', self.name) + fs.endWriteStruct() + + def read(self, node): + if (not node.empty()): + self.A = int(node.getNode('A').real()) + self.X = node.getNode('X').real() + self.name = node.getNode('name').string() + else: + self.A = self.X = 0 + self.name = '' + +class filestorage_io_test(NewOpenCVTests): + strings_data = ['image1.jpg', 'Awesomeness', '../data/baboon.jpg'] + R0 = np.eye(3,3) + T0 = np.zeros((3,1)) + + def write_data(self, fname): + fs = cv.FileStorage(fname, cv.FileStorage_WRITE) + R = self.R0 + T = self.T0 + m = MyData() + + fs.write('iterationNr', 100) + + fs.startWriteStruct('strings', cv.FileNode_SEQ) + for elem in self.strings_data: + fs.write('', elem) + fs.endWriteStruct() + + fs.startWriteStruct('Mapping', cv.FileNode_MAP) + fs.write('One', 1) + fs.write('Two', 2) + fs.endWriteStruct() + + fs.write('R_MAT', R) + fs.write('T_MAT', T) + + m.write(fs, 'MyData') + fs.release() + + def read_data_and_check(self, fname): + fs = cv.FileStorage(fname, cv.FileStorage_READ) + + n = fs.getNode('iterationNr') + itNr = int(n.real()) + self.assertEqual(itNr, 100) + + n = fs.getNode('strings') + self.assertTrue(n.isSeq()) + self.assertEqual(n.size(), len(self.strings_data)) + + for i in range(n.size()): + self.assertEqual(n.at(i).string(), self.strings_data[i]) + + n = fs.getNode('Mapping') + self.assertEqual(int(n.getNode('Two').real()), 2) + self.assertEqual(int(n.getNode('One').real()), 1) + + R = fs.getNode('R_MAT').mat() + T = fs.getNode('T_MAT').mat() + + self.assertEqual(cv.norm(R, self.R0, cv.NORM_INF), 0) + self.assertEqual(cv.norm(T, self.T0, cv.NORM_INF), 0) + + m0 = MyData() + m = MyData() + m.read(fs.getNode('MyData')) + self.assertEqual(m.A, m0.A) + self.assertEqual(m.X, m0.X) + self.assertEqual(m.name, m0.name) + + n = fs.getNode('NonExisting') + self.assertTrue(n.isNone()) + fs.release() + + def run_fs_test(self, ext): + fd, fname = tempfile.mkstemp(prefix="opencv_python_sample_filestorage", suffix=ext) + os.close(fd) + self.write_data(fname) + self.read_data_and_check(fname) + os.remove(fname) + + def test_xml(self): + self.run_fs_test(".xml") + + def test_yml(self): + self.run_fs_test(".yml") + + def test_json(self): + self.run_fs_test(".json") + +if __name__ == '__main__': + NewOpenCVTests.bootstrap() diff --git a/samples/python/tutorial_code/core/file_input_output/file_input_output.py b/samples/python/tutorial_code/core/file_input_output/file_input_output.py index e3adb216a5..66b3108dba 100644 --- a/samples/python/tutorial_code/core/file_input_output/file_input_output.py +++ b/samples/python/tutorial_code/core/file_input_output/file_input_output.py @@ -29,12 +29,12 @@ class MyData: return s ## [inside] - def write(self, fs): - fs.write('MyData','{') + def write(self, fs, name): + fs.startWriteStruct(name, cv.FileNode_MAP|cv.FileNode_FLOW) fs.write('A', self.A) fs.write('X', self.X) fs.write('name', self.name) - fs.write('MyData','}') + fs.endWriteStruct() def read(self, node): if (not node.empty()): @@ -74,25 +74,26 @@ def main(argv): ## [writeNum] ## [writeStr] - s.write('strings', '[') - s.write('image1.jpg','Awesomeness') - s.write('../data/baboon.jpg',']') + s.startWriteStruct('strings', cv.FileNode_SEQ) + for elem in ['image1.jpg', 'Awesomeness', '../data/baboon.jpg']: + s.write('', elem) + s.endWriteStruct() ## [writeStr] ## [writeMap] - s.write ('Mapping', '{') - s.write ('One', 1) - s.write ('Two', 2) - s.write ('Mapping', '}') + s.startWriteStruct('Mapping', cv.FileNode_MAP) + s.write('One', 1) + s.write('Two', 2) + s.endWriteStruct() ## [writeMap] ## [iomatw] - s.write ('R_MAT', R) - s.write ('T_MAT', T) + s.write('R_MAT', R) + s.write('T_MAT', T) ## [iomatw] ## [customIOw] - m.write(s) + m.write(s, 'MyData') ## [customIOw] ## [close] s.release()