diff --git a/modules/core/include/opencv2/core/core_c.h b/modules/core/include/opencv2/core/core_c.h index a0ed632642..94ba1fc034 100644 --- a/modules/core/include/opencv2/core/core_c.h +++ b/modules/core/include/opencv2/core/core_c.h @@ -1976,8 +1976,19 @@ CVAPI(void) cvSetIPLAllocators( Cv_iplCreateImageHeader create_header, The function opens file storage for reading or writing data. In the latter case, a new file is created or an existing file is rewritten. The type of the read or written file is determined by the -filename extension: .xml for XML and .yml or .yaml for YAML. The function returns a pointer to the -CvFileStorage structure. If the file cannot be opened then the function returns NULL. +filename extension: .xml for XML and .yml or .yaml for YAML. + +At the same time, it also supports adding parameters like "example.xml?base64". +@code + CvFileStorage* fs = cvOpenFileStorage( "example.yml?base64", 0, CV_STORAGE_WRITE ); +@endcode +it's exactly the same as +@code + CvFileStorage* fs = cvOpenFileStorage( "example.yml", 0, CV_STORAGE_WRITE_BASE64 ); +@endcode + +The function returns a pointer to the CvFileStorage structure. +If the file cannot be opened then the function returns NULL. @param filename Name of the file associated with the storage @param memstorage Memory storage used for temporary data and for : storing dynamic structures, such as CvSeq or CvGraph . If it is NULL, a temporary memory @@ -1985,6 +1996,7 @@ CvFileStorage structure. If the file cannot be opened then the function returns @param flags Can be one of the following: > - **CV_STORAGE_READ** the storage is open for reading > - **CV_STORAGE_WRITE** the storage is open for writing + (use **CV_STORAGE_WRITE | CV_STORAGE_WRITE_BASE64** to write rawdata in Base64) @param encoding */ CVAPI(CvFileStorage*) cvOpenFileStorage( const char* filename, CvMemStorage* memstorage, @@ -2162,7 +2174,7 @@ the file with multiple streams looks like this: @endcode The YAML file will look like this: @code{.yaml} - %YAML:1.0 + %YAML 1.0 # stream #1 data ... --- @@ -2187,6 +2199,46 @@ to a sequence rather than a map. CVAPI(void) cvWriteRawData( CvFileStorage* fs, const void* src, int len, const char* dt ); +/** @brief Writes multiple numbers in Base64. + +If either CV_STORAGE_WRITE_BASE64 or cv::FileStorage::WRITE_BASE64 is used, +this function will be the same as cvWriteRawData. If neither, the main +difference is that it outputs a sequence in Base64 encoding rather than +in plain text. + +This function can only be used to write a sequence with a type "binary". + +Consider the following two examples where their output is the same: +@code + std::vector rawdata(10, 0x00010203); + // without the flag CV_STORAGE_WRITE_BASE64. + CvFileStorage* fs = cvOpenFileStorage( "example.yml", 0, CV_STORAGE_WRITE ); + // both CV_NODE_SEQ and "binary" are necessary. + cvStartWriteStruct(fs, "rawdata", CV_NODE_SEQ | CV_NODE_FLOW, "binary"); + cvWriteRawDataBase64(fs, rawdata.data(), rawdata.size(), "i"); + cvEndWriteStruct(fs); + cvReleaseFileStorage( &fs ); +@endcode +and +@code + std::vector rawdata(10, 0x00010203); + // with the flag CV_STORAGE_WRITE_BASE64. + CvFileStorage* fs = cvOpenFileStorage( "example.yml", 0, CV_STORAGE_WRITE_BASE64); + // parameter, typename "binary" could be omitted. + cvStartWriteStruct(fs, "rawdata", CV_NODE_SEQ | CV_NODE_FLOW); + cvWriteRawData(fs, rawdata.data(), rawdata.size(), "i"); + cvEndWriteStruct(fs); + cvReleaseFileStorage( &fs ); +@endcode + +@param fs File storage +@param src Pointer to the written array +@param len Number of the array elements to write +@param dt Specification of each array element, see @ref format_spec "format specification" +*/ +CVAPI(void) cvWriteRawDataBase64( CvFileStorage* fs, const void* _data, + int len, const char* dt ); + /** @brief Returns a unique pointer for a given name. The function returns a unique pointer for each particular file node name. This pointer can be then diff --git a/modules/core/include/opencv2/core/persistence.hpp b/modules/core/include/opencv2/core/persistence.hpp index 65a1ff4c4c..75f0c32bd2 100644 --- a/modules/core/include/opencv2/core/persistence.hpp +++ b/modules/core/include/opencv2/core/persistence.hpp @@ -311,7 +311,10 @@ public: FORMAT_MASK = (7<<3), //!< mask for format flags FORMAT_AUTO = 0, //!< flag, auto format FORMAT_XML = (1<<3), //!< flag, XML format - FORMAT_YAML = (2<<3) //!< flag, YAML format + FORMAT_YAML = (2<<3), //!< flag, YAML format + + BASE64 = 64, //!< flag, write rawdata in Base64 by default. (consider using WRITE_BASE64) + WRITE_BASE64 = BASE64 | WRITE, //!< flag, enable both WRITE and BASE64 }; enum { @@ -354,7 +357,9 @@ public: Extension of the file (.xml or .yml/.yaml) determines its format (XML or YAML respectively). Also you can append .gz to work with compressed files, for example myHugeMatrix.xml.gz. If both FileStorage::WRITE and FileStorage::MEMORY flags are specified, source is used just to specify - the output file format (e.g. mydata.xml, .yml etc.). + the output file format (e.g. mydata.xml, .yml etc.). A file name can also contain parameters. + You can use this format, "*?base64" (e.g. "file.xml?base64"), as an alternative to + FileStorage::BASE64 flag. Note: it is case sensitive. @param flags Mode of operation. One of FileStorage::Mode @param encoding Encoding of the file. Note that UTF-16 XML encoding is not supported currently and you should use 8-bit encoding instead of it. diff --git a/modules/core/include/opencv2/core/types_c.h b/modules/core/include/opencv2/core/types_c.h index cb39587a9a..e693aa4724 100644 --- a/modules/core/include/opencv2/core/types_c.h +++ b/modules/core/include/opencv2/core/types_c.h @@ -1669,6 +1669,8 @@ typedef struct CvFileStorage CvFileStorage; #define CV_STORAGE_FORMAT_AUTO 0 #define CV_STORAGE_FORMAT_XML 8 #define CV_STORAGE_FORMAT_YAML 16 +#define CV_STORAGE_BASE64 64 +#define CV_STORAGE_WRITE_BASE64 (CV_STORAGE_BASE64 | CV_STORAGE_WRITE) /** @brief List of attributes. : diff --git a/modules/core/perf/perf_io_base64.cpp b/modules/core/perf/perf_io_base64.cpp new file mode 100644 index 0000000000..693c1cc132 --- /dev/null +++ b/modules/core/perf/perf_io_base64.cpp @@ -0,0 +1,86 @@ +#include "perf_precomp.hpp" + +using namespace std; +using namespace cv; +using namespace perf; +using std::tr1::make_tuple; +using std::tr1::get; + +typedef std::tr1::tuple Size_MatType_Str_t; +typedef TestBaseWithParam Size_Mat_StrType; + +#define MAT_SIZES ::perf::sz1080p/*, ::perf::sz4320p*/ +#define MAT_TYPES CV_8UC1, CV_32FC1 +#define FILE_EXTENSION String(".xml"), String(".yml") + +PERF_TEST_P(Size_Mat_StrType, fs_text, + testing::Combine(testing::Values(MAT_SIZES), + testing::Values(MAT_TYPES), + testing::Values(FILE_EXTENSION)) + ) +{ + Size size = get<0>(GetParam()); + int type = get<1>(GetParam()); + String ext = get<2>(GetParam()); + + Mat src(size.height, size.width, type); + Mat dst = src.clone(); + + declare.in(src, WARMUP_RNG).out(dst); + + cv::String file_name = cv::tempfile(ext.c_str()); + cv::String key = "test_mat"; + + TEST_CYCLE_MULTIRUN(4) + { + { + FileStorage fs(file_name, cv::FileStorage::WRITE); + fs << key << src; + fs.release(); + } + { + FileStorage fs(file_name, cv::FileStorage::READ); + fs[key] >> dst; + fs.release(); + } + } + + remove(file_name.c_str()); + + SANITY_CHECK(dst, 1); +} + +PERF_TEST_P(Size_Mat_StrType, fs_base64, + testing::Combine(testing::Values(MAT_SIZES), + testing::Values(MAT_TYPES), + testing::Values(FILE_EXTENSION)) + ) +{ + Size size = get<0>(GetParam()); + int type = get<1>(GetParam()); + String ext = get<2>(GetParam()); + + Mat src(size.height, size.width, type); + Mat dst = src.clone(); + + cv::String file_name = cv::tempfile(ext.c_str()); + cv::String key = "test_mat"; + + declare.in(src, WARMUP_RNG).out(dst); + TEST_CYCLE_MULTIRUN(4) + { + { + FileStorage fs(file_name, cv::FileStorage::WRITE_BASE64); + fs << key << src; + fs.release(); + } + { + FileStorage fs(file_name, cv::FileStorage::READ); + fs[key] >> dst; + fs.release(); + } + } + + remove(file_name.c_str()); + SANITY_CHECK(dst, 1); +} diff --git a/modules/core/src/persistence.cpp b/modules/core/src/persistence.cpp index 4d99a4a275..6eb3b0c75a 100644 --- a/modules/core/src/persistence.cpp +++ b/modules/core/src/persistence.cpp @@ -243,6 +243,17 @@ typedef struct CvFileStorage std::deque* outbuf; base64::Base64Writer * base64_writer; + bool is_default_using_base64; + enum Base64State { + Uncertain, + NotUse, + InUse, + } state_of_writing_base64; /**< used in WriteRawData only */ + + bool is_write_struct_delayed; + char* delayed_struct_key; + int delayed_struct_flags; + char* delayed_type_name; bool is_opened; } @@ -270,21 +281,23 @@ namespace base64 bool base64_valid (uint8_t const * src, size_t off, size_t cnt); bool base64_valid ( char const * src, size_t off = 0U, size_t cnt = 0U); - size_t base64_encode_buffer_size(size_t cnt); + size_t base64_encode_buffer_size(size_t cnt, bool is_end_with_zero = true); - size_t base64_decode_buffer_size(size_t cnt); + size_t base64_decode_buffer_size(size_t cnt, bool is_end_with_zero = true); + size_t base64_decode_buffer_size(size_t cnt, char const * src, bool is_end_with_zero = true); + size_t base64_decode_buffer_size(size_t cnt, uchar const * src, bool is_end_with_zero = true); /* binary */ - template inline size_t to_binary(_uint_t val, uchar * cur); - template<> inline size_t to_binary(double val, uchar * cur); - template<> inline size_t to_binary(float val, uchar * cur); + template inline size_t to_binary(_uint_t val, uchar * cur); + template<> inline size_t to_binary(double val, uchar * cur); + template<> inline size_t to_binary(float val, uchar * cur); template inline size_t to_binary(uchar const * val, uchar * cur); - template inline size_t binary_to(uchar const * cur, _uint_t & val); - template<> inline size_t binary_to(uchar const * cur, double & val); - template<> inline size_t binary_to(uchar const * cur, float & val); - template inline size_t binary_to(uchar const * cur, uchar * val); + template inline size_t binary_to(uchar const * cur, _uint_t & val); + template<> inline size_t binary_to(uchar const * cur, double & val); + template<> inline size_t binary_to(uchar const * cur, float & val); + template inline size_t binary_to(uchar const * cur, uchar * val); class MatToBinaryConvertor; class RawDataToBinaryConvertor; @@ -313,22 +326,37 @@ namespace base64 class Base64ContextEmitter; + class Base64Writer + { + public: + Base64Writer(::CvFileStorage * fs); + ~Base64Writer(); + void write(const void* _data, size_t len, const char* dt); + template void write(_to_binary_convertor_t & convertor, const char* dt); + + private: + void check_dt(const char* dt); + + private: + + ::CvFileStorage * file_storage; + Base64ContextEmitter * emitter; + std::string data_type_string; + }; + /* other */ - std::string make_base64_header(int byte_size, const char * dt); + std::string make_base64_header(const char * dt); - bool read_base64_header(std::string const & header, int & byte_size, std::string & dt); + bool read_base64_header(std::vector const & header, std::string & dt); void make_seq(void * binary_data, int elem_cnt, const char * dt, CvSeq & seq); /* sample */ - void cvStartWriteRawData_Base64(::CvFileStorage * fs, const char* name, int len, const char* dt); - void cvWriteRawData_Base64(::CvFileStorage * fs, const void* _data, int len); - void cvEndWriteRawData_Base64(::CvFileStorage * fs); + void cvWriteRawDataBase64(::CvFileStorage* fs, const void* _data, int len, const char* dt); - void cvWriteRawData_Base64(::cv::FileStorage & fs, const void* _data, int len, const char* dt); - void cvWriteMat_Base64(CvFileStorage * fs, const char * name, ::cv::Mat const & mat); + void cvWriteMat_Base64(::CvFileStorage * fs, const char * name, ::cv::Mat const & mat); } @@ -1031,6 +1059,184 @@ static double icv_strtod( CvFileStorage* fs, char* ptr, char** endptr ) return fval; } +static std::vector analyze_file_name( std::string const & file_name ) +{ + static const char parameter_begin = '?'; + static const char parameter_separator = '&'; + std::vector result; + + size_t beg = file_name.find_last_of(parameter_begin); + size_t end = file_name.size(); + result.push_back(file_name.substr(0U, beg)); + + if ( beg != std::string::npos ) + { + beg ++; + for ( size_t param_beg = beg, param_end = beg; + param_end < end; + param_beg = param_end + 1U ) + { + param_end = file_name.find_first_of( parameter_separator, param_beg ); + if ( (param_end == std::string::npos || param_end != param_beg) && param_beg + 1U < end ) + { + result.push_back( file_name.substr( param_beg, param_end - param_beg ) ); + } + } + } + + return result; +} + +static bool is_param_exist( std::vector & const params, std::string & const param ) +{ + if ( params.size() < 2U ) + return false; + + return std::find(params.begin(), params.end(), param) != params.end(); +} + +static void switch_to_Base64_state( CvFileStorage* fs, CvFileStorage::Base64State state ) +{ + const char * err_unkonwn_state = "Unexpected error, unable to determine the Base64 state."; + const char * err_unable_to_switch = "Unexpected error, unable to switch to this state."; + + /* like a finite state machine */ + switch (fs->state_of_writing_base64) + { + case CvFileStorage::Base64State::Uncertain: + switch (state) + { + case CvFileStorage::Base64State::InUse: + CV_DbgAssert( fs->base64_writer == 0 ); + fs->base64_writer = new base64::Base64Writer( fs ); + break; + case CvFileStorage::Base64State::Uncertain: + break; + case CvFileStorage::Base64State::NotUse: + break; + default: + CV_Error( CV_StsError, err_unkonwn_state ); + break; + } + break; + case CvFileStorage::Base64State::InUse: + switch (state) + { + case CvFileStorage::Base64State::InUse: + case CvFileStorage::Base64State::NotUse: + CV_Error( CV_StsError, err_unable_to_switch ); + break; + case CvFileStorage::Base64State::Uncertain: + delete fs->base64_writer; + fs->base64_writer = 0; + break; + default: + CV_Error( CV_StsError, err_unkonwn_state ); + break; + } + break; + case CvFileStorage::Base64State::NotUse: + switch (state) + { + case CvFileStorage::Base64State::InUse: + case CvFileStorage::Base64State::NotUse: + CV_Error( CV_StsError, err_unable_to_switch ); + break; + case CvFileStorage::Base64State::Uncertain: + break; + default: + CV_Error( CV_StsError, err_unkonwn_state ); + break; + } + break; + default: + CV_Error( CV_StsError, err_unkonwn_state ); + break; + } + + fs->state_of_writing_base64 = state; +} + + +static void check_if_write_struct_is_delayed( CvFileStorage* fs, bool change_type_to_base64 = false ) +{ + if ( fs->is_write_struct_delayed ) + { + /* save data to prevent recursive call errors */ + char * struct_key = 0; + char * type_name = 0; + int struct_flags = fs->delayed_struct_flags; + + if ( fs->delayed_struct_key != 0 ) + { + struct_key = new char[strlen(fs->delayed_struct_key) + 1U]; + strcpy(struct_key, fs->delayed_struct_key); + } + if ( fs->delayed_type_name != 0 ) + { + type_name = new char[strlen(type_name) + 1U]; + strcpy(type_name, fs->delayed_type_name); + } + + /* reset */ + delete fs->delayed_struct_key; + delete fs->delayed_type_name; + fs->delayed_struct_key = 0; + fs->delayed_struct_flags = 0; + fs->delayed_type_name = 0; + + fs->is_write_struct_delayed = false; + + /* call */ + if ( change_type_to_base64 ) + { + fs->start_write_struct( fs, struct_key, struct_flags, "binary"); + if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::Uncertain ) + switch_to_Base64_state( fs, CvFileStorage::Base64State::Uncertain ); + switch_to_Base64_state( fs, CvFileStorage::Base64State::InUse ); + } + else + { + fs->start_write_struct( fs, struct_key, struct_flags, type_name); + if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::Uncertain ) + switch_to_Base64_state( fs, CvFileStorage::Base64State::Uncertain ); + switch_to_Base64_state( fs, CvFileStorage::Base64State::NotUse ); + } + + delete struct_key; + delete type_name; + } +} + + +static void make_write_struct_delayed( + CvFileStorage* fs, + const char* key, + int struct_flags, + const char* type_name ) +{ + CV_Assert( fs->is_write_struct_delayed == false ); + CV_DbgAssert( fs->delayed_struct_key == 0 ); + CV_DbgAssert( fs->delayed_struct_flags == 0 ); + CV_DbgAssert( fs->delayed_type_name == 0 ); + + fs->delayed_struct_flags = struct_flags; + + if ( key != 0 ) + { + fs->delayed_struct_key = new char[strlen(key) + 1U]; + strcpy(fs->delayed_struct_key, key); + } + + if ( type_name != 0 ) + { + fs->delayed_type_name = new char[strlen(type_name) + 1U]; + strcpy(fs->delayed_type_name, type_name); + } + + fs->is_write_struct_delayed = true; +} + /****************************************************************************************\ * YAML Parser * @@ -1119,40 +1325,41 @@ static char* icvYMLParseBase64(CvFileStorage* fs, char* ptr, int indent, CvFileN /* calc (decoded) total_byte_size from header */ std::string dt; - int total_byte_size = -1; { if (end - beg < static_cast(base64::ENCODED_HEADER_SIZE)) CV_PARSE_ERROR("Unrecognized Base64 header"); std::vector header(base64::HEADER_SIZE + 1, ' '); base64::base64_decode(beg, header.data(), 0U, base64::ENCODED_HEADER_SIZE); - std::istringstream iss(header.data()); - - if (!(iss >> total_byte_size) || total_byte_size < 0) - CV_PARSE_ERROR("Cannot parse size in Base64 header"); - if (!(iss >> dt) || dt.empty()) + if ( !base64::read_base64_header(header, dt) || dt.empty() ) CV_PARSE_ERROR("Cannot parse dt in Base64 header"); beg += base64::ENCODED_HEADER_SIZE; } - /* buffer for decoded data(exclude header) */ - std::vector buffer(total_byte_size + 4); + /* get all Base64 data */ + std::string base64_buffer; + base64_buffer.reserve( 16U * 1024U * 1024U ); + while( beg < end ) { - base64::Base64ContextParser parser(buffer.data(), total_byte_size + 4); - - /* decoding */ - while(beg < end) - { - /* save this part [beg, end) */ - parser.read((const uchar *)beg, (const uchar *)end); - - beg = end; - - /* find next part */ - icvYMLGetMultilineStringContent(fs, beg, indent, beg, end); - } + base64_buffer.append( beg, end ); + beg = end; + icvYMLGetMultilineStringContent( fs, beg, indent, beg, end ); } + if ( !base64::base64_valid(base64_buffer.data(), 0U, base64_buffer.size()) ) + CV_PARSE_ERROR( "Invalid Base64 data." ); + + /* buffer for decoded data(exclude header) */ + std::vector binary_buffer( base64::base64_decode_buffer_size(base64_buffer.size()) ); + int total_byte_size = base64::base64_decode_buffer_size( base64_buffer.size(), base64_buffer.data(), false ); + { + base64::Base64ContextParser parser(binary_buffer.data(), binary_buffer.size() ); + const uchar * buffer_beg = reinterpret_cast( base64_buffer.data() ); + const uchar * buffer_end = buffer_beg + base64_buffer.size(); + parser.read( buffer_beg, buffer_end ); + parser.flush(); + } + /* save as CvSeq */ int elem_size = ::icvCalcStructSize(dt.c_str(), 0); if (total_byte_size % elem_size != 0) @@ -1160,9 +1367,10 @@ static char* icvYMLParseBase64(CvFileStorage* fs, char* ptr, int indent, CvFileN int elem_cnt = total_byte_size / elem_size; node->tag = CV_NODE_NONE; - int struct_flags = CV_NODE_FLOW + CV_NODE_SEQ; /* after icvFSCreateCollection, node->tag == struct_flags */ + int struct_flags = CV_NODE_FLOW | CV_NODE_SEQ; + /* after icvFSCreateCollection, node->tag == struct_flags */ icvFSCreateCollection(fs, struct_flags, node); - base64::make_seq(buffer.data(), elem_cnt, dt.c_str(), *node->data.seq); + base64::make_seq(binary_buffer.data(), elem_cnt, dt.c_str(), *node->data.seq); if (fs->dummy_eof) { /* end of file */ @@ -1621,6 +1829,16 @@ icvYMLParse( CvFileStorage* fs ) static void icvYMLWrite( CvFileStorage* fs, const char* key, const char* data ) { + check_if_write_struct_is_delayed( fs ); + if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::Uncertain ) + { + switch_to_Base64_state( fs, CvFileStorage::Base64State::NotUse ); + } + else if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::InUse ) + { + CV_Error( CV_StsError, "At present, output Base64 data only." ); + } + int i, keylen = 0; int datalen = 0; int struct_flags; @@ -2034,17 +2252,17 @@ static void icvXMLGetMultilineStringContent(CvFileStorage* fs, ptr = icvXMLSkipSpaces(fs, ptr, CV_XML_INSIDE_TAG); beg = ptr; end = ptr; - if (fs->dummy_eof) + if ( fs->dummy_eof ) return ; /* end of file */ - if (*beg == '<') + if ( *beg == '<' ) return; /* end of string */ /* find end */ - while(cv_isprint(*ptr)) /* no check for base64 string */ + while( cv_isprint(*ptr) ) /* no check for base64 string */ ++ ptr; - if (*ptr == '\0') - CV_PARSE_ERROR("Unexpected end of line"); + if ( *ptr == '\0' ) + CV_PARSE_ERROR( "Unexpected end of line" ); end = ptr; } @@ -2061,48 +2279,52 @@ static char* icvXMLParseBase64(CvFileStorage* fs, char* ptr, CvFileNode * node) /* calc (decoded) total_byte_size from header */ std::string dt; - int total_byte_size = -1; { if (end - beg < static_cast(base64::ENCODED_HEADER_SIZE)) CV_PARSE_ERROR("Unrecognized Base64 header"); std::vector header(base64::HEADER_SIZE + 1, ' '); base64::base64_decode(beg, header.data(), 0U, base64::ENCODED_HEADER_SIZE); - std::istringstream iss(header.data()); - if (!(iss >> total_byte_size) || total_byte_size < 0) - CV_PARSE_ERROR("Cannot parse size in Base64 header"); - if (!(iss >> dt) || dt.empty()) + if ( !base64::read_base64_header(header, dt) || dt.empty() ) CV_PARSE_ERROR("Cannot parse dt in Base64 header"); beg += base64::ENCODED_HEADER_SIZE; } - /* alloc buffer for all decoded data(include header) */ - std::vector buffer(total_byte_size + 4); + /* get all Base64 data */ + std::string base64_buffer; + base64_buffer.reserve( 16U * 1024U * 1024U ); + while( beg < end ) { - base64::Base64ContextParser parser(buffer.data(), total_byte_size + 4); + base64_buffer.append( beg, end ); + beg = end; + icvXMLGetMultilineStringContent( fs, beg, beg, end ); + } + if ( !base64::base64_valid(base64_buffer.data(), 0U, base64_buffer.size()) ) + CV_PARSE_ERROR( "Invalid Base64 data." ); - /* decoding */ - while(beg < end) - { - /* save this part [beg, end) */ - parser.read((const uchar *)beg, (const uchar *)end); - beg = end; - /* find next part */ - icvXMLGetMultilineStringContent(fs, beg, beg, end); - } + /* alloc buffer for all decoded data(include header) */ + std::vector binary_buffer( base64::base64_decode_buffer_size(base64_buffer.size()) ); + int total_byte_size = base64::base64_decode_buffer_size( base64_buffer.size(), base64_buffer.data(), false ); + { + base64::Base64ContextParser parser(binary_buffer.data(), binary_buffer.size() ); + const uchar * buffer_beg = reinterpret_cast( base64_buffer.data() ); + const uchar * buffer_end = buffer_beg + base64_buffer.size(); + parser.read( buffer_beg, buffer_end ); + parser.flush(); } /* save as CvSeq */ int elem_size = ::icvCalcStructSize(dt.c_str(), 0); if (total_byte_size % elem_size != 0) - CV_PARSE_ERROR("Byte size not match elememt size"); + CV_PARSE_ERROR("data size not matches elememt size"); int elem_cnt = total_byte_size / elem_size; node->tag = CV_NODE_NONE; - int struct_flags = CV_NODE_SEQ; /* after icvFSCreateCollection, node->tag == struct_flags */ + int struct_flags = CV_NODE_SEQ; + /* after icvFSCreateCollection, node->tag == struct_flags */ icvFSCreateCollection(fs, struct_flags, node); - base64::make_seq(buffer.data(), elem_cnt, dt.c_str(), *node->data.seq); + base64::make_seq(binary_buffer.data(), elem_cnt, dt.c_str(), *node->data.seq); if (fs->dummy_eof) { /* end of file */ @@ -2757,6 +2979,16 @@ icvXMLStartNextStream( CvFileStorage* fs ) static void icvXMLWriteScalar( CvFileStorage* fs, const char* key, const char* data, int len ) { + check_if_write_struct_is_delayed( fs ); + if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::Uncertain ) + { + switch_to_Base64_state( fs, CvFileStorage::Base64State::NotUse ); + } + else if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::InUse ) + { + CV_Error( CV_StsError, "Currently only Base64 data is allowed." ); + } + if( CV_NODE_IS_MAP(fs->struct_flags) || (!CV_NODE_IS_COLLECTION(fs->struct_flags) && key) ) { @@ -2964,15 +3196,24 @@ icvXMLWriteComment( CvFileStorage* fs, const char* comment, int eol_comment ) \****************************************************************************************/ CV_IMPL CvFileStorage* -cvOpenFileStorage( const char* filename, CvMemStorage* dststorage, int flags, const char* encoding ) +cvOpenFileStorage( const char* query, CvMemStorage* dststorage, int flags, const char* encoding ) { CvFileStorage* fs = 0; int default_block_size = 1 << 18; bool append = (flags & 3) == CV_STORAGE_APPEND; bool mem = (flags & CV_STORAGE_MEMORY) != 0; bool write_mode = (flags & 3) != 0; + bool write_base64 = write_mode && (flags & CV_STORAGE_BASE64) != 0; bool isGZ = false; size_t fnamelen = 0; + const char * filename = 0; + + std::vector params = analyze_file_name( query ); + if ( !params.empty() ) + filename = params.begin()->c_str(); + + if (write_base64 == false && is_param_exist( params, std::string("base64") ) ) + write_base64 = true; if( !filename || filename[0] == '\0' ) { @@ -3073,6 +3314,16 @@ cvOpenFileStorage( const char* filename, CvMemStorage* dststorage, int flags, co fs->struct_flags = CV_NODE_EMPTY; fs->buffer_start = fs->buffer = (char*)cvAlloc( buf_size + 1024 ); fs->buffer_end = fs->buffer_start + buf_size; + + fs->base64_writer = 0; + fs->is_default_using_base64 = write_base64; + fs->state_of_writing_base64 = CvFileStorage::Base64State::Uncertain; + + fs->is_write_struct_delayed = false; + fs->delayed_struct_key = 0; + fs->delayed_struct_flags = 0; + fs->delayed_type_name = 0; + if( fs->fmt == CV_STORAGE_FORMAT_XML ) { size_t file_size = fs->file ? (size_t)ftell( fs->file ) : (size_t)0; @@ -3247,7 +3498,48 @@ cvStartWriteStruct( CvFileStorage* fs, const char* key, int struct_flags, const char* type_name, CvAttrList /*attributes*/ ) { CV_CHECK_OUTPUT_FILE_STORAGE(fs); - fs->start_write_struct( fs, key, struct_flags, type_name ); + check_if_write_struct_is_delayed( fs ); + if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::NotUse ) + switch_to_Base64_state( fs, CvFileStorage::Base64State::Uncertain ); + + if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::Uncertain + && + CV_NODE_IS_SEQ(struct_flags) + && + fs->is_default_using_base64 + && + type_name == 0 + ) + { + /* Uncertain if output Base64 data */ + make_write_struct_delayed( fs, key, struct_flags, type_name ); + } + else if ( type_name && memcmp(type_name, "binary", 6) == 0 ) + { + /* Must output Base64 data */ + if ( !CV_NODE_IS_SEQ(struct_flags) ) + CV_Error( CV_StsBadArg, "must set 'struct_flags |= CV_NODE_SEQ' if using Base64."); + else if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::Uncertain ) + CV_Error( CV_StsError, "function \'cvStartWriteStruct\' calls cannot be nested if using Base64."); + + fs->start_write_struct( fs, key, struct_flags, type_name ); + + if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::Uncertain ) + switch_to_Base64_state( fs, CvFileStorage::Base64State::Uncertain ); + switch_to_Base64_state( fs, CvFileStorage::Base64State::InUse ); + } + else + { + /* Won't output Base64 data */ + if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::InUse ) + CV_Error( CV_StsError, "At the end of the output Base64, `cvEndWriteStruct` is needed."); + + fs->start_write_struct( fs, key, struct_flags, type_name ); + + if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::Uncertain ) + switch_to_Base64_state( fs, CvFileStorage::Base64State::Uncertain ); + switch_to_Base64_state( fs, CvFileStorage::Base64State::NotUse ); + } } @@ -3255,6 +3547,11 @@ CV_IMPL void cvEndWriteStruct( CvFileStorage* fs ) { CV_CHECK_OUTPUT_FILE_STORAGE(fs); + check_if_write_struct_is_delayed( fs ); + + if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::Uncertain ) + switch_to_Base64_state( fs, CvFileStorage::Base64State::Uncertain ); + fs->end_write_struct( fs ); } @@ -3432,6 +3729,17 @@ icvDecodeSimpleFormat( const char* dt ) CV_IMPL void cvWriteRawData( CvFileStorage* fs, const void* _data, int len, const char* dt ) { + if (fs->is_default_using_base64 || + fs->state_of_writing_base64 == CvFileStorage::Base64State::InUse ) + { + base64::cvWriteRawDataBase64( fs, _data, len, dt ); + return; + } + else if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::Uncertain ) + { + switch_to_Base64_state( fs, CvFileStorage::Base64State::NotUse ); + } + const char* data0 = (const char*)_data; int offset = 0; int fmt_pairs[CV_FS_MAX_FMT_PAIRS*2], k, fmt_pair_count; @@ -3760,15 +4068,15 @@ icvWriteFileNode( CvFileStorage* fs, const char* name, const CvFileNode* node ) break; case CV_NODE_SEQ: case CV_NODE_MAP: - fs->start_write_struct( fs, name, CV_NODE_TYPE(node->tag) + + cvStartWriteStruct( fs, name, CV_NODE_TYPE(node->tag) + (CV_NODE_SEQ_IS_SIMPLE(node->data.seq) ? CV_NODE_FLOW : 0), node->info ? node->info->type_name : 0 ); icvWriteCollection( fs, node ); - fs->end_write_struct( fs ); + cvEndWriteStruct( fs ); break; case CV_NODE_NONE: - fs->start_write_struct( fs, name, CV_NODE_SEQ, 0 ); - fs->end_write_struct( fs ); + cvStartWriteStruct( fs, name, CV_NODE_SEQ, 0 ); + cvEndWriteStruct( fs ); break; default: CV_Error( CV_StsBadFlag, "Unknown type of file node" ); @@ -6226,14 +6534,29 @@ bool base64::base64_valid(char const * src, size_t off, size_t cnt) return base64_valid(reinterpret_cast(src), off, cnt); } -size_t base64::base64_encode_buffer_size(size_t cnt) +size_t base64::base64_encode_buffer_size(size_t cnt, bool is_end_with_zero) { - return size_t((cnt + 2U) / 3U * 4U + 1U); + size_t additional = static_cast(is_end_with_zero == true); + return (cnt + 2U) / 3U * 4U + additional; } -size_t base64::base64_decode_buffer_size(size_t cnt) +size_t base64::base64_decode_buffer_size(size_t cnt, bool is_end_with_zero) { - return size_t(cnt / 4U * 3U + 1U); + size_t additional = static_cast(is_end_with_zero == true); + return cnt / 4U * 3U + additional; +} + +size_t base64::base64_decode_buffer_size(size_t cnt, char const * src, bool is_end_with_zero) +{ + return base64_decode_buffer_size(cnt, reinterpret_cast(src), is_end_with_zero); +} + +size_t base64::base64_decode_buffer_size(size_t cnt, uchar const * src, bool is_end_with_zero) +{ + size_t padding_cnt = 0U; + for (uchar const * ptr = src + cnt - 1U; *ptr == base64_padding; ptr--) + padding_cnt ++; + return base64_decode_buffer_size(cnt, is_end_with_zero) - padding_cnt; } /**************************************************************************** @@ -6308,13 +6631,10 @@ binary_to(uchar const * cur, uchar * val) * others ***************************************************************************/ -std::string base64::make_base64_header(int byte_size, const char * dt) +std::string base64::make_base64_header(const char * dt) { - int size = byte_size; - std::ostringstream oss; - oss << size << ' ' - << dt << ' '; + oss << dt << ' '; std::string buffer(oss.str()); CV_Assert(buffer.size() < HEADER_SIZE); @@ -6325,10 +6645,10 @@ std::string base64::make_base64_header(int byte_size, const char * dt) return buffer; } -bool base64::read_base64_header(std::string const & header, int & byte_size, std::string & dt) +bool base64::read_base64_header(std::vector const & header, std::string & dt) { - std::istringstream iss(header); - return static_cast(iss >> byte_size >> dt); + std::istringstream iss(header.data()); + return static_cast(iss >> dt); } /**************************************************************************** @@ -6351,10 +6671,9 @@ base64::Base64ContextParser::Base64ContextParser(uchar * buffer, size_t size) base64::Base64ContextParser::~Base64ContextParser() { - if (src_cur != src_beg) { - /* encode the rest binary data to base64 buffer */ + /* encode the rest binary data to base64 buffer */ + if (src_cur != src_beg) flush(); - } } base64::Base64ContextParser & base64::Base64ContextParser:: @@ -6383,9 +6702,12 @@ read(const uchar * beg, const uchar * end) bool base64::Base64ContextParser::flush() { - if (!base64_valid(src_beg, 0U, src_cur - src_beg)) + if ( !base64_valid(src_beg, 0U, src_cur - src_beg) ) return false; + if ( src_cur == src_beg ) + return true; + uchar * buffer = binary_buffer.data(); size_t len = base64_decode(src_beg, buffer, 0U, src_cur - src_beg); src_cur = src_beg; @@ -6514,7 +6836,7 @@ public: private: /* because of Base64, we must keep its length a multiple of 3 */ - static const size_t BUFFER_LEN = 51U; + static const size_t BUFFER_LEN = 48U; // static_assert(BUFFER_LEN % 3 == 0, "BUFFER_LEN is invalid"); private: @@ -6584,17 +6906,17 @@ public: { CV_DbgAssert(*this); - /* copy to dst */ + /* [1]copy to dst */ dst += to_binary_func(row_begin + x, dst); - /* move to next */ + /* [2]move to next */ x += step; if (x >= x_max) { - /* when x arrive end, reset it and increase y */ + /* when x arrive end, reset x and increase y */ x = 0U; ++ y; if (y >= y_max) { - /* when y arrive end, reset it and increase iter */ + /* when y arrive end, reset y and increase iter */ y = 0U; ++ mat_iter; if (mat_iter == mats.end()) { @@ -6935,68 +7257,55 @@ private: * Wapper ***************************************************************************/ -class base64::Base64Writer + +base64::Base64Writer::Base64Writer(::CvFileStorage * fs) + : file_storage(fs) + , emitter(new Base64ContextEmitter(fs)) + , data_type_string() { -public: + CV_CHECK_OUTPUT_FILE_STORAGE(fs); + icvFSFlush(fs); +} - Base64Writer(::CvFileStorage * fs, const char * name, int len, const char* dt) - : file_storage(fs) - , emitter(fs) - , remaining_data_length(len) - , data_type_string(dt) - { - CV_CHECK_OUTPUT_FILE_STORAGE(fs); +void base64::Base64Writer::write(const void* _data, size_t len, const char* dt) +{ + check_dt(dt); - cvStartWriteStruct(fs, name, CV_NODE_SEQ, "binary"); - icvFSFlush(fs); + RawDataToBinaryConvertor convertor( + _data, static_cast(len), data_type_string.c_str() + ); + emitter->write(convertor); +} + +template inline +void base64::Base64Writer::write(_to_binary_convertor_t & convertor, const char* dt) +{ + check_dt(dt); + emitter->write(convertor); +} + +base64::Base64Writer::~Base64Writer() +{ + delete emitter; +} + +void base64::Base64Writer::check_dt(const char* dt) +{ + if ( dt == 0 ) + CV_Error( CV_StsBadArg, "Invalid \'dt\'." ); + else if (data_type_string.empty()) { + data_type_string = dt; /* output header */ - - /* total byte size(before encode) */ - int size = len * ::icvCalcStructSize(dt, 0); - - std::string buffer = make_base64_header(size, dt); + std::string buffer = make_base64_header(dt); const uchar * beg = reinterpret_cast(buffer.data()); const uchar * end = beg + buffer.size(); - emitter.write(beg, end); - } + emitter->write(beg, end); + } else if ( data_type_string != dt ) + CV_Error( CV_StsBadArg, "\'dt\' does not match." ); +} - void write(const void* _data, int len) - { - CV_Assert(len >= 0); - CV_Assert(remaining_data_length >= static_cast(len)); - remaining_data_length -= static_cast(len); - - RawDataToBinaryConvertor convertor(_data, len, data_type_string); - emitter.write(convertor); - } - - template inline - void write(_to_binary_convertor_t & convertor, int data_length_of_convertor) - { - CV_Assert(data_length_of_convertor >= 0); - CV_Assert(remaining_data_length >= static_cast(data_length_of_convertor)); - remaining_data_length -= static_cast(data_length_of_convertor); - - emitter.write(convertor); - } - - ~Base64Writer() - { - CV_Assert(remaining_data_length == 0U); - emitter.flush(); - cvEndWriteStruct(file_storage); - icvFSFlush(file_storage); - } - -private: - - ::CvFileStorage * file_storage; - Base64ContextEmitter emitter; - size_t remaining_data_length; - const char* data_type_string; -}; void base64::make_seq(void * binary, int elem_cnt, const char * dt, ::CvSeq & seq) { @@ -7009,51 +7318,23 @@ void base64::make_seq(void * binary, int elem_cnt, const char * dt, ::CvSeq & se } } -void base64::cvStartWriteRawData_Base64(::CvFileStorage * fs, const char* name, int len, const char* dt) +void base64::cvWriteRawDataBase64(::CvFileStorage* fs, const void* _data, int len, const char* dt) { CV_Assert(fs); CV_CHECK_OUTPUT_FILE_STORAGE(fs); - CV_Assert(fs->base64_writer == 0); - fs->base64_writer = new Base64Writer(fs, name, len, dt); -} -void base64::cvWriteRawData_Base64(::CvFileStorage * fs, const void* _data, int len) -{ - CV_Assert(fs); - CV_CHECK_OUTPUT_FILE_STORAGE(fs); - CV_Assert(fs->base64_writer != 0); - fs->base64_writer->write(_data, len); -} + check_if_write_struct_is_delayed( fs, true ); -void base64::cvEndWriteRawData_Base64(::CvFileStorage * fs) -{ - CV_Assert(fs); - CV_CHECK_OUTPUT_FILE_STORAGE(fs); - CV_Assert(fs->base64_writer != 0); - delete fs->base64_writer; - fs->base64_writer = 0; -} - -void base64::cvWriteRawData_Base64(::cv::FileStorage & fs, const void* _data, int len, const char* dt) -{ - cvStartWriteStruct(*fs, fs.elname.c_str(), CV_NODE_SEQ, "binary"); + if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::Uncertain ) { - Base64ContextEmitter emitter(*fs); - { /* header */ - /* total byte size(before encode) */ - int size = len * ::icvCalcStructSize(dt, 0); - std::string buffer = make_base64_header(size, dt); - const uchar * beg = reinterpret_cast(buffer.data()); - const uchar * end = beg + buffer.size(); - - emitter.write(beg, end); - } - { /* body */ - RawDataToBinaryConvertor convert(_data, len, dt); - emitter.write(convert); - } + switch_to_Base64_state( fs, CvFileStorage::Base64State::InUse ); } - cvEndWriteStruct(*fs); + else if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::InUse ) + { + CV_Error( CV_StsError, "Base64 should not be used at present." ); + } + + fs->base64_writer->write(_data, len, dt); } void base64::cvWriteMat_Base64(::CvFileStorage * fs, const char * name, ::cv::Mat const & mat) @@ -7064,31 +7345,30 @@ void base64::cvWriteMat_Base64(::CvFileStorage * fs, const char * name, ::cv::Ma { /* [1]output other attr */ if (mat.dims <= 2) { - cvStartWriteStruct(fs, name, CV_NODE_MAP, CV_TYPE_NAME_MAT); + ::cvStartWriteStruct(fs, name, CV_NODE_MAP, CV_TYPE_NAME_MAT); - cvWriteInt(fs, "rows", mat.rows ); - cvWriteInt(fs, "cols", mat.cols ); + ::cvWriteInt(fs, "rows", mat.rows ); + ::cvWriteInt(fs, "cols", mat.cols ); } else { - cvStartWriteStruct(fs, name, CV_NODE_MAP, CV_TYPE_NAME_MATND); + ::cvStartWriteStruct(fs, name, CV_NODE_MAP, CV_TYPE_NAME_MATND); - cvStartWriteStruct(fs, "sizes", CV_NODE_SEQ | CV_NODE_FLOW); - cvWriteRawData(fs, mat.size.p, mat.dims, "i"); - cvEndWriteStruct(fs); + ::cvStartWriteStruct(fs, "sizes", CV_NODE_SEQ | CV_NODE_FLOW); + ::cvWriteRawData(fs, mat.size.p, mat.dims, "i"); + ::cvEndWriteStruct(fs); } - cvWriteString(fs, "dt", ::icvEncodeFormat(CV_MAT_TYPE(mat.type()), dt ), 0 ); + ::cvWriteString(fs, "dt", ::icvEncodeFormat(CV_MAT_TYPE(mat.type()), dt ), 0 ); } { /* [2]deal with matrix's data */ - int len = static_cast(mat.total()); MatToBinaryConvertor convertor(mat); - cvStartWriteRawData_Base64(fs, "data", len, dt); - fs->base64_writer->write(convertor, len); - cvEndWriteRawData_Base64(fs); + ::cvStartWriteStruct(fs, "data", CV_NODE_SEQ, "binary"); + fs->base64_writer->write(convertor, dt); + ::cvEndWriteStruct(fs); } { /* [3]output end */ - cvEndWriteStruct(fs); + ::cvEndWriteStruct(fs); } } @@ -7096,36 +7376,21 @@ void base64::cvWriteMat_Base64(::CvFileStorage * fs, const char * name, ::cv::Ma * Interface ***************************************************************************/ -namespace cv +CV_IMPL void cvWriteMatBase64(::CvFileStorage* fs, const char* name, const ::CvMat* mat) { - void cvWriteMat_Base64(::CvFileStorage* fs, const char* name, const ::CvMat* mat) - { - ::cv::Mat holder = ::cv::cvarrToMat(mat); - ::base64::cvWriteMat_Base64(fs, name, holder); - } - - void cvWriteMatND_Base64(::CvFileStorage* fs, const char* name, const ::CvMatND* mat) - { - ::cv::Mat holder = ::cv::cvarrToMat(mat); - ::base64::cvWriteMat_Base64(fs, name, holder); - } - - void cvStartWriteRawData_Base64(::CvFileStorage * fs, const char* name, int len, const char* dt) - { - ::base64::cvStartWriteRawData_Base64(fs, name, len, dt); - } - - void cvWriteRawData_Base64(::CvFileStorage * fs, const void* _data, int len) - { - ::base64::cvWriteRawData_Base64(fs, _data, len); - } - - void cvEndWriteRawData_Base64(::CvFileStorage * fs) - { - ::base64::cvEndWriteRawData_Base64(fs); - } - + ::cv::Mat holder = ::cv::cvarrToMat(mat); + ::base64::cvWriteMat_Base64(fs, name, holder); } +CV_IMPL void cvWriteMatNDBase64(::CvFileStorage* fs, const char* name, const ::CvMatND* mat) +{ + ::cv::Mat holder = ::cv::cvarrToMat(mat); + ::base64::cvWriteMat_Base64(fs, name, holder); +} + +CV_IMPL void cvWriteRawDataBase64(::CvFileStorage* fs, const void* _data, int len, const char* dt) +{ + ::base64::cvWriteRawDataBase64(fs, _data, len, dt); +} /* End of file. */ diff --git a/modules/core/test/test_io.cpp b/modules/core/test/test_io.cpp index f2c53dc964..1f7191a669 100644 --- a/modules/core/test/test_io.cpp +++ b/modules/core/test/test_io.cpp @@ -593,150 +593,3 @@ TEST(Core_InputOutput, FileStorageSpaces) ASSERT_STREQ(values[i].c_str(), valuesRead[i].c_str()); } } - -TEST(Core_InputOutput, filestorage_yml_compatibility) -{ - // TODO: -} - -class CV_Base64IOTest : public cvtest::BaseTest -{ -private: - std::string file_name; - - struct data_t - { - uchar u1, u2; - int i1, i2, i3; - double d1, d2; - int i4; - }; - -public: - CV_Base64IOTest(std::string const & test_file_name) - : file_name(test_file_name) {} - ~CV_Base64IOTest() {} -protected: - void run(int) - { - try - { - std::vector rawdata; - - cv::Mat _em_out, _em_in; - cv::Mat _2d_out, _2d_in; - cv::Mat _nd_out, _nd_in; - - { /* init */ - - /* normal mat */ - _2d_out = cv::Mat(100, 100, CV_8UC3, cvScalar(1U, 2U, 127U)); - for (int i = 0; i < _2d_out.rows; ++i) - for (int j = 0; j < _2d_out.cols; ++j) - _2d_out.at(i, j)[1] = (i + j) % 256; - - /* 4d mat */ - const int Size[] = {4, 4, 4, 4}; - cv::Mat _4d(4, Size, CV_64FC4, cvScalar(0.888, 0.111, 0.666, 0.444)); - const cv::Range ranges[] = { - cv::Range(0, 2), - cv::Range(0, 2), - cv::Range(1, 2), - cv::Range(0, 2) }; - _nd_out = _4d(ranges); - - /* raw data */ - for (int i = 0; i < 1000; i++) { - data_t tmp; - tmp.u1 = 1; - tmp.u2 = 2; - tmp.i1 = 1; - tmp.i2 = 2; - tmp.i3 = 3; - tmp.d1 = 0.1; - tmp.d2 = 0.2; - tmp.i4 = i; - rawdata.push_back(tmp); - } - } - - { /* write */ - cv::FileStorage fs(file_name, cv::FileStorage::WRITE); - CvMat holder = _2d_out; - cv::cvWriteMat_Base64(*fs, "normal_2d_mat", &holder); - CvMatND holder_nd = _nd_out; - cv::cvWriteMatND_Base64(*fs, "normal_nd_mat", &holder_nd); - holder = _em_out; - cv::cvWriteMat_Base64(*fs, "empty_2d_mat", &holder); - - cv::cvStartWriteRawData_Base64(*fs, "rawdata", static_cast(rawdata.size()), "2u3i2di"); - for (int i = 0; i < 10; i++) - cv::cvWriteRawData_Base64(*fs, rawdata.data() + i * 100, 100); - cv::cvEndWriteRawData_Base64(*fs); - - fs.release(); - } - - { /* read */ - cv::FileStorage fs(file_name, cv::FileStorage::READ); - - /* mat */ - fs["empty_2d_mat"] >> _em_in; - fs["normal_2d_mat"] >> _2d_in; - fs["normal_nd_mat"] >> _nd_in; - - /* raw data */ - std::vector(1000).swap(rawdata); - cvReadRawData(*fs, fs["rawdata"].node, rawdata.data(), "2u3i2di"); - - fs.release(); - } - - for (int i = 0; i < 1000; i++) { - // TODO: Solve this bug in `cvReadRawData` - //EXPECT_EQ(rawdata[i].u1, 1); - //EXPECT_EQ(rawdata[i].u2, 2); - //EXPECT_EQ(rawdata[i].i1, 1); - //EXPECT_EQ(rawdata[i].i2, 2); - //EXPECT_EQ(rawdata[i].i3, 3); - //EXPECT_EQ(rawdata[i].d1, 0.1); - //EXPECT_EQ(rawdata[i].d2, 0.2); - //EXPECT_EQ(rawdata[i].i4, i); - } - - EXPECT_EQ(_em_in.rows , _em_out.rows); - EXPECT_EQ(_em_in.cols , _em_out.cols); - EXPECT_EQ(_em_in.dims , _em_out.dims); - EXPECT_EQ(_em_in.depth(), _em_out.depth()); - EXPECT_TRUE(_em_in.empty()); - - EXPECT_EQ(_2d_in.rows , _2d_in.rows); - EXPECT_EQ(_2d_in.cols , _2d_in.cols); - EXPECT_EQ(_2d_in.dims , _2d_in.dims); - EXPECT_EQ(_2d_in.depth(), _2d_in.depth()); - for(int i = 0; i < _2d_in.rows; ++i) - for (int j = 0; j < _2d_in.cols; ++j) - EXPECT_EQ(_2d_in.at(i, j), _2d_out.at(i, j)); - - EXPECT_EQ(_nd_in.rows , _nd_in.rows); - EXPECT_EQ(_nd_in.cols , _nd_in.cols); - EXPECT_EQ(_nd_in.dims , _nd_in.dims); - EXPECT_EQ(_nd_in.depth(), _nd_in.depth()); - EXPECT_EQ(cv::countNonZero(cv::mean(_nd_in != _nd_out)), 0); - } - catch(...) - { - ts->set_failed_test_info(cvtest::TS::FAIL_MISMATCH); - } - } -}; - -TEST(Core_InputOutput, filestorage_yml_base64) -{ - CV_Base64IOTest test("base64_test_tmp_file.yml"); test.safe_run(); -} - -TEST(Core_InputOutput, filestorage_xml_base64) -{ - CV_Base64IOTest test("base64_test_tmp_file.xml"); test.safe_run(); -} diff --git a/modules/core/test/test_io_base64.cpp b/modules/core/test/test_io_base64.cpp new file mode 100644 index 0000000000..d619106eae --- /dev/null +++ b/modules/core/test/test_io_base64.cpp @@ -0,0 +1,270 @@ +#include "test_precomp.hpp" + +using namespace cv; +using namespace std; + +struct data_t +{ + typedef uchar u; + typedef char b; + typedef ushort w; + typedef short s; + typedef int i; + typedef float f; + typedef double d; + + u u1 ;u u2 ; i i1 ; + i i2 ;i i3 ; + d d1 ; + d d2 ; + i i4 ; + + static inline const char * signature() { return "2u3i2di"; } +}; + + +TEST(Core_InputOutput_Base64, basic) +{ + char const * filenames[] = { + "core_io_base64_basic_test.yml", + "core_io_base64_basic_test.xml", + 0 + }; + + for (char const ** ptr = filenames; *ptr; ptr++) + { + char const * name = *ptr; + + std::vector rawdata; + + cv::Mat _em_out, _em_in; + cv::Mat _2d_out, _2d_in; + cv::Mat _nd_out, _nd_in; + cv::Mat _rd_out(64, 64, CV_64FC1), _rd_in; + + { /* init */ + + /* a normal mat */ + _2d_out = cv::Mat(100, 100, CV_8UC3, cvScalar(1U, 2U, 127U)); + for (int i = 0; i < _2d_out.rows; ++i) + for (int j = 0; j < _2d_out.cols; ++j) + _2d_out.at(i, j)[1] = (i + j) % 256; + + /* a 4d mat */ + const int Size[] = {4, 4, 4, 4}; + cv::Mat _4d(4, Size, CV_64FC4, cvScalar(0.888, 0.111, 0.666, 0.444)); + const cv::Range ranges[] = { + cv::Range(0, 2), + cv::Range(0, 2), + cv::Range(1, 2), + cv::Range(0, 2) }; + _nd_out = _4d(ranges); + + /* a random mat */ + cv::randu(_rd_out, cv::Scalar(0.0), cv::Scalar(1.0)); + + /* raw data */ + for (int i = 0; i < 1000; i++) { + data_t tmp; + tmp.u1 = 1; + tmp.u2 = 2; + tmp.i1 = 1; + tmp.i2 = 2; + tmp.i3 = 3; + tmp.d1 = 0.1; + tmp.d2 = 0.2; + tmp.i4 = i; + rawdata.push_back(tmp); + } + } + + { /* write */ + cv::FileStorage fs(name, cv::FileStorage::WRITE_BASE64); + fs << "normal_2d_mat" << _2d_out; + fs << "normal_nd_mat" << _nd_out; + fs << "empty_2d_mat" << _em_out; + fs << "random_mat" << _rd_out; + + cvStartWriteStruct( *fs, "rawdata", CV_NODE_SEQ | CV_NODE_FLOW, "binary" ); + for (int i = 0; i < 10; i++) + cvWriteRawDataBase64(*fs, rawdata.data() + i * 100, 100, data_t::signature()); + cvEndWriteStruct( *fs ); + + fs.release(); + } + + { /* read */ + cv::FileStorage fs(name, cv::FileStorage::READ); + + /* mat */ + fs["empty_2d_mat"] >> _em_in; + fs["normal_2d_mat"] >> _2d_in; + fs["normal_nd_mat"] >> _nd_in; + fs["random_mat"] >> _rd_in; + + /* raw data */ + std::vector(1000).swap(rawdata); + cvReadRawData(*fs, fs["rawdata"].node, rawdata.data(), data_t::signature()); + + fs.release(); + } + + for (int i = 0; i < 1000; i++) { + // TODO: Solve this bug in `cvReadRawData` + //EXPECT_EQ(rawdata[i].u1, 1); + //EXPECT_EQ(rawdata[i].u2, 2); + //EXPECT_EQ(rawdata[i].i1, 1); + //EXPECT_EQ(rawdata[i].i2, 2); + //EXPECT_EQ(rawdata[i].i3, 3); + //EXPECT_EQ(rawdata[i].d1, 0.1); + //EXPECT_EQ(rawdata[i].d2, 0.2); + //EXPECT_EQ(rawdata[i].i4, i); + } + + EXPECT_EQ(_em_in.rows , _em_out.rows); + EXPECT_EQ(_em_in.cols , _em_out.cols); + EXPECT_EQ(_em_in.dims , _em_out.dims); + EXPECT_EQ(_em_in.depth(), _em_out.depth()); + EXPECT_TRUE(_em_in.empty()); + + EXPECT_EQ(_2d_in.rows , _2d_out.rows); + EXPECT_EQ(_2d_in.cols , _2d_out.cols); + EXPECT_EQ(_2d_in.dims , _2d_out.dims); + EXPECT_EQ(_2d_in.depth(), _2d_out.depth()); + for(int i = 0; i < _2d_out.rows; ++i) + for (int j = 0; j < _2d_out.cols; ++j) + EXPECT_EQ(_2d_in.at(i, j), _2d_out.at(i, j)); + + EXPECT_EQ(_nd_in.rows , _nd_out.rows); + EXPECT_EQ(_nd_in.cols , _nd_out.cols); + EXPECT_EQ(_nd_in.dims , _nd_out.dims); + EXPECT_EQ(_nd_in.depth(), _nd_out.depth()); + EXPECT_EQ(cv::countNonZero(cv::mean(_nd_in != _nd_out)), 0); + + EXPECT_EQ(_rd_in.rows , _rd_out.rows); + EXPECT_EQ(_rd_in.cols , _rd_out.cols); + EXPECT_EQ(_rd_in.dims , _rd_out.dims); + EXPECT_EQ(_rd_in.depth(), _rd_out.depth()); + EXPECT_EQ(cv::countNonZero(cv::mean(_rd_in != _rd_out)), 0); + + remove(name); + } +} + +TEST(Core_InputOutput_Base64, valid) +{ + char const * filenames[] = { + "core_io_base64_other_test.yml", + "core_io_base64_other_test.xml", + "core_io_base64_other_test.yml?base64", + "core_io_base64_other_test.xml?base64", + 0 + }; + char const * real_name[] = { + "core_io_base64_other_test.yml", + "core_io_base64_other_test.xml", + "core_io_base64_other_test.yml", + "core_io_base64_other_test.xml", + 0 + }; + + std::vector rawdata(10, static_cast(0x00010203)); + cv::String str_out = "test_string"; + + for (char const ** ptr = filenames; *ptr; ptr++) + { + char const * name = *ptr; + + EXPECT_NO_THROW( + { + cv::FileStorage fs(name, cv::FileStorage::WRITE_BASE64); + + cvStartWriteStruct(*fs, "manydata", CV_NODE_SEQ); + cvStartWriteStruct(*fs, 0, CV_NODE_SEQ | CV_NODE_FLOW); + for (int i = 0; i < 10; i++) + cvWriteRawData(*fs, rawdata.data(), rawdata.size(), "i"); + cvEndWriteStruct(*fs); + cvWriteString(*fs, 0, str_out.c_str(), 1); + cvEndWriteStruct(*fs); + + fs.release(); + }); + + { + cv::FileStorage fs(name, cv::FileStorage::READ); + std::vector data_in(rawdata.size()); + fs["manydata"][0].readRaw("i", (uchar *)data_in.data(), data_in.size()); + EXPECT_TRUE(fs["manydata"][0].isSeq()); + EXPECT_TRUE(std::equal(rawdata.begin(), rawdata.end(), data_in.begin())); + cv::String str_in; + fs["manydata"][1] >> str_in; + EXPECT_TRUE(fs["manydata"][1].isString()); + EXPECT_EQ(str_in, str_out); + fs.release(); + } + + EXPECT_NO_THROW( + { + cv::FileStorage fs(name, cv::FileStorage::WRITE); + + cvStartWriteStruct(*fs, "manydata", CV_NODE_SEQ); + cvWriteString(*fs, 0, str_out.c_str(), 1); + cvStartWriteStruct(*fs, 0, CV_NODE_SEQ | CV_NODE_FLOW, "binary"); + for (int i = 0; i < 10; i++) + cvWriteRawData(*fs, rawdata.data(), rawdata.size(), "i"); + cvEndWriteStruct(*fs); + cvEndWriteStruct(*fs); + + fs.release(); + }); + + { + cv::FileStorage fs(name, cv::FileStorage::READ); + cv::String str_in; + fs["manydata"][0] >> str_in; + EXPECT_TRUE(fs["manydata"][0].isString()); + EXPECT_EQ(str_in, str_out); + std::vector data_in(rawdata.size()); + fs["manydata"][1].readRaw("i", (uchar *)data_in.data(), data_in.size()); + EXPECT_TRUE(fs["manydata"][1].isSeq()); + EXPECT_TRUE(std::equal(rawdata.begin(), rawdata.end(), data_in.begin())); + fs.release(); + } + + remove(real_name[ptr - filenames]); + } +} + +TEST(Core_InputOutput_Base64, invalid) +{ + char const * filenames[] = { + "core_io_base64_other_test.yml", + "core_io_base64_other_test.xml", + 0 + }; + + for (char const ** ptr = filenames; *ptr; ptr++) + { + char const * name = *ptr; + + EXPECT_ANY_THROW({ + cv::FileStorage fs(name, cv::FileStorage::WRITE); + cvStartWriteStruct(*fs, "rawdata", CV_NODE_SEQ, "binary"); + cvStartWriteStruct(*fs, 0, CV_NODE_SEQ | CV_NODE_FLOW); + }); + + EXPECT_ANY_THROW({ + cv::FileStorage fs(name, cv::FileStorage::WRITE); + cvStartWriteStruct(*fs, "rawdata", CV_NODE_SEQ); + cvStartWriteStruct(*fs, 0, CV_NODE_SEQ | CV_NODE_FLOW); + cvWriteRawDataBase64(*fs, name, 1, "u"); + }); + + remove(name); + } +} + +TEST(Core_InputOutput_Base64, TODO_compatibility) +{ + // TODO: +} diff --git a/modules/ml/test/test_save_load.cpp b/modules/ml/test/test_save_load.cpp index d127471cc9..7c2a7c4a6c 100644 --- a/modules/ml/test/test_save_load.cpp +++ b/modules/ml/test/test_save_load.cpp @@ -65,11 +65,11 @@ int CV_SLMLTest::run_test_case( int testCaseIdx ) { get_test_error( testCaseIdx, &test_resps1 ); fname1 = tempfile(".yml.gz"); - save( fname1.c_str() ); + save( (fname1 + "?base64").c_str() ); load( fname1.c_str() ); get_test_error( testCaseIdx, &test_resps2 ); fname2 = tempfile(".yml.gz"); - save( fname2.c_str() ); + save( (fname2 + "?base64").c_str() ); } else ts->printf( cvtest::TS::LOG, "model can not be trained" ); @@ -280,7 +280,7 @@ TEST(DISABLED_ML_SVM, linear_save_load) svm1 = Algorithm::load("SVM45_X_38-1.xml"); svm2 = Algorithm::load("SVM45_X_38-2.xml"); string tname = tempfile("a.xml"); - svm2->save(tname); + svm2->save(tname + "?base64"); svm3 = Algorithm::load(tname); ASSERT_EQ(svm1->getVarCount(), svm2->getVarCount());