mirror/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2024-11-24 03:00:14 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Modify Base64 functions and add test and documentation

Browse Source 
Major changes:

- modify the Base64 functions to compatible with `cvWriteRawData` and so
on.
- add a Base64 flag for FileStorage and outputs raw data in Base64
automatically.
- complete all  testing and documentation.
This commit is contained in:
MYLS 2016-07-19 15:54:38 +08:00
parent a9b4cdd36f
commit 617df09143
8 changed files with 911 additions and 378 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
modules/core/include/opencv2/core/core_c.h
View File

@ -1976,8 +1976,19 @@ CVAPI(void) cvSetIPLAllocators( Cv_iplCreateImageHeader create_header,
The function opens file storage for reading or writing data. In the latter case, a new file is
created or an existing file is rewritten. The type of the read or written file is determined by the
filename extension: .xml for XML and .yml or .yaml for YAML. The function returns a pointer to the
CvFileStorage structure. If the file cannot be opened then the function returns NULL.
filename extension: .xml for XML and .yml or .yaml for YAML.
At the same time, it also supports adding parameters like "example.xml?base64".
@code
CvFileStorage* fs = cvOpenFileStorage( "example.yml?base64", 0, CV_STORAGE_WRITE );
@endcode
it's exactly the same as
@code
CvFileStorage* fs = cvOpenFileStorage( "example.yml", 0, CV_STORAGE_WRITE_BASE64 );
@endcode
The function returns a pointer to the CvFileStorage structure.
If the file cannot be opened then the function returns NULL.
@param filename Name of the file associated with the storage
@param memstorage Memory storage used for temporary data and for
: storing dynamic structures, such as CvSeq or CvGraph . If it is NULL, a temporary memory
@ -1985,6 +1996,7 @@ CvFileStorage structure. If the file cannot be opened then the function returns
@param flags Can be one of the following:
> - **CV_STORAGE_READ** the storage is open for reading
> - **CV_STORAGE_WRITE** the storage is open for writing
(use **CV_STORAGE_WRITE | CV_STORAGE_WRITE_BASE64** to write rawdata in Base64)
@param encoding
*/
CVAPI(CvFileStorage*) cvOpenFileStorage( const char* filename, CvMemStorage* memstorage,
@ -2162,7 +2174,7 @@ the file with multiple streams looks like this:
@endcode
The YAML file will look like this:
@code{.yaml}
%YAML:1.0
%YAML 1.0
# stream #1 data
...
---
@ -2187,6 +2199,46 @@ to a sequence rather than a map.
CVAPI(void) cvWriteRawData( CvFileStorage* fs, const void* src,
int len, const char* dt );
/** @brief Writes multiple numbers in Base64.
If either CV_STORAGE_WRITE_BASE64 or cv::FileStorage::WRITE_BASE64 is used,
this function will be the same as cvWriteRawData. If neither, the main
difference is that it outputs a sequence in Base64 encoding rather than
in plain text.
This function can only be used to write a sequence with a type "binary".
Consider the following two examples where their output is the same:
@code
std::vector<int> rawdata(10, 0x00010203);
// without the flag CV_STORAGE_WRITE_BASE64.
CvFileStorage* fs = cvOpenFileStorage( "example.yml", 0, CV_STORAGE_WRITE );
// both CV_NODE_SEQ and "binary" are necessary.
cvStartWriteStruct(fs, "rawdata", CV_NODE_SEQ | CV_NODE_FLOW, "binary");
cvWriteRawDataBase64(fs, rawdata.data(), rawdata.size(), "i");
cvEndWriteStruct(fs);
cvReleaseFileStorage( &fs );
@endcode
and
@code
std::vector<int> rawdata(10, 0x00010203);
// with the flag CV_STORAGE_WRITE_BASE64.
CvFileStorage* fs = cvOpenFileStorage( "example.yml", 0, CV_STORAGE_WRITE_BASE64);
// parameter, typename "binary" could be omitted.
cvStartWriteStruct(fs, "rawdata", CV_NODE_SEQ | CV_NODE_FLOW);
cvWriteRawData(fs, rawdata.data(), rawdata.size(), "i");
cvEndWriteStruct(fs);
cvReleaseFileStorage( &fs );
@endcode
@param fs File storage
@param src Pointer to the written array
@param len Number of the array elements to write
@param dt Specification of each array element, see @ref format_spec "format specification"
*/
CVAPI(void) cvWriteRawDataBase64( CvFileStorage* fs, const void* _data,
int len, const char* dt );
/** @brief Returns a unique pointer for a given name.
The function returns a unique pointer for each particular file node name. This pointer can be then

9
modules/core/include/opencv2/core/persistence.hpp
View File

@ -311,7 +311,10 @@ public:
FORMAT_MASK = (7<<3), //!< mask for format flags
FORMAT_AUTO = 0, //!< flag, auto format
FORMAT_XML = (1<<3), //!< flag, XML format
FORMAT_YAML = (2<<3) //!< flag, YAML format
FORMAT_YAML = (2<<3), //!< flag, YAML format
BASE64 = 64, //!< flag, write rawdata in Base64 by default. (consider using WRITE_BASE64)
WRITE_BASE64 = BASE64 | WRITE, //!< flag, enable both WRITE and BASE64
};
enum
{
@ -354,7 +357,9 @@ public:
Extension of the file (.xml or .yml/.yaml) determines its format (XML or YAML respectively).
Also you can append .gz to work with compressed files, for example myHugeMatrix.xml.gz. If both
FileStorage::WRITE and FileStorage::MEMORY flags are specified, source is used just to specify
the output file format (e.g. mydata.xml, .yml etc.).
the output file format (e.g. mydata.xml, .yml etc.). A file name can also contain parameters.
You can use this format, "*?base64" (e.g. "file.xml?base64"), as an alternative to
FileStorage::BASE64 flag. Note: it is case sensitive.
@param flags Mode of operation. One of FileStorage::Mode
@param encoding Encoding of the file. Note that UTF-16 XML encoding is not supported currently and
you should use 8-bit encoding instead of it.

2
modules/core/include/opencv2/core/types_c.h
View File

@ -1669,6 +1669,8 @@ typedef struct CvFileStorage CvFileStorage;
#define CV_STORAGE_FORMAT_AUTO 0
#define CV_STORAGE_FORMAT_XML 8
#define CV_STORAGE_FORMAT_YAML 16
#define CV_STORAGE_BASE64 64
#define CV_STORAGE_WRITE_BASE64 (CV_STORAGE_BASE64 | CV_STORAGE_WRITE)
/** @brief List of attributes. :

86
modules/core/perf/perf_io_base64.cpp Normal file
View File

@ -0,0 +1,86 @@
#include "perf_precomp.hpp"
using namespace std;
using namespace cv;
using namespace perf;
using std::tr1::make_tuple;
using std::tr1::get;
typedef std::tr1::tuple<cv::Size, MatType, String> Size_MatType_Str_t;
typedef TestBaseWithParam<Size_MatType_Str_t> Size_Mat_StrType;
#define MAT_SIZES ::perf::sz1080p/*, ::perf::sz4320p*/
#define MAT_TYPES CV_8UC1, CV_32FC1
#define FILE_EXTENSION String(".xml"), String(".yml")
PERF_TEST_P(Size_Mat_StrType, fs_text,
testing::Combine(testing::Values(MAT_SIZES),
testing::Values(MAT_TYPES),
testing::Values(FILE_EXTENSION))
)
{
Size size = get<0>(GetParam());
int type = get<1>(GetParam());
String ext = get<2>(GetParam());
Mat src(size.height, size.width, type);
Mat dst = src.clone();
declare.in(src, WARMUP_RNG).out(dst);
cv::String file_name = cv::tempfile(ext.c_str());
cv::String key = "test_mat";
TEST_CYCLE_MULTIRUN(4)
{
{
FileStorage fs(file_name, cv::FileStorage::WRITE);
fs << key << src;
fs.release();
}
{
FileStorage fs(file_name, cv::FileStorage::READ);
fs[key] >> dst;
fs.release();
}
}
remove(file_name.c_str());
SANITY_CHECK(dst, 1);
}
PERF_TEST_P(Size_Mat_StrType, fs_base64,
testing::Combine(testing::Values(MAT_SIZES),
testing::Values(MAT_TYPES),
testing::Values(FILE_EXTENSION))
)
{
Size size = get<0>(GetParam());
int type = get<1>(GetParam());
String ext = get<2>(GetParam());
Mat src(size.height, size.width, type);
Mat dst = src.clone();
cv::String file_name = cv::tempfile(ext.c_str());
cv::String key = "test_mat";
declare.in(src, WARMUP_RNG).out(dst);
TEST_CYCLE_MULTIRUN(4)
{
{
FileStorage fs(file_name, cv::FileStorage::WRITE_BASE64);
fs << key << src;
fs.release();
}
{
FileStorage fs(file_name, cv::FileStorage::READ);
fs[key] >> dst;
fs.release();
}
}
remove(file_name.c_str());
SANITY_CHECK(dst, 1);
}

711
modules/core/src/persistence.cpp
View File

@ -243,6 +243,17 @@ typedef struct CvFileStorage
std::deque<char>* outbuf;
base64::Base64Writer * base64_writer;
bool is_default_using_base64;
enum Base64State {
Uncertain,
NotUse,
InUse,
} state_of_writing_base64; /**< used in WriteRawData only */
bool is_write_struct_delayed;
char* delayed_struct_key;
int delayed_struct_flags;
char* delayed_type_name;
bool is_opened;
}
@ -270,21 +281,23 @@ namespace base64
bool base64_valid (uint8_t const * src, size_t off, size_t cnt);
bool base64_valid ( char const * src, size_t off = 0U, size_t cnt = 0U);
size_t base64_encode_buffer_size(size_t cnt);
size_t base64_encode_buffer_size(size_t cnt, bool is_end_with_zero = true);
size_t base64_decode_buffer_size(size_t cnt);
size_t base64_decode_buffer_size(size_t cnt, bool is_end_with_zero = true);
size_t base64_decode_buffer_size(size_t cnt, char const * src, bool is_end_with_zero = true);
size_t base64_decode_buffer_size(size_t cnt, uchar const * src, bool is_end_with_zero = true);
/* binary */
template<typename _uint_t> inline size_t to_binary(_uint_t val, uchar * cur);
template<> inline size_t to_binary(double val, uchar * cur);
template<> inline size_t to_binary(float val, uchar * cur);
template<typename _uint_t> inline size_t to_binary(_uint_t val, uchar * cur);
template<> inline size_t to_binary(double val, uchar * cur);
template<> inline size_t to_binary(float val, uchar * cur);
template<typename _primitive_t> inline size_t to_binary(uchar const * val, uchar * cur);
template<typename _uint_t> inline size_t binary_to(uchar const * cur, _uint_t & val);
template<> inline size_t binary_to(uchar const * cur, double & val);
template<> inline size_t binary_to(uchar const * cur, float & val);
template<typename _primitive_t> inline size_t binary_to(uchar const * cur, uchar * val);
template<typename _uint_t> inline size_t binary_to(uchar const * cur, _uint_t & val);
template<> inline size_t binary_to(uchar const * cur, double & val);
template<> inline size_t binary_to(uchar const * cur, float & val);
template<typename _primitive_t> inline size_t binary_to(uchar const * cur, uchar * val);
class MatToBinaryConvertor;
class RawDataToBinaryConvertor;
@ -313,22 +326,37 @@ namespace base64
class Base64ContextEmitter;
class Base64Writer
{
public:
Base64Writer(::CvFileStorage * fs);
~Base64Writer();
void write(const void* _data, size_t len, const char* dt);
template<typename _to_binary_convertor_t> void write(_to_binary_convertor_t & convertor, const char* dt);
private:
void check_dt(const char* dt);
private:
::CvFileStorage * file_storage;
Base64ContextEmitter * emitter;
std::string data_type_string;
};
/* other */
std::string make_base64_header(int byte_size, const char * dt);
std::string make_base64_header(const char * dt);
bool read_base64_header(std::string const & header, int & byte_size, std::string & dt);
bool read_base64_header(std::vector<char> const & header, std::string & dt);
void make_seq(void * binary_data, int elem_cnt, const char * dt, CvSeq & seq);
/* sample */
void cvStartWriteRawData_Base64(::CvFileStorage * fs, const char* name, int len, const char* dt);
void cvWriteRawData_Base64(::CvFileStorage * fs, const void* _data, int len);
void cvEndWriteRawData_Base64(::CvFileStorage * fs);
void cvWriteRawDataBase64(::CvFileStorage* fs, const void* _data, int len, const char* dt);
void cvWriteRawData_Base64(::cv::FileStorage & fs, const void* _data, int len, const char* dt);
void cvWriteMat_Base64(CvFileStorage * fs, const char * name, ::cv::Mat const & mat);
void cvWriteMat_Base64(::CvFileStorage * fs, const char * name, ::cv::Mat const & mat);
}
@ -1031,6 +1059,184 @@ static double icv_strtod( CvFileStorage* fs, char* ptr, char** endptr )
return fval;
}
static std::vector<std::string> analyze_file_name( std::string const & file_name )
{
static const char parameter_begin = '?';
static const char parameter_separator = '&';
std::vector<std::string> result;
size_t beg = file_name.find_last_of(parameter_begin);
size_t end = file_name.size();
result.push_back(file_name.substr(0U, beg));
if ( beg != std::string::npos )
{
beg ++;
for ( size_t param_beg = beg, param_end = beg;
param_end < end;
param_beg = param_end + 1U )
{
param_end = file_name.find_first_of( parameter_separator, param_beg );
if ( (param_end == std::string::npos || param_end != param_beg) && param_beg + 1U < end )
{
result.push_back( file_name.substr( param_beg, param_end - param_beg ) );
}
}
}
return result;
}
static bool is_param_exist( std::vector<std::string> & const params, std::string & const param )
{
if ( params.size() < 2U )
return false;
return std::find(params.begin(), params.end(), param) != params.end();
}
static void switch_to_Base64_state( CvFileStorage* fs, CvFileStorage::Base64State state )
{
const char * err_unkonwn_state = "Unexpected error, unable to determine the Base64 state.";
const char * err_unable_to_switch = "Unexpected error, unable to switch to this state.";
/* like a finite state machine */
switch (fs->state_of_writing_base64)
{
case CvFileStorage::Base64State::Uncertain:
switch (state)
{
case CvFileStorage::Base64State::InUse:
CV_DbgAssert( fs->base64_writer == 0 );
fs->base64_writer = new base64::Base64Writer( fs );
break;
case CvFileStorage::Base64State::Uncertain:
break;
case CvFileStorage::Base64State::NotUse:
break;
default:
CV_Error( CV_StsError, err_unkonwn_state );
break;
}
break;
case CvFileStorage::Base64State::InUse:
switch (state)
{
case CvFileStorage::Base64State::InUse:
case CvFileStorage::Base64State::NotUse:
CV_Error( CV_StsError, err_unable_to_switch );
break;
case CvFileStorage::Base64State::Uncertain:
delete fs->base64_writer;
fs->base64_writer = 0;
break;
default:
CV_Error( CV_StsError, err_unkonwn_state );
break;
}
break;
case CvFileStorage::Base64State::NotUse:
switch (state)
{
case CvFileStorage::Base64State::InUse:
case CvFileStorage::Base64State::NotUse:
CV_Error( CV_StsError, err_unable_to_switch );
break;
case CvFileStorage::Base64State::Uncertain:
break;
default:
CV_Error( CV_StsError, err_unkonwn_state );
break;
}
break;
default:
CV_Error( CV_StsError, err_unkonwn_state );
break;
}
fs->state_of_writing_base64 = state;
}
static void check_if_write_struct_is_delayed( CvFileStorage* fs, bool change_type_to_base64 = false )
{
if ( fs->is_write_struct_delayed )
{
/* save data to prevent recursive call errors */
char * struct_key = 0;
char * type_name = 0;
int struct_flags = fs->delayed_struct_flags;
if ( fs->delayed_struct_key != 0 )
{
struct_key = new char[strlen(fs->delayed_struct_key) + 1U];
strcpy(struct_key, fs->delayed_struct_key);
}
if ( fs->delayed_type_name != 0 )
{
type_name = new char[strlen(type_name) + 1U];
strcpy(type_name, fs->delayed_type_name);
}
/* reset */
delete fs->delayed_struct_key;
delete fs->delayed_type_name;
fs->delayed_struct_key = 0;
fs->delayed_struct_flags = 0;
fs->delayed_type_name = 0;
fs->is_write_struct_delayed = false;
/* call */
if ( change_type_to_base64 )
{
fs->start_write_struct( fs, struct_key, struct_flags, "binary");
if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::Uncertain )
switch_to_Base64_state( fs, CvFileStorage::Base64State::Uncertain );
switch_to_Base64_state( fs, CvFileStorage::Base64State::InUse );
}
else
{
fs->start_write_struct( fs, struct_key, struct_flags, type_name);
if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::Uncertain )
switch_to_Base64_state( fs, CvFileStorage::Base64State::Uncertain );
switch_to_Base64_state( fs, CvFileStorage::Base64State::NotUse );
}
delete struct_key;
delete type_name;
}
}
static void make_write_struct_delayed(
CvFileStorage* fs,
const char* key,
int struct_flags,
const char* type_name )
{
CV_Assert( fs->is_write_struct_delayed == false );
CV_DbgAssert( fs->delayed_struct_key == 0 );
CV_DbgAssert( fs->delayed_struct_flags == 0 );
CV_DbgAssert( fs->delayed_type_name == 0 );
fs->delayed_struct_flags = struct_flags;
if ( key != 0 )
{
fs->delayed_struct_key = new char[strlen(key) + 1U];
strcpy(fs->delayed_struct_key, key);
}
if ( type_name != 0 )
{
fs->delayed_type_name = new char[strlen(type_name) + 1U];
strcpy(fs->delayed_type_name, type_name);
}
fs->is_write_struct_delayed = true;
}
/****************************************************************************************\
* YAML Parser *
@ -1119,40 +1325,41 @@ static char* icvYMLParseBase64(CvFileStorage* fs, char* ptr, int indent, CvFileN
/* calc (decoded) total_byte_size from header */
std::string dt;
int total_byte_size = -1;
{
if (end - beg < static_cast<int>(base64::ENCODED_HEADER_SIZE))
CV_PARSE_ERROR("Unrecognized Base64 header");
std::vector<char> header(base64::HEADER_SIZE + 1, ' ');
base64::base64_decode(beg, header.data(), 0U, base64::ENCODED_HEADER_SIZE);
std::istringstream iss(header.data());
if (!(iss >> total_byte_size) || total_byte_size < 0)
CV_PARSE_ERROR("Cannot parse size in Base64 header");
if (!(iss >> dt) || dt.empty())
if ( !base64::read_base64_header(header, dt) || dt.empty() )
CV_PARSE_ERROR("Cannot parse dt in Base64 header");
beg += base64::ENCODED_HEADER_SIZE;
}
/* buffer for decoded data(exclude header) */
std::vector<uchar> buffer(total_byte_size + 4);
/* get all Base64 data */
std::string base64_buffer;
base64_buffer.reserve( 16U * 1024U * 1024U );
while( beg < end )
{
base64::Base64ContextParser parser(buffer.data(), total_byte_size + 4);
/* decoding */
while(beg < end)
{
/* save this part [beg, end) */
parser.read((const uchar *)beg, (const uchar *)end);
beg = end;
/* find next part */
icvYMLGetMultilineStringContent(fs, beg, indent, beg, end);
}
base64_buffer.append( beg, end );
beg = end;
icvYMLGetMultilineStringContent( fs, beg, indent, beg, end );
}
if ( !base64::base64_valid(base64_buffer.data(), 0U, base64_buffer.size()) )
CV_PARSE_ERROR( "Invalid Base64 data." );
/* buffer for decoded data(exclude header) */
std::vector<uchar> binary_buffer( base64::base64_decode_buffer_size(base64_buffer.size()) );
int total_byte_size = base64::base64_decode_buffer_size( base64_buffer.size(), base64_buffer.data(), false );
{
base64::Base64ContextParser parser(binary_buffer.data(), binary_buffer.size() );
const uchar * buffer_beg = reinterpret_cast<const uchar *>( base64_buffer.data() );
const uchar * buffer_end = buffer_beg + base64_buffer.size();
parser.read( buffer_beg, buffer_end );
parser.flush();
}
/* save as CvSeq */
int elem_size = ::icvCalcStructSize(dt.c_str(), 0);
if (total_byte_size % elem_size != 0)
@ -1160,9 +1367,10 @@ static char* icvYMLParseBase64(CvFileStorage* fs, char* ptr, int indent, CvFileN
int elem_cnt = total_byte_size / elem_size;
node->tag = CV_NODE_NONE;
int struct_flags = CV_NODE_FLOW + CV_NODE_SEQ; /* after icvFSCreateCollection, node->tag == struct_flags */
int struct_flags = CV_NODE_FLOW | CV_NODE_SEQ;
/* after icvFSCreateCollection, node->tag == struct_flags */
icvFSCreateCollection(fs, struct_flags, node);
base64::make_seq(buffer.data(), elem_cnt, dt.c_str(), *node->data.seq);
base64::make_seq(binary_buffer.data(), elem_cnt, dt.c_str(), *node->data.seq);
if (fs->dummy_eof) {
/* end of file */
@ -1621,6 +1829,16 @@ icvYMLParse( CvFileStorage* fs )
static void
icvYMLWrite( CvFileStorage* fs, const char* key, const char* data )
{
check_if_write_struct_is_delayed( fs );
if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::Uncertain )
{
switch_to_Base64_state( fs, CvFileStorage::Base64State::NotUse );
}
else if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::InUse )
{
CV_Error( CV_StsError, "At present, output Base64 data only." );
}
int i, keylen = 0;
int datalen = 0;
int struct_flags;
@ -2034,17 +2252,17 @@ static void icvXMLGetMultilineStringContent(CvFileStorage* fs,
ptr = icvXMLSkipSpaces(fs, ptr, CV_XML_INSIDE_TAG);
beg = ptr;
end = ptr;
if (fs->dummy_eof)
if ( fs->dummy_eof )
return ; /* end of file */
if (*beg == '<')
if ( *beg == '<' )
return; /* end of string */
/* find end */
while(cv_isprint(*ptr)) /* no check for base64 string */
while( cv_isprint(*ptr) ) /* no check for base64 string */
++ ptr;
if (*ptr == '\0')
CV_PARSE_ERROR("Unexpected end of line");
if ( *ptr == '\0' )
CV_PARSE_ERROR( "Unexpected end of line" );
end = ptr;
}
@ -2061,48 +2279,52 @@ static char* icvXMLParseBase64(CvFileStorage* fs, char* ptr, CvFileNode * node)
/* calc (decoded) total_byte_size from header */
std::string dt;
int total_byte_size = -1;
{
if (end - beg < static_cast<int>(base64::ENCODED_HEADER_SIZE))
CV_PARSE_ERROR("Unrecognized Base64 header");
std::vector<char> header(base64::HEADER_SIZE + 1, ' ');
base64::base64_decode(beg, header.data(), 0U, base64::ENCODED_HEADER_SIZE);
std::istringstream iss(header.data());
if (!(iss >> total_byte_size) || total_byte_size < 0)
CV_PARSE_ERROR("Cannot parse size in Base64 header");
if (!(iss >> dt) || dt.empty())
if ( !base64::read_base64_header(header, dt) || dt.empty() )
CV_PARSE_ERROR("Cannot parse dt in Base64 header");
beg += base64::ENCODED_HEADER_SIZE;
}
/* alloc buffer for all decoded data(include header) */
std::vector<uchar> buffer(total_byte_size + 4);
/* get all Base64 data */
std::string base64_buffer;
base64_buffer.reserve( 16U * 1024U * 1024U );
while( beg < end )
{
base64::Base64ContextParser parser(buffer.data(), total_byte_size + 4);
base64_buffer.append( beg, end );
beg = end;
icvXMLGetMultilineStringContent( fs, beg, beg, end );
}
if ( !base64::base64_valid(base64_buffer.data(), 0U, base64_buffer.size()) )
CV_PARSE_ERROR( "Invalid Base64 data." );
/* decoding */
while(beg < end)
{
/* save this part [beg, end) */
parser.read((const uchar *)beg, (const uchar *)end);
beg = end;
/* find next part */
icvXMLGetMultilineStringContent(fs, beg, beg, end);
}
/* alloc buffer for all decoded data(include header) */
std::vector<uchar> binary_buffer( base64::base64_decode_buffer_size(base64_buffer.size()) );
int total_byte_size = base64::base64_decode_buffer_size( base64_buffer.size(), base64_buffer.data(), false );
{
base64::Base64ContextParser parser(binary_buffer.data(), binary_buffer.size() );
const uchar * buffer_beg = reinterpret_cast<const uchar *>( base64_buffer.data() );
const uchar * buffer_end = buffer_beg + base64_buffer.size();
parser.read( buffer_beg, buffer_end );
parser.flush();
}
/* save as CvSeq */
int elem_size = ::icvCalcStructSize(dt.c_str(), 0);
if (total_byte_size % elem_size != 0)
CV_PARSE_ERROR("Byte size not match elememt size");
CV_PARSE_ERROR("data size not matches elememt size");
int elem_cnt = total_byte_size / elem_size;
node->tag = CV_NODE_NONE;
int struct_flags = CV_NODE_SEQ; /* after icvFSCreateCollection, node->tag == struct_flags */
int struct_flags = CV_NODE_SEQ;
/* after icvFSCreateCollection, node->tag == struct_flags */
icvFSCreateCollection(fs, struct_flags, node);
base64::make_seq(buffer.data(), elem_cnt, dt.c_str(), *node->data.seq);
base64::make_seq(binary_buffer.data(), elem_cnt, dt.c_str(), *node->data.seq);
if (fs->dummy_eof) {
/* end of file */
@ -2757,6 +2979,16 @@ icvXMLStartNextStream( CvFileStorage* fs )
static void
icvXMLWriteScalar( CvFileStorage* fs, const char* key, const char* data, int len )
{
check_if_write_struct_is_delayed( fs );
if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::Uncertain )
{
switch_to_Base64_state( fs, CvFileStorage::Base64State::NotUse );
}
else if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::InUse )
{
CV_Error( CV_StsError, "Currently only Base64 data is allowed." );
}
if( CV_NODE_IS_MAP(fs->struct_flags) ||
(!CV_NODE_IS_COLLECTION(fs->struct_flags) && key) )
{
@ -2964,15 +3196,24 @@ icvXMLWriteComment( CvFileStorage* fs, const char* comment, int eol_comment )
\****************************************************************************************/
CV_IMPL CvFileStorage*
cvOpenFileStorage( const char* filename, CvMemStorage* dststorage, int flags, const char* encoding )
cvOpenFileStorage( const char* query, CvMemStorage* dststorage, int flags, const char* encoding )
{
CvFileStorage* fs = 0;
int default_block_size = 1 << 18;
bool append = (flags & 3) == CV_STORAGE_APPEND;
bool mem = (flags & CV_STORAGE_MEMORY) != 0;
bool write_mode = (flags & 3) != 0;
bool write_base64 = write_mode && (flags & CV_STORAGE_BASE64) != 0;
bool isGZ = false;
size_t fnamelen = 0;
const char * filename = 0;
std::vector<std::string> params = analyze_file_name( query );
if ( !params.empty() )
filename = params.begin()->c_str();
if (write_base64 == false && is_param_exist( params, std::string("base64") ) )
write_base64 = true;
if( !filename || filename[0] == '\0' )
{
@ -3073,6 +3314,16 @@ cvOpenFileStorage( const char* filename, CvMemStorage* dststorage, int flags, co
fs->struct_flags = CV_NODE_EMPTY;
fs->buffer_start = fs->buffer = (char*)cvAlloc( buf_size + 1024 );
fs->buffer_end = fs->buffer_start + buf_size;
fs->base64_writer = 0;
fs->is_default_using_base64 = write_base64;
fs->state_of_writing_base64 = CvFileStorage::Base64State::Uncertain;
fs->is_write_struct_delayed = false;
fs->delayed_struct_key = 0;
fs->delayed_struct_flags = 0;
fs->delayed_type_name = 0;
if( fs->fmt == CV_STORAGE_FORMAT_XML )
{
size_t file_size = fs->file ? (size_t)ftell( fs->file ) : (size_t)0;
@ -3247,7 +3498,48 @@ cvStartWriteStruct( CvFileStorage* fs, const char* key, int struct_flags,
const char* type_name, CvAttrList /*attributes*/ )
{
CV_CHECK_OUTPUT_FILE_STORAGE(fs);
fs->start_write_struct( fs, key, struct_flags, type_name );
check_if_write_struct_is_delayed( fs );
if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::NotUse )
switch_to_Base64_state( fs, CvFileStorage::Base64State::Uncertain );
if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::Uncertain
&&
CV_NODE_IS_SEQ(struct_flags)
&&
fs->is_default_using_base64
&&
type_name == 0
)
{
/* Uncertain if output Base64 data */
make_write_struct_delayed( fs, key, struct_flags, type_name );
}
else if ( type_name && memcmp(type_name, "binary", 6) == 0 )
{
/* Must output Base64 data */
if ( !CV_NODE_IS_SEQ(struct_flags) )
CV_Error( CV_StsBadArg, "must set 'struct_flags |= CV_NODE_SEQ' if using Base64.");
else if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::Uncertain )
CV_Error( CV_StsError, "function \'cvStartWriteStruct\' calls cannot be nested if using Base64.");
fs->start_write_struct( fs, key, struct_flags, type_name );
if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::Uncertain )
switch_to_Base64_state( fs, CvFileStorage::Base64State::Uncertain );
switch_to_Base64_state( fs, CvFileStorage::Base64State::InUse );
}
else
{
/* Won't output Base64 data */
if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::InUse )
CV_Error( CV_StsError, "At the end of the output Base64, `cvEndWriteStruct` is needed.");
fs->start_write_struct( fs, key, struct_flags, type_name );
if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::Uncertain )
switch_to_Base64_state( fs, CvFileStorage::Base64State::Uncertain );
switch_to_Base64_state( fs, CvFileStorage::Base64State::NotUse );
}
}
@ -3255,6 +3547,11 @@ CV_IMPL void
cvEndWriteStruct( CvFileStorage* fs )
{
CV_CHECK_OUTPUT_FILE_STORAGE(fs);
check_if_write_struct_is_delayed( fs );
if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::Uncertain )
switch_to_Base64_state( fs, CvFileStorage::Base64State::Uncertain );
fs->end_write_struct( fs );
}
@ -3432,6 +3729,17 @@ icvDecodeSimpleFormat( const char* dt )
CV_IMPL void
cvWriteRawData( CvFileStorage* fs, const void* _data, int len, const char* dt )
{
if (fs->is_default_using_base64 ||
fs->state_of_writing_base64 == CvFileStorage::Base64State::InUse )
{
base64::cvWriteRawDataBase64( fs, _data, len, dt );
return;
}
else if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::Uncertain )
{
switch_to_Base64_state( fs, CvFileStorage::Base64State::NotUse );
}
const char* data0 = (const char*)_data;
int offset = 0;
int fmt_pairs[CV_FS_MAX_FMT_PAIRS*2], k, fmt_pair_count;
@ -3760,15 +4068,15 @@ icvWriteFileNode( CvFileStorage* fs, const char* name, const CvFileNode* node )
break;
case CV_NODE_SEQ:
case CV_NODE_MAP:
fs->start_write_struct( fs, name, CV_NODE_TYPE(node->tag) +
cvStartWriteStruct( fs, name, CV_NODE_TYPE(node->tag) +
(CV_NODE_SEQ_IS_SIMPLE(node->data.seq) ? CV_NODE_FLOW : 0),
node->info ? node->info->type_name : 0 );
icvWriteCollection( fs, node );
fs->end_write_struct( fs );
cvEndWriteStruct( fs );
break;
case CV_NODE_NONE:
fs->start_write_struct( fs, name, CV_NODE_SEQ, 0 );
fs->end_write_struct( fs );
cvStartWriteStruct( fs, name, CV_NODE_SEQ, 0 );
cvEndWriteStruct( fs );
break;
default:
CV_Error( CV_StsBadFlag, "Unknown type of file node" );
@ -6226,14 +6534,29 @@ bool base64::base64_valid(char const * src, size_t off, size_t cnt)
return base64_valid(reinterpret_cast<uint8_t const *>(src), off, cnt);
}
size_t base64::base64_encode_buffer_size(size_t cnt)
size_t base64::base64_encode_buffer_size(size_t cnt, bool is_end_with_zero)
{
return size_t((cnt + 2U) / 3U * 4U + 1U);
size_t additional = static_cast<size_t>(is_end_with_zero == true);
return (cnt + 2U) / 3U * 4U + additional;
}
size_t base64::base64_decode_buffer_size(size_t cnt)
size_t base64::base64_decode_buffer_size(size_t cnt, bool is_end_with_zero)
{
return size_t(cnt / 4U * 3U + 1U);
size_t additional = static_cast<size_t>(is_end_with_zero == true);
return cnt / 4U * 3U + additional;
}
size_t base64::base64_decode_buffer_size(size_t cnt, char const * src, bool is_end_with_zero)
{
return base64_decode_buffer_size(cnt, reinterpret_cast<uchar const *>(src), is_end_with_zero);
}
size_t base64::base64_decode_buffer_size(size_t cnt, uchar const * src, bool is_end_with_zero)
{
size_t padding_cnt = 0U;
for (uchar const * ptr = src + cnt - 1U; *ptr == base64_padding; ptr--)
padding_cnt ++;
return base64_decode_buffer_size(cnt, is_end_with_zero) - padding_cnt;
}
/****************************************************************************
@ -6308,13 +6631,10 @@ binary_to(uchar const * cur, uchar * val)
* others
***************************************************************************/
std::string base64::make_base64_header(int byte_size, const char * dt)
std::string base64::make_base64_header(const char * dt)
{
int size = byte_size;
std::ostringstream oss;
oss << size << ' '
<< dt << ' ';
oss << dt << ' ';
std::string buffer(oss.str());
CV_Assert(buffer.size() < HEADER_SIZE);
@ -6325,10 +6645,10 @@ std::string base64::make_base64_header(int byte_size, const char * dt)
return buffer;
}
bool base64::read_base64_header(std::string const & header, int & byte_size, std::string & dt)
bool base64::read_base64_header(std::vector<char> const & header, std::string & dt)
{
std::istringstream iss(header);
return static_cast<bool>(iss >> byte_size >> dt);
std::istringstream iss(header.data());
return static_cast<bool>(iss >> dt);
}
/****************************************************************************
@ -6351,10 +6671,9 @@ base64::Base64ContextParser::Base64ContextParser(uchar * buffer, size_t size)
base64::Base64ContextParser::~Base64ContextParser()
{
if (src_cur != src_beg) {
/* encode the rest binary data to base64 buffer */
/* encode the rest binary data to base64 buffer */
if (src_cur != src_beg)
flush();
}
}
base64::Base64ContextParser & base64::Base64ContextParser::
@ -6383,9 +6702,12 @@ read(const uchar * beg, const uchar * end)
bool base64::Base64ContextParser::flush()
{
if (!base64_valid(src_beg, 0U, src_cur - src_beg))
if ( !base64_valid(src_beg, 0U, src_cur - src_beg) )
return false;
if ( src_cur == src_beg )
return true;
uchar * buffer = binary_buffer.data();
size_t len = base64_decode(src_beg, buffer, 0U, src_cur - src_beg);
src_cur = src_beg;
@ -6514,7 +6836,7 @@ public:
private:
/* because of Base64, we must keep its length a multiple of 3 */
static const size_t BUFFER_LEN = 51U;
static const size_t BUFFER_LEN = 48U;
// static_assert(BUFFER_LEN % 3 == 0, "BUFFER_LEN is invalid");
private:
@ -6584,17 +6906,17 @@ public:
{
CV_DbgAssert(*this);
/* copy to dst */
/* [1]copy to dst */
dst += to_binary_func(row_begin + x, dst);
/* move to next */
/* [2]move to next */
x += step;
if (x >= x_max) {
/* when x arrive end, reset it and increase y */
/* when x arrive end, reset x and increase y */
x = 0U;
++ y;
if (y >= y_max) {
/* when y arrive end, reset it and increase iter */
/* when y arrive end, reset y and increase iter */
y = 0U;
++ mat_iter;
if (mat_iter == mats.end()) {
@ -6935,68 +7257,55 @@ private:
* Wapper
***************************************************************************/
class base64::Base64Writer
base64::Base64Writer::Base64Writer(::CvFileStorage * fs)
: file_storage(fs)
, emitter(new Base64ContextEmitter(fs))
, data_type_string()
{
public:
CV_CHECK_OUTPUT_FILE_STORAGE(fs);
icvFSFlush(fs);
}
Base64Writer(::CvFileStorage * fs, const char * name, int len, const char* dt)
: file_storage(fs)
, emitter(fs)
, remaining_data_length(len)
, data_type_string(dt)
{
CV_CHECK_OUTPUT_FILE_STORAGE(fs);
void base64::Base64Writer::write(const void* _data, size_t len, const char* dt)
{
check_dt(dt);
cvStartWriteStruct(fs, name, CV_NODE_SEQ, "binary");
icvFSFlush(fs);
RawDataToBinaryConvertor convertor(
_data, static_cast<int>(len), data_type_string.c_str()
);
emitter->write(convertor);
}
template<typename _to_binary_convertor_t> inline
void base64::Base64Writer::write(_to_binary_convertor_t & convertor, const char* dt)
{
check_dt(dt);
emitter->write(convertor);
}
base64::Base64Writer::~Base64Writer()
{
delete emitter;
}
void base64::Base64Writer::check_dt(const char* dt)
{
if ( dt == 0 )
CV_Error( CV_StsBadArg, "Invalid \'dt\'." );
else if (data_type_string.empty()) {
data_type_string = dt;
/* output header */
/* total byte size(before encode) */
int size = len * ::icvCalcStructSize(dt, 0);
std::string buffer = make_base64_header(size, dt);
std::string buffer = make_base64_header(dt);
const uchar * beg = reinterpret_cast<const uchar *>(buffer.data());
const uchar * end = beg + buffer.size();
emitter.write(beg, end);
}
emitter->write(beg, end);
} else if ( data_type_string != dt )
CV_Error( CV_StsBadArg, "\'dt\' does not match." );
}
void write(const void* _data, int len)
{
CV_Assert(len >= 0);
CV_Assert(remaining_data_length >= static_cast<size_t>(len));
remaining_data_length -= static_cast<size_t>(len);
RawDataToBinaryConvertor convertor(_data, len, data_type_string);
emitter.write(convertor);
}
template<typename _to_binary_convertor_t> inline
void write(_to_binary_convertor_t & convertor, int data_length_of_convertor)
{
CV_Assert(data_length_of_convertor >= 0);
CV_Assert(remaining_data_length >= static_cast<size_t>(data_length_of_convertor));
remaining_data_length -= static_cast<size_t>(data_length_of_convertor);
emitter.write(convertor);
}
~Base64Writer()
{
CV_Assert(remaining_data_length == 0U);
emitter.flush();
cvEndWriteStruct(file_storage);
icvFSFlush(file_storage);
}
private:
::CvFileStorage * file_storage;
Base64ContextEmitter emitter;
size_t remaining_data_length;
const char* data_type_string;
};
void base64::make_seq(void * binary, int elem_cnt, const char * dt, ::CvSeq & seq)
{
@ -7009,51 +7318,23 @@ void base64::make_seq(void * binary, int elem_cnt, const char * dt, ::CvSeq & se
}
}
void base64::cvStartWriteRawData_Base64(::CvFileStorage * fs, const char* name, int len, const char* dt)
void base64::cvWriteRawDataBase64(::CvFileStorage* fs, const void* _data, int len, const char* dt)
{
CV_Assert(fs);
CV_CHECK_OUTPUT_FILE_STORAGE(fs);
CV_Assert(fs->base64_writer == 0);
fs->base64_writer = new Base64Writer(fs, name, len, dt);
}
void base64::cvWriteRawData_Base64(::CvFileStorage * fs, const void* _data, int len)
{
CV_Assert(fs);
CV_CHECK_OUTPUT_FILE_STORAGE(fs);
CV_Assert(fs->base64_writer != 0);
fs->base64_writer->write(_data, len);
}
check_if_write_struct_is_delayed( fs, true );
void base64::cvEndWriteRawData_Base64(::CvFileStorage * fs)
{
CV_Assert(fs);
CV_CHECK_OUTPUT_FILE_STORAGE(fs);
CV_Assert(fs->base64_writer != 0);
delete fs->base64_writer;
fs->base64_writer = 0;
}
void base64::cvWriteRawData_Base64(::cv::FileStorage & fs, const void* _data, int len, const char* dt)
{
cvStartWriteStruct(*fs, fs.elname.c_str(), CV_NODE_SEQ, "binary");
if ( fs->state_of_writing_base64 == CvFileStorage::Base64State::Uncertain )
{
Base64ContextEmitter emitter(*fs);
{ /* header */
/* total byte size(before encode) */
int size = len * ::icvCalcStructSize(dt, 0);
std::string buffer = make_base64_header(size, dt);
const uchar * beg = reinterpret_cast<const uchar *>(buffer.data());
const uchar * end = beg + buffer.size();
emitter.write(beg, end);
}
{ /* body */
RawDataToBinaryConvertor convert(_data, len, dt);
emitter.write(convert);
}
switch_to_Base64_state( fs, CvFileStorage::Base64State::InUse );
}
cvEndWriteStruct(*fs);
else if ( fs->state_of_writing_base64 != CvFileStorage::Base64State::InUse )
{
CV_Error( CV_StsError, "Base64 should not be used at present." );
}
fs->base64_writer->write(_data, len, dt);
}
void base64::cvWriteMat_Base64(::CvFileStorage * fs, const char * name, ::cv::Mat const & mat)
@ -7064,31 +7345,30 @@ void base64::cvWriteMat_Base64(::CvFileStorage * fs, const char * name, ::cv::Ma
{ /* [1]output other attr */
if (mat.dims <= 2) {
cvStartWriteStruct(fs, name, CV_NODE_MAP, CV_TYPE_NAME_MAT);
::cvStartWriteStruct(fs, name, CV_NODE_MAP, CV_TYPE_NAME_MAT);
cvWriteInt(fs, "rows", mat.rows );
cvWriteInt(fs, "cols", mat.cols );
::cvWriteInt(fs, "rows", mat.rows );
::cvWriteInt(fs, "cols", mat.cols );
} else {
cvStartWriteStruct(fs, name, CV_NODE_MAP, CV_TYPE_NAME_MATND);
::cvStartWriteStruct(fs, name, CV_NODE_MAP, CV_TYPE_NAME_MATND);
cvStartWriteStruct(fs, "sizes", CV_NODE_SEQ | CV_NODE_FLOW);
cvWriteRawData(fs, mat.size.p, mat.dims, "i");
cvEndWriteStruct(fs);
::cvStartWriteStruct(fs, "sizes", CV_NODE_SEQ | CV_NODE_FLOW);
::cvWriteRawData(fs, mat.size.p, mat.dims, "i");
::cvEndWriteStruct(fs);
}
cvWriteString(fs, "dt", ::icvEncodeFormat(CV_MAT_TYPE(mat.type()), dt ), 0 );
::cvWriteString(fs, "dt", ::icvEncodeFormat(CV_MAT_TYPE(mat.type()), dt ), 0 );
}
{ /* [2]deal with matrix's data */
int len = static_cast<int>(mat.total());
MatToBinaryConvertor convertor(mat);
cvStartWriteRawData_Base64(fs, "data", len, dt);
fs->base64_writer->write(convertor, len);
cvEndWriteRawData_Base64(fs);
::cvStartWriteStruct(fs, "data", CV_NODE_SEQ, "binary");
fs->base64_writer->write(convertor, dt);
::cvEndWriteStruct(fs);
}
{ /* [3]output end */
cvEndWriteStruct(fs);
::cvEndWriteStruct(fs);
}
}
@ -7096,36 +7376,21 @@ void base64::cvWriteMat_Base64(::CvFileStorage * fs, const char * name, ::cv::Ma
* Interface
***************************************************************************/
namespace cv
CV_IMPL void cvWriteMatBase64(::CvFileStorage* fs, const char* name, const ::CvMat* mat)
{
void cvWriteMat_Base64(::CvFileStorage* fs, const char* name, const ::CvMat* mat)
{
::cv::Mat holder = ::cv::cvarrToMat(mat);
::base64::cvWriteMat_Base64(fs, name, holder);
}
void cvWriteMatND_Base64(::CvFileStorage* fs, const char* name, const ::CvMatND* mat)
{
::cv::Mat holder = ::cv::cvarrToMat(mat);
::base64::cvWriteMat_Base64(fs, name, holder);
}
void cvStartWriteRawData_Base64(::CvFileStorage * fs, const char* name, int len, const char* dt)
{
::base64::cvStartWriteRawData_Base64(fs, name, len, dt);
}
void cvWriteRawData_Base64(::CvFileStorage * fs, const void* _data, int len)
{
::base64::cvWriteRawData_Base64(fs, _data, len);
}
void cvEndWriteRawData_Base64(::CvFileStorage * fs)
{
::base64::cvEndWriteRawData_Base64(fs);
}
::cv::Mat holder = ::cv::cvarrToMat(mat);
::base64::cvWriteMat_Base64(fs, name, holder);
}
CV_IMPL void cvWriteMatNDBase64(::CvFileStorage* fs, const char* name, const ::CvMatND* mat)
{
::cv::Mat holder = ::cv::cvarrToMat(mat);
::base64::cvWriteMat_Base64(fs, name, holder);
}
CV_IMPL void cvWriteRawDataBase64(::CvFileStorage* fs, const void* _data, int len, const char* dt)
{
::base64::cvWriteRawDataBase64(fs, _data, len, dt);
}
/* End of file. */

147
modules/core/test/test_io.cpp
View File

@ -593,150 +593,3 @@ TEST(Core_InputOutput, FileStorageSpaces)
ASSERT_STREQ(values[i].c_str(), valuesRead[i].c_str());
}
}
TEST(Core_InputOutput, filestorage_yml_compatibility)
{
// TODO:
}
class CV_Base64IOTest : public cvtest::BaseTest
{
private:
std::string file_name;
struct data_t
{
uchar u1, u2;
int i1, i2, i3;
double d1, d2;
int i4;
};
public:
CV_Base64IOTest(std::string const & test_file_name)
: file_name(test_file_name) {}
~CV_Base64IOTest() {}
protected:
void run(int)
{
try
{
std::vector<data_t> rawdata;
cv::Mat _em_out, _em_in;
cv::Mat _2d_out, _2d_in;
cv::Mat _nd_out, _nd_in;
{ /* init */
/* normal mat */
_2d_out = cv::Mat(100, 100, CV_8UC3, cvScalar(1U, 2U, 127U));
for (int i = 0; i < _2d_out.rows; ++i)
for (int j = 0; j < _2d_out.cols; ++j)
_2d_out.at<cv::Vec3b>(i, j)[1] = (i + j) % 256;
/* 4d mat */
const int Size[] = {4, 4, 4, 4};
cv::Mat _4d(4, Size, CV_64FC4, cvScalar(0.888, 0.111, 0.666, 0.444));
const cv::Range ranges[] = {
cv::Range(0, 2),
cv::Range(0, 2),
cv::Range(1, 2),
cv::Range(0, 2) };
_nd_out = _4d(ranges);
/* raw data */
for (int i = 0; i < 1000; i++) {
data_t tmp;
tmp.u1 = 1;
tmp.u2 = 2;
tmp.i1 = 1;
tmp.i2 = 2;
tmp.i3 = 3;
tmp.d1 = 0.1;
tmp.d2 = 0.2;
tmp.i4 = i;
rawdata.push_back(tmp);
}
}
{ /* write */
cv::FileStorage fs(file_name, cv::FileStorage::WRITE);
CvMat holder = _2d_out;
cv::cvWriteMat_Base64(*fs, "normal_2d_mat", &holder);
CvMatND holder_nd = _nd_out;
cv::cvWriteMatND_Base64(*fs, "normal_nd_mat", &holder_nd);
holder = _em_out;
cv::cvWriteMat_Base64(*fs, "empty_2d_mat", &holder);
cv::cvStartWriteRawData_Base64(*fs, "rawdata", static_cast<int>(rawdata.size()), "2u3i2di");
for (int i = 0; i < 10; i++)
cv::cvWriteRawData_Base64(*fs, rawdata.data() + i * 100, 100);
cv::cvEndWriteRawData_Base64(*fs);
fs.release();
}
{ /* read */
cv::FileStorage fs(file_name, cv::FileStorage::READ);
/* mat */
fs["empty_2d_mat"] >> _em_in;
fs["normal_2d_mat"] >> _2d_in;
fs["normal_nd_mat"] >> _nd_in;
/* raw data */
std::vector<data_t>(1000).swap(rawdata);
cvReadRawData(*fs, fs["rawdata"].node, rawdata.data(), "2u3i2di");
fs.release();
}
for (int i = 0; i < 1000; i++) {
// TODO: Solve this bug in `cvReadRawData`
//EXPECT_EQ(rawdata[i].u1, 1);
//EXPECT_EQ(rawdata[i].u2, 2);
//EXPECT_EQ(rawdata[i].i1, 1);
//EXPECT_EQ(rawdata[i].i2, 2);
//EXPECT_EQ(rawdata[i].i3, 3);
//EXPECT_EQ(rawdata[i].d1, 0.1);
//EXPECT_EQ(rawdata[i].d2, 0.2);
//EXPECT_EQ(rawdata[i].i4, i);
}
EXPECT_EQ(_em_in.rows , _em_out.rows);
EXPECT_EQ(_em_in.cols , _em_out.cols);
EXPECT_EQ(_em_in.dims , _em_out.dims);
EXPECT_EQ(_em_in.depth(), _em_out.depth());
EXPECT_TRUE(_em_in.empty());
EXPECT_EQ(_2d_in.rows , _2d_in.rows);
EXPECT_EQ(_2d_in.cols , _2d_in.cols);
EXPECT_EQ(_2d_in.dims , _2d_in.dims);
EXPECT_EQ(_2d_in.depth(), _2d_in.depth());
for(int i = 0; i < _2d_in.rows; ++i)
for (int j = 0; j < _2d_in.cols; ++j)
EXPECT_EQ(_2d_in.at<cv::Vec3b>(i, j), _2d_out.at<cv::Vec3b>(i, j));
EXPECT_EQ(_nd_in.rows , _nd_in.rows);
EXPECT_EQ(_nd_in.cols , _nd_in.cols);
EXPECT_EQ(_nd_in.dims , _nd_in.dims);
EXPECT_EQ(_nd_in.depth(), _nd_in.depth());
EXPECT_EQ(cv::countNonZero(cv::mean(_nd_in != _nd_out)), 0);
}
catch(...)
{
ts->set_failed_test_info(cvtest::TS::FAIL_MISMATCH);
}
}
};
TEST(Core_InputOutput, filestorage_yml_base64)
{
CV_Base64IOTest test("base64_test_tmp_file.yml"); test.safe_run();
}
TEST(Core_InputOutput, filestorage_xml_base64)
{
CV_Base64IOTest test("base64_test_tmp_file.xml"); test.safe_run();
}

270
modules/core/test/test_io_base64.cpp Normal file
View File

@ -0,0 +1,270 @@
#include "test_precomp.hpp"
using namespace cv;
using namespace std;
struct data_t
{
typedef uchar u;
typedef char b;
typedef ushort w;
typedef short s;
typedef int i;
typedef float f;
typedef double d;
u u1 ;u u2 ; i i1 ;
i i2 ;i i3 ;
d d1 ;
d d2 ;
i i4 ;
static inline const char * signature() { return "2u3i2di"; }
};
TEST(Core_InputOutput_Base64, basic)
{
char const * filenames[] = {
"core_io_base64_basic_test.yml",
"core_io_base64_basic_test.xml",
0
};
for (char const ** ptr = filenames; *ptr; ptr++)
{
char const * name = *ptr;
std::vector<data_t> rawdata;
cv::Mat _em_out, _em_in;
cv::Mat _2d_out, _2d_in;
cv::Mat _nd_out, _nd_in;
cv::Mat _rd_out(64, 64, CV_64FC1), _rd_in;
{ /* init */
/* a normal mat */
_2d_out = cv::Mat(100, 100, CV_8UC3, cvScalar(1U, 2U, 127U));
for (int i = 0; i < _2d_out.rows; ++i)
for (int j = 0; j < _2d_out.cols; ++j)
_2d_out.at<cv::Vec3b>(i, j)[1] = (i + j) % 256;
/* a 4d mat */
const int Size[] = {4, 4, 4, 4};
cv::Mat _4d(4, Size, CV_64FC4, cvScalar(0.888, 0.111, 0.666, 0.444));
const cv::Range ranges[] = {
cv::Range(0, 2),
cv::Range(0, 2),
cv::Range(1, 2),
cv::Range(0, 2) };
_nd_out = _4d(ranges);
/* a random mat */
cv::randu(_rd_out, cv::Scalar(0.0), cv::Scalar(1.0));
/* raw data */
for (int i = 0; i < 1000; i++) {
data_t tmp;
tmp.u1 = 1;
tmp.u2 = 2;
tmp.i1 = 1;
tmp.i2 = 2;
tmp.i3 = 3;
tmp.d1 = 0.1;
tmp.d2 = 0.2;
tmp.i4 = i;
rawdata.push_back(tmp);
}
}
{ /* write */
cv::FileStorage fs(name, cv::FileStorage::WRITE_BASE64);
fs << "normal_2d_mat" << _2d_out;
fs << "normal_nd_mat" << _nd_out;
fs << "empty_2d_mat" << _em_out;
fs << "random_mat" << _rd_out;
cvStartWriteStruct( *fs, "rawdata", CV_NODE_SEQ | CV_NODE_FLOW, "binary" );
for (int i = 0; i < 10; i++)
cvWriteRawDataBase64(*fs, rawdata.data() + i * 100, 100, data_t::signature());
cvEndWriteStruct( *fs );
fs.release();
}
{ /* read */
cv::FileStorage fs(name, cv::FileStorage::READ);
/* mat */
fs["empty_2d_mat"] >> _em_in;
fs["normal_2d_mat"] >> _2d_in;
fs["normal_nd_mat"] >> _nd_in;
fs["random_mat"] >> _rd_in;
/* raw data */
std::vector<data_t>(1000).swap(rawdata);
cvReadRawData(*fs, fs["rawdata"].node, rawdata.data(), data_t::signature());
fs.release();
}
for (int i = 0; i < 1000; i++) {
// TODO: Solve this bug in `cvReadRawData`
//EXPECT_EQ(rawdata[i].u1, 1);
//EXPECT_EQ(rawdata[i].u2, 2);
//EXPECT_EQ(rawdata[i].i1, 1);
//EXPECT_EQ(rawdata[i].i2, 2);
//EXPECT_EQ(rawdata[i].i3, 3);
//EXPECT_EQ(rawdata[i].d1, 0.1);
//EXPECT_EQ(rawdata[i].d2, 0.2);
//EXPECT_EQ(rawdata[i].i4, i);
}
EXPECT_EQ(_em_in.rows , _em_out.rows);
EXPECT_EQ(_em_in.cols , _em_out.cols);
EXPECT_EQ(_em_in.dims , _em_out.dims);
EXPECT_EQ(_em_in.depth(), _em_out.depth());
EXPECT_TRUE(_em_in.empty());
EXPECT_EQ(_2d_in.rows , _2d_out.rows);
EXPECT_EQ(_2d_in.cols , _2d_out.cols);
EXPECT_EQ(_2d_in.dims , _2d_out.dims);
EXPECT_EQ(_2d_in.depth(), _2d_out.depth());
for(int i = 0; i < _2d_out.rows; ++i)
for (int j = 0; j < _2d_out.cols; ++j)
EXPECT_EQ(_2d_in.at<cv::Vec3b>(i, j), _2d_out.at<cv::Vec3b>(i, j));
EXPECT_EQ(_nd_in.rows , _nd_out.rows);
EXPECT_EQ(_nd_in.cols , _nd_out.cols);
EXPECT_EQ(_nd_in.dims , _nd_out.dims);
EXPECT_EQ(_nd_in.depth(), _nd_out.depth());
EXPECT_EQ(cv::countNonZero(cv::mean(_nd_in != _nd_out)), 0);
EXPECT_EQ(_rd_in.rows , _rd_out.rows);
EXPECT_EQ(_rd_in.cols , _rd_out.cols);
EXPECT_EQ(_rd_in.dims , _rd_out.dims);
EXPECT_EQ(_rd_in.depth(), _rd_out.depth());
EXPECT_EQ(cv::countNonZero(cv::mean(_rd_in != _rd_out)), 0);
remove(name);
}
}
TEST(Core_InputOutput_Base64, valid)
{
char const * filenames[] = {
"core_io_base64_other_test.yml",
"core_io_base64_other_test.xml",
"core_io_base64_other_test.yml?base64",
"core_io_base64_other_test.xml?base64",
0
};
char const * real_name[] = {
"core_io_base64_other_test.yml",
"core_io_base64_other_test.xml",
"core_io_base64_other_test.yml",
"core_io_base64_other_test.xml",
0
};
std::vector<int> rawdata(10, static_cast<int>(0x00010203));
cv::String str_out = "test_string";
for (char const ** ptr = filenames; *ptr; ptr++)
{
char const * name = *ptr;
EXPECT_NO_THROW(
{
cv::FileStorage fs(name, cv::FileStorage::WRITE_BASE64);
cvStartWriteStruct(*fs, "manydata", CV_NODE_SEQ);
cvStartWriteStruct(*fs, 0, CV_NODE_SEQ | CV_NODE_FLOW);
for (int i = 0; i < 10; i++)
cvWriteRawData(*fs, rawdata.data(), rawdata.size(), "i");
cvEndWriteStruct(*fs);
cvWriteString(*fs, 0, str_out.c_str(), 1);
cvEndWriteStruct(*fs);
fs.release();
});
{
cv::FileStorage fs(name, cv::FileStorage::READ);
std::vector<int> data_in(rawdata.size());
fs["manydata"][0].readRaw("i", (uchar *)data_in.data(), data_in.size());
EXPECT_TRUE(fs["manydata"][0].isSeq());
EXPECT_TRUE(std::equal(rawdata.begin(), rawdata.end(), data_in.begin()));
cv::String str_in;
fs["manydata"][1] >> str_in;
EXPECT_TRUE(fs["manydata"][1].isString());
EXPECT_EQ(str_in, str_out);
fs.release();
}
EXPECT_NO_THROW(
{
cv::FileStorage fs(name, cv::FileStorage::WRITE);
cvStartWriteStruct(*fs, "manydata", CV_NODE_SEQ);
cvWriteString(*fs, 0, str_out.c_str(), 1);
cvStartWriteStruct(*fs, 0, CV_NODE_SEQ | CV_NODE_FLOW, "binary");
for (int i = 0; i < 10; i++)
cvWriteRawData(*fs, rawdata.data(), rawdata.size(), "i");
cvEndWriteStruct(*fs);
cvEndWriteStruct(*fs);
fs.release();
});
{
cv::FileStorage fs(name, cv::FileStorage::READ);
cv::String str_in;
fs["manydata"][0] >> str_in;
EXPECT_TRUE(fs["manydata"][0].isString());
EXPECT_EQ(str_in, str_out);
std::vector<int> data_in(rawdata.size());
fs["manydata"][1].readRaw("i", (uchar *)data_in.data(), data_in.size());
EXPECT_TRUE(fs["manydata"][1].isSeq());
EXPECT_TRUE(std::equal(rawdata.begin(), rawdata.end(), data_in.begin()));
fs.release();
}
remove(real_name[ptr - filenames]);
}
}
TEST(Core_InputOutput_Base64, invalid)
{
char const * filenames[] = {
"core_io_base64_other_test.yml",
"core_io_base64_other_test.xml",
0
};
for (char const ** ptr = filenames; *ptr; ptr++)
{
char const * name = *ptr;
EXPECT_ANY_THROW({
cv::FileStorage fs(name, cv::FileStorage::WRITE);
cvStartWriteStruct(*fs, "rawdata", CV_NODE_SEQ, "binary");
cvStartWriteStruct(*fs, 0, CV_NODE_SEQ | CV_NODE_FLOW);
});
EXPECT_ANY_THROW({
cv::FileStorage fs(name, cv::FileStorage::WRITE);
cvStartWriteStruct(*fs, "rawdata", CV_NODE_SEQ);
cvStartWriteStruct(*fs, 0, CV_NODE_SEQ | CV_NODE_FLOW);
cvWriteRawDataBase64(*fs, name, 1, "u");
});
remove(name);
}
}
TEST(Core_InputOutput_Base64, TODO_compatibility)
{
// TODO:
}

6
modules/ml/test/test_save_load.cpp
View File

@ -65,11 +65,11 @@ int CV_SLMLTest::run_test_case( int testCaseIdx )
{
get_test_error( testCaseIdx, &test_resps1 );
fname1 = tempfile(".yml.gz");
save( fname1.c_str() );
save( (fname1 + "?base64").c_str() );
load( fname1.c_str() );
get_test_error( testCaseIdx, &test_resps2 );
fname2 = tempfile(".yml.gz");
save( fname2.c_str() );
save( (fname2 + "?base64").c_str() );
}
else
ts->printf( cvtest::TS::LOG, "model can not be trained" );
@ -280,7 +280,7 @@ TEST(DISABLED_ML_SVM, linear_save_load)
svm1 = Algorithm::load<SVM>("SVM45_X_38-1.xml");
svm2 = Algorithm::load<SVM>("SVM45_X_38-2.xml");
string tname = tempfile("a.xml");
svm2->save(tname);
svm2->save(tname + "?base64");
svm3 = Algorithm::load<SVM>(tname);
ASSERT_EQ(svm1->getVarCount(), svm2->getVarCount());