mirror/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2024-11-30 22:40:17 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #26399 from dkurt:dk/file_storage_new_data

Browse Source 
int64 data type in FileStorage #26399

### Pull Request Readiness Checklist

resolves #23333

Proposed approach is not perfect in terms of complexity and potential bugs. Instead of changing `INT` raw size from `4` to `8`, we check int64 value can be fitted to int32 or not.

Collections such as cv::Mat rely on data type symbol.

This PR is addressed to 5.x branch first to cover `CV_64S` Mat. Later, it can be backported to 4.x

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [x] There is a reference to the original bug report and related work
- [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [x] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
Dmitry Kurtaev 2024-11-08 08:27:59 +03:00 committed by GitHub
parent 3fac9a9d69
commit a7bb17b092
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 143 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
modules/core/include/opencv2/core/persistence.hpp
View File

@ -365,6 +365,8 @@ public:
*/
CV_WRAP void write(const String& name, int val);
/// @overload
CV_WRAP void write(const String& name, int64_t val);
/// @overload
CV_WRAP void write(const String& name, double val);
/// @overload
CV_WRAP void write(const String& name, const String& val);
@ -530,6 +532,8 @@ public:
CV_WRAP size_t rawSize() const;
//! returns the node content as an integer. If the node stores floating-point number, it is rounded.
operator int() const;
//! returns the node content as a signed 64bit integer. If the node stores floating-point number, it is rounded.
operator int64_t() const;
//! returns the node content as float
operator float() const;
//! returns the node content as double
@ -654,6 +658,7 @@ protected:
/////////////////// XML & YAML I/O implementation //////////////////
CV_EXPORTS void write( FileStorage& fs, const String& name, int value );
CV_EXPORTS void write( FileStorage& fs, const String& name, int64_t value );
CV_EXPORTS void write( FileStorage& fs, const String& name, float value );
CV_EXPORTS void write( FileStorage& fs, const String& name, double value );
CV_EXPORTS void write( FileStorage& fs, const String& name, const String& value );
@ -665,11 +670,13 @@ CV_EXPORTS void write( FileStorage& fs, const String& name, const std::vector<DM
#endif
CV_EXPORTS void writeScalar( FileStorage& fs, int value );
CV_EXPORTS void writeScalar( FileStorage& fs, int64_t value );
CV_EXPORTS void writeScalar( FileStorage& fs, float value );
CV_EXPORTS void writeScalar( FileStorage& fs, double value );
CV_EXPORTS void writeScalar( FileStorage& fs, const String& value );
CV_EXPORTS void read(const FileNode& node, int& value, int default_value);
CV_EXPORTS void read(const FileNode& node, int64_t& value, int64_t default_value);
CV_EXPORTS void read(const FileNode& node, float& value, float default_value);
CV_EXPORTS void read(const FileNode& node, double& value, double default_value);
CV_EXPORTS void read(const FileNode& node, std::string& value, const std::string& default_value);

105
modules/core/src/persistence.cpp
View File

@ -176,7 +176,7 @@ char* floatToString( char* buf, size_t bufSize, float value, bool halfprecision,
return buf;
}
static const char symbols[] = "ucwsifdhHbLUn";
static const char symbols[] = "ucwsifdhHbUIn";
static char typeSymbol(int depth)
{
@ -354,6 +354,20 @@ static inline int readInt(const uchar* p)
#endif
}
static inline int64_t readLong(const uchar* p)
{
// On little endian CPUs, both branches produce the same result. On big endian, only the else branch does.
#if CV_LITTLE_ENDIAN_MEM_ACCESS
int64_t val;
memcpy(&val, p, sizeof(val));
return val;
#else
unsigned val0 = (unsigned)(p[0] | (p[1] << 8) | (p[2] << 16) | (p[3] << 24));
unsigned val1 = (unsigned)(p[4] | (p[5] << 8) | (p[6] << 16) | (p[7] << 24));
return val0 | ((int64_t)val1 << 32);
#endif
}
static inline double readReal(const uchar* p)
{
// On little endian CPUs, both branches produce the same result. On big endian, only the else branch does.
@ -370,16 +384,15 @@ static inline double readReal(const uchar* p)
#endif
}
static inline void writeInt(uchar* p, int ival)
template <typename T>
static inline void writeInt(uchar* p, T ival)
{
// On little endian CPUs, both branches produce the same result. On big endian, only the else branch does.
#if CV_LITTLE_ENDIAN_MEM_ACCESS
memcpy(p, &ival, sizeof(ival));
#else
p[0] = (uchar)ival;
p[1] = (uchar)(ival >> 8);
p[2] = (uchar)(ival >> 16);
p[3] = (uchar)(ival >> 24);
for (size_t i = 0, j = 0; i < sizeof(ival); ++i, j += 8)
p[i] = (uchar)(ival >> j);
#endif
}
@ -1083,6 +1096,11 @@ void FileStorage::Impl::write(const String &key, int value) {
getEmitter().write(key.c_str(), value);
}
void FileStorage::Impl::write(const String &key, int64_t value) {
CV_Assert(write_mode);
getEmitter().write(key.c_str(), value);
}
void FileStorage::Impl::write(const String &key, double value) {
CV_Assert(write_mode);
getEmitter().write(key.c_str(), value);
@ -1455,7 +1473,7 @@ void FileStorage::Impl::convertToCollection(int type, FileNode &node) {
bool named = node.isNamed();
uchar *ptr = node.ptr() + 1 + (named ? 4 : 0);
int ival = 0;
int64_t ival = 0;
double fval = 0;
std::string sval;
bool add_first_scalar = false;
@ -1468,7 +1486,7 @@ void FileStorage::Impl::convertToCollection(int type, FileNode &node) {
// otherwise we don't know where to get the element names from
CV_Assert(type == FileNode::SEQ);
if (node_type == FileNode::INT) {
ival = readInt(ptr);
ival = readLong(ptr);
add_first_scalar = true;
} else if (node_type == FileNode::REAL) {
fval = readReal(ptr);
@ -1830,7 +1848,7 @@ char *FileStorage::Impl::parseBase64(char *ptr, int indent, FileNode &collection
int fmt_pairs[CV_FS_MAX_FMT_PAIRS * 2];
int fmt_pair_count = fs::decodeFormat(dt, fmt_pairs, CV_FS_MAX_FMT_PAIRS);
int ival = 0;
int64_t ival = 0;
double fval = 0;
for (;;) {
@ -2070,6 +2088,11 @@ void writeScalar( FileStorage& fs, int value )
fs.p->write(String(), value);
}
void writeScalar( FileStorage& fs, int64_t value )
{
fs.p->write(String(), value);
}
void writeScalar( FileStorage& fs, float value )
{
fs.p->write(String(), (double)value);
@ -2090,6 +2113,11 @@ void write( FileStorage& fs, const String& name, int value )
fs.p->write(name, value);
}
void write( FileStorage& fs, const String& name, int64_t value )
{
fs.p->write(name, value);
}
void write( FileStorage& fs, const String& name, float value )
{
fs.p->write(name, (double)value);
@ -2106,6 +2134,7 @@ void write( FileStorage& fs, const String& name, const String& value )
}
void FileStorage::write(const String& name, int val) { p->write(name, val); }
void FileStorage::write(const String& name, int64_t val) { p->write(name, val); }
void FileStorage::write(const String& name, double val) { p->write(name, val); }
void FileStorage::write(const String& name, const String& val) { p->write(name, val); }
void FileStorage::write(const String& name, const Mat& val) { cv::write(*this, name, val); }
@ -2326,6 +2355,27 @@ FileNode::operator int() const
return 0x7fffffff;
}
FileNode::operator int64_t() const
{
const uchar* p = ptr();
if(!p)
return 0;
int tag = *p;
int type = (tag & TYPE_MASK);
p += (tag & NAMED) ? 5 : 1;
if( type == INT )
{
return readLong(p);
}
else if( type == REAL )
{
return cvRound(readReal(p));
}
else
return 0x7fffffff;
}
FileNode::operator float() const
{
const uchar* p = ptr();
@ -2450,7 +2500,13 @@ void FileNode::setValue( int type, const void* value, int len )
sz += 4;
if( type == INT )
sz += 4;
{
int64_t ival = *(const int64_t*)value;
if (ival > INT_MAX || ival < INT_MIN)
sz += 8;
else
sz += 4;
}
else if( type == REAL )
sz += 8;
else if( type == STRING )
@ -2470,8 +2526,11 @@ void FileNode::setValue( int type, const void* value, int len )
if( type == INT )
{
int ival = *(const int*)value;
writeInt(p, ival);
int64_t ival = *(const int64_t*)value;
if (sz > 8)
writeInt(p, ival);
else
writeInt(p, static_cast<int>(ival));
}
else if( type == REAL )
{
@ -2622,12 +2681,12 @@ FileNodeIterator& FileNodeIterator::readRaw( const String& fmt, void* _data0, si
offset = alignSize( offset, elem_size );
uchar* data = data0 + offset;
for( int i = 0; i < count; i++, ++(*this) )
for( int i = 0; i < count; i++ )
{
FileNode node = *(*this);
if( node.isInt() )
{
int ival = (int)node;
int64_t ival = static_cast<int64_t>(elem_size == 8 ? (int64_t)node : (int)node);
switch( elem_type )
{
case CV_8U:
@ -2651,11 +2710,11 @@ FileNodeIterator& FileNodeIterator::readRaw( const String& fmt, void* _data0, si
data += sizeof(short);
break;
case CV_32U:
*(unsigned*)data = (unsigned)std::max(ival, 0);
*(unsigned*)data = (unsigned)std::max(ival, (int64_t)0);
data += sizeof(unsigned);
break;
case CV_32S:
*(int*)data = ival;
*(int*)data = (int)ival;
data += sizeof(int);
break;
case CV_32F:
@ -2746,6 +2805,11 @@ FileNodeIterator& FileNodeIterator::readRaw( const String& fmt, void* _data0, si
}
else
CV_Error( Error::StsError, "readRawData can only be used to read plain sequences of numbers" );
++(*this);
if (elem_type == CV_64S)
{
ofs += 4;
}
}
offset = (int)(data - data0);
}
@ -2785,6 +2849,15 @@ void read(const FileNode& node, int& val, int default_val)
}
}
void read(const FileNode& node, int64_t& val, int64_t default_val)
{
val = default_val;
if( !node.empty() )
{
val = (int64_t)node;
}
}
void read(const FileNode& node, double& val, double default_val)
{
val = default_val;

1
modules/core/src/persistence.hpp
View File

@ -194,6 +194,7 @@ public:
int struct_flags, const char* type_name=0 ) = 0;
virtual void endWriteStruct(const FStructData& current_struct) = 0;
virtual void write(const char* key, int value) = 0;
virtual void write(const char* key, int64_t value) = 0;
virtual void write(const char* key, double value) = 0;
virtual void write(const char* key, const char* value, bool quote) = 0;
virtual void writeScalar(const char* key, const char* value) = 0;

2
modules/core/src/persistence_impl.hpp
View File

@ -69,6 +69,8 @@ public:
void write( const String& key, int value );
void write( const String& key, int64_t value );
void write( const String& key, double value );
void write( const String& key, const String& value );

10
modules/core/src/persistence_json.cpp
View File

@ -81,6 +81,12 @@ public:
writeScalar( key, fs::itoa( value, buf, 10 ));
}
void write(const char* key, int64_t value)
{
char buf[128];
writeScalar( key, fs::itoa( value, buf, 10, true ));
}
void write( const char* key, double value )
{
char buf[128];
@ -596,7 +602,7 @@ public:
}
else
{
int ival = (int)strtol( beg, &ptr, 0 );
int64_t ival = strtoll( beg, &ptr, 0 );
CV_PERSISTENCE_CHECK_END_OF_BUFFER_BUG_CPP();
node.setValue(FileNode::INT, &ival);
@ -623,7 +629,7 @@ public:
else if( (len == 4 && memcmp( beg, "true", 4 ) == 0) ||
(len == 5 && memcmp( beg, "false", 5 ) == 0) )
{
int ival = *beg == 't' ? 1 : 0;
int64_t ival = *beg == 't' ? 1 : 0;
node.setValue(FileNode::INT, &ival);
}
else

8
modules/core/src/persistence_xml.cpp
View File

@ -145,6 +145,12 @@ public:
writeScalar( key, ptr);
}
void write(const char* key, int64_t value)
{
char buf[128], *ptr = fs::itoa( value, buf, 10, true );
writeScalar( key, ptr);
}
void write( const char* key, double value )
{
char buf[128];
@ -556,7 +562,7 @@ public:
}
else
{
int ival = (int)strtol( ptr, &endptr, 0 );
int64_t ival = strtoll( ptr, &endptr, 0 );
elem->setValue(FileNode::INT, &ival);
}

8
modules/core/src/persistence_yml.cpp
View File

@ -107,6 +107,12 @@ public:
writeScalar( key, fs::itoa( value, buf, 10 ));
}
void write(const char* key, int64_t value)
{
char buf[128];
writeScalar( key, fs::itoa( value, buf, 10, true ));
}
void write( const char* key, double value )
{
char buf[128];
@ -567,7 +573,7 @@ public:
else
{
force_int:
int ival = (int)strtol( ptr, &endptr, 0 );
int64_t ival = strtoll( ptr, &endptr, 0 );
node.setValue(FileNode::INT, &ival);
}

21
modules/core/test/test_io.cpp
View File

@ -2066,6 +2066,27 @@ TEST_P(FileStorage_exact_type, mat_1d)
testExactMat(Mat({1}, CV_32S, Scalar(8)), GetParam());
}
TEST_P(FileStorage_exact_type, long_int)
{
for (const int64_t expected : std::vector<int64_t>{INT64_MAX, INT64_MIN, -1, 1, 0})
{
int64_t value = fsWriteRead(expected, GetParam());
EXPECT_EQ(value, expected);
}
}
TEST_P(FileStorage_exact_type, long_int_mat)
{
Mat src(2, 4, CV_64SC(3));
int64_t* data = src.ptr<int64_t>();
for (size_t i = 0; i < src.total() * src.channels(); ++i)
{
data[i] = INT64_MAX - static_cast<int64_t>(std::rand());
}
Mat dst = fsWriteRead(src, GetParam());
EXPECT_EQ(cv::norm(src, dst, NORM_INF), 0.0);
}
INSTANTIATE_TEST_CASE_P(Core_InputOutput,
FileStorage_exact_type, Values(".yml", ".xml", ".json")
);

1
modules/python/src2/typing_stubs_generation/predefined_types.py
View File

@ -27,6 +27,7 @@ _PREDEFINED_TYPES = (
PrimitiveTypeNode.int_("int32_t"),
PrimitiveTypeNode.int_("uint32_t"),
PrimitiveTypeNode.int_("size_t"),
PrimitiveTypeNode.int_("int64_t"),
PrimitiveTypeNode.float_("float"),
PrimitiveTypeNode.float_("double"),
PrimitiveTypeNode.bool_("bool"),