From 8aa6da61dc992fec50c5e06b4d72b71008cb2b62 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 20 Jul 2020 13:57:19 +0200 Subject: [PATCH] :construction: support for UBJSON high-precision numbers #2286 --- include/nlohmann/detail/exceptions.hpp | 2 +- .../nlohmann/detail/output/binary_writer.hpp | 46 +++++++++++++----- single_include/nlohmann/json.hpp | 48 ++++++++++++++----- test/src/unit-ubjson.cpp | 25 ++++++---- 4 files changed, 86 insertions(+), 35 deletions(-) diff --git a/include/nlohmann/detail/exceptions.hpp b/include/nlohmann/detail/exceptions.hpp index 9ead68557..dd92897d5 100644 --- a/include/nlohmann/detail/exceptions.hpp +++ b/include/nlohmann/detail/exceptions.hpp @@ -286,7 +286,7 @@ json.exception.out_of_range.403 | key 'foo' not found | The provided key was not json.exception.out_of_range.404 | unresolved reference token 'foo' | A reference token in a JSON Pointer could not be resolved. json.exception.out_of_range.405 | JSON pointer has no parent | The JSON Patch operations 'remove' and 'add' can not be applied to the root element of the JSON value. json.exception.out_of_range.406 | number overflow parsing '10E1000' | A parsed number could not be stored as without changing it to NaN or INF. -json.exception.out_of_range.407 | number overflow serializing '9223372036854775808' | UBJSON and BSON only support integer numbers up to 9223372036854775807. | +json.exception.out_of_range.407 | number overflow serializing '9223372036854775808' | UBJSON and BSON only support integer numbers up to 9223372036854775807. (until version 3.8.0) | json.exception.out_of_range.408 | excessive array size: 8658170730974374167 | The size (following `#`) of an UBJSON array or object exceeds the maximal capacity. | json.exception.out_of_range.409 | BSON key cannot contain code point U+0000 (at byte 2) | Key identifiers to be serialized to BSON cannot contain code point U+0000, since the key is stored as zero-terminated c-string | diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 342cb4784..b707f772b 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -1319,7 +1319,17 @@ class binary_writer } else { - JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(n) + " cannot be represented by UBJSON as it does not fit int64")); + if (add_prefix) + { + oa->write_character(to_char_type('H')); // high-precision number + } + + const auto number = BasicJsonType(n).dump(); + write_number_with_ubjson_prefix(number.size(), true); + for (std::size_t i = 0; i < number.size(); ++i) + { + oa->write_character(to_char_type(number[i])); + } } } @@ -1373,19 +1383,23 @@ class binary_writer // LCOV_EXCL_START else { - JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(n) + " cannot be represented by UBJSON as it does not fit int64")); + if (add_prefix) + { + oa->write_character(to_char_type('H')); // high-precision number + } + + const auto number = BasicJsonType(n).dump(); + write_number_with_ubjson_prefix(number.size(), true); + for (std::size_t i = 0; i < number.size(); ++i) + { + oa->write_character(to_char_type(number[i])); + } } // LCOV_EXCL_STOP } /*! @brief determine the type prefix of container values - - @note This function does not need to be 100% accurate when it comes to - integer limits. In case a number exceeds the limits of int64_t, - this will be detected by a later call to function - write_number_with_ubjson_prefix. Therefore, we return 'L' for any - value that does not fit the previous limits. */ CharType ubjson_prefix(const BasicJsonType& j) const noexcept { @@ -1415,8 +1429,12 @@ class binary_writer { return 'l'; } - // no check and assume int64_t (see note above) - return 'L'; + if ((std::numeric_limits::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits::max)()) + { + return 'L'; + } + // anything else is treated as high-precision number + return 'H'; } case value_t::number_unsigned: @@ -1437,8 +1455,12 @@ class binary_writer { return 'l'; } - // no check and assume int64_t (see note above) - return 'L'; + if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + { + return 'L'; + } + // anything else is treated as high-precision number + return 'H'; } case value_t::number_float: diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 4ecaac4e1..48504a85c 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -2469,7 +2469,7 @@ json.exception.out_of_range.403 | key 'foo' not found | The provided key was not json.exception.out_of_range.404 | unresolved reference token 'foo' | A reference token in a JSON Pointer could not be resolved. json.exception.out_of_range.405 | JSON pointer has no parent | The JSON Patch operations 'remove' and 'add' can not be applied to the root element of the JSON value. json.exception.out_of_range.406 | number overflow parsing '10E1000' | A parsed number could not be stored as without changing it to NaN or INF. -json.exception.out_of_range.407 | number overflow serializing '9223372036854775808' | UBJSON and BSON only support integer numbers up to 9223372036854775807. | +json.exception.out_of_range.407 | number overflow serializing '9223372036854775808' | UBJSON and BSON only support integer numbers up to 9223372036854775807. (until version 3.8.0) | json.exception.out_of_range.408 | excessive array size: 8658170730974374167 | The size (following `#`) of an UBJSON array or object exceeds the maximal capacity. | json.exception.out_of_range.409 | BSON key cannot contain code point U+0000 (at byte 2) | Key identifiers to be serialized to BSON cannot contain code point U+0000, since the key is stored as zero-terminated c-string | @@ -13889,7 +13889,17 @@ class binary_writer } else { - JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(n) + " cannot be represented by UBJSON as it does not fit int64")); + if (add_prefix) + { + oa->write_character(to_char_type('H')); // high-precision number + } + + const auto number = BasicJsonType(n).dump(); + write_number_with_ubjson_prefix(number.size(), true); + for (std::size_t i = 0; i < number.size(); ++i) + { + oa->write_character(to_char_type(number[i])); + } } } @@ -13943,19 +13953,23 @@ class binary_writer // LCOV_EXCL_START else { - JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(n) + " cannot be represented by UBJSON as it does not fit int64")); + if (add_prefix) + { + oa->write_character(to_char_type('H')); // high-precision number + } + + const auto number = BasicJsonType(n).dump(); + write_number_with_ubjson_prefix(number.size(), true); + for (std::size_t i = 0; i < number.size(); ++i) + { + oa->write_character(to_char_type(number[i])); + } } // LCOV_EXCL_STOP } /*! @brief determine the type prefix of container values - - @note This function does not need to be 100% accurate when it comes to - integer limits. In case a number exceeds the limits of int64_t, - this will be detected by a later call to function - write_number_with_ubjson_prefix. Therefore, we return 'L' for any - value that does not fit the previous limits. */ CharType ubjson_prefix(const BasicJsonType& j) const noexcept { @@ -13985,8 +13999,12 @@ class binary_writer { return 'l'; } - // no check and assume int64_t (see note above) - return 'L'; + if ((std::numeric_limits::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits::max)()) + { + return 'L'; + } + // anything else is treated as high-precision number + return 'H'; } case value_t::number_unsigned: @@ -14007,8 +14025,12 @@ class binary_writer { return 'l'; } - // no check and assume int64_t (see note above) - return 'L'; + if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + { + return 'L'; + } + // anything else is treated as high-precision number + return 'H'; } case value_t::number_float: diff --git a/test/src/unit-ubjson.cpp b/test/src/unit-ubjson.cpp index ea15ba1ce..c8b5f6bc3 100644 --- a/test/src/unit-ubjson.cpp +++ b/test/src/unit-ubjson.cpp @@ -32,6 +32,7 @@ SOFTWARE. #include using nlohmann::json; +#include #include #include #include @@ -803,6 +804,21 @@ TEST_CASE("UBJSON") std::vector vec3 = {'H', 2, '1', '0'}; CHECK_THROWS_WITH_AS(json::from_ubjson(vec3), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing UBJSON size: expected length type specification (U, i, I, l, L) after '#'; last byte: 0x02", json::parse_error); } + + SECTION("serialization") + { + // number that does not fit int64 + json j = 11111111111111111111ULL; + CHECK(j.is_number_unsigned()); + + // number will be serialized to high-precision number + const auto vec = json::to_ubjson(j); + std::vector expected = {'H', 'i', 0x14, '1', '1', '1', '1', '1', '1', '1', '1', '1', '1', '1', '1', '1', '1', '1', '1', '1', '1', '1', '1'}; + CHECK(vec == expected); + + // roundtrip + CHECK(json::from_ubjson(vec) == j); + } } } @@ -1576,15 +1592,6 @@ TEST_CASE("UBJSON") } } - SECTION("number out of range") - { - // larger than max int64 - json j = 9223372036854775808llu; - json _; - CHECK_THROWS_AS(_ = json::to_ubjson(j), json::out_of_range&); - CHECK_THROWS_WITH(_ = json::to_ubjson(j), "[json.exception.out_of_range.407] integer number 9223372036854775808 cannot be represented by UBJSON as it does not fit int64"); - } - SECTION("excessive size") { SECTION("array")