From f0e73163f242cd631848d007a3765fe6b94c5bcd Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Sat, 20 Jun 2020 15:25:54 +0200 Subject: [PATCH 1/2] json_pointer.hpp: Mention more exception in documentation Forgotten in dcd3a6c6 (move the catch of std::invalid_argument into array_index(), 2020-03-23). --- include/nlohmann/detail/json_pointer.hpp | 2 ++ single_include/nlohmann/json.hpp | 2 ++ 2 files changed, 4 insertions(+) diff --git a/include/nlohmann/detail/json_pointer.hpp b/include/nlohmann/detail/json_pointer.hpp index 421c5ec8f..18b66e234 100644 --- a/include/nlohmann/detail/json_pointer.hpp +++ b/include/nlohmann/detail/json_pointer.hpp @@ -325,6 +325,8 @@ class json_pointer @return integer representation of @a s + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index begins not with a digit @throw out_of_range.404 if string @a s could not be converted to an integer */ static int array_index(const std::string& s) diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 33d0c3577..22fc0e330 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -11357,6 +11357,8 @@ class json_pointer @return integer representation of @a s + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index begins not with a digit @throw out_of_range.404 if string @a s could not be converted to an integer */ static int array_index(const std::string& s) From ecbb2756fd7629a8dba64be581e19488d0be3061 Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Sat, 20 Jun 2020 15:36:28 +0200 Subject: [PATCH 2/2] json_pointer::array_index: Use unsigned values for the array index when parsing The current code uses std::stoi to convert the input string to an int array_index. This limits the maximum addressable array size to ~2GB on most platforms. But all callers immediately convert the result of array_index to BasicJsonType::size_type. So let's parse it as unsigned long long, which allows us to have as big arrays as available memory. And also makes the call sites nicer to read. One complication arises on platforms where size_type is smaller than unsigned long long. We need to bail out on these if the parsed array index does not fit into size_type. --- include/nlohmann/detail/json_pointer.hpp | 33 +++++++++++++-------- include/nlohmann/json.hpp | 4 +-- single_include/nlohmann/json.hpp | 37 +++++++++++++++--------- test/src/unit-json_pointer.cpp | 29 +++++++++++++++---- 4 files changed, 69 insertions(+), 34 deletions(-) diff --git a/include/nlohmann/detail/json_pointer.hpp b/include/nlohmann/detail/json_pointer.hpp index 18b66e234..562802920 100644 --- a/include/nlohmann/detail/json_pointer.hpp +++ b/include/nlohmann/detail/json_pointer.hpp @@ -3,6 +3,7 @@ #include // all_of #include // assert #include // isdigit +#include // max #include // accumulate #include // string #include // move @@ -328,9 +329,12 @@ class json_pointer @throw parse_error.106 if an array index begins with '0' @throw parse_error.109 if an array index begins not with a digit @throw out_of_range.404 if string @a s could not be converted to an integer + @throw out_of_range.410 if an array index exceeds size_type */ - static int array_index(const std::string& s) + static typename BasicJsonType::size_type array_index(const std::string& s) { + using size_type = typename BasicJsonType::size_type; + // error condition (cf. RFC 6901, Sect. 4) if (JSON_HEDLEY_UNLIKELY(s.size() > 1 and s[0] == '0')) { @@ -346,10 +350,10 @@ class json_pointer } std::size_t processed_chars = 0; - int res = 0; + unsigned long long res = 0; JSON_TRY { - res = std::stoi(s, &processed_chars); + res = std::stoull(s, &processed_chars); } JSON_CATCH(std::out_of_range&) { @@ -362,7 +366,14 @@ class json_pointer JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + s + "'")); } - return res; + // only triggered on special platforms (like 32bit), see also + // https://github.com/nlohmann/json/pull/2203 + if (res >= static_cast((std::numeric_limits::max)())) + { + JSON_THROW(detail::out_of_range::create(410, "array index " + s + " exceeds size_type")); // LCOV_EXCL_LINE + } + + return static_cast(res); } json_pointer top() const @@ -421,7 +432,7 @@ class json_pointer case detail::value_t::array: { // create an entry in the array - result = &result->operator[](static_cast(array_index(reference_token))); + result = &result->operator[](array_index(reference_token)); break; } @@ -499,8 +510,7 @@ class json_pointer else { // convert array index to number; unchecked access - ptr = &ptr->operator[]( - static_cast(array_index(reference_token))); + ptr = &ptr->operator[](array_index(reference_token)); } break; } @@ -544,7 +554,7 @@ class json_pointer } // note: at performs range check - ptr = &ptr->at(static_cast(array_index(reference_token))); + ptr = &ptr->at(array_index(reference_token)); break; } @@ -594,8 +604,7 @@ class json_pointer } // use unchecked array access - ptr = &ptr->operator[]( - static_cast(array_index(reference_token))); + ptr = &ptr->operator[](array_index(reference_token)); break; } @@ -638,7 +647,7 @@ class json_pointer } // note: at performs range check - ptr = &ptr->at(static_cast(array_index(reference_token))); + ptr = &ptr->at(array_index(reference_token)); break; } @@ -702,7 +711,7 @@ class json_pointer } } - const auto idx = static_cast(array_index(reference_token)); + const auto idx = array_index(reference_token); if (idx >= ptr->size()) { // index out of range diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 790ecd4b2..048307b3a 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -8179,7 +8179,7 @@ class basic_json else { const auto idx = json_pointer::array_index(last_path); - if (JSON_HEDLEY_UNLIKELY(static_cast(idx) > parent.size())) + if (JSON_HEDLEY_UNLIKELY(idx > parent.size())) { // avoid undefined behavior JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range")); @@ -8222,7 +8222,7 @@ class basic_json else if (parent.is_array()) { // note erase performs range check - parent.erase(static_cast(json_pointer::array_index(last_path))); + parent.erase(json_pointer::array_index(last_path)); } }; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 22fc0e330..b919d2318 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -11032,6 +11032,7 @@ class json_reverse_iterator : public std::reverse_iterator #include // all_of #include // assert #include // isdigit +#include // max #include // accumulate #include // string #include // move @@ -11360,9 +11361,12 @@ class json_pointer @throw parse_error.106 if an array index begins with '0' @throw parse_error.109 if an array index begins not with a digit @throw out_of_range.404 if string @a s could not be converted to an integer + @throw out_of_range.410 if an array index exceeds size_type */ - static int array_index(const std::string& s) + static typename BasicJsonType::size_type array_index(const std::string& s) { + using size_type = typename BasicJsonType::size_type; + // error condition (cf. RFC 6901, Sect. 4) if (JSON_HEDLEY_UNLIKELY(s.size() > 1 and s[0] == '0')) { @@ -11378,10 +11382,10 @@ class json_pointer } std::size_t processed_chars = 0; - int res = 0; + unsigned long long res = 0; JSON_TRY { - res = std::stoi(s, &processed_chars); + res = std::stoull(s, &processed_chars); } JSON_CATCH(std::out_of_range&) { @@ -11394,7 +11398,14 @@ class json_pointer JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + s + "'")); } - return res; + // only triggered on special platforms (like 32bit), see also + // https://github.com/nlohmann/json/pull/2203 + if (res >= static_cast((std::numeric_limits::max)())) + { + JSON_THROW(detail::out_of_range::create(410, "array index " + s + " exceeds size_type")); // LCOV_EXCL_LINE + } + + return static_cast(res); } json_pointer top() const @@ -11453,7 +11464,7 @@ class json_pointer case detail::value_t::array: { // create an entry in the array - result = &result->operator[](static_cast(array_index(reference_token))); + result = &result->operator[](array_index(reference_token)); break; } @@ -11531,8 +11542,7 @@ class json_pointer else { // convert array index to number; unchecked access - ptr = &ptr->operator[]( - static_cast(array_index(reference_token))); + ptr = &ptr->operator[](array_index(reference_token)); } break; } @@ -11576,7 +11586,7 @@ class json_pointer } // note: at performs range check - ptr = &ptr->at(static_cast(array_index(reference_token))); + ptr = &ptr->at(array_index(reference_token)); break; } @@ -11626,8 +11636,7 @@ class json_pointer } // use unchecked array access - ptr = &ptr->operator[]( - static_cast(array_index(reference_token))); + ptr = &ptr->operator[](array_index(reference_token)); break; } @@ -11670,7 +11679,7 @@ class json_pointer } // note: at performs range check - ptr = &ptr->at(static_cast(array_index(reference_token))); + ptr = &ptr->at(array_index(reference_token)); break; } @@ -11734,7 +11743,7 @@ class json_pointer } } - const auto idx = static_cast(array_index(reference_token)); + const auto idx = array_index(reference_token); if (idx >= ptr->size()) { // index out of range @@ -23971,7 +23980,7 @@ class basic_json else { const auto idx = json_pointer::array_index(last_path); - if (JSON_HEDLEY_UNLIKELY(static_cast(idx) > parent.size())) + if (JSON_HEDLEY_UNLIKELY(idx > parent.size())) { // avoid undefined behavior JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range")); @@ -24014,7 +24023,7 @@ class basic_json else if (parent.is_array()) { // note erase performs range check - parent.erase(static_cast(json_pointer::array_index(last_path))); + parent.erase(json_pointer::array_index(last_path)); } }; diff --git a/test/src/unit-json_pointer.cpp b/test/src/unit-json_pointer.cpp index ca11efa42..89a41d76b 100644 --- a/test/src/unit-json_pointer.cpp +++ b/test/src/unit-json_pointer.cpp @@ -348,12 +348,29 @@ TEST_CASE("JSON pointers") CHECK_THROWS_WITH(j_const["/1+1"_json_pointer] == 1, "[json.exception.out_of_range.404] unresolved reference token '1+1'"); - CHECK_THROWS_AS(j["/111111111111111111111111"_json_pointer] = 1, json::out_of_range&); - CHECK_THROWS_WITH(j["/111111111111111111111111"_json_pointer] = 1, - "[json.exception.out_of_range.404] unresolved reference token '111111111111111111111111'"); - CHECK_THROWS_AS(j_const["/111111111111111111111111"_json_pointer] == 1, json::out_of_range&); - CHECK_THROWS_WITH(j_const["/111111111111111111111111"_json_pointer] == 1, - "[json.exception.out_of_range.404] unresolved reference token '111111111111111111111111'"); + { + auto too_large_index = std::to_string((std::numeric_limits::max)()) + "1"; + json::json_pointer jp(std::string("/") + too_large_index); + std::string throw_msg = std::string("[json.exception.out_of_range.404] unresolved reference token '") + too_large_index + "'"; + + CHECK_THROWS_AS(j[jp] = 1, json::out_of_range&); + CHECK_THROWS_WITH(j[jp] = 1, throw_msg.c_str()); + CHECK_THROWS_AS(j_const[jp] == 1, json::out_of_range&); + CHECK_THROWS_WITH(j_const[jp] == 1, throw_msg.c_str()); + } + + if (sizeof(typename json::size_type) < sizeof(unsigned long long)) + { + auto size_type_max_uul = static_cast((std::numeric_limits::max)()); + auto too_large_index = std::to_string(size_type_max_uul); + json::json_pointer jp(std::string("/") + too_large_index); + std::string throw_msg = std::string("[json.exception.out_of_range.410] array index ") + too_large_index + " exceeds size_type"; + + CHECK_THROWS_AS(j[jp] = 1, json::out_of_range&); + CHECK_THROWS_WITH(j[jp] = 1, throw_msg.c_str()); + CHECK_THROWS_AS(j_const[jp] == 1, json::out_of_range&); + CHECK_THROWS_WITH(j_const[jp] == 1, throw_msg.c_str()); + } CHECK_THROWS_AS(j.at("/one"_json_pointer) = 1, json::parse_error&); CHECK_THROWS_WITH(j.at("/one"_json_pointer) = 1,