From 22929fe18960c1a6d9c6fa18b666a1961cb38e36 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Sun, 18 Mar 2018 15:13:53 +0100 Subject: [PATCH] :construction: started a SAX/DOM/callback parser --- include/nlohmann/detail/input/json_sax.hpp | 216 +++++++++++++++++++ include/nlohmann/detail/input/parser.hpp | 21 ++ include/nlohmann/json.hpp | 2 + single_include/nlohmann/json.hpp | 240 +++++++++++++++++++++ 4 files changed, 479 insertions(+) diff --git a/include/nlohmann/detail/input/json_sax.hpp b/include/nlohmann/detail/input/json_sax.hpp index 8bbaa9c9d..e19088a93 100644 --- a/include/nlohmann/detail/input/json_sax.hpp +++ b/include/nlohmann/detail/input/json_sax.hpp @@ -4,6 +4,7 @@ #include #include +#include #include namespace nlohmann @@ -310,6 +311,221 @@ class json_sax_dom_parser : public json_sax const bool allow_exceptions = true; }; +template +class json_sax_dom_callback_parser : public json_sax +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using parser_callback_t = typename BasicJsonType::parser_callback_t; + using parse_event_t = typename BasicJsonType::parse_event_t; + + json_sax_dom_callback_parser(BasicJsonType& r, + const parser_callback_t cb = nullptr, + const bool allow_exceptions_ = true) + : root(r), callback(cb), allow_exceptions(allow_exceptions_) + { + keep_stack.push_back(true); + } + + bool null() override + { + handle_value(nullptr); + return true; + } + + bool boolean(bool val) override + { + handle_value(val); + return true; + } + + bool number_integer(number_integer_t val) override + { + handle_value(val); + return true; + } + + bool number_unsigned(number_unsigned_t val) override + { + handle_value(val); + return true; + } + + bool number_float(number_float_t val, const string_t&) override + { + handle_value(val); + return true; + } + + bool string(string_t&& val) override + { + handle_value(val); + return true; + } + + bool start_object(std::size_t len) override + { + const bool keep = callback(ref_stack.size() + 1, parse_event_t::object_start, discarded); + keep_stack.push_back(keep); + + ref_stack.push_back(handle_value(BasicJsonType::value_t::object)); + + if (JSON_UNLIKELY(len != json_sax::no_limit and len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, + "excessive object size: " + std::to_string(len))); + } + + return true; + } + + bool key(string_t&& val) override + { + BasicJsonType k = BasicJsonType(std::forward < string_t&& > (val)); + const bool keep = callback(ref_stack.size(), parse_event_t::key, k); + + // add null at given key and store the reference for later + object_element = &(ref_stack.back()->m_value.object->operator[](val)); + return true; + } + + bool end_object() override + { + const bool keep = callback(ref_stack.size() - 1, parse_event_t::object_end, *ref_stack.back()); + if (not keep) + { + *ref_stack.back() = discarded; + } + + ref_stack.pop_back(); + keep_stack.pop_back(); + return true; + } + + bool start_array(std::size_t len) override + { + const bool keep = callback(ref_stack.size() + 1, parse_event_t::array_start, discarded); + keep_stack.push_back(keep); + + ref_stack.push_back(handle_value(BasicJsonType::value_t::array)); + + if (JSON_UNLIKELY(len != json_sax::no_limit and len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, + "excessive array size: " + std::to_string(len))); + } + + return true; + } + + bool end_array() override + { + const bool keep = callback(ref_stack.size() - 1, parse_event_t::array_end, *ref_stack.back()); + if (not keep) + { + *ref_stack.back() = discarded; + } + + ref_stack.pop_back(); + keep_stack.pop_back(); + return true; + } + + bool binary(const std::vector&) override + { + return true; + } + + bool parse_error(std::size_t, const std::string&, + const detail::exception& ex) override + { + errored = true; + if (allow_exceptions) + { + // determine the proper exception type from the id + switch ((ex.id / 100) % 100) + { + case 1: + JSON_THROW(*reinterpret_cast(&ex)); + case 2: + JSON_THROW(*reinterpret_cast(&ex)); // LCOV_EXCL_LINE + case 3: + JSON_THROW(*reinterpret_cast(&ex)); // LCOV_EXCL_LINE + case 4: + JSON_THROW(*reinterpret_cast(&ex)); + case 5: + JSON_THROW(*reinterpret_cast(&ex)); // LCOV_EXCL_LINE + default: + assert(false); // LCOV_EXCL_LINE + } + } + return false; + } + + bool is_errored() const + { + return errored; + } + + private: + /*! + @invariant If the ref stack is empty, then the passed value will be the new + root. + @invariant If the ref stack contains a value, then it is an array or an + object to which we can add elements + */ + template + BasicJsonType* handle_value(Value&& v) + { + if (ref_stack.empty()) + { + root = BasicJsonType(std::forward(v)); + return &root; + } + else + { + switch (ref_stack.back()->m_type) + { + case value_t::array: + { + ref_stack.back()->m_value.array->push_back(BasicJsonType(std::forward(v))); + return &(ref_stack.back()->m_value.array->back()); + } + + case value_t::object: + { + assert(object_element); + *object_element = BasicJsonType(std::forward(v)); + return object_element; + } + + default: + assert(false); // LCOV_EXCL_LINE + } + } + } + + /// the parsed JSON value + BasicJsonType& root; + /// stack to model hierarchy of values + std::vector ref_stack; + /// stack to manage which values to keep + std::vector keep_stack; + /// helper to hold the reference for the next object element + BasicJsonType* object_element = nullptr; + /// whether a syntax error occurred + bool errored = false; + /// callback function + const parser_callback_t callback = nullptr; + /// whether to throw exceptions in case of errors + const bool allow_exceptions = true; + /// a discarded value for the callback + BasicJsonType discarded = BasicJsonType::value_t::discarded; +}; + template class json_sax_acceptor : public json_sax { diff --git a/include/nlohmann/detail/input/parser.hpp b/include/nlohmann/detail/input/parser.hpp index 83bd75f17..c0710cf8e 100644 --- a/include/nlohmann/detail/input/parser.hpp +++ b/include/nlohmann/detail/input/parser.hpp @@ -83,6 +83,27 @@ class parser { if (callback) { + /* + json_sax_dom_callback_parser sdp(result, callback, allow_exceptions); + sax_parse_internal(&sdp); + result.assert_invariant(); + + // in strict mode, input must be completely read + if (strict and (get_token() != token_type::end_of_input)) + { + sdp.parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input))); + } + + // in case of an error, return discarded value + if (sdp.is_errored()) + { + result = value_t::discarded; + return; + } + */ + parse_internal(true, result); result.assert_invariant(); diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 022845976..bdc905f14 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -173,6 +173,8 @@ class basic_json friend class ::nlohmann::detail::binary_reader; template friend class ::nlohmann::detail::json_sax_dom_parser; + template + friend class ::nlohmann::detail::json_sax_dom_callback_parser; /// workaround type for MSVC using basic_json_t = NLOHMANN_BASIC_JSON_TPL; diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 503029031..f9d997b9c 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -3139,6 +3139,8 @@ scan_number_done: #include #include +// #include + // #include @@ -3446,6 +3448,221 @@ class json_sax_dom_parser : public json_sax const bool allow_exceptions = true; }; +template +class json_sax_dom_callback_parser : public json_sax +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using parser_callback_t = typename BasicJsonType::parser_callback_t; + using parse_event_t = typename BasicJsonType::parse_event_t; + + json_sax_dom_callback_parser(BasicJsonType& r, + const parser_callback_t cb = nullptr, + const bool allow_exceptions_ = true) + : root(r), callback(cb), allow_exceptions(allow_exceptions_) + { + keep_stack.push_back(true); + } + + bool null() override + { + handle_value(nullptr); + return true; + } + + bool boolean(bool val) override + { + handle_value(val); + return true; + } + + bool number_integer(number_integer_t val) override + { + handle_value(val); + return true; + } + + bool number_unsigned(number_unsigned_t val) override + { + handle_value(val); + return true; + } + + bool number_float(number_float_t val, const string_t&) override + { + handle_value(val); + return true; + } + + bool string(string_t&& val) override + { + handle_value(val); + return true; + } + + bool start_object(std::size_t len) override + { + const bool keep = callback(ref_stack.size() + 1, parse_event_t::object_start, discarded); + keep_stack.push_back(keep); + + ref_stack.push_back(handle_value(BasicJsonType::value_t::object)); + + if (JSON_UNLIKELY(len != json_sax::no_limit and len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, + "excessive object size: " + std::to_string(len))); + } + + return true; + } + + bool key(string_t&& val) override + { + BasicJsonType k = BasicJsonType(std::forward < string_t&& > (val)); + const bool keep = callback(ref_stack.size(), parse_event_t::key, k); + + // add null at given key and store the reference for later + object_element = &(ref_stack.back()->m_value.object->operator[](val)); + return true; + } + + bool end_object() override + { + const bool keep = callback(ref_stack.size() - 1, parse_event_t::object_end, *ref_stack.back()); + if (not keep) + { + *ref_stack.back() = discarded; + } + + ref_stack.pop_back(); + keep_stack.pop_back(); + return true; + } + + bool start_array(std::size_t len) override + { + const bool keep = callback(ref_stack.size() + 1, parse_event_t::array_start, discarded); + keep_stack.push_back(keep); + + ref_stack.push_back(handle_value(BasicJsonType::value_t::array)); + + if (JSON_UNLIKELY(len != json_sax::no_limit and len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, + "excessive array size: " + std::to_string(len))); + } + + return true; + } + + bool end_array() override + { + const bool keep = callback(ref_stack.size() - 1, parse_event_t::array_end, *ref_stack.back()); + if (not keep) + { + *ref_stack.back() = discarded; + } + + ref_stack.pop_back(); + keep_stack.pop_back(); + return true; + } + + bool binary(const std::vector&) override + { + return true; + } + + bool parse_error(std::size_t, const std::string&, + const detail::exception& ex) override + { + errored = true; + if (allow_exceptions) + { + // determine the proper exception type from the id + switch ((ex.id / 100) % 100) + { + case 1: + JSON_THROW(*reinterpret_cast(&ex)); + case 2: + JSON_THROW(*reinterpret_cast(&ex)); // LCOV_EXCL_LINE + case 3: + JSON_THROW(*reinterpret_cast(&ex)); // LCOV_EXCL_LINE + case 4: + JSON_THROW(*reinterpret_cast(&ex)); + case 5: + JSON_THROW(*reinterpret_cast(&ex)); // LCOV_EXCL_LINE + default: + assert(false); // LCOV_EXCL_LINE + } + } + return false; + } + + bool is_errored() const + { + return errored; + } + + private: + /*! + @invariant If the ref stack is empty, then the passed value will be the new + root. + @invariant If the ref stack contains a value, then it is an array or an + object to which we can add elements + */ + template + BasicJsonType* handle_value(Value&& v) + { + if (ref_stack.empty()) + { + root = BasicJsonType(std::forward(v)); + return &root; + } + else + { + switch (ref_stack.back()->m_type) + { + case value_t::array: + { + ref_stack.back()->m_value.array->push_back(BasicJsonType(std::forward(v))); + return &(ref_stack.back()->m_value.array->back()); + } + + case value_t::object: + { + assert(object_element); + *object_element = BasicJsonType(std::forward(v)); + return object_element; + } + + default: + assert(false); // LCOV_EXCL_LINE + } + } + } + + /// the parsed JSON value + BasicJsonType& root; + /// stack to model hierarchy of values + std::vector ref_stack; + /// stack to manage which values to keep + std::vector keep_stack; + /// helper to hold the reference for the next object element + BasicJsonType* object_element = nullptr; + /// whether a syntax error occurred + bool errored = false; + /// callback function + const parser_callback_t callback = nullptr; + /// whether to throw exceptions in case of errors + const bool allow_exceptions = true; + /// a discarded value for the callback + BasicJsonType discarded = BasicJsonType::value_t::discarded; +}; + template class json_sax_acceptor : public json_sax { @@ -3599,6 +3816,27 @@ class parser { if (callback) { + /* + json_sax_dom_callback_parser sdp(result, callback, allow_exceptions); + sax_parse_internal(&sdp); + result.assert_invariant(); + + // in strict mode, input must be completely read + if (strict and (get_token() != token_type::end_of_input)) + { + sdp.parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), exception_message(token_type::end_of_input))); + } + + // in case of an error, return discarded value + if (sdp.is_errored()) + { + result = value_t::discarded; + return; + } + */ + parse_internal(true, result); result.assert_invariant(); @@ -10411,6 +10649,8 @@ class basic_json friend class ::nlohmann::detail::binary_reader; template friend class ::nlohmann::detail::json_sax_dom_parser; + template + friend class ::nlohmann::detail::json_sax_dom_callback_parser; /// workaround type for MSVC using basic_json_t = NLOHMANN_BASIC_JSON_TPL;