From 166371bcfd3d2f4b865aed47cf147b5e493ac6fc Mon Sep 17 00:00:00 2001 From: Damien Merenne Date: Thu, 29 Dec 2016 10:55:40 +0100 Subject: [PATCH 01/12] Add partial parsing api. --- json11.cpp | 39 ++++++++++++++++++++++++++++++--------- json11.hpp | 20 ++++++++++++++++++++ 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/json11.cpp b/json11.cpp index cf54e08..6b0108b 100644 --- a/json11.cpp +++ b/json11.cpp @@ -326,21 +326,24 @@ static inline bool in_range(long x, long lower, long upper) { return (x >= lower && x <= upper); } -namespace { -/* JsonParser +/* JsonParserPriv * * Object that tracks all state of an in-progress parse. */ -struct JsonParser final { +struct JsonParserPriv final { /* State */ - const string &str; - size_t i; + string str; + size_t i = 0; string &err; - bool failed; + bool failed = false; const JsonParse strategy; + JsonParserPriv(string str, string &err, const JsonParse strategy): + str(str), err(err), strategy(strategy) { + } + /* fail(msg, err_ret = Json()) * * Mark this parse as failed. @@ -719,10 +722,28 @@ struct JsonParser final { return fail("expected value, got " + esc(ch)); } }; -}//namespace { + +JsonParser::JsonParser(): + parser(new JsonParserPriv("", error, JsonParse::STANDARD)) { +} + +JsonParser::JsonParser(JsonParse strategy): + parser(new JsonParserPriv("", error, strategy)) { +} + +JsonParser::~JsonParser() { +} + +void JsonParser::consume(const std::string &in) { + parser->str = in; +} + +Json JsonParser::json() { + return parser->parse_json(0); +} Json Json::parse(const string &in, string &err, JsonParse strategy) { - JsonParser parser { in, 0, err, false, strategy }; + JsonParserPriv parser { in, err, strategy }; Json result = parser.parse_json(0); // Check for any trailing garbage @@ -738,7 +759,7 @@ vector Json::parse_multi(const string &in, std::string::size_type &parser_stop_pos, string &err, JsonParse strategy) { - JsonParser parser { in, 0, err, false, strategy }; + JsonParserPriv parser { in, err, strategy }; parser_stop_pos = 0; vector json_vec; while (parser.i != in.size() && !parser.failed) { diff --git a/json11.hpp b/json11.hpp index 5202ef9..74c7df1 100644 --- a/json11.hpp +++ b/json11.hpp @@ -229,4 +229,24 @@ class JsonValue { virtual ~JsonValue() {} }; +struct JsonParserPriv; + +class JsonParser { +public: + JsonParser(); + JsonParser(JsonParse strategy); + ~JsonParser(); + + Json json(); + void consume(const std::string &in); + + const std::string &last_error() const { + return error; + } + +private: + std::string error; + std::unique_ptr parser; +}; + } // namespace json11 From 55f568f3e07e961424ad56daacf170a9e55c6107 Mon Sep 17 00:00:00 2001 From: Damien Merenne Date: Thu, 29 Dec 2016 11:34:19 +0100 Subject: [PATCH 02/12] Differenciate end of input failures from other failures. For partial input parsing, we need to know when failure is caused by missing data so that we can retry parsing later when we have more data. This commit adds a need_data boolean flag alongs the failed one. It adds a stop function to set that flag and propagate the failure. It replaces checks for failure with a check on both failed and need_data flags. It also adds a eos function to check for end of input. It also replaces the fail calls due to end of input by stop calls. --- json11.cpp | 147 ++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 122 insertions(+), 25 deletions(-) diff --git a/json11.cpp b/json11.cpp index 6b0108b..f4ce4ad 100644 --- a/json11.cpp +++ b/json11.cpp @@ -338,6 +338,8 @@ struct JsonParserPriv final { size_t i = 0; string &err; bool failed = false; + bool need_data = false; + bool eof = false; const JsonParse strategy; JsonParserPriv(string str, string &err, const JsonParse strategy): @@ -360,6 +362,32 @@ struct JsonParserPriv final { return err_ret; } + /* stop(msg, err_ret = Json()) + * + * Mark this parse as needing more data. + */ + Json stop(string &&msg) { + return stop(move(msg), Json()); + } + + template + T stop(string &&msg, const T err_ret) { + if (!failed && !need_data) + err = std::move(msg); + if (eof) + failed = true; + need_data = true; + return err_ret; + } + + /* eos() + * + * Return true if we are at the end of the parsing string + */ + bool eos() { + return i == str.size(); + } + /* consume_whitespace() * * Advance until the current character is non-whitespace. @@ -377,34 +405,38 @@ struct JsonParserPriv final { bool comment_found = false; if (str[i] == '/') { i++; - if (i == str.size()) - return fail("unexpected end of input inside comment", false); + if (eos()) + return stop("unexpected end of input inside comment", false); if (str[i] == '/') { // inline comment i++; - if (i == str.size()) - return fail("unexpected end of input inside inline comment", false); + if (eos()) + return stop("unexpected end of input inside inline comment", false); // advance until next line while (str[i] != '\n') { i++; - if (i == str.size()) - return fail("unexpected end of input inside inline comment", false); + if (eos()) { + if (eof) + break; + else + return stop("unexpected end of input inside inline comment", false); + } } comment_found = true; } else if (str[i] == '*') { // multiline comment i++; if (i > str.size()-2) - return fail("unexpected end of input inside multi-line comment", false); + return stop("unexpected end of input inside multi-line comment", false); // advance until closing tokens while (!(str[i] == '*' && str[i+1] == '/')) { i++; if (i > str.size()-2) - return fail( + return stop( "unexpected end of input inside multi-line comment", false); } i += 2; - if (i == str.size()) - return fail( + if (eos()) + return stop( "unexpected end of input inside multi-line comment", false); comment_found = true; } @@ -437,8 +469,11 @@ struct JsonParserPriv final { */ char get_next_token() { consume_garbage(); - if (i == str.size()) - return fail("unexpected end of input", (char)0); + if (need_data) { + return '\0'; + } else if (eos()) { + return stop("unexpected end of input", '\0'); + } return str[i++]; } @@ -476,8 +511,8 @@ struct JsonParserPriv final { string out; long last_escaped_codepoint = -1; while (true) { - if (i == str.size()) - return fail("unexpected end of input in string", ""); + if (eos()) + return stop("unexpected end of input in string", ""); char ch = str[i++]; @@ -498,8 +533,8 @@ struct JsonParserPriv final { } // Handle escapes - if (i == str.size()) - return fail("unexpected end of input in string", ""); + if (eos()) + return stop("unexpected end of input in string", ""); ch = str[i++]; @@ -574,12 +609,24 @@ struct JsonParserPriv final { // Integer part if (str[i] == '0') { i++; + if (eos()) + return stop("end of input while parsing number"); if (in_range(str[i], '0', '9')) return fail("leading 0s not permitted in numbers"); } else if (in_range(str[i], '1', '9')) { i++; - while (in_range(str[i], '0', '9')) + if (eos()) + return stop("end of input while parsing number"); + while (in_range(str[i], '0', '9')) { i++; + if (eos()) { + if (eof) + break; + else + return stop("end of input while parsing number"); + } + } + } else { return fail("invalid " + esc(str[i]) + " in number"); } @@ -592,25 +639,47 @@ struct JsonParserPriv final { // Decimal part if (str[i] == '.') { i++; + if (eos()) + return stop("end of input while parsing number"); + if (!in_range(str[i], '0', '9')) return fail("at least one digit required in fractional part"); - while (in_range(str[i], '0', '9')) + while (in_range(str[i], '0', '9')) { i++; + if (eos()) { + if (eof) + break; + else + return stop("end of input while parsing number"); + } + } } // Exponent part if (str[i] == 'e' || str[i] == 'E') { i++; + if (eos()) + return stop("end of input while parsing number"); - if (str[i] == '+' || str[i] == '-') + if (str[i] == '+' || str[i] == '-') { i++; + if (eos()) + return stop("end of input while parsing number"); + } if (!in_range(str[i], '0', '9')) return fail("at least one digit required in exponent"); - while (in_range(str[i], '0', '9')) + while (in_range(str[i], '0', '9')) { i++; + if (eos()) { + if (eof) + break; + else + return stop("end of input while parsing number"); + } + } } return std::strtod(str.c_str() + start_pos, nullptr); @@ -624,6 +693,10 @@ struct JsonParserPriv final { Json expect(const string &expected, Json res) { assert(i != 0); i--; + + if (str.length() - i < expected.length()) + return stop("end of input, while parsing " + expected); + if (str.compare(i, expected.length(), expected) == 0) { i += expected.length(); return res; @@ -642,7 +715,7 @@ struct JsonParserPriv final { } char ch = get_next_token(); - if (failed) + if (need_data) return Json(); if (ch == '-' || (ch >= '0' && ch <= '9')) { @@ -669,22 +742,32 @@ struct JsonParserPriv final { return data; while (1) { + if (need_data) + return Json(); + if (ch != '"') return fail("expected '\"' in object, got " + esc(ch)); string key = parse_string(); if (failed) + if (failed || need_data) return Json(); ch = get_next_token(); + if (need_data) + return Json(); + if (ch != ':') return fail("expected ':' in object, got " + esc(ch)); data[std::move(key)] = parse_json(depth + 1); - if (failed) + if (failed || need_data) return Json(); ch = get_next_token(); + if (need_data) + return Json(); + if (ch == '}') break; if (ch != ',') @@ -702,12 +785,18 @@ struct JsonParserPriv final { return data; while (1) { + if (need_data) + return Json(); + i--; data.push_back(parse_json(depth + 1)); - if (failed) + if (failed || need_data) return Json(); ch = get_next_token(); + if (need_data) + return Json(); + if (ch == ']') break; if (ch != ',') @@ -739,11 +828,13 @@ void JsonParser::consume(const std::string &in) { } Json JsonParser::json() { + parser->eof = true; return parser->parse_json(0); } Json Json::parse(const string &in, string &err, JsonParse strategy) { JsonParserPriv parser { in, err, strategy }; + parser.eof = true; Json result = parser.parse_json(0); // Check for any trailing garbage @@ -751,6 +842,11 @@ Json Json::parse(const string &in, string &err, JsonParse strategy) { if (parser.i != in.size()) return parser.fail("unexpected trailing " + esc(in[parser.i])); + if (parser.need_data) { + /* when doing full parsing, this is an error */ + return Json(); + } + return result; } @@ -760,13 +856,14 @@ vector Json::parse_multi(const string &in, string &err, JsonParse strategy) { JsonParserPriv parser { in, err, strategy }; + parser.eof = true; parser_stop_pos = 0; vector json_vec; - while (parser.i != in.size() && !parser.failed) { + while (parser.i != in.size() && !parser.failed && !parser.need_data) { json_vec.push_back(parser.parse_json(0)); // Check for another object parser.consume_garbage(); - if (!parser.failed) + if (!parser.failed && !parser.need_data) parser_stop_pos = parser.i; } return json_vec; From cd6e3a958dffc9e118f4be14889806834f8d5d77 Mon Sep 17 00:00:00 2001 From: Damien Merenne Date: Thu, 29 Dec 2016 12:23:11 +0100 Subject: [PATCH 03/12] Make json object stack explicit. As of now, the program stack is implicitly used to build the json object tree. For partial parsing support, we cannot use the program stack as parsing can be called at anytime. This commit adds a values stack and uses it to transmit value back along the object tree. All parsing functions now return void and instead pushes their result on the stack. The functions needing the result of other parsing functions (mostly for arrays and objects) read values from the stack and pop them. --- json11.cpp | 178 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 116 insertions(+), 62 deletions(-) diff --git a/json11.cpp b/json11.cpp index f4ce4ad..5b2ed7e 100644 --- a/json11.cpp +++ b/json11.cpp @@ -25,6 +25,7 @@ #include #include #include +#include namespace json11 { @@ -341,6 +342,7 @@ struct JsonParserPriv final { bool need_data = false; bool eof = false; const JsonParse strategy; + std::stack values; JsonParserPriv(string str, string &err, const JsonParse strategy): str(str), err(err), strategy(strategy) { @@ -350,34 +352,35 @@ struct JsonParserPriv final { * * Mark this parse as failed. */ - Json fail(string &&msg) { - return fail(move(msg), Json()); + void fail(string &&msg) { + fail(move(msg), Json()); } template - T fail(string &&msg, const T err_ret) { - if (!failed) + void fail(string &&msg, const T err_ret) { + if (!failed) { err = std::move(msg); + } + + /* if we are at end of file, push the error value anyway */ + if (!failed || (need_data && eof)) + values.push(err_ret); + failed = true; - return err_ret; } /* stop(msg, err_ret = Json()) * * Mark this parse as needing more data. */ - Json stop(string &&msg) { - return stop(move(msg), Json()); - } - - template - T stop(string &&msg, const T err_ret) { + void stop(string &&msg) { if (!failed && !need_data) err = std::move(msg); - if (eof) + if (eof) { + values.push(Json()); failed = true; + } need_data = true; - return err_ret; } /* eos() @@ -401,16 +404,16 @@ struct JsonParserPriv final { * * Advance comments (c-style inline and multiline). */ - bool consume_comment() { + void consume_comment() { bool comment_found = false; if (str[i] == '/') { i++; if (eos()) - return stop("unexpected end of input inside comment", false); + return stop("unexpected end of input inside comment"); if (str[i] == '/') { // inline comment i++; if (eos()) - return stop("unexpected end of input inside inline comment", false); + return stop("unexpected end of input inside inline comment"); // advance until next line while (str[i] != '\n') { i++; @@ -418,7 +421,7 @@ struct JsonParserPriv final { if (eof) break; else - return stop("unexpected end of input inside inline comment", false); + return stop("unexpected end of input inside inline comment"); } } comment_found = true; @@ -426,24 +429,24 @@ struct JsonParserPriv final { else if (str[i] == '*') { // multiline comment i++; if (i > str.size()-2) - return stop("unexpected end of input inside multi-line comment", false); + return stop("unexpected end of input inside multi-line comment"); // advance until closing tokens while (!(str[i] == '*' && str[i+1] == '/')) { i++; if (i > str.size()-2) return stop( - "unexpected end of input inside multi-line comment", false); + "unexpected end of input inside multi-line comment"); } i += 2; if (eos()) return stop( - "unexpected end of input inside multi-line comment", false); + "unexpected end of input inside multi-line comment"); comment_found = true; } else return fail("malformed comment", false); } - return comment_found; + values.push(comment_found); } /* consume_garbage() @@ -455,7 +458,12 @@ struct JsonParserPriv final { if(strategy == JsonParse::COMMENTS) { bool comment_found = false; do { - comment_found = consume_comment(); + consume_comment(); + if (need_data) + break; + + comment_found = values.top().bool_value(); + values.pop(); consume_whitespace(); } while(comment_found); @@ -472,7 +480,8 @@ struct JsonParserPriv final { if (need_data) { return '\0'; } else if (eos()) { - return stop("unexpected end of input", '\0'); + stop("unexpected end of input"); + return '\0'; } return str[i++]; @@ -507,18 +516,19 @@ struct JsonParserPriv final { * * Parse a string, starting at the current position. */ - string parse_string() { + void parse_string() { string out; long last_escaped_codepoint = -1; while (true) { if (eos()) - return stop("unexpected end of input in string", ""); + return stop("unexpected end of input in string"); char ch = str[i++]; if (ch == '"') { encode_utf8(last_escaped_codepoint, out); - return out; + values.push(out); + return; } if (in_range(ch, 0, 0x1f)) @@ -534,7 +544,7 @@ struct JsonParserPriv final { // Handle escapes if (eos()) - return stop("unexpected end of input in string", ""); + return stop("unexpected end of input in string"); ch = str[i++]; @@ -600,7 +610,7 @@ struct JsonParserPriv final { * * Parse a double. */ - Json parse_number() { + void parse_number() { size_t start_pos = i; if (str[i] == '-') @@ -633,7 +643,7 @@ struct JsonParserPriv final { if (str[i] != '.' && str[i] != 'e' && str[i] != 'E' && (i - start_pos) <= static_cast(std::numeric_limits::digits10)) { - return std::atoi(str.c_str() + start_pos); + return values.push(std::atoi(str.c_str() + start_pos)); } // Decimal part @@ -682,7 +692,7 @@ struct JsonParserPriv final { } } - return std::strtod(str.c_str() + start_pos, nullptr); + return values.push(std::strtod(str.c_str() + start_pos, nullptr)); } /* expect(str, res) @@ -690,7 +700,7 @@ struct JsonParserPriv final { * Expect that 'str' starts at the character that was just read. If it does, advance * the input and return res. If not, flag an error. */ - Json expect(const string &expected, Json res) { + void expect(const string &expected, Json res) { assert(i != 0); i--; @@ -699,7 +709,7 @@ struct JsonParserPriv final { if (str.compare(i, expected.length(), expected) == 0) { i += expected.length(); - return res; + return values.push(res); } else { return fail("parse error: expected " + expected + ", got " + str.substr(i, expected.length())); } @@ -709,14 +719,14 @@ struct JsonParserPriv final { * * Parse a JSON object. */ - Json parse_json(int depth) { + void parse_json(int depth) { if (depth > max_depth) { return fail("exceeded maximum nesting depth"); } char ch = get_next_token(); if (need_data) - return Json(); + return; if (ch == '-' || (ch >= '0' && ch <= '9')) { i--; @@ -739,73 +749,99 @@ struct JsonParserPriv final { map data; ch = get_next_token(); if (ch == '}') - return data; + return values.push(data); while (1) { if (need_data) - return Json(); + return; if (ch != '"') return fail("expected '\"' in object, got " + esc(ch)); - string key = parse_string(); + parse_string(); + if (need_data) + return; + + string key = values.top().string_value(); + values.pop(); + if (failed) - if (failed || need_data) - return Json(); + return values.push(Json()); ch = get_next_token(); if (need_data) - return Json(); + return; if (ch != ':') return fail("expected ':' in object, got " + esc(ch)); - data[std::move(key)] = parse_json(depth + 1); - if (failed || need_data) - return Json(); + parse_json(depth + 1); + if (need_data) + return; + + Json value = values.top(); + values.pop(); + + if (failed) + return values.push(Json()); + + data[std::move(key)] = value; ch = get_next_token(); if (need_data) - return Json(); + return; - if (ch == '}') + if (ch == '}') { + values.push(data); break; + } + if (ch != ',') return fail("expected ',' in object, got " + esc(ch)); ch = get_next_token(); } - return data; + return; } if (ch == '[') { vector data; ch = get_next_token(); if (ch == ']') - return data; + return values.push(data); while (1) { if (need_data) - return Json(); + return; i--; - data.push_back(parse_json(depth + 1)); - if (failed || need_data) - return Json(); + parse_json(depth + 1); + if (need_data) + return; + + Json value = values.top(); + values.pop(); + + if (failed) + return values.push(Json()); + data.push_back(value); ch = get_next_token(); if (need_data) - return Json(); + return; - if (ch == ']') + if (ch == ']') { + values.push(data); break; + } + if (ch != ',') return fail("expected ',' in list, got " + esc(ch)); ch = get_next_token(); (void)ch; } - return data; + return; } return fail("expected value, got " + esc(ch)); @@ -825,29 +861,37 @@ JsonParser::~JsonParser() { void JsonParser::consume(const std::string &in) { parser->str = in; + parser->eof = true; + parser->parse_json(0); } Json JsonParser::json() { - parser->eof = true; - return parser->parse_json(0); + return parser->values.top(); } Json Json::parse(const string &in, string &err, JsonParse strategy) { JsonParserPriv parser { in, err, strategy }; parser.eof = true; - Json result = parser.parse_json(0); + parser.parse_json(0); // Check for any trailing garbage parser.consume_garbage(); - if (parser.i != in.size()) - return parser.fail("unexpected trailing " + esc(in[parser.i])); + if (parser.i != in.size()) { + parser.fail("unexpected trailing " + esc(in[parser.i])); + return Json(); + } if (parser.need_data) { /* when doing full parsing, this is an error */ - return Json(); + parser.failed = true; + parser.values.push(Json()); } - return result; +#ifndef NDEBUG + if (!parser.failed) + assert(parser.values.size() == 1); +#endif + return parser.values.top(); } // Documented in json11.hpp @@ -860,7 +904,17 @@ vector Json::parse_multi(const string &in, parser_stop_pos = 0; vector json_vec; while (parser.i != in.size() && !parser.failed && !parser.need_data) { - json_vec.push_back(parser.parse_json(0)); + parser.parse_json(0); + if (parser.need_data) { + parser.failed = true; + parser.values.push(Json()); + } +#ifndef NDEBUG + if (!parser.failed) + assert(parser.values.size() == 1); +#endif + json_vec.push_back(parser.values.top()); + parser.values.pop(); // Check for another object parser.consume_garbage(); if (!parser.failed && !parser.need_data) From 17716064a46412ebcf4c618465e848086a0200a2 Mon Sep 17 00:00:00 2001 From: Damien Merenne Date: Thu, 29 Dec 2016 12:31:58 +0100 Subject: [PATCH 04/12] Use value stack for checking maximum depth. As we now have an explicit stack, we don't need the depth variable argument to count the recursion levels. We can simply use the value stack size instead. --- json11.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/json11.cpp b/json11.cpp index 5b2ed7e..1402987 100644 --- a/json11.cpp +++ b/json11.cpp @@ -719,8 +719,8 @@ struct JsonParserPriv final { * * Parse a JSON object. */ - void parse_json(int depth) { - if (depth > max_depth) { + void parse_json() { + if (values.size() > max_depth) { return fail("exceeded maximum nesting depth"); } @@ -775,7 +775,7 @@ struct JsonParserPriv final { if (ch != ':') return fail("expected ':' in object, got " + esc(ch)); - parse_json(depth + 1); + parse_json(); if (need_data) return; @@ -815,7 +815,7 @@ struct JsonParserPriv final { return; i--; - parse_json(depth + 1); + parse_json(); if (need_data) return; @@ -862,7 +862,7 @@ JsonParser::~JsonParser() { void JsonParser::consume(const std::string &in) { parser->str = in; parser->eof = true; - parser->parse_json(0); + parser->parse_json(); } Json JsonParser::json() { @@ -872,7 +872,7 @@ Json JsonParser::json() { Json Json::parse(const string &in, string &err, JsonParse strategy) { JsonParserPriv parser { in, err, strategy }; parser.eof = true; - parser.parse_json(0); + parser.parse_json(); // Check for any trailing garbage parser.consume_garbage(); @@ -904,7 +904,7 @@ vector Json::parse_multi(const string &in, parser_stop_pos = 0; vector json_vec; while (parser.i != in.size() && !parser.failed && !parser.need_data) { - parser.parse_json(0); + parser.parse_json(); if (parser.need_data) { parser.failed = true; parser.values.push(Json()); From 4cbea0b62676296916cef2e6c56d7eaae419039a Mon Sep 17 00:00:00 2001 From: Damien Merenne Date: Thu, 29 Dec 2016 12:43:50 +0100 Subject: [PATCH 05/12] Split json value parse function. For partial support, when continuing parsing, we need to restart in the same state as we left. For that, we need to skip some actions if they were already done. Left as is, the code would be littered with ifs so refactor the parse_json function into multiple function, one per type of objects. --- json11.cpp | 219 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 128 insertions(+), 91 deletions(-) diff --git a/json11.cpp b/json11.cpp index 1402987..69642a1 100644 --- a/json11.cpp +++ b/json11.cpp @@ -715,134 +715,171 @@ struct JsonParserPriv final { } } - /* parse_json() + /* parse_true() * - * Parse a JSON object. + * Parse a json true value. */ - void parse_json() { - if (values.size() > max_depth) { - return fail("exceeded maximum nesting depth"); - } + void parse_true() { + expect("true", true); + } + + /* parse_false() + * + * Parse a json false value. + */ + void parse_false() { + expect("false", false); + } + /* parse_null() + * + * Parse a json null value. + */ + void parse_null() { + expect("null", Json()); + } + + /* parse_object() + * + * Parse a json object value. + */ + void parse_object() { + map data; char ch = get_next_token(); - if (need_data) - return; + if (ch == '}') + return values.push(data); - if (ch == '-' || (ch >= '0' && ch <= '9')) { - i--; - return parse_number(); - } + while (1) { + if (need_data) + return; - if (ch == 't') - return expect("true", true); + if (ch != '"') + return fail("expected '\"' in object, got " + esc(ch)); - if (ch == 'f') - return expect("false", false); + parse_string(); + if (need_data) + return; - if (ch == 'n') - return expect("null", Json()); + string key = values.top().string_value(); + values.pop(); - if (ch == '"') - return parse_string(); + if (failed) + return values.push(Json()); - if (ch == '{') { - map data; ch = get_next_token(); - if (ch == '}') - return values.push(data); + if (need_data) + return; - while (1) { - if (need_data) - return; + if (ch != ':') + return fail("expected ':' in object, got " + esc(ch)); - if (ch != '"') - return fail("expected '\"' in object, got " + esc(ch)); + parse_json(); + if (need_data) + return; - parse_string(); - if (need_data) - return; + Json value = values.top(); + values.pop(); - string key = values.top().string_value(); - values.pop(); + if (failed) + return values.push(Json()); - if (failed) - return values.push(Json()); + data[std::move(key)] = value; - ch = get_next_token(); - if (need_data) - return; + ch = get_next_token(); + if (need_data) + return; - if (ch != ':') - return fail("expected ':' in object, got " + esc(ch)); + if (ch == '}') { + values.push(data); + break; + } - parse_json(); - if (need_data) - return; + if (ch != ',') + return fail("expected ',' in object, got " + esc(ch)); - Json value = values.top(); - values.pop(); + ch = get_next_token(); + } + } - if (failed) - return values.push(Json()); + /* parse_array() + * + * Parse a json array value. + */ + void parse_array() { + vector data; + char ch = get_next_token(); - data[std::move(key)] = value; + if (ch == ']') + return values.push(data); - ch = get_next_token(); - if (need_data) - return; + while (1) { + if (need_data) + return; - if (ch == '}') { - values.push(data); - break; - } + i--; + parse_json(); + if (need_data) + return; - if (ch != ',') - return fail("expected ',' in object, got " + esc(ch)); + Json value = values.top(); + values.pop(); - ch = get_next_token(); + if (failed) + return values.push(Json()); + data.push_back(value); + + ch = get_next_token(); + if (need_data) + return; + + if (ch == ']') { + values.push(data); + break; } - return; - } - if (ch == '[') { - vector data; + if (ch != ',') + return fail("expected ',' in list, got " + esc(ch)); + ch = get_next_token(); - if (ch == ']') - return values.push(data); + (void)ch; + } + } - while (1) { - if (need_data) - return; + /* parse_json() + * + * Parse any JSON value. + */ + void parse_json() { + if (values.size() > max_depth) { + return fail("exceeded maximum nesting depth"); + } - i--; - parse_json(); - if (need_data) - return; + char ch = get_next_token(); + if (need_data) + return; - Json value = values.top(); - values.pop(); + if (ch == '-' || (ch >= '0' && ch <= '9')) { + i--; + return parse_number(); + } - if (failed) - return values.push(Json()); - data.push_back(value); + if (ch == 't') + return parse_true(); - ch = get_next_token(); - if (need_data) - return; + if (ch == 'f') + return parse_false(); - if (ch == ']') { - values.push(data); - break; - } + if (ch == 'n') + return parse_null(); - if (ch != ',') - return fail("expected ',' in list, got " + esc(ch)); + if (ch == '"') + return parse_string(); - ch = get_next_token(); - (void)ch; - } - return; - } + if (ch == '{') + return parse_object(); + + if (ch == '[') + return parse_array(); return fail("expected value, got " + esc(ch)); } From 4e0e31ceae6feb7228c8c2b52799385852ae0326 Mon Sep 17 00:00:00 2001 From: Damien Merenne Date: Thu, 29 Dec 2016 14:15:38 +0100 Subject: [PATCH 06/12] Add explicit state management. For partial parsing support, we need to remember what we were doing so that we can continue parsing. This commit adds an enum with all our parsing states and a states stack to store the current states. --- json11.cpp | 133 ++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 122 insertions(+), 11 deletions(-) diff --git a/json11.cpp b/json11.cpp index 69642a1..487f7de 100644 --- a/json11.cpp +++ b/json11.cpp @@ -344,8 +344,30 @@ struct JsonParserPriv final { const JsonParse strategy; std::stack values; + enum State { + EXPECT_VALUE, + VALUE_STRING, + VALUE_NUMBER, + VALUE_TRUE, + VALUE_FALSE, + VALUE_NULL, + VALUE_COMMENT, + VALUE_OBJECT, + OBJECT_KEY_OR_END, + OBJECT_COMMA_OR_END, + OBJECT_KEY, + OBJECT_COLON, + OBJECT_VALUE, + VALUE_ARRAY, + ARRAY_VALUE_OR_END, + ARRAY_COMMA_OR_END, + ARRAY_VALUE + }; + std::stack states; + JsonParserPriv(string str, string &err, const JsonParse strategy): str(str), err(err), strategy(strategy) { + push_state(EXPECT_VALUE); } /* fail(msg, err_ret = Json()) @@ -359,6 +381,7 @@ struct JsonParserPriv final { template void fail(string &&msg, const T err_ret) { if (!failed) { + pop_state(); err = std::move(msg); } @@ -367,6 +390,7 @@ struct JsonParserPriv final { values.push(err_ret); failed = true; + assert(states.size() > 0); } /* stop(msg, err_ret = Json()) @@ -391,6 +415,33 @@ struct JsonParserPriv final { return i == str.size(); } + /* set_state() + * + * Set current parsing state. + */ + void set_state(State state) { + states.pop(); + states.push(state); + } + + /* push_state() + * + * push new current parsing state. + */ + void push_state(State state) { + states.push(state); + } + + /* pop_state() + * + * Set current parsing state. + */ + void pop_state() { + states.pop(); + } + +#define assert_state(S) assert(states.top() == S) + /* consume_whitespace() * * Advance until the current character is non-whitespace. @@ -405,6 +456,7 @@ struct JsonParserPriv final { * Advance comments (c-style inline and multiline). */ void consume_comment() { + assert_state(VALUE_COMMENT); bool comment_found = false; if (str[i] == '/') { i++; @@ -446,6 +498,7 @@ struct JsonParserPriv final { else return fail("malformed comment", false); } + pop_state(); values.push(comment_found); } @@ -458,6 +511,7 @@ struct JsonParserPriv final { if(strategy == JsonParse::COMMENTS) { bool comment_found = false; do { + push_state(VALUE_COMMENT); consume_comment(); if (need_data) break; @@ -517,6 +571,8 @@ struct JsonParserPriv final { * Parse a string, starting at the current position. */ void parse_string() { + assert_state(VALUE_STRING); + string out; long last_escaped_codepoint = -1; while (true) { @@ -527,6 +583,7 @@ struct JsonParserPriv final { if (ch == '"') { encode_utf8(last_escaped_codepoint, out); + pop_state(); values.push(out); return; } @@ -611,6 +668,7 @@ struct JsonParserPriv final { * Parse a double. */ void parse_number() { + assert_state(VALUE_NUMBER); size_t start_pos = i; if (str[i] == '-') @@ -643,6 +701,7 @@ struct JsonParserPriv final { if (str[i] != '.' && str[i] != 'e' && str[i] != 'E' && (i - start_pos) <= static_cast(std::numeric_limits::digits10)) { + pop_state(); return values.push(std::atoi(str.c_str() + start_pos)); } @@ -692,6 +751,7 @@ struct JsonParserPriv final { } } + pop_state(); return values.push(std::strtod(str.c_str() + start_pos, nullptr)); } @@ -709,6 +769,7 @@ struct JsonParserPriv final { if (str.compare(i, expected.length(), expected) == 0) { i += expected.length(); + pop_state(); return values.push(res); } else { return fail("parse error: expected " + expected + ", got " + str.substr(i, expected.length())); @@ -720,6 +781,7 @@ struct JsonParserPriv final { * Parse a json true value. */ void parse_true() { + assert_state(VALUE_TRUE); expect("true", true); } @@ -728,6 +790,7 @@ struct JsonParserPriv final { * Parse a json false value. */ void parse_false() { + assert_state(VALUE_FALSE); expect("false", false); } @@ -736,6 +799,7 @@ struct JsonParserPriv final { * Parse a json null value. */ void parse_null() { + assert_state(VALUE_NULL); expect("null", Json()); } @@ -744,10 +808,16 @@ struct JsonParserPriv final { * Parse a json object value. */ void parse_object() { + assert(states.top() >= VALUE_OBJECT && states.top() <= OBJECT_VALUE); + map data; + + set_state(OBJECT_KEY_OR_END); char ch = get_next_token(); - if (ch == '}') + if (ch == '}') { + pop_state(); return values.push(data); + } while (1) { if (need_data) @@ -756,6 +826,8 @@ struct JsonParserPriv final { if (ch != '"') return fail("expected '\"' in object, got " + esc(ch)); + set_state(OBJECT_KEY); + push_state(VALUE_STRING); parse_string(); if (need_data) return; @@ -766,6 +838,7 @@ struct JsonParserPriv final { if (failed) return values.push(Json()); + set_state(OBJECT_COLON); ch = get_next_token(); if (need_data) return; @@ -773,6 +846,8 @@ struct JsonParserPriv final { if (ch != ':') return fail("expected ':' in object, got " + esc(ch)); + set_state(OBJECT_VALUE); + push_state(EXPECT_VALUE); parse_json(); if (need_data) return; @@ -785,12 +860,14 @@ struct JsonParserPriv final { data[std::move(key)] = value; + set_state(OBJECT_COMMA_OR_END); ch = get_next_token(); if (need_data) return; if (ch == '}') { values.push(data); + pop_state(); break; } @@ -806,17 +883,27 @@ struct JsonParserPriv final { * Parse a json array value. */ void parse_array() { + assert(states.top() >= VALUE_ARRAY && states.top() <= ARRAY_VALUE); + vector data; + + set_state(ARRAY_VALUE_OR_END); char ch = get_next_token(); - if (ch == ']') + if (ch == ']') { + pop_state(); return values.push(data); + } while (1) { if (need_data) return; i--; + + set_state(ARRAY_VALUE); + push_state(EXPECT_VALUE); + parse_json(); if (need_data) return; @@ -828,12 +915,14 @@ struct JsonParserPriv final { return values.push(Json()); data.push_back(value); + set_state(ARRAY_COMMA_OR_END); ch = get_next_token(); if (need_data) return; if (ch == ']') { values.push(data); + pop_state(); break; } @@ -850,6 +939,8 @@ struct JsonParserPriv final { * Parse any JSON value. */ void parse_json() { + assert_state(EXPECT_VALUE); + if (values.size() > max_depth) { return fail("exceeded maximum nesting depth"); } @@ -860,26 +951,39 @@ struct JsonParserPriv final { if (ch == '-' || (ch >= '0' && ch <= '9')) { i--; + set_state(VALUE_NUMBER); return parse_number(); } - if (ch == 't') + if (ch == 't') { + set_state(VALUE_TRUE); return parse_true(); + } - if (ch == 'f') + if (ch == 'f') { + set_state(VALUE_FALSE); return parse_false(); + } - if (ch == 'n') + if (ch == 'n') { + set_state(VALUE_NULL); return parse_null(); + } - if (ch == '"') + if (ch == '"') { + set_state(VALUE_STRING); return parse_string(); + } - if (ch == '{') + if (ch == '{') { + set_state(VALUE_OBJECT); return parse_object(); + } - if (ch == '[') + if (ch == '[') { + set_state(VALUE_ARRAY); return parse_array(); + } return fail("expected value, got " + esc(ch)); } @@ -908,13 +1012,15 @@ Json JsonParser::json() { Json Json::parse(const string &in, string &err, JsonParse strategy) { JsonParserPriv parser { in, err, strategy }; + assert(parser.states.size() == 1); + parser.eof = true; parser.parse_json(); // Check for any trailing garbage parser.consume_garbage(); if (parser.i != in.size()) { - parser.fail("unexpected trailing " + esc(in[parser.i])); + err = "unexpected trailing " + esc(in[parser.i]); return Json(); } @@ -925,8 +1031,10 @@ Json Json::parse(const string &in, string &err, JsonParse strategy) { } #ifndef NDEBUG - if (!parser.failed) + if (!parser.failed) { assert(parser.values.size() == 1); + assert(parser.states.empty()); + } #endif return parser.values.top(); } @@ -954,8 +1062,11 @@ vector Json::parse_multi(const string &in, parser.values.pop(); // Check for another object parser.consume_garbage(); - if (!parser.failed && !parser.need_data) + if (!parser.failed && !parser.need_data) { + assert(parser.states.empty()); + parser.push_state(JsonParserPriv::EXPECT_VALUE); parser_stop_pos = parser.i; + } } return json_vec; } From c9d94366a1e34aea79a76cd04d462a5e77d03457 Mon Sep 17 00:00:00 2001 From: Damien Merenne Date: Thu, 29 Dec 2016 15:58:44 +0100 Subject: [PATCH 07/12] Store builded object on the stack. For partial building, we need to temporary store the object being build as we maybe interrupted. This commits change object and array parsing to use the values stack has a temporary place for storing the object being build. --- json11.cpp | 44 ++++++++++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/json11.cpp b/json11.cpp index 487f7de..a2e257b 100644 --- a/json11.cpp +++ b/json11.cpp @@ -810,21 +810,23 @@ struct JsonParserPriv final { void parse_object() { assert(states.top() >= VALUE_OBJECT && states.top() <= OBJECT_VALUE); - map data; + values.push(map()); set_state(OBJECT_KEY_OR_END); char ch = get_next_token(); if (ch == '}') { pop_state(); - return values.push(data); + return; } while (1) { if (need_data) return; - if (ch != '"') + if (ch != '"') { + values.pop(); return fail("expected '\"' in object, got " + esc(ch)); + } set_state(OBJECT_KEY); push_state(VALUE_STRING); @@ -835,16 +837,20 @@ struct JsonParserPriv final { string key = values.top().string_value(); values.pop(); - if (failed) + if (failed) { + values.pop(); return values.push(Json()); + } set_state(OBJECT_COLON); ch = get_next_token(); if (need_data) return; - if (ch != ':') + if (ch != ':') { + values.pop(); return fail("expected ':' in object, got " + esc(ch)); + } set_state(OBJECT_VALUE); push_state(EXPECT_VALUE); @@ -855,10 +861,14 @@ struct JsonParserPriv final { Json value = values.top(); values.pop(); - if (failed) + if (failed) { + values.pop(); return values.push(Json()); + } + map data = values.top().object_items(); data[std::move(key)] = value; + values.top() = data; set_state(OBJECT_COMMA_OR_END); ch = get_next_token(); @@ -866,14 +876,14 @@ struct JsonParserPriv final { return; if (ch == '}') { - values.push(data); pop_state(); break; } - if (ch != ',') + if (ch != ',') { + values.pop(); return fail("expected ',' in object, got " + esc(ch)); - + } ch = get_next_token(); } } @@ -885,14 +895,14 @@ struct JsonParserPriv final { void parse_array() { assert(states.top() >= VALUE_ARRAY && states.top() <= ARRAY_VALUE); - vector data; + values.push(vector()); set_state(ARRAY_VALUE_OR_END); char ch = get_next_token(); if (ch == ']') { pop_state(); - return values.push(data); + return; } while (1) { @@ -911,9 +921,14 @@ struct JsonParserPriv final { Json value = values.top(); values.pop(); - if (failed) + if (failed) { + values.pop(); return values.push(Json()); + } + + vector data = values.top().array_items(); data.push_back(value); + values.top() = data; set_state(ARRAY_COMMA_OR_END); ch = get_next_token(); @@ -921,13 +936,14 @@ struct JsonParserPriv final { return; if (ch == ']') { - values.push(data); pop_state(); break; } - if (ch != ',') + if (ch != ',') { + values.pop(); return fail("expected ',' in list, got " + esc(ch)); + } ch = get_next_token(); (void)ch; From 44d93d851bd1e77aa2862521cc728323581b5304 Mon Sep 17 00:00:00 2001 From: Damien Merenne Date: Thu, 29 Dec 2016 18:55:12 +0100 Subject: [PATCH 08/12] Store parser position. For parsing partial json, we need to be able to restart parsing from a valid position in the input stream. This commit stores the current position when switching state and restores it when more data is needed so that when parsing again, the parser restarts at the right position. --- json11.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/json11.cpp b/json11.cpp index a2e257b..7cc183e 100644 --- a/json11.cpp +++ b/json11.cpp @@ -343,6 +343,7 @@ struct JsonParserPriv final { bool eof = false; const JsonParse strategy; std::stack values; + size_t last_position; enum State { EXPECT_VALUE, @@ -404,6 +405,9 @@ struct JsonParserPriv final { values.push(Json()); failed = true; } + else + i = last_position; + need_data = true; } @@ -420,6 +424,7 @@ struct JsonParserPriv final { * Set current parsing state. */ void set_state(State state) { + last_position = i; states.pop(); states.push(state); } @@ -429,6 +434,7 @@ struct JsonParserPriv final { * push new current parsing state. */ void push_state(State state) { + last_position = i; states.push(state); } @@ -437,6 +443,7 @@ struct JsonParserPriv final { * Set current parsing state. */ void pop_state() { + last_position = i; states.pop(); } From ab45c1774a98ac8009ca1cef2171b1624092e80b Mon Sep 17 00:00:00 2001 From: Damien Merenne Date: Thu, 29 Dec 2016 19:35:00 +0100 Subject: [PATCH 09/12] Implement partial parsing support. This commit implements a consume method that takes a chunk of json, append it to the current data and parses it. --- json11.cpp | 294 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 181 insertions(+), 113 deletions(-) diff --git a/json11.cpp b/json11.cpp index 7cc183e..326129e 100644 --- a/json11.cpp +++ b/json11.cpp @@ -573,6 +573,27 @@ struct JsonParserPriv final { } } + /* parse_comment() + * + * Parse json comments. + */ + void parse_comment() { + assert_state(VALUE_COMMENT); + + bool comment_found = false; + + consume_comment(); + if (need_data) + return; + + comment_found = values.top().bool_value(); + values.pop(); + consume_whitespace(); + + if (comment_found) + consume_garbage(); + } + /* parse_string() * * Parse a string, starting at the current position. @@ -817,40 +838,71 @@ struct JsonParserPriv final { void parse_object() { assert(states.top() >= VALUE_OBJECT && states.top() <= OBJECT_VALUE); - values.push(map()); + switch (states.top()) { + case VALUE_OBJECT: + values.push(map()); + set_state(OBJECT_KEY_OR_END); + break; - set_state(OBJECT_KEY_OR_END); - char ch = get_next_token(); - if (ch == '}') { - pop_state(); - return; - } + case OBJECT_KEY_OR_END: { + char ch = get_next_token(); - while (1) { if (need_data) - return; + break; + + if (ch == '}') + return pop_state(); if (ch != '"') { values.pop(); - return fail("expected '\"' in object, got " + esc(ch)); + fail("expected '\"' in object, got " + esc(ch)); + return; } - set_state(OBJECT_KEY); + set_state(OBJECT_COLON); push_state(VALUE_STRING); - parse_string(); + + break; + } + case OBJECT_COMMA_OR_END: { + char ch = get_next_token(); + if (need_data) + break; + + if (ch == '}') { + pop_state(); return; + } - string key = values.top().string_value(); - values.pop(); + if (ch != ',') { + values.pop(); + fail("expected ',' or '}' in object, got " + esc(ch)); + return; + } + + set_state(OBJECT_KEY); - if (failed) { + break; + } + case OBJECT_KEY: { + char ch = get_next_token(); + + if (need_data) + break; + + if (ch != '"') { values.pop(); - return values.push(Json()); + return fail("expected '\"' in object, got " + esc(ch)); } set_state(OBJECT_COLON); - ch = get_next_token(); + push_state(VALUE_STRING); + + break; + } + case OBJECT_COLON: { + char ch = get_next_token(); if (need_data) return; @@ -861,37 +913,22 @@ struct JsonParserPriv final { set_state(OBJECT_VALUE); push_state(EXPECT_VALUE); - parse_json(); - if (need_data) - return; - + break; + } + case OBJECT_VALUE: { Json value = values.top(); values.pop(); - - if (failed) { - values.pop(); - return values.push(Json()); - } - + string key = values.top().string_value(); + values.pop(); map data = values.top().object_items(); data[std::move(key)] = value; values.top() = data; set_state(OBJECT_COMMA_OR_END); - ch = get_next_token(); - if (need_data) - return; - - if (ch == '}') { - pop_state(); - break; - } - - if (ch != ',') { - values.pop(); - return fail("expected ',' in object, got " + esc(ch)); - } - ch = get_next_token(); + break; + } + default: + assert(false); } } @@ -902,58 +939,61 @@ struct JsonParserPriv final { void parse_array() { assert(states.top() >= VALUE_ARRAY && states.top() <= ARRAY_VALUE); - values.push(vector()); + switch (states.top()) { + case VALUE_ARRAY: + values.push(vector()); + set_state(ARRAY_VALUE_OR_END); + break; - set_state(ARRAY_VALUE_OR_END); - char ch = get_next_token(); - - if (ch == ']') { - pop_state(); - return; - } + case ARRAY_VALUE_OR_END: { + char ch = get_next_token(); - while (1) { if (need_data) return; + if (ch == ']') + return pop_state(); + i--; set_state(ARRAY_VALUE); push_state(EXPECT_VALUE); - parse_json(); + break; + } + case ARRAY_COMMA_OR_END: { + char ch = get_next_token(); if (need_data) return; - Json value = values.top(); - values.pop(); - - if (failed) { + if (ch == ']') { + pop_state(); + return; + } + if (ch != ',') { values.pop(); - return values.push(Json()); + fail("expected ',' in list, got " + esc(ch)); + return; } + set_state(ARRAY_VALUE_OR_END); + + break; + } + case ARRAY_VALUE: { + Json value = values.top(); + values.pop(); + vector data = values.top().array_items(); data.push_back(value); values.top() = data; set_state(ARRAY_COMMA_OR_END); - ch = get_next_token(); - if (need_data) - return; - - if (ch == ']') { - pop_state(); - break; - } - - if (ch != ',') { - values.pop(); - return fail("expected ',' in list, got " + esc(ch)); - } - ch = get_next_token(); - (void)ch; + break; + } + default: + assert(false); } } @@ -974,42 +1014,79 @@ struct JsonParserPriv final { if (ch == '-' || (ch >= '0' && ch <= '9')) { i--; - set_state(VALUE_NUMBER); - return parse_number(); + return set_state(VALUE_NUMBER); } - if (ch == 't') { - set_state(VALUE_TRUE); - return parse_true(); - } + if (ch == 't') + return set_state(VALUE_TRUE); - if (ch == 'f') { - set_state(VALUE_FALSE); - return parse_false(); - } + if (ch == 'f') + return set_state(VALUE_FALSE); - if (ch == 'n') { - set_state(VALUE_NULL); - return parse_null(); - } + if (ch == 'n') + return set_state(VALUE_NULL); - if (ch == '"') { - set_state(VALUE_STRING); - return parse_string(); - } + if (ch == '"') + return set_state(VALUE_STRING); - if (ch == '{') { - set_state(VALUE_OBJECT); - return parse_object(); - } + if (ch == '{') + return set_state(VALUE_OBJECT); - if (ch == '[') { - set_state(VALUE_ARRAY); - return parse_array(); - } + if (ch == '[') + return set_state(VALUE_ARRAY); return fail("expected value, got " + esc(ch)); } + + void consume(const std::string &in = std::string()) { + need_data = false; + str += in; + + /* try to parse as much as possible */ + while (!states.empty()) { + switch (states.top()) { + case EXPECT_VALUE: + parse_json(); + break; + case VALUE_OBJECT: + case OBJECT_KEY_OR_END: + case OBJECT_COMMA_OR_END: + case OBJECT_KEY: + case OBJECT_COLON: + case OBJECT_VALUE: + parse_object(); + break; + case VALUE_ARRAY: + case ARRAY_VALUE_OR_END: + case ARRAY_COMMA_OR_END: + case ARRAY_VALUE: + parse_array(); + break; + case VALUE_STRING: + parse_string(); + break; + case VALUE_NUMBER: + parse_number(); + break; + case VALUE_TRUE: + parse_true(); + break; + case VALUE_FALSE: + parse_false(); + break; + case VALUE_NULL: + parse_null(); + break; + case VALUE_COMMENT: + parse_comment(); + break; + } + + if (failed || need_data) + break; + } + } + }; JsonParser::JsonParser(): @@ -1024,21 +1101,22 @@ JsonParser::~JsonParser() { } void JsonParser::consume(const std::string &in) { - parser->str = in; - parser->eof = true; - parser->parse_json(); + parser->consume(in); } Json JsonParser::json() { + parser->eof = true; + parser->consume(); + if (!parser->failed && !parser->need_data) + error.clear(); return parser->values.top(); } Json Json::parse(const string &in, string &err, JsonParse strategy) { JsonParserPriv parser { in, err, strategy }; assert(parser.states.size() == 1); - parser.eof = true; - parser.parse_json(); + parser.consume(); // Check for any trailing garbage parser.consume_garbage(); @@ -1047,12 +1125,6 @@ Json Json::parse(const string &in, string &err, JsonParse strategy) { return Json(); } - if (parser.need_data) { - /* when doing full parsing, this is an error */ - parser.failed = true; - parser.values.push(Json()); - } - #ifndef NDEBUG if (!parser.failed) { assert(parser.values.size() == 1); @@ -1072,11 +1144,7 @@ vector Json::parse_multi(const string &in, parser_stop_pos = 0; vector json_vec; while (parser.i != in.size() && !parser.failed && !parser.need_data) { - parser.parse_json(); - if (parser.need_data) { - parser.failed = true; - parser.values.push(Json()); - } + parser.consume(); #ifndef NDEBUG if (!parser.failed) assert(parser.values.size() == 1); From e6b1df6623962b862326564307548ee2ccb17458 Mon Sep 17 00:00:00 2001 From: Damien Merenne Date: Fri, 6 Jan 2017 15:02:28 +0100 Subject: [PATCH 10/12] Add reset method to be able to reuse parser. --- json11.cpp | 4 ++++ json11.hpp | 2 ++ 2 files changed, 6 insertions(+) diff --git a/json11.cpp b/json11.cpp index 326129e..5b65d46 100644 --- a/json11.cpp +++ b/json11.cpp @@ -1112,6 +1112,10 @@ Json JsonParser::json() { return parser->values.top(); } +void JsonParser::reset() { + parser.reset(new JsonParserPriv("", error, parser->strategy)); +} + Json Json::parse(const string &in, string &err, JsonParse strategy) { JsonParserPriv parser { in, err, strategy }; assert(parser.states.size() == 1); diff --git a/json11.hpp b/json11.hpp index 74c7df1..3096e4f 100644 --- a/json11.hpp +++ b/json11.hpp @@ -239,6 +239,8 @@ class JsonParser { Json json(); void consume(const std::string &in); + void reset(); + const std::string &last_error() const { return error; From 5ea385440aa033210c7de1b76ff0848e4085f870 Mon Sep 17 00:00:00 2001 From: Damien Merenne Date: Fri, 6 Jan 2017 15:03:34 +0100 Subject: [PATCH 11/12] Add chunk parsing tests. --- test.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test.cpp b/test.cpp index 3224f5c..45cc794 100644 --- a/test.cpp +++ b/test.cpp @@ -250,6 +250,23 @@ JSON11_TEST_CASE(json11_test) { std::vector points = { { 1, 2 }, { 10, 20 }, { 100, 200 } }; std::string points_json = Json(points).dump(); printf("%s\n", points_json.c_str()); + + /* Test chunk parsing */ + JsonParser parser = { JsonParse::COMMENTS }; + for (auto c: simple_test) { + parser.consume(std::string(1, c)); + } + my_json = parser.json(); + JSON11_TEST_ASSERT(my_json == json); + + parser.reset(); + for (auto c: comment_test) { + parser.consume(std::string(1, c)); + } + + my_json = parser.json(); + JSON11_TEST_ASSERT(parser.last_error().empty()); + JSON11_TEST_ASSERT(my_json == json_comment); } #if JSON11_TEST_STANDALONE_MAIN From 7d97cfc92ca804e13c9a60b1c6a89b2afc41e853 Mon Sep 17 00:00:00 2001 From: Damien Merenne Date: Fri, 6 Jan 2017 15:15:29 +0100 Subject: [PATCH 12/12] Add documentation for chunk parsing. --- README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.md b/README.md index 21f4e8b..c62727b 100644 --- a/README.md +++ b/README.md @@ -39,4 +39,27 @@ JSON values can have their values queried and inspected: Json json = Json::array { Json::object { { "k", "v" } } }; std::string str = json[0]["k"].string_value(); +## Parsing + +To parse a string, you can simply call + + Json json = Json::parse("{ \"key\": \"value1\" }"); + +If you need to parse json data in multiple step, because you are +using non-blocking io for example, you can create a parser object. It will hold +the parsing state and you can give it the data as it comes along: + + JsonParser parser; + std::string data = read_data(); + while (!data.empty()) { + parse.consume(data); + } + + if (!parser.last_error().empty()) { + /* an error occured */ + ... + } + + Json json = parser.json(); + More documentation is still to come. For now, see json11.hpp.