diff options
Diffstat (limited to 'src/google/protobuf/util/internal/json_stream_parser.cc')
| -rw-r--r-- | src/google/protobuf/util/internal/json_stream_parser.cc | 179 |
1 files changed, 110 insertions, 69 deletions
diff --git a/src/google/protobuf/util/internal/json_stream_parser.cc b/src/google/protobuf/util/internal/json_stream_parser.cc index 39be7b03..1c63b31d 100644 --- a/src/google/protobuf/util/internal/json_stream_parser.cc +++ b/src/google/protobuf/util/internal/json_stream_parser.cc @@ -36,15 +36,14 @@ #include <cstdlib> #include <cstring> #include <memory> -#ifndef _SHARED_PTR_H -#include <google/protobuf/stubs/shared_ptr.h> -#endif #include <google/protobuf/stubs/logging.h> #include <google/protobuf/stubs/common.h> #include <google/protobuf/util/internal/object_writer.h> #include <google/protobuf/util/internal/json_escaping.h> #include <google/protobuf/stubs/strutil.h> +#include <google/protobuf/stubs/mathlimits.h> + namespace google { namespace protobuf { @@ -54,6 +53,7 @@ namespace util { // this file. using util::Status; namespace error { +using util::error::CANCELLED; using util::error::INTERNAL; using util::error::INVALID_ARGUMENT; } // namespace error @@ -107,7 +107,9 @@ JsonStreamParser::JsonStreamParser(ObjectWriter* ow) parsed_storage_(), string_open_(0), chunk_storage_(), - coerce_to_utf8_(false) { + coerce_to_utf8_(false), + allow_empty_null_(false), + loose_float_number_conversion_(false) { // Initialize the stack with a single value to be parsed. stack_.push(VALUE); } @@ -125,7 +127,7 @@ util::Status JsonStreamParser::Parse(StringPiece json) { // Don't point chunk to leftover_ because leftover_ will be updated in // ParseChunk(chunk). chunk_storage_.swap(leftover_); - json.AppendToString(&chunk_storage_); + StrAppend(&chunk_storage_, json); chunk = StringPiece(chunk_storage_); } @@ -136,11 +138,11 @@ util::Status JsonStreamParser::Parse(StringPiece json) { // Any leftover characters are stashed in leftover_ for later parsing when // there is more data available. - chunk.substr(n).AppendToString(&leftover_); + StrAppend(&leftover_, chunk.substr(n)); return status; } else { - chunk.CopyToString(&leftover_); - return util::Status::OK; + leftover_.assign(chunk.data(), chunk.size()); + return util::Status(); } } @@ -148,11 +150,11 @@ util::Status JsonStreamParser::FinishParse() { // If we do not expect anything and there is nothing left to parse we're all // done. if (stack_.empty() && leftover_.empty()) { - return util::Status::OK; + return util::Status(); } // Storage for UTF8-coerced string. - google::protobuf::scoped_array<char> utf8; + std::unique_ptr<char[]> utf8; if (coerce_to_utf8_) { utf8.reset(new char[leftover_.size()]); char* coerced = internal::UTF8CoerceToStructurallyValid(leftover_, utf8.get(), ' '); @@ -179,7 +181,7 @@ util::Status JsonStreamParser::FinishParse() { util::Status JsonStreamParser::ParseChunk(StringPiece chunk) { // Do not do any work if the chunk is empty. - if (chunk.empty()) return util::Status::OK; + if (chunk.empty()) return util::Status(); p_ = json_ = chunk; @@ -201,7 +203,7 @@ util::Status JsonStreamParser::ParseChunk(StringPiece chunk) { // unparsed data left, we save it for later parse. leftover_ = p_.ToString(); } - return util::Status::OK; + return util::Status(); } util::Status JsonStreamParser::RunParser() { @@ -242,20 +244,20 @@ util::Status JsonStreamParser::RunParser() { } if (!result.ok()) { // If we were cancelled, save our state and try again later. - if (!finishing_ && result == util::Status::CANCELLED) { + if (!finishing_ && result == util::Status(error::CANCELLED, "")) { stack_.push(type); // If we have a key we still need to render, make sure to save off the // contents in our own storage. if (!key_.empty() && key_storage_.empty()) { - key_.AppendToString(&key_storage_); + StrAppend(&key_storage_, key_); key_ = StringPiece(key_storage_); } - result = util::Status::OK; + result = util::Status(); } return result; } } - return util::Status::OK; + return util::Status(); } util::Status JsonStreamParser::ParseValue(TokenType type) { @@ -277,12 +279,16 @@ util::Status JsonStreamParser::ParseValue(TokenType type) { case UNKNOWN: return ReportUnknown("Expected a value."); default: { + if (allow_empty_null_ && IsEmptyNullAllowed(type)) { + return ParseEmptyNull(); + } + // Special case for having been cut off while parsing, wait for more data. // This handles things like 'fals' being at the end of the string, we // don't know if the next char would be e, completing it, or something // else, making it invalid. if (!finishing_ && p_.length() < false_len) { - return util::Status::CANCELLED; + return util::Status(error::CANCELLED, ""); } return ReportFailure("Unexpected token."); } @@ -293,8 +299,8 @@ util::Status JsonStreamParser::ParseString() { util::Status result = ParseStringHelper(); if (result.ok()) { ow_->RenderString(key_, parsed_); - key_.clear(); - parsed_.clear(); + key_ = StringPiece(); + parsed_ = StringPiece(); parsed_storage_.clear(); } return result; @@ -315,13 +321,12 @@ util::Status JsonStreamParser::ParseStringHelper() { // We're about to handle an escape, copy all bytes from last to data. if (last < data) { parsed_storage_.append(last, data - last); - last = data; } // If we ran out of string after the \, cancel or report an error // depending on if we expect more data later. if (p_.length() == 1) { if (!finishing_) { - return util::Status::CANCELLED; + return util::Status(error::CANCELLED, ""); } return ReportFailure("Closing quote expected in string."); } @@ -371,7 +376,6 @@ util::Status JsonStreamParser::ParseStringHelper() { } else { if (last < data) { parsed_storage_.append(last, data - last); - last = data; } parsed_ = StringPiece(parsed_storage_); } @@ -379,7 +383,7 @@ util::Status JsonStreamParser::ParseStringHelper() { // start fresh. string_open_ = 0; Advance(); - return util::Status::OK; + return util::Status(); } // Normal character, just advance past it. Advance(); @@ -390,7 +394,7 @@ util::Status JsonStreamParser::ParseStringHelper() { } // If we didn't find the closing quote but we expect more data, cancel for now if (!finishing_) { - return util::Status::CANCELLED; + return util::Status(error::CANCELLED, ""); } // End of string reached without a closing quote, report an error. string_open_ = 0; @@ -407,7 +411,7 @@ util::Status JsonStreamParser::ParseStringHelper() { util::Status JsonStreamParser::ParseUnicodeEscape() { if (p_.length() < kUnicodeEscapedLength) { if (!finishing_) { - return util::Status::CANCELLED; + return util::Status(error::CANCELLED, ""); } return ReportFailure("Illegal hex string."); } @@ -424,7 +428,7 @@ util::Status JsonStreamParser::ParseUnicodeEscape() { code <= JsonEscaping::kMaxHighSurrogate) { if (p_.length() < 2 * kUnicodeEscapedLength) { if (!finishing_) { - return util::Status::CANCELLED; + return util::Status(error::CANCELLED, ""); } if (!coerce_to_utf8_) { return ReportFailure("Missing low surrogate."); @@ -461,7 +465,7 @@ util::Status JsonStreamParser::ParseUnicodeEscape() { // Advance past the [final] code unit escape. p_.remove_prefix(kUnicodeEscapedLength); parsed_storage_.append(buf, len); - return util::Status::OK; + return util::Status(); } util::Status JsonStreamParser::ParseNumber() { @@ -471,17 +475,17 @@ util::Status JsonStreamParser::ParseNumber() { switch (number.type) { case NumberResult::DOUBLE: ow_->RenderDouble(key_, number.double_val); - key_.clear(); + key_ = StringPiece(); break; case NumberResult::INT: ow_->RenderInt64(key_, number.int_val); - key_.clear(); + key_ = StringPiece(); break; case NumberResult::UINT: ow_->RenderUint64(key_, number.uint_val); - key_.clear(); + key_ = StringPiece(); break; default: @@ -491,6 +495,19 @@ util::Status JsonStreamParser::ParseNumber() { return result; } +util::Status JsonStreamParser::ParseDoubleHelper( + const string& number, NumberResult* result) { + if (!safe_strtod(number, &result->double_val)) { + return ReportFailure("Unable to parse number."); + } + if (!loose_float_number_conversion_ && + !MathLimits<double>::IsFinite(result->double_val)) { + return ReportFailure("Number exceeds the range of double."); + } + result->type = NumberResult::DOUBLE; + return util::Status(); +} + util::Status JsonStreamParser::ParseNumberHelper(NumberResult* result) { const char* data = p_.data(); int length = p_.length(); @@ -518,7 +535,7 @@ util::Status JsonStreamParser::ParseNumberHelper(NumberResult* result) { // If the entire input is a valid number, and we may have more content in the // future, we abort for now and resume when we know more. if (index == length && !finishing_) { - return util::Status::CANCELLED; + return util::Status(error::CANCELLED, ""); } // Create a string containing just the number, so we can use safe_strtoX @@ -526,12 +543,11 @@ util::Status JsonStreamParser::ParseNumberHelper(NumberResult* result) { // Floating point number, parse as a double. if (floating) { - if (!safe_strtod(number, &result->double_val)) { - return ReportFailure("Unable to parse number."); + util::Status status = ParseDoubleHelper(number, result); + if (status.ok()) { + p_.remove_prefix(index); } - result->type = NumberResult::DOUBLE; - p_.remove_prefix(index); - return util::Status::OK; + return status; } // Positive non-floating point number, parse as a uint64. @@ -540,12 +556,18 @@ util::Status JsonStreamParser::ParseNumberHelper(NumberResult* result) { if (number.length() >= 2 && number[0] == '0') { return ReportFailure("Octal/hex numbers are not valid JSON values."); } - if (!safe_strtou64(number, &result->uint_val)) { - return ReportFailure("Unable to parse number."); + if (safe_strtou64(number, &result->uint_val)) { + result->type = NumberResult::UINT; + p_.remove_prefix(index); + return util::Status(); + } else { + // If the value is too large, parse it as double. + util::Status status = ParseDoubleHelper(number, result); + if (status.ok()) { + p_.remove_prefix(index); + } + return status; } - result->type = NumberResult::UINT; - p_.remove_prefix(index); - return util::Status::OK; } // Octal/Hex numbers are not valid JSON values. @@ -553,21 +575,27 @@ util::Status JsonStreamParser::ParseNumberHelper(NumberResult* result) { return ReportFailure("Octal/hex numbers are not valid JSON values."); } // Negative non-floating point number, parse as an int64. - if (!safe_strto64(number, &result->int_val)) { - return ReportFailure("Unable to parse number."); + if (safe_strto64(number, &result->int_val)) { + result->type = NumberResult::INT; + p_.remove_prefix(index); + return util::Status(); + } else { + // If the value is too large, parse it as double. + util::Status status = ParseDoubleHelper(number, result); + if (status.ok()) { + p_.remove_prefix(index); + } + return status; } - result->type = NumberResult::INT; - p_.remove_prefix(index); - return util::Status::OK; } util::Status JsonStreamParser::HandleBeginObject() { GOOGLE_DCHECK_EQ('{', *p_.data()); Advance(); ow_->StartObject(key_); - key_.clear(); + key_ = StringPiece(); stack_.push(ENTRY); - return util::Status::OK; + return util::Status(); } util::Status JsonStreamParser::ParseObjectMid(TokenType type) { @@ -579,13 +607,13 @@ util::Status JsonStreamParser::ParseObjectMid(TokenType type) { if (type == END_OBJECT) { Advance(); ow_->EndObject(); - return util::Status::OK; + return util::Status(); } // Found a comma, advance past it and get ready for an entry. if (type == VALUE_SEPARATOR) { Advance(); stack_.push(ENTRY); - return util::Status::OK; + return util::Status(); } // Illegal token after key:value pair. return ReportFailure("Expected , or } after key:value pair."); @@ -600,7 +628,7 @@ util::Status JsonStreamParser::ParseEntry(TokenType type) { if (type == END_OBJECT) { ow_->EndObject(); Advance(); - return util::Status::OK; + return util::Status(); } util::Status result; @@ -615,7 +643,7 @@ util::Status JsonStreamParser::ParseEntry(TokenType type) { } else { key_ = parsed_; } - parsed_.clear(); + parsed_ = StringPiece(); } } else if (type == BEGIN_KEY) { // Key is a bare key (back compat), create a StringPiece pointing to it. @@ -639,7 +667,7 @@ util::Status JsonStreamParser::ParseEntryMid(TokenType type) { if (type == ENTRY_SEPARATOR) { Advance(); stack_.push(VALUE); - return util::Status::OK; + return util::Status(); } return ReportFailure("Expected : between key:value pair."); } @@ -648,9 +676,9 @@ util::Status JsonStreamParser::HandleBeginArray() { GOOGLE_DCHECK_EQ('[', *p_.data()); Advance(); ow_->StartList(key_); - key_.clear(); + key_ = StringPiece(); stack_.push(ARRAY_VALUE); - return util::Status::OK; + return util::Status(); } util::Status JsonStreamParser::ParseArrayValue(TokenType type) { @@ -661,14 +689,15 @@ util::Status JsonStreamParser::ParseArrayValue(TokenType type) { if (type == END_ARRAY) { ow_->EndList(); Advance(); - return util::Status::OK; + return util::Status(); } // The ParseValue call may push something onto the stack so we need to make - // sure an ARRAY_MID is after it, so we push it on now. + // sure an ARRAY_MID is after it, so we push it on now. Also, the parsing of + // empty-null array value is relying on this ARRAY_MID token. stack_.push(ARRAY_MID); util::Status result = ParseValue(type); - if (result == util::Status::CANCELLED) { + if (result == util::Status(error::CANCELLED, "")) { // If we were cancelled, pop back off the ARRAY_MID so we don't try to // push it on again when we try over. stack_.pop(); @@ -684,14 +713,14 @@ util::Status JsonStreamParser::ParseArrayMid(TokenType type) { if (type == END_ARRAY) { ow_->EndList(); Advance(); - return util::Status::OK; + return util::Status(); } // Found a comma, advance past it and expect an array value next. if (type == VALUE_SEPARATOR) { Advance(); stack_.push(ARRAY_VALUE); - return util::Status::OK; + return util::Status(); } // Illegal token after array value. return ReportFailure("Expected , or ] after array value."); @@ -699,23 +728,35 @@ util::Status JsonStreamParser::ParseArrayMid(TokenType type) { util::Status JsonStreamParser::ParseTrue() { ow_->RenderBool(key_, true); - key_.clear(); + key_ = StringPiece(); p_.remove_prefix(true_len); - return util::Status::OK; + return util::Status(); } util::Status JsonStreamParser::ParseFalse() { ow_->RenderBool(key_, false); - key_.clear(); + key_ = StringPiece(); p_.remove_prefix(false_len); - return util::Status::OK; + return util::Status(); } util::Status JsonStreamParser::ParseNull() { ow_->RenderNull(key_); - key_.clear(); + key_ = StringPiece(); p_.remove_prefix(null_len); - return util::Status::OK; + return util::Status(); +} + +util::Status JsonStreamParser::ParseEmptyNull() { + ow_->RenderNull(key_); + key_ = StringPiece(); + return util::Status(); +} + +bool JsonStreamParser::IsEmptyNullAllowed(TokenType type) { + if (stack_.empty()) return false; + return (stack_.top() == ARRAY_MID && type == VALUE_SEPARATOR) || + stack_.top() == OBJ_MID; } util::Status JsonStreamParser::ReportFailure(StringPiece message) { @@ -735,7 +776,7 @@ util::Status JsonStreamParser::ReportFailure(StringPiece message) { util::Status JsonStreamParser::ReportUnknown(StringPiece message) { // If we aren't finishing the parse, cancel parsing and try later. if (!finishing_) { - return util::Status::CANCELLED; + return util::Status(error::CANCELLED, ""); } if (p_.empty()) { return ReportFailure(StrCat("Unexpected end of string. ", message)); @@ -765,11 +806,11 @@ util::Status JsonStreamParser::ParseKey() { // we can't know if the key was complete or not. if (!finishing_ && p_.empty()) { p_ = original; - return util::Status::CANCELLED; + return util::Status(error::CANCELLED, ""); } // Since we aren't using the key storage, clear it out. key_storage_.clear(); - return util::Status::OK; + return util::Status(); } JsonStreamParser::TokenType JsonStreamParser::GetNextTokenType() { |