diff options
Diffstat (limited to 'src/google/protobuf/util/internal')
38 files changed, 3564 insertions, 946 deletions
diff --git a/src/google/protobuf/util/internal/constants.h b/src/google/protobuf/util/internal/constants.h index 0cb8f6e1..a018a09e 100644 --- a/src/google/protobuf/util/internal/constants.h +++ b/src/google/protobuf/util/internal/constants.h @@ -43,13 +43,23 @@ namespace converter { const char kTypeServiceBaseUrl[] = "type.googleapis.com"; // Format string for RFC3339 timestamp formatting. -const char kRfc3339TimeFormat[] = "%Y-%m-%dT%H:%M:%S"; +const char kRfc3339TimeFormat[] = "%E4Y-%m-%dT%H:%M:%S"; -// Minimum seconds allowed in a google.protobuf.TimeStamp or Duration value. -const int64 kMinSeconds = -315576000000; +// Same as above, but the year value is not zero-padded i.e. this accepts +// timestamps like "1-01-0001T23:59:59Z" instead of "0001-01-0001T23:59:59Z". +const char kRfc3339TimeFormatNoPadding[] = "%Y-%m-%dT%H:%M:%S"; -// Maximum seconds allowed in a google.protobuf.TimeStamp or Duration value. -const int64 kMaxSeconds = 315576000000; +// Minimun seconds allowed in a google.protobuf.Timestamp value. +const int64 kTimestampMinSeconds = -62135596800LL; + +// Maximum seconds allowed in a google.protobuf.Timestamp value. +const int64 kTimestampMaxSeconds = 253402300799LL; + +// Minimum seconds allowed in a google.protobuf.Duration value. +const int64 kDurationMinSeconds = -315576000000LL; + +// Maximum seconds allowed in a google.protobuf.Duration value. +const int64 kDurationMaxSeconds = 315576000000LL; // Nano seconds in a second. const int32 kNanosPerSecond = 1000000000; diff --git a/src/google/protobuf/util/internal/datapiece.cc b/src/google/protobuf/util/internal/datapiece.cc index b557429f..59bc28ae 100644 --- a/src/google/protobuf/util/internal/datapiece.cc +++ b/src/google/protobuf/util/internal/datapiece.cc @@ -47,6 +47,7 @@ using google::protobuf::EnumDescriptor; using google::protobuf::EnumValueDescriptor; ; ; +; using util::error::Code; using util::Status; using util::StatusOr; @@ -63,9 +64,9 @@ StatusOr<To> ValidateNumberConversion(To after, From before) { MathUtil::Sign<From>(before) == MathUtil::Sign<To>(after)) { return after; } else { - return InvalidArgument(::google::protobuf::internal::is_integral<From>::value + return InvalidArgument(std::is_integral<From>::value ? ValueAsString(before) - : ::google::protobuf::internal::is_same<From, double>::value + : std::is_same<From, double>::value ? DoubleAsString(before) : FloatAsString(before)); } @@ -76,7 +77,7 @@ StatusOr<To> ValidateNumberConversion(To after, From before) { // except conversion between double and float. template <typename To, typename From> StatusOr<To> NumberConvertAndCheck(From before) { - if (::google::protobuf::internal::is_same<From, To>::value) return before; + if (std::is_same<From, To>::value) return before; To after = static_cast<To>(before); return ValidateNumberConversion(after, before); @@ -86,26 +87,31 @@ StatusOr<To> NumberConvertAndCheck(From before) { // point types (double, float) only. template <typename To, typename From> StatusOr<To> FloatingPointToIntConvertAndCheck(From before) { - if (::google::protobuf::internal::is_same<From, To>::value) return before; + if (std::is_same<From, To>::value) return before; To after = static_cast<To>(before); return ValidateNumberConversion(after, before); } // For conversion between double and float only. -template <typename To, typename From> -StatusOr<To> FloatingPointConvertAndCheck(From before) { - if (MathLimits<From>::IsNaN(before)) { - return std::numeric_limits<To>::quiet_NaN(); - } +StatusOr<double> FloatToDouble(float before) { + // Casting float to double should just work as double has more precision + // than float. + return static_cast<double>(before); +} - To after = static_cast<To>(before); - if (MathUtil::AlmostEquals<To>(after, before)) { - return after; +StatusOr<float> DoubleToFloat(double before) { + if (MathLimits<double>::IsNaN(before)) { + return std::numeric_limits<float>::quiet_NaN(); + } else if (!MathLimits<double>::IsFinite(before)) { + // Converting a double +inf/-inf to float should just work. + return static_cast<float>(before); + } else if (before > std::numeric_limits<float>::max() || + before < -std::numeric_limits<float>::max()) { + // Double value outside of the range of float. + return InvalidArgument(DoubleAsString(before)); } else { - return InvalidArgument(::google::protobuf::internal::is_same<From, double>::value - ? DoubleAsString(before) - : FloatAsString(before)); + return static_cast<float>(before); } } @@ -161,20 +167,27 @@ StatusOr<uint64> DataPiece::ToUint64() const { StatusOr<double> DataPiece::ToDouble() const { if (type_ == TYPE_FLOAT) { - return FloatingPointConvertAndCheck<double, float>(float_); + return FloatToDouble(float_); } if (type_ == TYPE_STRING) { if (str_ == "Infinity") return std::numeric_limits<double>::infinity(); if (str_ == "-Infinity") return -std::numeric_limits<double>::infinity(); if (str_ == "NaN") return std::numeric_limits<double>::quiet_NaN(); - return StringToNumber<double>(safe_strtod); + StatusOr<double> value = StringToNumber<double>(safe_strtod); + if (value.ok() && !MathLimits<double>::IsFinite(value.ValueOrDie())) { + // safe_strtod converts out-of-range values to +inf/-inf, but we want + // to report them as errors. + return InvalidArgument(StrCat("\"", str_, "\"")); + } else { + return value; + } } return GenericConvert<double>(); } StatusOr<float> DataPiece::ToFloat() const { if (type_ == TYPE_DOUBLE) { - return FloatingPointConvertAndCheck<float, double>(double_); + return DoubleToFloat(double_); } if (type_ == TYPE_STRING) { if (str_ == "Infinity") return std::numeric_limits<float>::infinity(); @@ -248,11 +261,8 @@ StatusOr<string> DataPiece::ToBytes() const { if (type_ == TYPE_BYTES) return str_.ToString(); if (type_ == TYPE_STRING) { string decoded; - if (!WebSafeBase64Unescape(str_, &decoded)) { - if (!Base64Unescape(str_, &decoded)) { - return InvalidArgument( - ValueAsStringOrDefault("Invalid data in input.")); - } + if (!DecodeBase64(str_, &decoded)) { + return InvalidArgument(ValueAsStringOrDefault("Invalid data in input.")); } return decoded; } else { @@ -261,7 +271,9 @@ StatusOr<string> DataPiece::ToBytes() const { } } -StatusOr<int> DataPiece::ToEnum(const google::protobuf::Enum* enum_type) const { +StatusOr<int> DataPiece::ToEnum(const google::protobuf::Enum* enum_type, + bool use_lower_camel_for_enums, + bool ignore_unknown_enum_values) const { if (type_ == TYPE_NULL) return google::protobuf::NULL_VALUE; if (type_ == TYPE_STRING) { @@ -269,21 +281,39 @@ StatusOr<int> DataPiece::ToEnum(const google::protobuf::Enum* enum_type) const { string enum_name = str_.ToString(); const google::protobuf::EnumValue* value = FindEnumValueByNameOrNull(enum_type, enum_name); - if (value != NULL) return value->number(); + if (value != nullptr) return value->number(); + + // Check if int version of enum is sent as string. + StatusOr<int32> int_value = ToInt32(); + if (int_value.ok()) { + if (const google::protobuf::EnumValue* enum_value = + FindEnumValueByNumberOrNull(enum_type, int_value.ValueOrDie())) { + return enum_value->number(); + } + } + // Next try a normalized name. for (string::iterator it = enum_name.begin(); it != enum_name.end(); ++it) { *it = *it == '-' ? '_' : ascii_toupper(*it); } value = FindEnumValueByNameOrNull(enum_type, enum_name); - if (value != NULL) return value->number(); - } else { - StatusOr<int32> value = ToInt32(); - if (value.ok()) { - if (const google::protobuf::EnumValue* enum_value = - FindEnumValueByNumberOrNull(enum_type, value.ValueOrDie())) { - return enum_value->number(); - } + if (value != nullptr) return value->number(); + + // If use_lower_camel_for_enums is true try with enum name without + // underscore. This will also accept camel case names as the enum_name has + // been normalized before. + if (use_lower_camel_for_enums) { + value = FindEnumValueByNameWithoutUnderscoreOrNull(enum_type, enum_name); + if (value != nullptr) return value->number(); } + + // If ignore_unknown_enum_values is true an unknown enum value is treated + // as the default + if (ignore_unknown_enum_values) return enum_type->enumvalue(0).number(); + } else { + // We don't need to check whether the value is actually declared in the + // enum because we preserve unknown enum values as well. + return ToInt32(); } return InvalidArgument( ValueAsStringOrDefault("Cannot find enum with given value.")); @@ -313,11 +343,70 @@ StatusOr<To> DataPiece::GenericConvert() const { template <typename To> StatusOr<To> DataPiece::StringToNumber(bool (*func)(StringPiece, To*)) const { + if (str_.size() > 0 && (str_[0] == ' ' || str_[str_.size() - 1] == ' ')) { + return InvalidArgument(StrCat("\"", str_, "\"")); + } To result; if (func(str_, &result)) return result; return InvalidArgument(StrCat("\"", str_.ToString(), "\"")); } +bool DataPiece::DecodeBase64(StringPiece src, string* dest) const { + // Try web-safe decode first, if it fails, try the non-web-safe decode. + if (WebSafeBase64Unescape(src, dest)) { + if (use_strict_base64_decoding_) { + // In strict mode, check if the escaped version gives us the same value as + // unescaped. + string encoded; + // WebSafeBase64Escape does no padding by default. + WebSafeBase64Escape(*dest, &encoded); + // Remove trailing padding '=' characters before comparison. + StringPiece src_no_padding = StringPiece(src).substr( + 0, StringEndsWith(src, "=") ? src.find_last_not_of('=') + 1 + : src.length()); + return encoded == src_no_padding; + } + return true; + } + + if (Base64Unescape(src, dest)) { + if (use_strict_base64_decoding_) { + string encoded; + Base64Escape( + reinterpret_cast<const unsigned char*>(dest->data()), dest->length(), + &encoded, false); + StringPiece src_no_padding = StringPiece(src).substr( + 0, StringEndsWith(src, "=") ? src.find_last_not_of('=') + 1 + : src.length()); + return encoded == src_no_padding; + } + return true; + } + + return false; +} + +void DataPiece::InternalCopy(const DataPiece& other) { + type_ = other.type_; + use_strict_base64_decoding_ = other.use_strict_base64_decoding_; + switch (type_) { + case TYPE_INT32: + case TYPE_INT64: + case TYPE_UINT32: + case TYPE_UINT64: + case TYPE_DOUBLE: + case TYPE_FLOAT: + case TYPE_BOOL: + case TYPE_ENUM: + case TYPE_NULL: + case TYPE_BYTES: + case TYPE_STRING: { + str_ = other.str_; + break; + } + } +} + } // namespace converter } // namespace util } // namespace protobuf diff --git a/src/google/protobuf/util/internal/datapiece.h b/src/google/protobuf/util/internal/datapiece.h index f22bfe70..95b133da 100644 --- a/src/google/protobuf/util/internal/datapiece.h +++ b/src/google/protobuf/util/internal/datapiece.h @@ -76,23 +76,36 @@ class LIBPROTOBUF_EXPORT DataPiece { }; // Constructors and Destructor - explicit DataPiece(const int32 value) : type_(TYPE_INT32), i32_(value) {} - explicit DataPiece(const int64 value) : type_(TYPE_INT64), i64_(value) {} - explicit DataPiece(const uint32 value) : type_(TYPE_UINT32), u32_(value) {} - explicit DataPiece(const uint64 value) : type_(TYPE_UINT64), u64_(value) {} - explicit DataPiece(const double value) : type_(TYPE_DOUBLE), double_(value) {} - explicit DataPiece(const float value) : type_(TYPE_FLOAT), float_(value) {} - explicit DataPiece(const bool value) : type_(TYPE_BOOL), bool_(value) {} - explicit DataPiece(StringPiece value) + explicit DataPiece(const int32 value) + : type_(TYPE_INT32), i32_(value), use_strict_base64_decoding_(false) {} + explicit DataPiece(const int64 value) + : type_(TYPE_INT64), i64_(value), use_strict_base64_decoding_(false) {} + explicit DataPiece(const uint32 value) + : type_(TYPE_UINT32), u32_(value), use_strict_base64_decoding_(false) {} + explicit DataPiece(const uint64 value) + : type_(TYPE_UINT64), u64_(value), use_strict_base64_decoding_(false) {} + explicit DataPiece(const double value) + : type_(TYPE_DOUBLE), + double_(value), + use_strict_base64_decoding_(false) {} + explicit DataPiece(const float value) + : type_(TYPE_FLOAT), float_(value), use_strict_base64_decoding_(false) {} + explicit DataPiece(const bool value) + : type_(TYPE_BOOL), bool_(value), use_strict_base64_decoding_(false) {} + DataPiece(StringPiece value, bool use_strict_base64_decoding) : type_(TYPE_STRING), - str_(StringPiecePod::CreateFromStringPiece(value)) {} + str_(StringPiecePod::CreateFromStringPiece(value)), + use_strict_base64_decoding_(use_strict_base64_decoding) {} // Constructor for bytes. The second parameter is not used. - explicit DataPiece(StringPiece value, bool dummy) - : type_(TYPE_BYTES), str_(StringPiecePod::CreateFromStringPiece(value)) {} - DataPiece(const DataPiece& r) : type_(r.type_), str_(r.str_) {} + DataPiece(StringPiece value, bool dummy, bool use_strict_base64_decoding) + : type_(TYPE_BYTES), + str_(StringPiecePod::CreateFromStringPiece(value)), + use_strict_base64_decoding_(use_strict_base64_decoding) {} + + DataPiece(const DataPiece& r) : type_(r.type_) { InternalCopy(r); } + DataPiece& operator=(const DataPiece& x) { - type_ = x.type_; - str_ = x.str_; + InternalCopy(x); return *this; } @@ -104,6 +117,8 @@ class LIBPROTOBUF_EXPORT DataPiece { // Accessors Type type() const { return type_; } + bool use_strict_base64_decoding() { return use_strict_base64_decoding_; } + StringPiece str() const { GOOGLE_LOG_IF(DFATAL, type_ != TYPE_STRING) << "Not a string type."; return str_; @@ -144,16 +159,21 @@ class LIBPROTOBUF_EXPORT DataPiece { // string, first attempts conversion by name, trying names as follows: // 1) the directly provided string value. // 2) the value upper-cased and replacing '-' by '_' + // 3) if use_lower_camel_for_enums is true it also attempts by comparing + // enum name without underscore with the value upper cased above. // If the value is not a string, attempts to convert to a 32-bit integer. // If none of these succeeds, returns a conversion error status. - util::StatusOr<int> ToEnum(const google::protobuf::Enum* enum_type) const; + util::StatusOr<int> ToEnum(const google::protobuf::Enum* enum_type, + bool use_lower_camel_for_enums, + bool ignore_unknown_enum_values) const; private: // Disallow implicit constructor. DataPiece(); // Helper to create NULL or ENUM types. - DataPiece(Type type, int32 val) : type_(type), i32_(val) {} + DataPiece(Type type, int32 val) + : type_(type), i32_(val), use_strict_base64_decoding_(false) {} // For numeric conversion between // int32, int64, uint32, uint64, double, float and bool @@ -165,32 +185,16 @@ class LIBPROTOBUF_EXPORT DataPiece { template <typename To> util::StatusOr<To> StringToNumber(bool (*func)(StringPiece, To*)) const; - // Data type for this piece of data. - Type type_; - - // StringPiece is not a POD and can not be used in an union (pre C++11). We - // need a POD version of it. - struct StringPiecePod { - const char* data; - int size; + // Decodes a base64 string. Returns true on success. + bool DecodeBase64(StringPiece src, string* dest) const; - // Create from a StringPiece. - static StringPiecePod CreateFromStringPiece(StringPiece str) { - StringPiecePod pod; - pod.data = str.data(); - pod.size = str.size(); - return pod; - } + // Helper function to initialize this DataPiece with 'other'. + void InternalCopy(const DataPiece& other); - // Cast to StringPiece. - operator StringPiece() const { return StringPiece(data, size); } + // Data type for this piece of data. + Type type_; - bool operator==(const char* value) const { - return StringPiece(data, size) == StringPiece(value); - } - - string ToString() const { return string(data, size); } - }; + typedef ::google::protobuf::internal::StringPiecePod StringPiecePod; // Stored piece of data. union { @@ -203,6 +207,9 @@ class LIBPROTOBUF_EXPORT DataPiece { bool bool_; StringPiecePod str_; }; + + // Uses a stricter version of base64 decoding for byte fields. + bool use_strict_base64_decoding_; }; } // namespace converter diff --git a/src/google/protobuf/util/internal/default_value_objectwriter.cc b/src/google/protobuf/util/internal/default_value_objectwriter.cc index a63e560d..b41feb7a 100644 --- a/src/google/protobuf/util/internal/default_value_objectwriter.cc +++ b/src/google/protobuf/util/internal/default_value_objectwriter.cc @@ -51,7 +51,7 @@ template <typename T> T ConvertTo(StringPiece value, StatusOr<T> (DataPiece::*converter_fn)() const, T default_value) { if (value.empty()) return default_value; - StatusOr<T> result = (DataPiece(value).*converter_fn)(); + StatusOr<T> result = (DataPiece(value, true).*converter_fn)(); return result.ok() ? result.ValueOrDie() : default_value; } } // namespace @@ -62,8 +62,12 @@ DefaultValueObjectWriter::DefaultValueObjectWriter( : typeinfo_(TypeInfo::NewTypeInfo(type_resolver)), own_typeinfo_(true), type_(type), - current_(NULL), - root_(NULL), + current_(nullptr), + root_(nullptr), + suppress_empty_list_(false), + preserve_proto_field_names_(false), + use_ints_for_enums_(false), + field_scrub_callback_(nullptr), ow_(ow) {} DefaultValueObjectWriter::~DefaultValueObjectWriter() { @@ -77,7 +81,7 @@ DefaultValueObjectWriter::~DefaultValueObjectWriter() { DefaultValueObjectWriter* DefaultValueObjectWriter::RenderBool(StringPiece name, bool value) { - if (current_ == NULL) { + if (current_ == nullptr) { ow_->RenderBool(name, value); } else { RenderDataPiece(name, DataPiece(value)); @@ -87,7 +91,7 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::RenderBool(StringPiece name, DefaultValueObjectWriter* DefaultValueObjectWriter::RenderInt32( StringPiece name, int32 value) { - if (current_ == NULL) { + if (current_ == nullptr) { ow_->RenderInt32(name, value); } else { RenderDataPiece(name, DataPiece(value)); @@ -97,7 +101,7 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::RenderInt32( DefaultValueObjectWriter* DefaultValueObjectWriter::RenderUint32( StringPiece name, uint32 value) { - if (current_ == NULL) { + if (current_ == nullptr) { ow_->RenderUint32(name, value); } else { RenderDataPiece(name, DataPiece(value)); @@ -107,7 +111,7 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::RenderUint32( DefaultValueObjectWriter* DefaultValueObjectWriter::RenderInt64( StringPiece name, int64 value) { - if (current_ == NULL) { + if (current_ == nullptr) { ow_->RenderInt64(name, value); } else { RenderDataPiece(name, DataPiece(value)); @@ -117,7 +121,7 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::RenderInt64( DefaultValueObjectWriter* DefaultValueObjectWriter::RenderUint64( StringPiece name, uint64 value) { - if (current_ == NULL) { + if (current_ == nullptr) { ow_->RenderUint64(name, value); } else { RenderDataPiece(name, DataPiece(value)); @@ -127,7 +131,7 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::RenderUint64( DefaultValueObjectWriter* DefaultValueObjectWriter::RenderDouble( StringPiece name, double value) { - if (current_ == NULL) { + if (current_ == nullptr) { ow_->RenderDouble(name, value); } else { RenderDataPiece(name, DataPiece(value)); @@ -137,7 +141,7 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::RenderDouble( DefaultValueObjectWriter* DefaultValueObjectWriter::RenderFloat( StringPiece name, float value) { - if (current_ == NULL) { + if (current_ == nullptr) { ow_->RenderBool(name, value); } else { RenderDataPiece(name, DataPiece(value)); @@ -147,30 +151,33 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::RenderFloat( DefaultValueObjectWriter* DefaultValueObjectWriter::RenderString( StringPiece name, StringPiece value) { - if (current_ == NULL) { + if (current_ == nullptr) { ow_->RenderString(name, value); } else { // Since StringPiece is essentially a pointer, takes a copy of "value" to // avoid ownership issues. string_values_.push_back(new string(value.ToString())); - RenderDataPiece(name, DataPiece(*string_values_.back())); + RenderDataPiece(name, DataPiece(*string_values_.back(), true)); } return this; } DefaultValueObjectWriter* DefaultValueObjectWriter::RenderBytes( StringPiece name, StringPiece value) { - if (current_ == NULL) { + if (current_ == nullptr) { ow_->RenderBytes(name, value); } else { - RenderDataPiece(name, DataPiece(value)); + // Since StringPiece is essentially a pointer, takes a copy of "value" to + // avoid ownership issues. + string_values_.push_back(new string(value.ToString())); + RenderDataPiece(name, DataPiece(*string_values_.back(), false, true)); } return this; } DefaultValueObjectWriter* DefaultValueObjectWriter::RenderNull( StringPiece name) { - if (current_ == NULL) { + if (current_ == nullptr) { ow_->RenderNull(name); } else { RenderDataPiece(name, DataPiece::NullData()); @@ -178,21 +185,66 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::RenderNull( return this; } -DefaultValueObjectWriter::Node::Node(const string& name, - const google::protobuf::Type* type, - NodeKind kind, const DataPiece& data, - bool is_placeholder) +void DefaultValueObjectWriter::RegisterFieldScrubCallBack( + FieldScrubCallBackPtr field_scrub_callback) { + field_scrub_callback_.reset(field_scrub_callback.release()); +} + +DefaultValueObjectWriter::Node* DefaultValueObjectWriter::CreateNewNode( + const string& name, const google::protobuf::Type* type, NodeKind kind, + const DataPiece& data, bool is_placeholder, const std::vector<string>& path, + bool suppress_empty_list, FieldScrubCallBack* field_scrub_callback) { + return new Node(name, type, kind, data, is_placeholder, path, + suppress_empty_list, field_scrub_callback); +} + +DefaultValueObjectWriter::Node* DefaultValueObjectWriter::CreateNewNode( + const string& name, const google::protobuf::Type* type, NodeKind kind, + const DataPiece& data, bool is_placeholder, const std::vector<string>& path, + bool suppress_empty_list, bool preserve_proto_field_names, bool use_ints_for_enums, + FieldScrubCallBack* field_scrub_callback) { + return new Node(name, type, kind, data, is_placeholder, path, + suppress_empty_list, preserve_proto_field_names, use_ints_for_enums, + field_scrub_callback); +} + +DefaultValueObjectWriter::Node::Node( + const string& name, const google::protobuf::Type* type, NodeKind kind, + const DataPiece& data, bool is_placeholder, const std::vector<string>& path, + bool suppress_empty_list, FieldScrubCallBack* field_scrub_callback) : name_(name), type_(type), kind_(kind), is_any_(false), data_(data), - is_placeholder_(is_placeholder) {} + is_placeholder_(is_placeholder), + path_(path), + suppress_empty_list_(suppress_empty_list), + preserve_proto_field_names_(false), + use_ints_for_enums_(false), + field_scrub_callback_(field_scrub_callback) {} + +DefaultValueObjectWriter::Node::Node( + const string& name, const google::protobuf::Type* type, NodeKind kind, + const DataPiece& data, bool is_placeholder, const std::vector<string>& path, + bool suppress_empty_list, bool preserve_proto_field_names, bool use_ints_for_enums, + FieldScrubCallBack* field_scrub_callback) + : name_(name), + type_(type), + kind_(kind), + is_any_(false), + data_(data), + is_placeholder_(is_placeholder), + path_(path), + suppress_empty_list_(suppress_empty_list), + preserve_proto_field_names_(preserve_proto_field_names), + use_ints_for_enums_(use_ints_for_enums), + field_scrub_callback_(field_scrub_callback) {} DefaultValueObjectWriter::Node* DefaultValueObjectWriter::Node::FindChild( StringPiece name) { if (name.empty() || kind_ != OBJECT) { - return NULL; + return nullptr; } for (int i = 0; i < children_.size(); ++i) { Node* child = children_[i]; @@ -200,7 +252,7 @@ DefaultValueObjectWriter::Node* DefaultValueObjectWriter::Node::FindChild( return child; } } - return NULL; + return nullptr; } void DefaultValueObjectWriter::Node::WriteTo(ObjectWriter* ow) { @@ -220,6 +272,9 @@ void DefaultValueObjectWriter::Node::WriteTo(ObjectWriter* ow) { // Write out lists. If we didn't have any list in response, write out empty // list. if (kind_ == LIST) { + // Suppress empty lists if requested. + if (suppress_empty_list_ && is_placeholder_) return; + ow->StartList(name_); WriteChildren(ow); ow->EndList(); @@ -265,7 +320,7 @@ const google::protobuf::Type* DefaultValueObjectWriter::Node::GetMapValueType( } break; } - return NULL; + return nullptr; } void DefaultValueObjectWriter::Node::PopulateChildren( @@ -276,7 +331,7 @@ void DefaultValueObjectWriter::Node::PopulateChildren( // TODO(tsun): remove "kStructValueType" from the list. It's being checked // now because of a bug in the tool-chain that causes the "oneof_index" // of kStructValueType to not be set correctly. - if (type_ == NULL || type_->name() == kAnyType || + if (type_ == nullptr || type_->name() == kAnyType || type_->name() == kStructType || type_->name() == kTimestampType || type_->name() == kDurationType || type_->name() == kStructValueType) { return; @@ -291,17 +346,30 @@ void DefaultValueObjectWriter::Node::PopulateChildren( for (int i = 0; i < type_->fields_size(); ++i) { const google::protobuf::Field& field = type_->fields(i); + + // This code is checking if the field to be added to the tree should be + // scrubbed or not by calling the field_scrub_callback_ callback function. + std::vector<string> path; + if (!path_.empty()) { + path.insert(path.begin(), path_.begin(), path_.end()); + } + path.push_back(field.name()); + if (field_scrub_callback_ != nullptr && + field_scrub_callback_->Run(path, &field)) { + continue; + } + hash_map<string, int>::iterator found = orig_children_map.find(field.name()); // If the child field has already been set, we just add it to the new list // of children. if (found != orig_children_map.end()) { new_children.push_back(children_[found->second]); - children_[found->second] = NULL; + children_[found->second] = nullptr; continue; } - const google::protobuf::Type* field_type = NULL; + const google::protobuf::Type* field_type = nullptr; bool is_map = false; NodeKind kind = PRIMITIVE; @@ -334,25 +402,28 @@ void DefaultValueObjectWriter::Node::PopulateChildren( } // If oneof_index() != 0, the child field is part of a "oneof", which means - // the child field is optional and we shouldn't populate its default value. - if (field.oneof_index() != 0) continue; + // the child field is optional and we shouldn't populate its default + // primitive value. + if (field.oneof_index() != 0 && kind == PRIMITIVE) continue; // If the child field is of primitive type, sets its data to the default // value of its type. - google::protobuf::scoped_ptr<Node> child(new Node( - field.json_name(), field_type, kind, - kind == PRIMITIVE ? CreateDefaultDataPieceForField(field, typeinfo) + std::unique_ptr<Node> child(new Node( + preserve_proto_field_names_ ? field.name() : field.json_name(), + field_type, kind, + kind == PRIMITIVE ? CreateDefaultDataPieceForField(field, typeinfo, use_ints_for_enums_) : DataPiece::NullData(), - true)); + true, path, suppress_empty_list_, preserve_proto_field_names_, use_ints_for_enums_, + field_scrub_callback_)); new_children.push_back(child.release()); } // Adds all leftover nodes in children_ to the beginning of new_child. for (int i = 0; i < children_.size(); ++i) { - if (children_[i] == NULL) { + if (children_[i] == nullptr) { continue; } new_children.insert(new_children.begin(), children_[i]); - children_[i] = NULL; + children_[i] = nullptr; } children_.swap(new_children); } @@ -360,15 +431,16 @@ void DefaultValueObjectWriter::Node::PopulateChildren( void DefaultValueObjectWriter::MaybePopulateChildrenOfAny(Node* node) { // If this is an "Any" node with "@type" already given and no other children // have been added, populates its children. - if (node != NULL && node->is_any() && node->type() != NULL && + if (node != nullptr && node->is_any() && node->type() != nullptr && node->type()->name() != kAnyType && node->number_of_children() == 1) { node->PopulateChildren(typeinfo_); } } DataPiece DefaultValueObjectWriter::FindEnumDefault( - const google::protobuf::Field& field, const TypeInfo* typeinfo) { - if (!field.default_value().empty()) return DataPiece(field.default_value()); + const google::protobuf::Field& field, const TypeInfo* typeinfo, bool use_ints_for_enums) { + if (!field.default_value().empty()) + return DataPiece(field.default_value(), true); const google::protobuf::Enum* enum_type = typeinfo->GetEnumByTypeUrl(field.type_url()); @@ -379,12 +451,12 @@ DataPiece DefaultValueObjectWriter::FindEnumDefault( } // We treat the first value as the default if none is specified. return enum_type->enumvalue_size() > 0 - ? DataPiece(enum_type->enumvalue(0).name()) + ? (use_ints_for_enums ? DataPiece(enum_type->enumvalue(0).number()) : DataPiece(enum_type->enumvalue(0).name(), true)) : DataPiece::NullData(); } DataPiece DefaultValueObjectWriter::CreateDefaultDataPieceForField( - const google::protobuf::Field& field, const TypeInfo* typeinfo) { + const google::protobuf::Field& field, const TypeInfo* typeinfo, bool use_ints_for_enums) { switch (field.kind()) { case google::protobuf::Field_Kind_TYPE_DOUBLE: { return DataPiece(ConvertTo<double>( @@ -416,10 +488,10 @@ DataPiece DefaultValueObjectWriter::CreateDefaultDataPieceForField( ConvertTo<bool>(field.default_value(), &DataPiece::ToBool, false)); } case google::protobuf::Field_Kind_TYPE_STRING: { - return DataPiece(field.default_value()); + return DataPiece(field.default_value(), true); } case google::protobuf::Field_Kind_TYPE_BYTES: { - return DataPiece(field.default_value(), false); + return DataPiece(field.default_value(), false, true); } case google::protobuf::Field_Kind_TYPE_UINT32: case google::protobuf::Field_Kind_TYPE_FIXED32: { @@ -427,7 +499,7 @@ DataPiece DefaultValueObjectWriter::CreateDefaultDataPieceForField( field.default_value(), &DataPiece::ToUint32, static_cast<uint32>(0))); } case google::protobuf::Field_Kind_TYPE_ENUM: { - return FindEnumDefault(field, typeinfo); + return FindEnumDefault(field, typeinfo, use_ints_for_enums); } default: { return DataPiece::NullData(); } } @@ -435,23 +507,30 @@ DataPiece DefaultValueObjectWriter::CreateDefaultDataPieceForField( DefaultValueObjectWriter* DefaultValueObjectWriter::StartObject( StringPiece name) { - if (current_ == NULL) { - root_.reset(new Node(name.ToString(), &type_, OBJECT, DataPiece::NullData(), - false)); + if (current_ == nullptr) { + std::vector<string> path; + root_.reset(CreateNewNode(string(name), &type_, OBJECT, + DataPiece::NullData(), false, path, + suppress_empty_list_, preserve_proto_field_names_, use_ints_for_enums_, + field_scrub_callback_.get())); root_->PopulateChildren(typeinfo_); current_ = root_.get(); return this; } MaybePopulateChildrenOfAny(current_); Node* child = current_->FindChild(name); - if (current_->kind() == LIST || current_->kind() == MAP || child == NULL) { + if (current_->kind() == LIST || current_->kind() == MAP || child == nullptr) { // If current_ is a list or a map node, we should create a new child and use // the type of current_ as the type of the new child. - google::protobuf::scoped_ptr<Node> node(new Node( - name.ToString(), ((current_->kind() == LIST || current_->kind() == MAP) - ? current_->type() - : NULL), - OBJECT, DataPiece::NullData(), false)); + std::unique_ptr<Node> node( + CreateNewNode(string(name), + ((current_->kind() == LIST || current_->kind() == MAP) + ? current_->type() + : nullptr), + OBJECT, DataPiece::NullData(), false, + child == nullptr ? current_->path() : child->path(), + suppress_empty_list_, preserve_proto_field_names_, use_ints_for_enums_, + field_scrub_callback_.get())); child = node.get(); current_->AddChild(node.release()); } @@ -479,17 +558,23 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::EndObject() { DefaultValueObjectWriter* DefaultValueObjectWriter::StartList( StringPiece name) { - if (current_ == NULL) { - root_.reset( - new Node(name.ToString(), &type_, LIST, DataPiece::NullData(), false)); + if (current_ == nullptr) { + std::vector<string> path; + root_.reset(CreateNewNode(string(name), &type_, LIST, DataPiece::NullData(), + false, path, suppress_empty_list_, + preserve_proto_field_names_, use_ints_for_enums_, + field_scrub_callback_.get())); current_ = root_.get(); return this; } MaybePopulateChildrenOfAny(current_); Node* child = current_->FindChild(name); - if (child == NULL || child->kind() != LIST) { - google::protobuf::scoped_ptr<Node> node( - new Node(name.ToString(), NULL, LIST, DataPiece::NullData(), false)); + if (child == nullptr || child->kind() != LIST) { + std::unique_ptr<Node> node( + CreateNewNode(string(name), nullptr, LIST, DataPiece::NullData(), false, + child == nullptr ? current_->path() : child->path(), + suppress_empty_list_, preserve_proto_field_names_, use_ints_for_enums_, + field_scrub_callback_.get())); child = node.get(); current_->AddChild(node.release()); } @@ -502,8 +587,8 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::StartList( void DefaultValueObjectWriter::WriteRoot() { root_->WriteTo(ow_); - root_.reset(NULL); - current_ = NULL; + root_.reset(nullptr); + current_ = nullptr; } DefaultValueObjectWriter* DefaultValueObjectWriter::EndList() { @@ -519,37 +604,43 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::EndList() { void DefaultValueObjectWriter::RenderDataPiece(StringPiece name, const DataPiece& data) { MaybePopulateChildrenOfAny(current_); - util::StatusOr<string> data_string = data.ToString(); - if (current_->type() != NULL && current_->type()->name() == kAnyType && - name == "@type" && data_string.ok()) { - const string& string_value = data_string.ValueOrDie(); - // If the type of current_ is "Any" and its "@type" field is being set here, - // sets the type of current_ to be the type specified by the "@type". - util::StatusOr<const google::protobuf::Type*> found_type = - typeinfo_->ResolveTypeUrl(string_value); - if (!found_type.ok()) { - GOOGLE_LOG(WARNING) << "Failed to resolve type '" << string_value << "'."; - } else { - current_->set_type(found_type.ValueOrDie()); - } - current_->set_is_any(true); - // If the "@type" field is placed after other fields, we should populate - // other children of primitive type now. Otherwise, we should wait until the - // first value field is rendered before we populate the children, because - // the "value" field of a Any message could be omitted. - if (current_->number_of_children() > 1 && current_->type() != NULL) { - current_->PopulateChildren(typeinfo_); + if (current_->type() != nullptr && current_->type()->name() == kAnyType && + name == "@type") { + util::StatusOr<string> data_string = data.ToString(); + if (data_string.ok()) { + const string& string_value = data_string.ValueOrDie(); + // If the type of current_ is "Any" and its "@type" field is being set + // here, sets the type of current_ to be the type specified by the + // "@type". + util::StatusOr<const google::protobuf::Type*> found_type = + typeinfo_->ResolveTypeUrl(string_value); + if (!found_type.ok()) { + GOOGLE_LOG(WARNING) << "Failed to resolve type '" << string_value << "'."; + } else { + current_->set_type(found_type.ValueOrDie()); + } + current_->set_is_any(true); + // If the "@type" field is placed after other fields, we should populate + // other children of primitive type now. Otherwise, we should wait until + // the first value field is rendered before we populate the children, + // because the "value" field of a Any message could be omitted. + if (current_->number_of_children() > 1 && current_->type() != nullptr) { + current_->PopulateChildren(typeinfo_); + } } } Node* child = current_->FindChild(name); - if (child == NULL || child->kind() != PRIMITIVE) { + if (child == nullptr || child->kind() != PRIMITIVE) { // No children are found, creates a new child. - google::protobuf::scoped_ptr<Node> node( - new Node(name.ToString(), NULL, PRIMITIVE, data, false)); - child = node.get(); + std::unique_ptr<Node> node( + CreateNewNode(string(name), nullptr, PRIMITIVE, data, false, + child == nullptr ? current_->path() : child->path(), + suppress_empty_list_, preserve_proto_field_names_, use_ints_for_enums_, + field_scrub_callback_.get())); current_->AddChild(node.release()); } else { child->set_data(data); + child->set_is_placeholder(false); } } diff --git a/src/google/protobuf/util/internal/default_value_objectwriter.h b/src/google/protobuf/util/internal/default_value_objectwriter.h index 695b9dd8..6e71f9c8 100644 --- a/src/google/protobuf/util/internal/default_value_objectwriter.h +++ b/src/google/protobuf/util/internal/default_value_objectwriter.h @@ -32,12 +32,10 @@ #define GOOGLE_PROTOBUF_UTIL_CONVERTER_DEFAULT_VALUE_OBJECTWRITER_H__ #include <memory> -#ifndef _SHARED_PTR_H -#include <google/protobuf/stubs/shared_ptr.h> -#endif #include <stack> #include <vector> +#include <google/protobuf/stubs/callback.h> #include <google/protobuf/stubs/common.h> #include <google/protobuf/util/internal/type_info.h> #include <google/protobuf/util/internal/datapiece.h> @@ -59,6 +57,25 @@ namespace converter { // with their default values (0 for numbers, "" for strings, etc). class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter { public: + // A Callback function to check whether a field needs to be scrubbed. + // + // Returns true if the field should not be present in the output. Returns + // false otherwise. + // + // The 'path' parameter is a vector of path to the field from root. For + // example: if a nested field "a.b.c" (b is the parent message field of c and + // a is the parent message field of b), then the vector should contain { "a", + // "b", "c" }. + // + // The Field* should point to the google::protobuf::Field of "c". + typedef ResultCallback2<bool /*return*/, + const std::vector<string>& /*path of the field*/, + const google::protobuf::Field* /*field*/> + FieldScrubCallBack; + + // A unique pointer to a DefaultValueObjectWriter::FieldScrubCallBack. + typedef std::unique_ptr<FieldScrubCallBack> FieldScrubCallBackPtr; + DefaultValueObjectWriter(TypeResolver* type_resolver, const google::protobuf::Type& type, ObjectWriter* ow); @@ -98,7 +115,26 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter { virtual DefaultValueObjectWriter* RenderNull(StringPiece name); - private: + // Register the callback for scrubbing of fields. Owership of + // field_scrub_callback pointer is also transferred to this class + void RegisterFieldScrubCallBack(FieldScrubCallBackPtr field_scrub_callback); + + // If set to true, empty lists are suppressed from output when default values + // are written. + void set_suppress_empty_list(bool value) { suppress_empty_list_ = value; } + + // If set to true, original proto field names are used + void set_preserve_proto_field_names(bool value) { + preserve_proto_field_names_ = value; + } + + // If set to true, enums are rendered as ints from output when default values + // are written. + void set_print_enums_as_ints(bool value) { + use_ints_for_enums_ = value; + } + + protected: enum NodeKind { PRIMITIVE = 0, OBJECT = 1, @@ -111,7 +147,14 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter { class LIBPROTOBUF_EXPORT Node { public: Node(const string& name, const google::protobuf::Type* type, NodeKind kind, - const DataPiece& data, bool is_placeholder); + const DataPiece& data, bool is_placeholder, + const std::vector<string>& path, bool suppress_empty_list, + FieldScrubCallBack* field_scrub_callback); + Node(const string& name, const google::protobuf::Type* type, NodeKind kind, + const DataPiece& data, bool is_placeholder, + const std::vector<string>& path, bool suppress_empty_list, + bool preserve_proto_field_names, bool use_ints_for_enums, + FieldScrubCallBack* field_scrub_callback); virtual ~Node() { for (int i = 0; i < children_.size(); ++i) { delete children_[i]; @@ -127,27 +170,29 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter { // Populates children of this Node based on its type. If there are already // children created, they will be merged to the result. Caller should pass // in TypeInfo for looking up types of the children. - void PopulateChildren(const TypeInfo* typeinfo); + virtual void PopulateChildren(const TypeInfo* typeinfo); // If this node is a leaf (has data), writes the current node to the // ObjectWriter; if not, then recursively writes the children to the // ObjectWriter. - void WriteTo(ObjectWriter* ow); + virtual void WriteTo(ObjectWriter* ow); // Accessors const string& name() const { return name_; } - const google::protobuf::Type* type() { return type_; } + const std::vector<string>& path() const { return path_; } + + const google::protobuf::Type* type() const { return type_; } void set_type(const google::protobuf::Type* type) { type_ = type; } - NodeKind kind() { return kind_; } + NodeKind kind() const { return kind_; } - int number_of_children() { return children_.size(); } + int number_of_children() const { return children_.size(); } void set_data(const DataPiece& data) { data_ = data; } - bool is_any() { return is_any_; } + bool is_any() const { return is_any_; } void set_is_any(bool is_any) { is_any_ = is_any; } @@ -155,7 +200,7 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter { is_placeholder_ = is_placeholder; } - private: + protected: // Returns the Value Type of a map given the Type of the map entry and a // TypeInfo instance. const google::protobuf::Type* GetMapValueType( @@ -181,20 +226,62 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter { // the parent node's StartObject()/StartList() method is called with this // node's name. bool is_placeholder_; + + // Path of the field of this node + std::vector<string> path_; + + // Whether to suppress empty list output. + bool suppress_empty_list_; + + // Whether to preserve original proto field names + bool preserve_proto_field_names_; + + // Whether to always print enums as ints + bool use_ints_for_enums_; + + // Pointer to function for determining whether a field needs to be scrubbed + // or not. This callback is owned by the creator of this node. + FieldScrubCallBack* field_scrub_callback_; + + private: + GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(Node); }; + // Creates a new Node and returns it. Caller owns memory of returned object. + virtual Node* CreateNewNode(const string& name, + const google::protobuf::Type* type, NodeKind kind, + const DataPiece& data, bool is_placeholder, + const std::vector<string>& path, + bool suppress_empty_list, + FieldScrubCallBack* field_scrub_callback); + + // Creates a new Node and returns it. Caller owns memory of returned object. + virtual Node* CreateNewNode(const string& name, + const google::protobuf::Type* type, NodeKind kind, + const DataPiece& data, bool is_placeholder, + const std::vector<string>& path, + bool suppress_empty_list, + bool preserve_proto_field_names, + bool use_ints_for_enums, + FieldScrubCallBack* field_scrub_callback); + + // Creates a DataPiece containing the default value of the type of the field. + static DataPiece CreateDefaultDataPieceForField( + const google::protobuf::Field& field, const TypeInfo* typeinfo, bool use_ints_for_enums); + + protected: + // Returns a pointer to current Node in tree. + Node* current() { return current_; } + + private: // Populates children of "node" if it is an "any" Node and its real type has // been given. void MaybePopulateChildrenOfAny(Node* node); // Writes the root_ node to ow_ and resets the root_ and current_ pointer to - // NULL. + // nullptr. void WriteRoot(); - // Creates a DataPiece containing the default value of the type of the field. - static DataPiece CreateDefaultDataPieceForField( - const google::protobuf::Field& field, const TypeInfo* typeinfo); - // Adds or replaces the data_ of a primitive child node. void RenderDataPiece(StringPiece name, const DataPiece& data); @@ -202,7 +289,8 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter { // there is no default. For proto3, where we cannot specify an explicit // default, a zero value will always be returned. static DataPiece FindEnumDefault(const google::protobuf::Field& field, - const TypeInfo* typeinfo); + const TypeInfo* typeinfo, + bool use_ints_for_enums); // Type information for all the types used in the descriptor. Used to find // google::protobuf::Type of nested messages/enums. @@ -212,15 +300,28 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter { // google::protobuf::Type of the root message type. const google::protobuf::Type& type_; // Holds copies of strings passed to RenderString. - vector<string*> string_values_; + std::vector<string*> string_values_; // The current Node. Owned by its parents. Node* current_; // The root Node. - google::protobuf::scoped_ptr<Node> root_; + std::unique_ptr<Node> root_; // The stack to hold the path of Nodes from current_ to root_; std::stack<Node*> stack_; + // Whether to suppress output of empty lists. + bool suppress_empty_list_; + + // Whether to preserve original proto field names + bool preserve_proto_field_names_; + + // Whether to always print enums as ints + bool use_ints_for_enums_; + + // Unique Pointer to function for determining whether a field needs to be + // scrubbed or not. + FieldScrubCallBackPtr field_scrub_callback_; + ObjectWriter* ow_; GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(DefaultValueObjectWriter); diff --git a/src/google/protobuf/util/internal/default_value_objectwriter_test.cc b/src/google/protobuf/util/internal/default_value_objectwriter_test.cc index 8254c0fa..0c4af61b 100644 --- a/src/google/protobuf/util/internal/default_value_objectwriter_test.cc +++ b/src/google/protobuf/util/internal/default_value_objectwriter_test.cc @@ -60,7 +60,7 @@ class BaseDefaultValueObjectWriterTest TypeInfoTestHelper helper_; MockObjectWriter mock_; ExpectingObjectWriter expects_; - google::protobuf::scoped_ptr<DefaultValueObjectWriter> testing_; + std::unique_ptr<DefaultValueObjectWriter> testing_; }; // Tests to cover some basic DefaultValueObjectWriter use cases. More tests are @@ -149,6 +149,39 @@ TEST_P(DefaultValueObjectWriterTest, ShouldRetainUnknownField) { } +class DefaultValueObjectWriterSuppressListTest + : public BaseDefaultValueObjectWriterTest { + protected: + DefaultValueObjectWriterSuppressListTest() + : BaseDefaultValueObjectWriterTest(DefaultValueTest::descriptor()) { + testing_->set_suppress_empty_list(true); + } + ~DefaultValueObjectWriterSuppressListTest() override {} +}; + +INSTANTIATE_TEST_CASE_P(DifferentTypeInfoSourceTest, + DefaultValueObjectWriterSuppressListTest, + ::testing::Values( + testing::USE_TYPE_RESOLVER)); + +TEST_P(DefaultValueObjectWriterSuppressListTest, Empty) { + // Set expectation. Emtpy lists should be suppressed. + expects_.StartObject("") + ->RenderDouble("doubleValue", 0.0) + ->RenderFloat("floatValue", 0.0) + ->RenderInt64("int64Value", 0) + ->RenderUint64("uint64Value", 0) + ->RenderInt32("int32Value", 0) + ->RenderUint32("uint32Value", 0) + ->RenderBool("boolValue", false) + ->RenderString("stringValue", "") + ->RenderBytes("bytesValue", "") + ->RenderString("enumValue", "ENUM_FIRST") + ->EndObject(); + + // Actual testing + testing_->StartObject("")->EndObject(); +} } // namespace testing } // namespace converter } // namespace util diff --git a/src/google/protobuf/util/internal/error_listener.h b/src/google/protobuf/util/internal/error_listener.h index 3f063936..a19bd3f7 100644 --- a/src/google/protobuf/util/internal/error_listener.h +++ b/src/google/protobuf/util/internal/error_listener.h @@ -33,9 +33,6 @@ #include <algorithm> #include <memory> -#ifndef _SHARED_PTR_H -#include <google/protobuf/stubs/shared_ptr.h> -#endif #include <string> #include <vector> @@ -57,7 +54,7 @@ class LIBPROTOBUF_EXPORT ErrorListener { // Reports an invalid name at the given location. virtual void InvalidName(const LocationTrackerInterface& loc, - StringPiece unknown_name, StringPiece message) = 0; + StringPiece invalid_name, StringPiece message) = 0; // Reports an invalid value for a field. virtual void InvalidValue(const LocationTrackerInterface& loc, @@ -82,7 +79,7 @@ class LIBPROTOBUF_EXPORT NoopErrorListener : public ErrorListener { virtual ~NoopErrorListener() {} virtual void InvalidName(const LocationTrackerInterface& loc, - StringPiece unknown_name, StringPiece message) {} + StringPiece invalid_name, StringPiece message) {} virtual void InvalidValue(const LocationTrackerInterface& loc, StringPiece type_name, StringPiece value) {} diff --git a/src/google/protobuf/util/internal/field_mask_utility.cc b/src/google/protobuf/util/internal/field_mask_utility.cc index f0e8fc88..778a4510 100644 --- a/src/google/protobuf/util/internal/field_mask_utility.cc +++ b/src/google/protobuf/util/internal/field_mask_utility.cc @@ -30,6 +30,7 @@ #include <google/protobuf/util/internal/field_mask_utility.h> +#include <google/protobuf/util/internal/utility.h> #include <google/protobuf/stubs/strutil.h> #include <google/protobuf/stubs/status_macros.h> @@ -44,11 +45,6 @@ inline util::Status CallPathSink(PathSinkCallback path_sink, return path_sink->Run(arg); } -util::Status CreatePublicError(util::error::Code code, - const string& message) { - return util::Status(code, message); -} - // Appends a FieldMask path segment to a prefix. string AppendPathSegmentToPrefix(StringPiece prefix, StringPiece segment) { if (prefix.empty()) { @@ -58,7 +54,7 @@ string AppendPathSegmentToPrefix(StringPiece prefix, StringPiece segment) { return prefix.ToString(); } // If the segment is a map key, appends it to the prefix without the ".". - if (segment.starts_with("[\"")) { + if (StringStartsWith(segment, "[\"")) { return StrCat(prefix, segment); } return StrCat(prefix, ".", segment); @@ -112,7 +108,7 @@ string ConvertFieldMaskPath(const StringPiece path, util::Status DecodeCompactFieldMaskPaths(StringPiece paths, PathSinkCallback path_sink) { - stack<string> prefix; + std::stack<string> prefix; int length = paths.length(); int previous_position = 0; bool in_map_key = false; @@ -216,7 +212,7 @@ util::Status DecodeCompactFieldMaskPaths(StringPiece paths, StrCat("Invalid FieldMask '", paths, "'. Cannot find matching ')' for all '('.")); } - return util::Status::OK; + return util::Status(); } } // namespace converter diff --git a/src/google/protobuf/util/internal/json_escaping.cc b/src/google/protobuf/util/internal/json_escaping.cc index 24bd554e..06b9a7f2 100644 --- a/src/google/protobuf/util/internal/json_escaping.cc +++ b/src/google/protobuf/util/internal/json_escaping.cc @@ -84,30 +84,6 @@ static const char kCommonEscapes[160][7] = { "\\u009c", "\\u009d", "\\u009e", "\\u009f" }; -// Determines if the given char value is a unicode high-surrogate code unit. -// Such values do not represent characters by themselves, but are used in the -// representation of supplementary characters in the utf-16 encoding. -inline bool IsHighSurrogate(uint16 c) { - // Optimized form of: - // return c >= kMinHighSurrogate && c <= kMaxHighSurrogate; - // (Reduced from 3 ALU instructions to 2 ALU instructions) - return (c & ~(JsonEscaping::kMaxHighSurrogate - - JsonEscaping::kMinHighSurrogate)) - == JsonEscaping::kMinHighSurrogate; -} - -// Determines if the given char value is a unicode low-surrogate code unit. -// Such values do not represent characters by themselves, but are used in the -// representation of supplementary characters in the utf-16 encoding. -inline bool IsLowSurrogate(uint16 c) { - // Optimized form of: - // return c >= kMinLowSurrogate && c <= kMaxLowSurrogate; - // (Reduced from 3 ALU instructions to 2 ALU instructions) - return (c & ~(JsonEscaping::kMaxLowSurrogate - - JsonEscaping::kMinLowSurrogate)) - == JsonEscaping::kMinLowSurrogate; -} - // Determines if the given char value is a unicode surrogate code unit (either // high-surrogate or low-surrogate). inline bool IsSurrogate(uint32 c) { @@ -117,36 +93,12 @@ inline bool IsSurrogate(uint32 c) { return (c & 0xfffff800) == JsonEscaping::kMinHighSurrogate; } -// Returns true if the given unicode code point cp is -// in the supplementary character range. -inline bool IsSupplementalCodePoint(uint32 cp) { - // Optimized form of: - // return kMinSupplementaryCodePoint <= cp && cp <= kMaxCodePoint; - // (Reduced from 3 ALU instructions to 2 ALU instructions) - return (cp & ~(JsonEscaping::kMinSupplementaryCodePoint - 1)) - < JsonEscaping::kMaxCodePoint; -} - // Returns true if the given unicode code point cp is a valid // unicode code point (i.e. in the range 0 <= cp <= kMaxCodePoint). inline bool IsValidCodePoint(uint32 cp) { return cp <= JsonEscaping::kMaxCodePoint; } -// Converts the specified surrogate pair to its supplementary code point value. -// It is the callers' responsibility to validate the specified surrogate pair. -inline uint32 ToCodePoint(uint16 high, uint16 low) { - // Optimized form of: - // return ((high - kMinHighSurrogate) << 10) - // + (low - kMinLowSurrogate) - // + kMinSupplementaryCodePoint; - // (Reduced from 5 ALU instructions to 3 ALU instructions) - return (high << 10) + low + - (JsonEscaping::kMinSupplementaryCodePoint - - (static_cast<unsigned>(JsonEscaping::kMinHighSurrogate) << 10) - - JsonEscaping::kMinLowSurrogate); -} - // Returns the low surrogate for the given unicode code point. The result is // meaningless if the given code point is not a supplementary character. inline uint16 ToLowSurrogate(uint32 cp) { @@ -255,7 +207,7 @@ StringPiece ToHex(uint16 cp, char* buffer) { buffer[3] = kHex[cp & 0x0f]; cp >>= 4; buffer[2] = kHex[cp & 0x0f]; - return StringPiece(buffer, 0, 6); + return StringPiece(buffer, 6); } // Stores the 32-bit unicode code point as its hexadecimal digits in buffer @@ -336,19 +288,19 @@ StringPiece EscapeCodePoint(uint32 cp, char* buffer, bool force_output) { cp >>= 6; if (cp <= 0x1f) { buffer[4] = cp | 0xc0; - sp.set(buffer + 4, 2); + sp = StringPiece(buffer + 4, 2); return sp; } buffer[4] = (cp & 0x3f) | 0x80; cp >>= 6; if (cp <= 0x0f) { buffer[3] = cp | 0xe0; - sp.set(buffer + 3, 3); + sp = StringPiece(buffer + 3, 3); return sp; } buffer[3] = (cp & 0x3f) | 0x80; buffer[2] = ((cp >> 6) & 0x07) | 0xf0; - sp.set(buffer + 2, 4); + sp = StringPiece(buffer + 2, 4); } return sp; } diff --git a/src/google/protobuf/util/internal/json_escaping.h b/src/google/protobuf/util/internal/json_escaping.h index e3e329fc..5495c57f 100644 --- a/src/google/protobuf/util/internal/json_escaping.h +++ b/src/google/protobuf/util/internal/json_escaping.h @@ -28,8 +28,8 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#ifndef NET_PROTO2_UTIL_CONVERTER_STRINGS_JSON_ESCAPING_H_ -#define NET_PROTO2_UTIL_CONVERTER_STRINGS_JSON_ESCAPING_H_ +#ifndef GOOGLE_PROTOBUF_UTIL_INTERNAL__JSON_ESCAPING_H__ +#define GOOGLE_PROTOBUF_UTIL_INTERNAL__JSON_ESCAPING_H__ #include <google/protobuf/stubs/common.h> #include <google/protobuf/stubs/bytestream.h> @@ -87,5 +87,5 @@ class JsonEscaping { } // namespace util } // namespace protobuf -#endif // NET_PROTO2_UTIL_CONVERTER_STRINGS_JSON_ESCAPING_H_ } // namespace google +#endif // GOOGLE_PROTOBUF_UTIL_INTERNAL__JSON_ESCAPING_H__ diff --git a/src/google/protobuf/util/internal/json_objectwriter.cc b/src/google/protobuf/util/internal/json_objectwriter.cc index 94d2ab7a..a431177a 100644 --- a/src/google/protobuf/util/internal/json_objectwriter.cc +++ b/src/google/protobuf/util/internal/json_objectwriter.cc @@ -46,6 +46,7 @@ namespace util { namespace converter { using strings::ArrayByteSource; +; JsonObjectWriter::~JsonObjectWriter() { if (!element_->is_root()) { @@ -56,7 +57,7 @@ JsonObjectWriter::~JsonObjectWriter() { JsonObjectWriter* JsonObjectWriter::StartObject(StringPiece name) { WritePrefix(name); WriteChar('{'); - Push(); + PushObject(); return this; } @@ -70,7 +71,7 @@ JsonObjectWriter* JsonObjectWriter::EndObject() { JsonObjectWriter* JsonObjectWriter::StartList(StringPiece name) { WritePrefix(name); WriteChar('['); - Push(); + PushArray(); return this; } @@ -81,13 +82,11 @@ JsonObjectWriter* JsonObjectWriter::EndList() { return this; } -JsonObjectWriter* JsonObjectWriter::RenderBool(StringPiece name, - bool value) { +JsonObjectWriter* JsonObjectWriter::RenderBool(StringPiece name, bool value) { return RenderSimple(name, value ? "true" : "false"); } -JsonObjectWriter* JsonObjectWriter::RenderInt32(StringPiece name, - int32 value) { +JsonObjectWriter* JsonObjectWriter::RenderInt32(StringPiece name, int32 value) { return RenderSimple(name, SimpleItoa(value)); } @@ -96,8 +95,7 @@ JsonObjectWriter* JsonObjectWriter::RenderUint32(StringPiece name, return RenderSimple(name, SimpleItoa(value)); } -JsonObjectWriter* JsonObjectWriter::RenderInt64(StringPiece name, - int64 value) { +JsonObjectWriter* JsonObjectWriter::RenderInt64(StringPiece name, int64 value) { WritePrefix(name); WriteChar('"'); stream_->WriteString(SimpleItoa(value)); @@ -124,8 +122,7 @@ JsonObjectWriter* JsonObjectWriter::RenderDouble(StringPiece name, return RenderString(name, DoubleAsString(value)); } -JsonObjectWriter* JsonObjectWriter::RenderFloat(StringPiece name, - float value) { +JsonObjectWriter* JsonObjectWriter::RenderFloat(StringPiece name, float value) { if (MathLimits<float>::IsFinite(value)) { return RenderSimple(name, SimpleFtoa(value)); } @@ -148,7 +145,12 @@ JsonObjectWriter* JsonObjectWriter::RenderBytes(StringPiece name, StringPiece value) { WritePrefix(name); string base64; - Base64Escape(value, &base64); + + if (use_websafe_base64_for_bytes_) + WebSafeBase64EscapeWithPadding(value.ToString(), &base64); + else + Base64Escape(value, &base64); + WriteChar('"'); // TODO(wpoon): Consider a ByteSink solution that writes the base64 bytes // directly to the stream, rather than first putting them @@ -162,14 +164,20 @@ JsonObjectWriter* JsonObjectWriter::RenderNull(StringPiece name) { return RenderSimple(name, "null"); } +JsonObjectWriter* JsonObjectWriter::RenderNullAsEmpty(StringPiece name) { + return RenderSimple(name, ""); +} + void JsonObjectWriter::WritePrefix(StringPiece name) { bool not_first = !element()->is_first(); if (not_first) WriteChar(','); if (not_first || !element()->is_root()) NewLine(); - if (!name.empty()) { + if (!name.empty() || element()->is_json_object()) { WriteChar('"'); - ArrayByteSource source(name); - JsonEscaping::Escape(&source, &sink_); + if (!name.empty()) { + ArrayByteSource source(name); + JsonEscaping::Escape(&source, &sink_); + } stream_->WriteString("\":"); if (!indent_string_.empty()) WriteChar(' '); } diff --git a/src/google/protobuf/util/internal/json_objectwriter.h b/src/google/protobuf/util/internal/json_objectwriter.h index 761a0a10..81644dab 100644 --- a/src/google/protobuf/util/internal/json_objectwriter.h +++ b/src/google/protobuf/util/internal/json_objectwriter.h @@ -32,9 +32,6 @@ #define GOOGLE_PROTOBUF_UTIL_CONVERTER_JSON_OBJECTWRITER_H__ #include <memory> -#ifndef _SHARED_PTR_H -#include <google/protobuf/stubs/shared_ptr.h> -#endif #include <string> #include <google/protobuf/io/coded_stream.h> @@ -89,10 +86,11 @@ class LIBPROTOBUF_EXPORT JsonObjectWriter : public StructuredObjectWriter { public: JsonObjectWriter(StringPiece indent_string, google::protobuf::io::CodedOutputStream* out) - : element_(new Element(NULL)), - stream_(out), sink_(out), - indent_string_(indent_string.ToString()) { - } + : element_(new Element(/*parent=*/nullptr, /*is_json_object=*/false)), + stream_(out), + sink_(out), + indent_string_(indent_string.ToString()), + use_websafe_base64_for_bytes_(false) {} virtual ~JsonObjectWriter(); // ObjectWriter methods. @@ -110,11 +108,19 @@ class LIBPROTOBUF_EXPORT JsonObjectWriter : public StructuredObjectWriter { virtual JsonObjectWriter* RenderString(StringPiece name, StringPiece value); virtual JsonObjectWriter* RenderBytes(StringPiece name, StringPiece value); virtual JsonObjectWriter* RenderNull(StringPiece name); + virtual JsonObjectWriter* RenderNullAsEmpty(StringPiece name); + + void set_use_websafe_base64_for_bytes(bool value) { + use_websafe_base64_for_bytes_ = value; + } protected: class LIBPROTOBUF_EXPORT Element : public BaseElement { public: - explicit Element(Element* parent) : BaseElement(parent), is_first_(true) {} + Element(Element* parent, bool is_json_object) + : BaseElement(parent), + is_first_(true), + is_json_object_(is_json_object) {} // Called before each field of the Element is to be processed. // Returns true if this is the first call (processing the first field). @@ -126,8 +132,13 @@ class LIBPROTOBUF_EXPORT JsonObjectWriter : public StructuredObjectWriter { return false; } + // Whether we are currently renderring inside a JSON object (i.e., between + // StartObject() and EndObject()). + bool is_json_object() const { return is_json_object_; } + private: bool is_first_; + bool is_json_object_; GOOGLE_DISALLOW_IMPLICIT_CONSTRUCTORS(Element); }; @@ -161,8 +172,15 @@ class LIBPROTOBUF_EXPORT JsonObjectWriter : public StructuredObjectWriter { return this; } - // Pushes a new element to the stack. - void Push() { element_.reset(new Element(element_.release())); } + // Pushes a new JSON array element to the stack. + void PushArray() { + element_.reset(new Element(element_.release(), /*is_json_object=*/false)); + } + + // Pushes a new JSON object element to the stack. + void PushObject() { + element_.reset(new Element(element_.release(), /*is_json_object=*/true)); + } // Pops an element off of the stack and deletes the popped element. void Pop() { @@ -190,11 +208,15 @@ class LIBPROTOBUF_EXPORT JsonObjectWriter : public StructuredObjectWriter { // Writes an individual character to the output. void WriteChar(const char c) { stream_->WriteRaw(&c, sizeof(c)); } - google::protobuf::scoped_ptr<Element> element_; + std::unique_ptr<Element> element_; google::protobuf::io::CodedOutputStream* stream_; ByteSinkWrapper sink_; const string indent_string_; + // Whether to use regular or websafe base64 encoding for byte fields. Defaults + // to regular base64 encoding. + bool use_websafe_base64_for_bytes_; + GOOGLE_DISALLOW_IMPLICIT_CONSTRUCTORS(JsonObjectWriter); }; diff --git a/src/google/protobuf/util/internal/json_objectwriter_test.cc b/src/google/protobuf/util/internal/json_objectwriter_test.cc index 9d820162..0dc710c7 100644 --- a/src/google/protobuf/util/internal/json_objectwriter_test.cc +++ b/src/google/protobuf/util/internal/json_objectwriter_test.cc @@ -47,7 +47,7 @@ class JsonObjectWriterTest : public ::testing::Test { JsonObjectWriterTest() : str_stream_(new StringOutputStream(&output_)), out_stream_(new CodedOutputStream(str_stream_)), - ow_(NULL) {} + ow_(nullptr) {} virtual ~JsonObjectWriterTest() { delete ow_; @@ -58,7 +58,7 @@ class JsonObjectWriterTest : public ::testing::Test { string output_; StringOutputStream* const str_stream_; CodedOutputStream* const out_stream_; - ObjectWriter* ow_; + JsonObjectWriter* ow_; }; TEST_F(JsonObjectWriterTest, EmptyRootObject) { @@ -95,6 +95,12 @@ TEST_F(JsonObjectWriterTest, EmptyList) { output_.substr(0, out_stream_->ByteCount())); } +TEST_F(JsonObjectWriterTest, EmptyObjectKey) { + ow_ = new JsonObjectWriter("", out_stream_); + ow_->StartObject("")->RenderString("", "value")->EndObject(); + EXPECT_EQ("{\"\":\"value\"}", output_.substr(0, out_stream_->ByteCount())); +} + TEST_F(JsonObjectWriterTest, ObjectInObject) { ow_ = new JsonObjectWriter("", out_stream_); ow_->StartObject("") @@ -283,6 +289,30 @@ TEST_F(JsonObjectWriterTest, Stringification) { output_.substr(0, out_stream_->ByteCount())); } +TEST_F(JsonObjectWriterTest, TestRegularByteEncoding) { + ow_ = new JsonObjectWriter("", out_stream_); + ow_->StartObject("") + ->RenderBytes("bytes", "\x03\xef\xc0") + ->EndObject(); + + // Test that we get regular (non websafe) base64 encoding on byte fields by + // default. + EXPECT_EQ("{\"bytes\":\"A+/A\"}", + output_.substr(0, out_stream_->ByteCount())); +} + +TEST_F(JsonObjectWriterTest, TestWebsafeByteEncoding) { + ow_ = new JsonObjectWriter("", out_stream_); + ow_->set_use_websafe_base64_for_bytes(true); + ow_->StartObject("") + ->RenderBytes("bytes", "\x03\xef\xc0\x10") + ->EndObject(); + + // Test that we get websafe base64 encoding when explicitly asked. + EXPECT_EQ("{\"bytes\":\"A-_AEA==\"}", + output_.substr(0, out_stream_->ByteCount())); +} + } // namespace converter } // namespace util } // namespace protobuf diff --git a/src/google/protobuf/util/internal/json_stream_parser.cc b/src/google/protobuf/util/internal/json_stream_parser.cc index df916751..1c63b31d 100644 --- a/src/google/protobuf/util/internal/json_stream_parser.cc +++ b/src/google/protobuf/util/internal/json_stream_parser.cc @@ -36,14 +36,14 @@ #include <cstdlib> #include <cstring> #include <memory> -#ifndef _SHARED_PTR_H -#include <google/protobuf/stubs/shared_ptr.h> -#endif #include <google/protobuf/stubs/logging.h> #include <google/protobuf/stubs/common.h> -#include <google/protobuf/stubs/strutil.h> #include <google/protobuf/util/internal/object_writer.h> +#include <google/protobuf/util/internal/json_escaping.h> +#include <google/protobuf/stubs/strutil.h> +#include <google/protobuf/stubs/mathlimits.h> + namespace google { namespace protobuf { @@ -53,13 +53,14 @@ namespace util { // this file. using util::Status; namespace error { +using util::error::CANCELLED; using util::error::INTERNAL; using util::error::INVALID_ARGUMENT; } // namespace error namespace converter { -// Number of digits in a unicode escape sequence (/uXXXX) +// Number of digits in an escaped UTF-16 code unit ('\\' 'u' X X X X) static const int kUnicodeEscapedLength = 6; // Length of the true, false, and null literals. @@ -106,7 +107,9 @@ JsonStreamParser::JsonStreamParser(ObjectWriter* ow) parsed_storage_(), string_open_(0), chunk_storage_(), - coerce_to_utf8_(false) { + coerce_to_utf8_(false), + allow_empty_null_(false), + loose_float_number_conversion_(false) { // Initialize the stack with a single value to be parsed. stack_.push(VALUE); } @@ -124,7 +127,7 @@ util::Status JsonStreamParser::Parse(StringPiece json) { // Don't point chunk to leftover_ because leftover_ will be updated in // ParseChunk(chunk). chunk_storage_.swap(leftover_); - json.AppendToString(&chunk_storage_); + StrAppend(&chunk_storage_, json); chunk = StringPiece(chunk_storage_); } @@ -135,11 +138,11 @@ util::Status JsonStreamParser::Parse(StringPiece json) { // Any leftover characters are stashed in leftover_ for later parsing when // there is more data available. - chunk.substr(n).AppendToString(&leftover_); + StrAppend(&leftover_, chunk.substr(n)); return status; } else { - chunk.CopyToString(&leftover_); - return util::Status::OK; + leftover_.assign(chunk.data(), chunk.size()); + return util::Status(); } } @@ -147,11 +150,11 @@ util::Status JsonStreamParser::FinishParse() { // If we do not expect anything and there is nothing left to parse we're all // done. if (stack_.empty() && leftover_.empty()) { - return util::Status::OK; + return util::Status(); } // Storage for UTF8-coerced string. - google::protobuf::scoped_array<char> utf8; + std::unique_ptr<char[]> utf8; if (coerce_to_utf8_) { utf8.reset(new char[leftover_.size()]); char* coerced = internal::UTF8CoerceToStructurallyValid(leftover_, utf8.get(), ' '); @@ -178,7 +181,7 @@ util::Status JsonStreamParser::FinishParse() { util::Status JsonStreamParser::ParseChunk(StringPiece chunk) { // Do not do any work if the chunk is empty. - if (chunk.empty()) return util::Status::OK; + if (chunk.empty()) return util::Status(); p_ = json_ = chunk; @@ -200,7 +203,7 @@ util::Status JsonStreamParser::ParseChunk(StringPiece chunk) { // unparsed data left, we save it for later parse. leftover_ = p_.ToString(); } - return util::Status::OK; + return util::Status(); } util::Status JsonStreamParser::RunParser() { @@ -241,20 +244,20 @@ util::Status JsonStreamParser::RunParser() { } if (!result.ok()) { // If we were cancelled, save our state and try again later. - if (!finishing_ && result == util::Status::CANCELLED) { + if (!finishing_ && result == util::Status(error::CANCELLED, "")) { stack_.push(type); // If we have a key we still need to render, make sure to save off the // contents in our own storage. if (!key_.empty() && key_storage_.empty()) { - key_.AppendToString(&key_storage_); + StrAppend(&key_storage_, key_); key_ = StringPiece(key_storage_); } - result = util::Status::OK; + result = util::Status(); } return result; } } - return util::Status::OK; + return util::Status(); } util::Status JsonStreamParser::ParseValue(TokenType type) { @@ -276,12 +279,16 @@ util::Status JsonStreamParser::ParseValue(TokenType type) { case UNKNOWN: return ReportUnknown("Expected a value."); default: { + if (allow_empty_null_ && IsEmptyNullAllowed(type)) { + return ParseEmptyNull(); + } + // Special case for having been cut off while parsing, wait for more data. // This handles things like 'fals' being at the end of the string, we // don't know if the next char would be e, completing it, or something // else, making it invalid. if (!finishing_ && p_.length() < false_len) { - return util::Status::CANCELLED; + return util::Status(error::CANCELLED, ""); } return ReportFailure("Unexpected token."); } @@ -292,8 +299,8 @@ util::Status JsonStreamParser::ParseString() { util::Status result = ParseStringHelper(); if (result.ok()) { ow_->RenderString(key_, parsed_); - key_.clear(); - parsed_.clear(); + key_ = StringPiece(); + parsed_ = StringPiece(); parsed_storage_.clear(); } return result; @@ -314,13 +321,12 @@ util::Status JsonStreamParser::ParseStringHelper() { // We're about to handle an escape, copy all bytes from last to data. if (last < data) { parsed_storage_.append(last, data - last); - last = data; } // If we ran out of string after the \, cancel or report an error // depending on if we expect more data later. if (p_.length() == 1) { if (!finishing_) { - return util::Status::CANCELLED; + return util::Status(error::CANCELLED, ""); } return ReportFailure("Closing quote expected in string."); } @@ -370,7 +376,6 @@ util::Status JsonStreamParser::ParseStringHelper() { } else { if (last < data) { parsed_storage_.append(last, data - last); - last = data; } parsed_ = StringPiece(parsed_storage_); } @@ -378,7 +383,7 @@ util::Status JsonStreamParser::ParseStringHelper() { // start fresh. string_open_ = 0; Advance(); - return util::Status::OK; + return util::Status(); } // Normal character, just advance past it. Advance(); @@ -389,7 +394,7 @@ util::Status JsonStreamParser::ParseStringHelper() { } // If we didn't find the closing quote but we expect more data, cancel for now if (!finishing_) { - return util::Status::CANCELLED; + return util::Status(error::CANCELLED, ""); } // End of string reached without a closing quote, report an error. string_open_ = 0; @@ -406,7 +411,7 @@ util::Status JsonStreamParser::ParseStringHelper() { util::Status JsonStreamParser::ParseUnicodeEscape() { if (p_.length() < kUnicodeEscapedLength) { if (!finishing_) { - return util::Status::CANCELLED; + return util::Status(error::CANCELLED, ""); } return ReportFailure("Illegal hex string."); } @@ -419,12 +424,48 @@ util::Status JsonStreamParser::ParseUnicodeEscape() { } code = (code << 4) + hex_digit_to_int(p_.data()[i]); } + if (code >= JsonEscaping::kMinHighSurrogate && + code <= JsonEscaping::kMaxHighSurrogate) { + if (p_.length() < 2 * kUnicodeEscapedLength) { + if (!finishing_) { + return util::Status(error::CANCELLED, ""); + } + if (!coerce_to_utf8_) { + return ReportFailure("Missing low surrogate."); + } + } else if (p_.data()[kUnicodeEscapedLength] == '\\' && + p_.data()[kUnicodeEscapedLength + 1] == 'u') { + uint32 low_code = 0; + for (int i = kUnicodeEscapedLength + 2; i < 2 * kUnicodeEscapedLength; + ++i) { + if (!isxdigit(p_.data()[i])) { + return ReportFailure("Invalid escape sequence."); + } + low_code = (low_code << 4) + hex_digit_to_int(p_.data()[i]); + } + if (low_code >= JsonEscaping::kMinLowSurrogate && + low_code <= JsonEscaping::kMaxLowSurrogate) { + // Convert UTF-16 surrogate pair to 21-bit Unicode codepoint. + code = (((code & 0x3FF) << 10) | (low_code & 0x3FF)) + + JsonEscaping::kMinSupplementaryCodePoint; + // Advance past the first code unit escape. + p_.remove_prefix(kUnicodeEscapedLength); + } else if (!coerce_to_utf8_) { + return ReportFailure("Invalid low surrogate."); + } + } else if (!coerce_to_utf8_) { + return ReportFailure("Missing low surrogate."); + } + } + if (!coerce_to_utf8_ && !IsValidCodePoint(code)) { + return ReportFailure("Invalid unicode code point."); + } char buf[UTFmax]; int len = EncodeAsUTF8Char(code, buf); - // Advance past the unicode escape. + // Advance past the [final] code unit escape. p_.remove_prefix(kUnicodeEscapedLength); parsed_storage_.append(buf, len); - return util::Status::OK; + return util::Status(); } util::Status JsonStreamParser::ParseNumber() { @@ -434,17 +475,17 @@ util::Status JsonStreamParser::ParseNumber() { switch (number.type) { case NumberResult::DOUBLE: ow_->RenderDouble(key_, number.double_val); - key_.clear(); + key_ = StringPiece(); break; case NumberResult::INT: ow_->RenderInt64(key_, number.int_val); - key_.clear(); + key_ = StringPiece(); break; case NumberResult::UINT: ow_->RenderUint64(key_, number.uint_val); - key_.clear(); + key_ = StringPiece(); break; default: @@ -454,6 +495,19 @@ util::Status JsonStreamParser::ParseNumber() { return result; } +util::Status JsonStreamParser::ParseDoubleHelper( + const string& number, NumberResult* result) { + if (!safe_strtod(number, &result->double_val)) { + return ReportFailure("Unable to parse number."); + } + if (!loose_float_number_conversion_ && + !MathLimits<double>::IsFinite(result->double_val)) { + return ReportFailure("Number exceeds the range of double."); + } + result->type = NumberResult::DOUBLE; + return util::Status(); +} + util::Status JsonStreamParser::ParseNumberHelper(NumberResult* result) { const char* data = p_.data(); int length = p_.length(); @@ -473,7 +527,7 @@ util::Status JsonStreamParser::ParseNumberHelper(NumberResult* result) { floating = true; continue; } - if (c == '+' || c == '-') continue; + if (c == '+' || c == '-' || c == 'x') continue; // Not a valid number character, break out. break; } @@ -481,7 +535,7 @@ util::Status JsonStreamParser::ParseNumberHelper(NumberResult* result) { // If the entire input is a valid number, and we may have more content in the // future, we abort for now and resume when we know more. if (index == length && !finishing_) { - return util::Status::CANCELLED; + return util::Status(error::CANCELLED, ""); } // Create a string containing just the number, so we can use safe_strtoX @@ -489,40 +543,59 @@ util::Status JsonStreamParser::ParseNumberHelper(NumberResult* result) { // Floating point number, parse as a double. if (floating) { - if (!safe_strtod(number, &result->double_val)) { - return ReportFailure("Unable to parse number."); + util::Status status = ParseDoubleHelper(number, result); + if (status.ok()) { + p_.remove_prefix(index); } - result->type = NumberResult::DOUBLE; - p_.remove_prefix(index); - return util::Status::OK; + return status; } // Positive non-floating point number, parse as a uint64. if (!negative) { - if (!safe_strtou64(number, &result->uint_val)) { - return ReportFailure("Unable to parse number."); + // Octal/Hex numbers are not valid JSON values. + if (number.length() >= 2 && number[0] == '0') { + return ReportFailure("Octal/hex numbers are not valid JSON values."); + } + if (safe_strtou64(number, &result->uint_val)) { + result->type = NumberResult::UINT; + p_.remove_prefix(index); + return util::Status(); + } else { + // If the value is too large, parse it as double. + util::Status status = ParseDoubleHelper(number, result); + if (status.ok()) { + p_.remove_prefix(index); + } + return status; } - result->type = NumberResult::UINT; - p_.remove_prefix(index); - return util::Status::OK; } + // Octal/Hex numbers are not valid JSON values. + if (number.length() >= 3 && number[1] == '0') { + return ReportFailure("Octal/hex numbers are not valid JSON values."); + } // Negative non-floating point number, parse as an int64. - if (!safe_strto64(number, &result->int_val)) { - return ReportFailure("Unable to parse number."); + if (safe_strto64(number, &result->int_val)) { + result->type = NumberResult::INT; + p_.remove_prefix(index); + return util::Status(); + } else { + // If the value is too large, parse it as double. + util::Status status = ParseDoubleHelper(number, result); + if (status.ok()) { + p_.remove_prefix(index); + } + return status; } - result->type = NumberResult::INT; - p_.remove_prefix(index); - return util::Status::OK; } util::Status JsonStreamParser::HandleBeginObject() { GOOGLE_DCHECK_EQ('{', *p_.data()); Advance(); ow_->StartObject(key_); - key_.clear(); + key_ = StringPiece(); stack_.push(ENTRY); - return util::Status::OK; + return util::Status(); } util::Status JsonStreamParser::ParseObjectMid(TokenType type) { @@ -534,13 +607,13 @@ util::Status JsonStreamParser::ParseObjectMid(TokenType type) { if (type == END_OBJECT) { Advance(); ow_->EndObject(); - return util::Status::OK; + return util::Status(); } // Found a comma, advance past it and get ready for an entry. if (type == VALUE_SEPARATOR) { Advance(); stack_.push(ENTRY); - return util::Status::OK; + return util::Status(); } // Illegal token after key:value pair. return ReportFailure("Expected , or } after key:value pair."); @@ -555,7 +628,7 @@ util::Status JsonStreamParser::ParseEntry(TokenType type) { if (type == END_OBJECT) { ow_->EndObject(); Advance(); - return util::Status::OK; + return util::Status(); } util::Status result; @@ -570,7 +643,7 @@ util::Status JsonStreamParser::ParseEntry(TokenType type) { } else { key_ = parsed_; } - parsed_.clear(); + parsed_ = StringPiece(); } } else if (type == BEGIN_KEY) { // Key is a bare key (back compat), create a StringPiece pointing to it. @@ -594,7 +667,7 @@ util::Status JsonStreamParser::ParseEntryMid(TokenType type) { if (type == ENTRY_SEPARATOR) { Advance(); stack_.push(VALUE); - return util::Status::OK; + return util::Status(); } return ReportFailure("Expected : between key:value pair."); } @@ -603,9 +676,9 @@ util::Status JsonStreamParser::HandleBeginArray() { GOOGLE_DCHECK_EQ('[', *p_.data()); Advance(); ow_->StartList(key_); - key_.clear(); + key_ = StringPiece(); stack_.push(ARRAY_VALUE); - return util::Status::OK; + return util::Status(); } util::Status JsonStreamParser::ParseArrayValue(TokenType type) { @@ -616,14 +689,15 @@ util::Status JsonStreamParser::ParseArrayValue(TokenType type) { if (type == END_ARRAY) { ow_->EndList(); Advance(); - return util::Status::OK; + return util::Status(); } // The ParseValue call may push something onto the stack so we need to make - // sure an ARRAY_MID is after it, so we push it on now. + // sure an ARRAY_MID is after it, so we push it on now. Also, the parsing of + // empty-null array value is relying on this ARRAY_MID token. stack_.push(ARRAY_MID); util::Status result = ParseValue(type); - if (result == util::Status::CANCELLED) { + if (result == util::Status(error::CANCELLED, "")) { // If we were cancelled, pop back off the ARRAY_MID so we don't try to // push it on again when we try over. stack_.pop(); @@ -639,14 +713,14 @@ util::Status JsonStreamParser::ParseArrayMid(TokenType type) { if (type == END_ARRAY) { ow_->EndList(); Advance(); - return util::Status::OK; + return util::Status(); } // Found a comma, advance past it and expect an array value next. if (type == VALUE_SEPARATOR) { Advance(); stack_.push(ARRAY_VALUE); - return util::Status::OK; + return util::Status(); } // Illegal token after array value. return ReportFailure("Expected , or ] after array value."); @@ -654,31 +728,44 @@ util::Status JsonStreamParser::ParseArrayMid(TokenType type) { util::Status JsonStreamParser::ParseTrue() { ow_->RenderBool(key_, true); - key_.clear(); + key_ = StringPiece(); p_.remove_prefix(true_len); - return util::Status::OK; + return util::Status(); } util::Status JsonStreamParser::ParseFalse() { ow_->RenderBool(key_, false); - key_.clear(); + key_ = StringPiece(); p_.remove_prefix(false_len); - return util::Status::OK; + return util::Status(); } util::Status JsonStreamParser::ParseNull() { ow_->RenderNull(key_); - key_.clear(); + key_ = StringPiece(); p_.remove_prefix(null_len); - return util::Status::OK; + return util::Status(); +} + +util::Status JsonStreamParser::ParseEmptyNull() { + ow_->RenderNull(key_); + key_ = StringPiece(); + return util::Status(); +} + +bool JsonStreamParser::IsEmptyNullAllowed(TokenType type) { + if (stack_.empty()) return false; + return (stack_.top() == ARRAY_MID && type == VALUE_SEPARATOR) || + stack_.top() == OBJ_MID; } util::Status JsonStreamParser::ReportFailure(StringPiece message) { static const int kContextLength = 20; const char* p_start = p_.data(); const char* json_start = json_.data(); - const char* begin = max(p_start - kContextLength, json_start); - const char* end = min(p_start + kContextLength, json_start + json_.size()); + const char* begin = std::max(p_start - kContextLength, json_start); + const char* end = + std::min(p_start + kContextLength, json_start + json_.size()); StringPiece segment(begin, end - begin); string location(p_start - begin, ' '); location.push_back('^'); @@ -689,7 +776,7 @@ util::Status JsonStreamParser::ReportFailure(StringPiece message) { util::Status JsonStreamParser::ReportUnknown(StringPiece message) { // If we aren't finishing the parse, cancel parsing and try later. if (!finishing_) { - return util::Status::CANCELLED; + return util::Status(error::CANCELLED, ""); } if (p_.empty()) { return ReportFailure(StrCat("Unexpected end of string. ", message)); @@ -706,8 +793,8 @@ void JsonStreamParser::SkipWhitespace() { void JsonStreamParser::Advance() { // Advance by moving one UTF8 character while making sure we don't go beyond // the length of StringPiece. - p_.remove_prefix( - min<int>(p_.length(), UTF8FirstLetterNumBytes(p_.data(), p_.length()))); + p_.remove_prefix(std::min<int>( + p_.length(), UTF8FirstLetterNumBytes(p_.data(), p_.length()))); } util::Status JsonStreamParser::ParseKey() { @@ -719,11 +806,11 @@ util::Status JsonStreamParser::ParseKey() { // we can't know if the key was complete or not. if (!finishing_ && p_.empty()) { p_ = original; - return util::Status::CANCELLED; + return util::Status(error::CANCELLED, ""); } // Since we aren't using the key storage, clear it out. key_storage_.clear(); - return util::Status::OK; + return util::Status(); } JsonStreamParser::TokenType JsonStreamParser::GetNextTokenType() { diff --git a/src/google/protobuf/util/internal/json_stream_parser.h b/src/google/protobuf/util/internal/json_stream_parser.h index 0278c28f..31933b67 100644 --- a/src/google/protobuf/util/internal/json_stream_parser.h +++ b/src/google/protobuf/util/internal/json_stream_parser.h @@ -154,6 +154,9 @@ class LIBPROTOBUF_EXPORT JsonStreamParser { // component. util::Status ParseNumberHelper(NumberResult* result); + // Parse a number as double into a NumberResult. + util::Status ParseDoubleHelper(const string& number, NumberResult* result); + // Handles a { during parsing of a value. util::Status HandleBeginObject(); @@ -179,6 +182,10 @@ class LIBPROTOBUF_EXPORT JsonStreamParser { util::Status ParseTrue(); util::Status ParseFalse(); util::Status ParseNull(); + util::Status ParseEmptyNull(); + + // Whether an empty-null is allowed in the current state. + bool IsEmptyNullAllowed(TokenType type); // Report a failure as a util::Status. util::Status ReportFailure(StringPiece message); @@ -247,6 +254,13 @@ class LIBPROTOBUF_EXPORT JsonStreamParser { // Whether to allow non UTF-8 encoded input and replace invalid code points. bool coerce_to_utf8_; + // Whether allows empty string represented null array value or object entry + // value. + bool allow_empty_null_; + + // Whether allows out-of-range floating point numbers or reject them. + bool loose_float_number_conversion_; + GOOGLE_DISALLOW_IMPLICIT_CONSTRUCTORS(JsonStreamParser); }; diff --git a/src/google/protobuf/util/internal/json_stream_parser_test.cc b/src/google/protobuf/util/internal/json_stream_parser_test.cc index 3414826e..a11e9be0 100644 --- a/src/google/protobuf/util/internal/json_stream_parser_test.cc +++ b/src/google/protobuf/util/internal/json_stream_parser_test.cc @@ -81,12 +81,16 @@ using util::Status; // For each test we split the input string on every possible character to ensure // the parser is able to handle arbitrarily split input for all cases. We also // do a final test of the entire test case one character at a time. +// +// It is verified that expected calls to the mocked objects are in sequence. class JsonStreamParserTest : public ::testing::Test { protected: JsonStreamParserTest() : mock_(), ow_(&mock_) {} virtual ~JsonStreamParserTest() {} - util::Status RunTest(StringPiece json, int split, bool coerce_utf8 = false) { + util::Status RunTest(StringPiece json, int split, bool coerce_utf8 = false, + bool allow_empty_null = false, + bool loose_float_number_conversion = false) { JsonStreamParser parser(&mock_); // Special case for split == length, test parsing one character at a time. @@ -116,22 +120,33 @@ class JsonStreamParserTest : public ::testing::Test { return result; } - void DoTest(StringPiece json, int split, bool coerce_utf8 = false) { - util::Status result = RunTest(json, split, coerce_utf8); + void DoTest(StringPiece json, int split, bool coerce_utf8 = false, + bool allow_empty_null = false, + bool loose_float_number_conversion = false) { + util::Status result = RunTest(json, split, coerce_utf8, allow_empty_null, + loose_float_number_conversion); if (!result.ok()) { GOOGLE_LOG(WARNING) << result; } EXPECT_OK(result); } - void DoErrorTest(StringPiece json, int split, StringPiece error_prefix) { - util::Status result = RunTest(json, split); + void DoErrorTest(StringPiece json, int split, StringPiece error_prefix, + bool coerce_utf8 = false, bool allow_empty_null = false) { + util::Status result = + RunTest(json, split, coerce_utf8, allow_empty_null); EXPECT_EQ(util::error::INVALID_ARGUMENT, result.error_code()); StringPiece error_message(result.error_message()); EXPECT_EQ(error_prefix, error_message.substr(0, error_prefix.size())); } +#ifndef _MSC_VER + // TODO(xiaofeng): We have to disable InSequence check for MSVC because it + // causes stack overflow due to its use of a linked list that is desctructed + // recursively. + ::testing::InSequence in_sequence_; +#endif // !_MSC_VER MockObjectWriter mock_; ExpectingObjectWriter ow_; }; @@ -230,6 +245,32 @@ TEST_F(JsonStreamParserTest, SimpleUnsignedInt) { } } +TEST_F(JsonStreamParserTest, OctalNumberIsInvalid) { + StringPiece str = "01234"; + for (int i = 0; i <= str.length(); ++i) { + DoErrorTest(str, i, "Octal/hex numbers are not valid JSON values."); + } + str = "-01234"; + for (int i = 0; i <= str.length(); ++i) { + DoErrorTest(str, i, "Octal/hex numbers are not valid JSON values."); + } +} + +TEST_F(JsonStreamParserTest, HexNumberIsInvalid) { + StringPiece str = "0x1234"; + for (int i = 0; i <= str.length(); ++i) { + DoErrorTest(str, i, "Octal/hex numbers are not valid JSON values."); + } + str = "-0x1234"; + for (int i = 0; i <= str.length(); ++i) { + DoErrorTest(str, i, "Octal/hex numbers are not valid JSON values."); + } + str = "12x34"; + for (int i = 0; i <= str.length(); ++i) { + DoErrorTest(str, i, "Unable to parse number."); + } +} + // - single and double quoted strings TEST_F(JsonStreamParserTest, EmptyDoubleQuotedString) { StringPiece str = "\"\""; @@ -281,18 +322,27 @@ TEST_F(JsonStreamParserTest, ObjectKeyTypes) { } } -// - array containing array, object, values (true, false, null, num, string) -TEST_F(JsonStreamParserTest, ArrayValues) { - StringPiece str = - "[true, false, null, 'a string', \"another string\", [22, -127, 45.3, " - "-1056.4, 11779497823553162765], {'key': true}]"; +// - array containing primitive values (true, false, null, num, string) +TEST_F(JsonStreamParserTest, ArrayPrimitiveValues) { + StringPiece str = "[true, false, null, 'one', \"two\"]"; for (int i = 0; i <= str.length(); ++i) { ow_.StartList("") ->RenderBool("", true) ->RenderBool("", false) ->RenderNull("") - ->RenderString("", "a string") - ->RenderString("", "another string") + ->RenderString("", "one") + ->RenderString("", "two") + ->EndList(); + DoTest(str, i); + } +} + +// - array containing array, object +TEST_F(JsonStreamParserTest, ArrayComplexValues) { + StringPiece str = + "[[22, -127, 45.3, -1056.4, 11779497823553162765], {'key': true}]"; + for (int i = 0; i <= str.length(); ++i) { + ow_.StartList("") ->StartList("") ->RenderUint64("", 22) ->RenderInt64("", -127) @@ -308,6 +358,7 @@ TEST_F(JsonStreamParserTest, ArrayValues) { } } + // - object containing array, object, value (true, false, null, num, string) TEST_F(JsonStreamParserTest, ObjectValues) { StringPiece str = @@ -351,19 +402,62 @@ TEST_F(JsonStreamParserTest, RejectNonUtf8WhenNotCoerced) { DoErrorTest("\xFF{}", 0, "Encountered non UTF-8 code points."); } -#ifndef _MSC_VER // - unicode handling in strings TEST_F(JsonStreamParserTest, UnicodeEscaping) { StringPiece str = "[\"\\u0639\\u0631\\u0628\\u0649\"]"; for (int i = 0; i <= str.length(); ++i) { - // TODO(xiaofeng): Figure out what default encoding to use for JSON strings. - // In protobuf we use UTF-8 for strings, but for JSON we probably should - // allow different encodings? - ow_.StartList("")->RenderString("", "\u0639\u0631\u0628\u0649")->EndList(); + ow_.StartList("") + ->RenderString("", "\xD8\xB9\xD8\xB1\xD8\xA8\xD9\x89") + ->EndList(); DoTest(str, i); } } -#endif + +// - unicode UTF-16 surrogate pair handling in strings +TEST_F(JsonStreamParserTest, UnicodeSurrogatePairEscaping) { + StringPiece str = + "[\"\\u0bee\\ud800\\uddf1\\uD80C\\uDDA4\\uD83d\\udC1D\\uD83C\\uDF6F\"]"; + for (int i = 0; i <= str.length(); ++i) { + ow_.StartList("") + ->RenderString("", + "\xE0\xAF\xAE\xF0\x90\x87\xB1\xF0\x93\x86\xA4\xF0" + "\x9F\x90\x9D\xF0\x9F\x8D\xAF") + ->EndList(); + DoTest(str, i); + } +} + + +TEST_F(JsonStreamParserTest, UnicodeEscapingInvalidCodePointWhenNotCoerced) { + // A low surrogate alone. + StringPiece str = "[\"\\ude36\"]"; + for (int i = 0; i <= str.length(); ++i) { + DoErrorTest(str, i, "Invalid unicode code point."); + } +} + +TEST_F(JsonStreamParserTest, UnicodeEscapingMissingLowSurrogateWhenNotCoerced) { + // A high surrogate alone. + StringPiece str = "[\"\\ud83d\"]"; + for (int i = 0; i <= str.length(); ++i) { + DoErrorTest(str, i, "Missing low surrogate."); + } + // A high surrogate with some trailing characters. + str = "[\"\\ud83d|ude36\"]"; + for (int i = 0; i <= str.length(); ++i) { + DoErrorTest(str, i, "Missing low surrogate."); + } + // A high surrogate with half a low surrogate. + str = "[\"\\ud83d\\ude--\"]"; + for (int i = 0; i <= str.length(); ++i) { + DoErrorTest(str, i, "Invalid escape sequence."); + } + // Two high surrogates. + str = "[\"\\ud83d\\ud83d\"]"; + for (int i = 0; i <= str.length(); ++i) { + DoErrorTest(str, i, "Invalid low surrogate."); + } +} // - ascii escaping (\b, \f, \n, \r, \t, \v) TEST_F(JsonStreamParserTest, AsciiEscaping) { @@ -600,34 +694,51 @@ TEST_F(JsonStreamParserTest, ExtraCharactersAfterObject) { } } -// numbers too large -TEST_F(JsonStreamParserTest, PositiveNumberTooBig) { - StringPiece str = "[18446744073709551616]"; // 2^64 +TEST_F(JsonStreamParserTest, PositiveNumberTooBigIsDouble) { + StringPiece str = "18446744073709551616"; // 2^64 for (int i = 0; i <= str.length(); ++i) { - ow_.StartList(""); - DoErrorTest(str, i, "Unable to parse number."); + ow_.RenderDouble("", 18446744073709552000.0); + DoTest(str, i); } } -TEST_F(JsonStreamParserTest, NegativeNumberTooBig) { - StringPiece str = "[-18446744073709551616]"; +TEST_F(JsonStreamParserTest, NegativeNumberTooBigIsDouble) { + StringPiece str = "-18446744073709551616"; for (int i = 0; i <= str.length(); ++i) { - ow_.StartList(""); - DoErrorTest(str, i, "Unable to parse number."); + ow_.RenderDouble("", -18446744073709551616.0); + DoTest(str, i); } } -/* -TODO(sven): Fail parsing when parsing a double that is too large. - TEST_F(JsonStreamParserTest, DoubleTooBig) { - StringPiece str = "[184464073709551232321616.45]"; + StringPiece str = "[1.89769e+308]"; for (int i = 0; i <= str.length(); ++i) { ow_.StartList(""); - DoErrorTest(str, i, "Unable to parse number"); + DoErrorTest(str, i, "Number exceeds the range of double."); + } + str = "[-1.89769e+308]"; + for (int i = 0; i <= str.length(); ++i) { + ow_.StartList(""); + DoErrorTest(str, i, "Number exceeds the range of double."); + } +} + + +// invalid bare backslash. +TEST_F(JsonStreamParserTest, UnfinishedEscape) { + StringPiece str = "\"\\"; + for (int i = 0; i <= str.length(); ++i) { + DoErrorTest(str, i, "Closing quote expected in string."); + } +} + +// invalid bare backslash u. +TEST_F(JsonStreamParserTest, UnfinishedUnicodeEscape) { + StringPiece str = "\"\\u"; + for (int i = 0; i <= str.length(); ++i) { + DoErrorTest(str, i, "Illegal hex string."); } } -*/ // invalid unicode sequence. TEST_F(JsonStreamParserTest, UnicodeEscapeCutOff) { @@ -637,6 +748,15 @@ TEST_F(JsonStreamParserTest, UnicodeEscapeCutOff) { } } +// invalid unicode sequence (valid in modern EcmaScript but not in JSON). +TEST_F(JsonStreamParserTest, BracketedUnicodeEscape) { + StringPiece str = "\"\\u{1f36f}\""; + for (int i = 0; i <= str.length(); ++i) { + DoErrorTest(str, i, "Invalid escape sequence."); + } +} + + TEST_F(JsonStreamParserTest, UnicodeEscapeInvalidCharacters) { StringPiece str = "\"\\u12$4hello"; for (int i = 0; i <= str.length(); ++i) { @@ -644,6 +764,14 @@ TEST_F(JsonStreamParserTest, UnicodeEscapeInvalidCharacters) { } } +// invalid unicode sequence in low half surrogate: g is not a hex digit. +TEST_F(JsonStreamParserTest, UnicodeEscapeLowHalfSurrogateInvalidCharacters) { + StringPiece str = "\"\\ud800\\udcfg\""; + for (int i = 0; i <= str.length(); ++i) { + DoErrorTest(str, i, "Invalid escape sequence."); + } +} + // Extra commas with an object or array. TEST_F(JsonStreamParserTest, ExtraCommaInObject) { StringPiece str = "{'k1': true,,'k2': false}"; diff --git a/src/google/protobuf/util/internal/object_writer.h b/src/google/protobuf/util/internal/object_writer.h index e695f45e..5781aa1e 100644 --- a/src/google/protobuf/util/internal/object_writer.h +++ b/src/google/protobuf/util/internal/object_writer.h @@ -101,14 +101,32 @@ class LIBPROTOBUF_EXPORT ObjectWriter { // Renders a Null value. virtual ObjectWriter* RenderNull(StringPiece name) = 0; + // Renders a DataPiece object to a ObjectWriter. static void RenderDataPieceTo(const DataPiece& data, StringPiece name, ObjectWriter* ow); + // Indicates whether this ObjectWriter has completed writing the root message, + // usually this means writing of one complete object. Subclasses must override + // this behavior appropriately. + virtual bool done() { return false; } + + void set_use_strict_base64_decoding(bool value) { + use_strict_base64_decoding_ = value; + } + + bool use_strict_base64_decoding() const { + return use_strict_base64_decoding_; + } + protected: - ObjectWriter() {} + ObjectWriter() : use_strict_base64_decoding_(true) {} private: + // If set to true, we use the stricter version of base64 decoding for byte + // fields by making sure decoded version encodes back to the original string. + bool use_strict_base64_decoding_; + // Do not add any data members to this class. GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ObjectWriter); }; diff --git a/src/google/protobuf/util/internal/proto_writer.cc b/src/google/protobuf/util/internal/proto_writer.cc index 47e0009e..b7a52db6 100644 --- a/src/google/protobuf/util/internal/proto_writer.cc +++ b/src/google/protobuf/util/internal/proto_writer.cc @@ -64,7 +64,9 @@ ProtoWriter::ProtoWriter(TypeResolver* type_resolver, typeinfo_(TypeInfo::NewTypeInfo(type_resolver)), own_typeinfo_(true), done_(false), - element_(NULL), + ignore_unknown_fields_(false), + use_lower_camel_for_enums_(false), + element_(nullptr), size_insert_(), output_(output), buffer_(), @@ -81,7 +83,9 @@ ProtoWriter::ProtoWriter(const TypeInfo* typeinfo, typeinfo_(typeinfo), own_typeinfo_(false), done_(false), - element_(NULL), + ignore_unknown_fields_(false), + use_lower_camel_for_enums_(false), + element_(nullptr), size_insert_(), output_(output), buffer_(), @@ -95,14 +99,14 @@ ProtoWriter::~ProtoWriter() { if (own_typeinfo_) { delete typeinfo_; } - if (element_ == NULL) return; + if (element_ == nullptr) return; // Cleanup explicitly in order to avoid destructor stack overflow when input // is deeply nested. // Cast to BaseElement to avoid doing additional checks (like missing fields) // during pop(). - google::protobuf::scoped_ptr<BaseElement> element( + std::unique_ptr<BaseElement> element( static_cast<BaseElement*>(element_.get())->pop<BaseElement>()); - while (element != NULL) { + while (element != nullptr) { element.reset(element->pop<BaseElement>()); } } @@ -262,8 +266,10 @@ inline Status WriteString(int field_number, const DataPiece& data, // Writes an ENUM field, including tag, to the stream. inline Status WriteEnum(int field_number, const DataPiece& data, const google::protobuf::Enum* enum_type, - CodedOutputStream* stream) { - StatusOr<int> e = data.ToEnum(enum_type); + CodedOutputStream* stream, + bool use_lower_camel_for_enums, + bool ignore_unknown_values) { + StatusOr<int> e = data.ToEnum(enum_type, use_lower_camel_for_enums, ignore_unknown_values); if (e.ok()) { WireFormatLite::WriteEnum(field_number, e.ValueOrDie(), stream); } @@ -289,14 +295,20 @@ std::set<const google::protobuf::Field*> GetRequiredFields( ProtoWriter::ProtoElement::ProtoElement(const TypeInfo* typeinfo, const google::protobuf::Type& type, ProtoWriter* enclosing) - : BaseElement(NULL), + : BaseElement(nullptr), ow_(enclosing), - parent_field_(NULL), + parent_field_(nullptr), typeinfo_(typeinfo), + proto3_(type.syntax() == google::protobuf::SYNTAX_PROTO3), type_(type), - required_fields_(GetRequiredFields(type)), size_index_(-1), - array_index_(-1) {} + array_index_(-1), + // oneof_indices_ values are 1-indexed (0 means not present). + oneof_indices_(type.oneofs_size() + 1) { + if (!proto3_) { + required_fields_ = GetRequiredFields(type_); + } +} ProtoWriter::ProtoElement::ProtoElement(ProtoWriter::ProtoElement* parent, const google::protobuf::Field* field, @@ -306,22 +318,28 @@ ProtoWriter::ProtoElement::ProtoElement(ProtoWriter::ProtoElement* parent, ow_(this->parent()->ow_), parent_field_(field), typeinfo_(this->parent()->typeinfo_), + proto3_(type.syntax() == google::protobuf::SYNTAX_PROTO3), type_(type), size_index_( !is_list && field->kind() == google::protobuf::Field_Kind_TYPE_MESSAGE ? ow_->size_insert_.size() : -1), - array_index_(is_list ? 0 : -1) { + array_index_(is_list ? 0 : -1), + // oneof_indices_ values are 1-indexed (0 means not present). + oneof_indices_(type_.oneofs_size() + 1) { if (!is_list) { if (ow_->IsRepeated(*field)) { // Update array_index_ if it is an explicit list. if (this->parent()->array_index_ >= 0) this->parent()->array_index_++; - } else { + } else if (!proto3_) { + // For required fields tracking. this->parent()->RegisterField(field); } if (field->kind() == google::protobuf::Field_Kind_TYPE_MESSAGE) { - required_fields_ = GetRequiredFields(type_); + if (!proto3_) { + required_fields_ = GetRequiredFields(type_); + } int start_pos = ow_->stream_->ByteCount(); // length of serialized message is the final buffer position minus // starting buffer position, plus length adjustments for size fields @@ -334,12 +352,14 @@ ProtoWriter::ProtoElement::ProtoElement(ProtoWriter::ProtoElement* parent, } ProtoWriter::ProtoElement* ProtoWriter::ProtoElement::pop() { - // Calls the registered error listener for any required field(s) not yet - // seen. - for (set<const google::protobuf::Field*>::iterator it = - required_fields_.begin(); - it != required_fields_.end(); ++it) { - ow_->MissingField((*it)->name()); + if (!proto3_) { + // Calls the registered error listener for any required field(s) not yet + // seen. + for (std::set<const google::protobuf::Field*>::iterator it = + required_fields_.begin(); + it != required_fields_.end(); ++it) { + ow_->MissingField((*it)->name()); + } } // Computes the total number of proto bytes used by a message, also adjusts // the size of all parent messages by the length of this size field. @@ -355,7 +375,7 @@ ProtoWriter::ProtoElement* ProtoWriter::ProtoElement::pop() { // all enclosing messages. int size = ow_->size_insert_[size_index_].size; int length = CodedOutputStream::VarintSize32(size); - for (ProtoElement* e = parent(); e != NULL; e = e->parent()) { + for (ProtoElement* e = parent(); e != nullptr; e = e->parent()) { // Only nested messages have size field, lists do not have size field. if (e->size_index_ >= 0) { ow_->size_insert_[e->size_index_].size += length; @@ -375,7 +395,7 @@ void ProtoWriter::ProtoElement::RegisterField( } string ProtoWriter::ProtoElement::ToString() const { - if (parent() == NULL) return ""; + if (parent() == nullptr) return ""; string loc = parent()->ToString(); if (!ow_->IsRepeated(*parent_field_) || parent()->parent_field_ != parent_field_) { @@ -399,11 +419,11 @@ string ProtoWriter::ProtoElement::ToString() const { } bool ProtoWriter::ProtoElement::IsOneofIndexTaken(int32 index) { - return ContainsKey(oneof_indices_, index); + return oneof_indices_[index]; } void ProtoWriter::ProtoElement::TakeOneofIndex(int32 index) { - InsertIfNotPresent(&oneof_indices_, index); + oneof_indices_[index] = true; } void ProtoWriter::InvalidName(StringPiece unknown_name, StringPiece message) { @@ -420,7 +440,7 @@ void ProtoWriter::MissingField(StringPiece missing_name) { ProtoWriter* ProtoWriter::StartObject(StringPiece name) { // Starting the root message. Create the root ProtoElement and return. - if (element_ == NULL) { + if (element_ == nullptr) { if (!name.empty()) { InvalidName(name, "Root element should not be named."); } @@ -428,9 +448,9 @@ ProtoWriter* ProtoWriter::StartObject(StringPiece name) { return this; } - const google::protobuf::Field* field = NULL; + const google::protobuf::Field* field = nullptr; field = BeginNamed(name, false); - if (field == NULL) return this; + if (field == nullptr) return this; // Check to see if this field is a oneof and that no oneof in that group has // already been set. @@ -440,16 +460,14 @@ ProtoWriter* ProtoWriter::StartObject(StringPiece name) { } const google::protobuf::Type* type = LookupType(field); - if (type == NULL) { + if (type == nullptr) { ++invalid_depth_; InvalidName(name, StrCat("Missing descriptor for field: ", field->type_url())); return this; } - WriteTag(*field); - element_.reset(new ProtoElement(element_.release(), field, *type, false)); - return this; + return StartObjectField(*field, *type); } ProtoWriter* ProtoWriter::EndObject() { @@ -458,14 +476,14 @@ ProtoWriter* ProtoWriter::EndObject() { return this; } - if (element_ != NULL) { + if (element_ != nullptr) { element_.reset(element_->pop()); } // If ending the root element, // then serialize the full message with calculated sizes. - if (element_ == NULL) { + if (element_ == nullptr) { WriteRootMessage(); } return this; @@ -473,7 +491,7 @@ ProtoWriter* ProtoWriter::EndObject() { ProtoWriter* ProtoWriter::StartList(StringPiece name) { const google::protobuf::Field* field = BeginNamed(name, true); - if (field == NULL) return this; + if (field == nullptr) return this; if (!ValidOneof(*field, name)) { ++invalid_depth_; @@ -481,21 +499,20 @@ ProtoWriter* ProtoWriter::StartList(StringPiece name) { } const google::protobuf::Type* type = LookupType(field); - if (type == NULL) { + if (type == nullptr) { ++invalid_depth_; InvalidName(name, StrCat("Missing descriptor for field: ", field->type_url())); return this; } - element_.reset(new ProtoElement(element_.release(), field, *type, true)); - return this; + return StartListField(*field, *type); } ProtoWriter* ProtoWriter::EndList() { if (invalid_depth_ > 0) { --invalid_depth_; - } else if (element_ != NULL) { + } else if (element_ != nullptr) { element_.reset(element_->pop()); } return this; @@ -507,96 +524,150 @@ ProtoWriter* ProtoWriter::RenderDataPiece(StringPiece name, if (invalid_depth_ > 0) return this; const google::protobuf::Field* field = Lookup(name); - if (field == NULL) return this; + if (field == nullptr) return this; if (!ValidOneof(*field, name)) return this; const google::protobuf::Type* type = LookupType(field); - if (type == NULL) { + if (type == nullptr) { InvalidName(name, StrCat("Missing descriptor for field: ", field->type_url())); return this; } + return RenderPrimitiveField(*field, *type, data); +} + +bool ProtoWriter::ValidOneof(const google::protobuf::Field& field, + StringPiece unnormalized_name) { + if (element_ == nullptr) return true; + + if (field.oneof_index() > 0) { + if (element_->IsOneofIndexTaken(field.oneof_index())) { + InvalidValue( + "oneof", + StrCat("oneof field '", + element_->type().oneofs(field.oneof_index() - 1), + "' is already set. Cannot set '", unnormalized_name, "'")); + return false; + } + element_->TakeOneofIndex(field.oneof_index()); + } + return true; +} + +bool ProtoWriter::IsRepeated(const google::protobuf::Field& field) { + return field.cardinality() == + google::protobuf::Field_Cardinality_CARDINALITY_REPEATED; +} + +ProtoWriter* ProtoWriter::StartObjectField(const google::protobuf::Field& field, + const google::protobuf::Type& type) { + WriteTag(field); + element_.reset(new ProtoElement(element_.release(), &field, type, false)); + return this; +} + +ProtoWriter* ProtoWriter::StartListField(const google::protobuf::Field& field, + const google::protobuf::Type& type) { + element_.reset(new ProtoElement(element_.release(), &field, type, true)); + return this; +} + +ProtoWriter* ProtoWriter::RenderPrimitiveField( + const google::protobuf::Field& field, const google::protobuf::Type& type, + const DataPiece& data) { + Status status; + // Pushing a ProtoElement and then pop it off at the end for 2 purposes: // error location reporting and required field accounting. - element_.reset(new ProtoElement(element_.release(), field, *type, false)); - - if (field->kind() == google::protobuf::Field_Kind_TYPE_UNKNOWN || - field->kind() == google::protobuf::Field_Kind_TYPE_MESSAGE) { - InvalidValue(field->type_url().empty() - ? google::protobuf::Field_Kind_Name(field->kind()) - : field->type_url(), + // + // For proto3, since there is no required field tracking, we only need to push + // ProtoElement for error cases. + if (!element_->proto3()) { + element_.reset(new ProtoElement(element_.release(), &field, type, false)); + } + + if (field.kind() == google::protobuf::Field_Kind_TYPE_UNKNOWN || + field.kind() == google::protobuf::Field_Kind_TYPE_MESSAGE) { + // Push a ProtoElement for location reporting purposes. + if (element_->proto3()) { + element_.reset(new ProtoElement(element_.release(), &field, type, false)); + } + InvalidValue(field.type_url().empty() + ? google::protobuf::Field_Kind_Name(field.kind()) + : field.type_url(), data.ValueAsStringOrDefault("")); element_.reset(element()->pop()); return this; } - switch (field->kind()) { + switch (field.kind()) { case google::protobuf::Field_Kind_TYPE_INT32: { - status = WriteInt32(field->number(), data, stream_.get()); + status = WriteInt32(field.number(), data, stream_.get()); break; } case google::protobuf::Field_Kind_TYPE_SFIXED32: { - status = WriteSFixed32(field->number(), data, stream_.get()); + status = WriteSFixed32(field.number(), data, stream_.get()); break; } case google::protobuf::Field_Kind_TYPE_SINT32: { - status = WriteSInt32(field->number(), data, stream_.get()); + status = WriteSInt32(field.number(), data, stream_.get()); break; } case google::protobuf::Field_Kind_TYPE_FIXED32: { - status = WriteFixed32(field->number(), data, stream_.get()); + status = WriteFixed32(field.number(), data, stream_.get()); break; } case google::protobuf::Field_Kind_TYPE_UINT32: { - status = WriteUInt32(field->number(), data, stream_.get()); + status = WriteUInt32(field.number(), data, stream_.get()); break; } case google::protobuf::Field_Kind_TYPE_INT64: { - status = WriteInt64(field->number(), data, stream_.get()); + status = WriteInt64(field.number(), data, stream_.get()); break; } case google::protobuf::Field_Kind_TYPE_SFIXED64: { - status = WriteSFixed64(field->number(), data, stream_.get()); + status = WriteSFixed64(field.number(), data, stream_.get()); break; } case google::protobuf::Field_Kind_TYPE_SINT64: { - status = WriteSInt64(field->number(), data, stream_.get()); + status = WriteSInt64(field.number(), data, stream_.get()); break; } case google::protobuf::Field_Kind_TYPE_FIXED64: { - status = WriteFixed64(field->number(), data, stream_.get()); + status = WriteFixed64(field.number(), data, stream_.get()); break; } case google::protobuf::Field_Kind_TYPE_UINT64: { - status = WriteUInt64(field->number(), data, stream_.get()); + status = WriteUInt64(field.number(), data, stream_.get()); break; } case google::protobuf::Field_Kind_TYPE_DOUBLE: { - status = WriteDouble(field->number(), data, stream_.get()); + status = WriteDouble(field.number(), data, stream_.get()); break; } case google::protobuf::Field_Kind_TYPE_FLOAT: { - status = WriteFloat(field->number(), data, stream_.get()); + status = WriteFloat(field.number(), data, stream_.get()); break; } case google::protobuf::Field_Kind_TYPE_BOOL: { - status = WriteBool(field->number(), data, stream_.get()); + status = WriteBool(field.number(), data, stream_.get()); break; } case google::protobuf::Field_Kind_TYPE_BYTES: { - status = WriteBytes(field->number(), data, stream_.get()); + status = WriteBytes(field.number(), data, stream_.get()); break; } case google::protobuf::Field_Kind_TYPE_STRING: { - status = WriteString(field->number(), data, stream_.get()); + status = WriteString(field.number(), data, stream_.get()); break; } case google::protobuf::Field_Kind_TYPE_ENUM: { - status = WriteEnum(field->number(), data, - typeinfo_->GetEnumByTypeUrl(field->type_url()), - stream_.get()); + status = WriteEnum(field.number(), data, + typeinfo_->GetEnumByTypeUrl(field.type_url()), + stream_.get(), use_lower_camel_for_enums_, + ignore_unknown_fields_); break; } default: // TYPE_GROUP or TYPE_MESSAGE @@ -604,53 +675,37 @@ ProtoWriter* ProtoWriter::RenderDataPiece(StringPiece name, } if (!status.ok()) { - InvalidValue(google::protobuf::Field_Kind_Name(field->kind()), + // Push a ProtoElement for location reporting purposes. + if (element_->proto3()) { + element_.reset(new ProtoElement(element_.release(), &field, type, false)); + } + InvalidValue(google::protobuf::Field_Kind_Name(field.kind()), status.error_message()); + element_.reset(element()->pop()); + return this; } - element_.reset(element()->pop()); - return this; -} - -bool ProtoWriter::ValidOneof(const google::protobuf::Field& field, - StringPiece unnormalized_name) { - if (element_ == NULL) return true; - - if (field.oneof_index() > 0) { - if (element_->IsOneofIndexTaken(field.oneof_index())) { - InvalidValue( - "oneof", - StrCat("oneof field '", - element_->type().oneofs(field.oneof_index() - 1), - "' is already set. Cannot set '", unnormalized_name, "'")); - return false; - } - element_->TakeOneofIndex(field.oneof_index()); - } - return true; -} + if (!element_->proto3()) element_.reset(element()->pop()); -bool ProtoWriter::IsRepeated(const google::protobuf::Field& field) { - return field.cardinality() == - google::protobuf::Field_Cardinality_CARDINALITY_REPEATED; + return this; } const google::protobuf::Field* ProtoWriter::BeginNamed(StringPiece name, bool is_list) { if (invalid_depth_ > 0) { ++invalid_depth_; - return NULL; + return nullptr; } const google::protobuf::Field* field = Lookup(name); - if (field == NULL) { + if (field == nullptr) { ++invalid_depth_; // InvalidName() already called in Lookup(). - return NULL; + return nullptr; } if (is_list && !IsRepeated(*field)) { ++invalid_depth_; InvalidName(name, "Proto field is not repeating, cannot start list."); - return NULL; + return nullptr; } return field; } @@ -658,23 +713,25 @@ const google::protobuf::Field* ProtoWriter::BeginNamed(StringPiece name, const google::protobuf::Field* ProtoWriter::Lookup( StringPiece unnormalized_name) { ProtoElement* e = element(); - if (e == NULL) { + if (e == nullptr) { InvalidName(unnormalized_name, "Root element must be a message."); - return NULL; + return nullptr; } if (unnormalized_name.empty()) { // Objects in repeated field inherit the same field descriptor. - if (e->parent_field() == NULL) { + if (e->parent_field() == nullptr) { InvalidName(unnormalized_name, "Proto fields must have a name."); } else if (!IsRepeated(*e->parent_field())) { InvalidName(unnormalized_name, "Proto fields must have a name."); - return NULL; + return nullptr; } return e->parent_field(); } const google::protobuf::Field* field = typeinfo_->FindField(&e->type(), unnormalized_name); - if (field == NULL) InvalidName(unnormalized_name, "Cannot find field."); + if (field == nullptr && !ignore_unknown_fields_) { + InvalidName(unnormalized_name, "Cannot find field."); + } return field; } @@ -691,7 +748,7 @@ void ProtoWriter::WriteRootMessage() { int curr_pos = 0; // Calls the destructor of CodedOutputStream to remove any uninitialized // memory from the Cord before we read it. - stream_.reset(NULL); + stream_.reset(nullptr); const void* data; int length; google::protobuf::io::ArrayInputStream input_stream(buffer_.data(), buffer_.size()); diff --git a/src/google/protobuf/util/internal/proto_writer.h b/src/google/protobuf/util/internal/proto_writer.h index e631e56f..f2b4f42f 100644 --- a/src/google/protobuf/util/internal/proto_writer.h +++ b/src/google/protobuf/util/internal/proto_writer.h @@ -32,8 +32,8 @@ #define GOOGLE_PROTOBUF_UTIL_CONVERTER_PROTO_WRITER_H__ #include <deque> -#include <google/protobuf/stubs/hash.h> #include <string> +#include <vector> #include <google/protobuf/stubs/common.h> #include <google/protobuf/io/coded_stream.h> @@ -45,6 +45,7 @@ #include <google/protobuf/util/internal/structured_objectwriter.h> #include <google/protobuf/util/type_resolver.h> #include <google/protobuf/stubs/bytestream.h> +#include <google/protobuf/stubs/hash.h> namespace google { namespace protobuf { @@ -80,41 +81,44 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter { virtual ~ProtoWriter(); // ObjectWriter methods. - virtual ProtoWriter* StartObject(StringPiece name); - virtual ProtoWriter* EndObject(); - virtual ProtoWriter* StartList(StringPiece name); - virtual ProtoWriter* EndList(); - virtual ProtoWriter* RenderBool(StringPiece name, bool value) { + ProtoWriter* StartObject(StringPiece name) override; + ProtoWriter* EndObject() override; + ProtoWriter* StartList(StringPiece name) override; + ProtoWriter* EndList() override; + ProtoWriter* RenderBool(StringPiece name, bool value) override { return RenderDataPiece(name, DataPiece(value)); } - virtual ProtoWriter* RenderInt32(StringPiece name, int32 value) { + ProtoWriter* RenderInt32(StringPiece name, int32 value) override { return RenderDataPiece(name, DataPiece(value)); } - virtual ProtoWriter* RenderUint32(StringPiece name, uint32 value) { + ProtoWriter* RenderUint32(StringPiece name, uint32 value) override { return RenderDataPiece(name, DataPiece(value)); } - virtual ProtoWriter* RenderInt64(StringPiece name, int64 value) { + ProtoWriter* RenderInt64(StringPiece name, int64 value) override { return RenderDataPiece(name, DataPiece(value)); } - virtual ProtoWriter* RenderUint64(StringPiece name, uint64 value) { + ProtoWriter* RenderUint64(StringPiece name, uint64 value) override { return RenderDataPiece(name, DataPiece(value)); } - virtual ProtoWriter* RenderDouble(StringPiece name, double value) { + ProtoWriter* RenderDouble(StringPiece name, double value) override { return RenderDataPiece(name, DataPiece(value)); } - virtual ProtoWriter* RenderFloat(StringPiece name, float value) { + ProtoWriter* RenderFloat(StringPiece name, float value) override { return RenderDataPiece(name, DataPiece(value)); } - virtual ProtoWriter* RenderString(StringPiece name, StringPiece value) { - return RenderDataPiece(name, DataPiece(value)); + ProtoWriter* RenderString(StringPiece name, StringPiece value) override { + return RenderDataPiece(name, + DataPiece(value, use_strict_base64_decoding())); } - virtual ProtoWriter* RenderBytes(StringPiece name, StringPiece value) { - return RenderDataPiece(name, DataPiece(value, false)); + ProtoWriter* RenderBytes(StringPiece name, StringPiece value) override { + return RenderDataPiece( + name, DataPiece(value, false, use_strict_base64_decoding())); } - virtual ProtoWriter* RenderNull(StringPiece name) { + ProtoWriter* RenderNull(StringPiece name) override { return RenderDataPiece(name, DataPiece::NullData()); } + // Renders a DataPiece 'value' into a field whose wire type is determined // from the given field 'name'. virtual ProtoWriter* RenderDataPiece(StringPiece name, @@ -122,11 +126,11 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter { // Returns the location tracker to use for tracking locations for errors. const LocationTrackerInterface& location() { - return element_ != NULL ? *element_ : *tracker_; + return element_ != nullptr ? *element_ : *tracker_; } // When true, we finished writing to output a complete message. - bool done() const { return done_; } + bool done() override { return done_; } // Returns the proto stream object. google::protobuf::io::CodedOutputStream* stream() { return stream_.get(); } @@ -140,6 +144,14 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter { const TypeInfo* typeinfo() { return typeinfo_; } + void set_ignore_unknown_fields(bool ignore_unknown_fields) { + ignore_unknown_fields_ = ignore_unknown_fields; + } + + void set_use_lower_camel_for_enums(bool use_lower_camel_for_enums) { + use_lower_camel_for_enums_ = use_lower_camel_for_enums; + } + protected: class LIBPROTOBUF_EXPORT ProtoElement : public BaseElement, public LocationTrackerInterface { public: @@ -161,7 +173,7 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter { ProtoElement* pop(); // Accessors - // parent_field() may be NULL if we are at root. + // parent_field() may be nullptr if we are at root. const google::protobuf::Field* parent_field() const { return parent_field_; } @@ -177,25 +189,30 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter { return static_cast<ProtoElement*>(BaseElement::parent()); } - // Returns true if the index is already taken by a preceeding oneof input. + // Returns true if the index is already taken by a preceding oneof input. bool IsOneofIndexTaken(int32 index); // Marks the oneof 'index' as taken. Future inputs to this oneof will // generate an error. void TakeOneofIndex(int32 index); + bool proto3() { return proto3_; } + private: // Used for access to variables of the enclosing instance of // ProtoWriter. ProtoWriter* ow_; // Describes the element as a field in the parent message. - // parent_field_ is NULL if and only if this element is the root element. + // parent_field_ is nullptr if and only if this element is the root element. const google::protobuf::Field* parent_field_; // TypeInfo to lookup types. const TypeInfo* typeinfo_; + // Whether the type_ is proto3 or not. + bool proto3_; + // Additional variables if this element is a message: // (Root element is always a message). // type_ : the type of this element. @@ -211,7 +228,7 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter { // Set of oneof indices already seen for the type_. Used to validate // incoming messages so no more than one oneof is set. - hash_set<int32> oneof_indices_; + std::vector<bool> oneof_indices_; GOOGLE_DISALLOW_IMPLICIT_CONSTRUCTORS(ProtoElement); }; @@ -225,7 +242,7 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter { ProtoWriter(const TypeInfo* typeinfo, const google::protobuf::Type& type, strings::ByteSink* output, ErrorListener* listener); - virtual ProtoElement* element() { return element_.get(); } + ProtoElement* element() override { return element_.get(); } // Helper methods for calling ErrorListener. See error_listener.h. void InvalidName(StringPiece unknown_name, StringPiece message); @@ -238,10 +255,11 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter { // Lookup the field in the current element. Looks in the base descriptor // and in any extension. This will report an error if the field cannot be - // found or if multiple matching extensions are found. + // found when ignore_unknown_names_ is false or if multiple matching + // extensions are found. const google::protobuf::Field* Lookup(StringPiece name); - // Lookup the field type in the type descriptor. Returns NULL if the type + // Lookup the field type in the type descriptor. Returns nullptr if the type // is not known. const google::protobuf::Type* LookupType( const google::protobuf::Field* field); @@ -266,6 +284,19 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter { // Returns true if the field is repeated. bool IsRepeated(const google::protobuf::Field& field); + // Starts an object given the field and the enclosing type. + ProtoWriter* StartObjectField(const google::protobuf::Field& field, + const google::protobuf::Type& type); + + // Starts a list given the field and the enclosing type. + ProtoWriter* StartListField(const google::protobuf::Field& field, + const google::protobuf::Type& type); + + // Renders a primitve field given the field and the enclosing type. + ProtoWriter* RenderPrimitiveField(const google::protobuf::Field& field, + const google::protobuf::Type& type, + const DataPiece& value); + private: // Variables for describing the structure of the input tree: // master_type_: descriptor for the whole protobuf message. @@ -278,12 +309,19 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter { // Indicates whether we finished writing root message completely. bool done_; + // If true, don't report unknown field names and enum values to the listener. + bool ignore_unknown_fields_; + + // If true, check if enum name in camel case or without underscore matches the + // field name. + bool use_lower_camel_for_enums_; + // Variable for internal state processing: // element_ : the current element. // size_insert_: sizes of nested messages. // pos - position to insert the size field. // size - size value to be inserted. - google::protobuf::scoped_ptr<ProtoElement> element_; + std::unique_ptr<ProtoElement> element_; std::deque<SizeInfo> size_insert_; // Variables for output generation: @@ -294,7 +332,7 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter { strings::ByteSink* output_; string buffer_; google::protobuf::io::StringOutputStream adapter_; - google::protobuf::scoped_ptr<google::protobuf::io::CodedOutputStream> stream_; + std::unique_ptr<google::protobuf::io::CodedOutputStream> stream_; // Variables for error tracking and reporting: // listener_ : a place to report any errors found. @@ -302,7 +340,7 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter { // tracker_ : the root location tracker interface. ErrorListener* listener_; int invalid_depth_; - google::protobuf::scoped_ptr<LocationTrackerInterface> tracker_; + std::unique_ptr<LocationTrackerInterface> tracker_; GOOGLE_DISALLOW_IMPLICIT_CONSTRUCTORS(ProtoWriter); }; diff --git a/src/google/protobuf/util/internal/protostream_objectsource.cc b/src/google/protobuf/util/internal/protostream_objectsource.cc index 034d616f..56e6db12 100644 --- a/src/google/protobuf/util/internal/protostream_objectsource.cc +++ b/src/google/protobuf/util/internal/protostream_objectsource.cc @@ -70,19 +70,45 @@ using util::Status; using util::StatusOr; namespace { -// Finds a field with the given number. NULL if none found. + +static int kDefaultMaxRecursionDepth = 64; + +// Finds a field with the given number. nullptr if none found. const google::protobuf::Field* FindFieldByNumber( const google::protobuf::Type& type, int number); // Returns true if the field is packable. bool IsPackable(const google::protobuf::Field& field); -// Finds an enum value with the given number. NULL if none found. +// Finds an enum value with the given number. nullptr if none found. const google::protobuf::EnumValue* FindEnumValueByNumber( const google::protobuf::Enum& tech_enum, int number); // Utility function to format nanos. -const string FormatNanos(uint32 nanos); +const string FormatNanos(uint32 nanos, bool with_trailing_zeros); + +StatusOr<string> MapKeyDefaultValueAsString( + const google::protobuf::Field& field) { + switch (field.kind()) { + case google::protobuf::Field_Kind_TYPE_BOOL: + return string("false"); + case google::protobuf::Field_Kind_TYPE_INT32: + case google::protobuf::Field_Kind_TYPE_INT64: + case google::protobuf::Field_Kind_TYPE_UINT32: + case google::protobuf::Field_Kind_TYPE_UINT64: + case google::protobuf::Field_Kind_TYPE_SINT32: + case google::protobuf::Field_Kind_TYPE_SINT64: + case google::protobuf::Field_Kind_TYPE_SFIXED32: + case google::protobuf::Field_Kind_TYPE_SFIXED64: + case google::protobuf::Field_Kind_TYPE_FIXED32: + case google::protobuf::Field_Kind_TYPE_FIXED64: + return string("0"); + case google::protobuf::Field_Kind_TYPE_STRING: + return string(); + default: + return Status(util::error::INTERNAL, "Invalid map key type."); + } +} } // namespace @@ -92,15 +118,34 @@ ProtoStreamObjectSource::ProtoStreamObjectSource( : stream_(stream), typeinfo_(TypeInfo::NewTypeInfo(type_resolver)), own_typeinfo_(true), - type_(type) { - GOOGLE_LOG_IF(DFATAL, stream == NULL) << "Input stream is NULL."; + type_(type), + use_lower_camel_for_enums_(false), + use_ints_for_enums_(false), + preserve_proto_field_names_(false), + recursion_depth_(0), + max_recursion_depth_(kDefaultMaxRecursionDepth), + render_unknown_fields_(false), + render_unknown_enum_values_(true), + add_trailing_zeros_for_timestamp_and_duration_(false) { + GOOGLE_LOG_IF(DFATAL, stream == nullptr) << "Input stream is nullptr."; } ProtoStreamObjectSource::ProtoStreamObjectSource( google::protobuf::io::CodedInputStream* stream, const TypeInfo* typeinfo, const google::protobuf::Type& type) - : stream_(stream), typeinfo_(typeinfo), own_typeinfo_(false), type_(type) { - GOOGLE_LOG_IF(DFATAL, stream == NULL) << "Input stream is NULL."; + : stream_(stream), + typeinfo_(typeinfo), + own_typeinfo_(false), + type_(type), + use_lower_camel_for_enums_(false), + use_ints_for_enums_(false), + preserve_proto_field_names_(false), + recursion_depth_(0), + max_recursion_depth_(kDefaultMaxRecursionDepth), + render_unknown_fields_(false), + render_unknown_enum_values_(true), + add_trailing_zeros_for_timestamp_and_duration_(false) { + GOOGLE_LOG_IF(DFATAL, stream == nullptr) << "Input stream is nullptr."; } ProtoStreamObjectSource::~ProtoStreamObjectSource() { @@ -120,7 +165,7 @@ const google::protobuf::Field* ProtoStreamObjectSource::FindAndVerifyField( const google::protobuf::Field* field = FindFieldByNumber(type, tag >> 3); // Verify if the field corresponds to the wire type in tag. // If there is any discrepancy, mark the field as not found. - if (field != NULL) { + if (field != nullptr) { WireFormatLite::WireType expected_type = WireFormatLite::WireTypeForFieldType( static_cast<WireFormatLite::FieldType>(field->kind())); @@ -128,7 +173,7 @@ const google::protobuf::Field* ProtoStreamObjectSource::FindAndVerifyField( if (actual_type != expected_type && (!IsPackable(*field) || actual_type != WireFormatLite::WIRETYPE_LENGTH_DELIMITED)) { - field = NULL; + field = nullptr; } } return field; @@ -141,14 +186,15 @@ Status ProtoStreamObjectSource::WriteMessage(const google::protobuf::Type& type, ObjectWriter* ow) const { const TypeRenderer* type_renderer = FindTypeRenderer(type.name()); - if (type_renderer != NULL) { + if (type_renderer != nullptr) { return (*type_renderer)(this, type, name, ow); } - const google::protobuf::Field* field = NULL; + const google::protobuf::Field* field = nullptr; string field_name; // last_tag set to dummy value that is different from tag. uint32 tag = stream_->ReadTag(), last_tag = tag + 1; + google::protobuf::UnknownFieldSet unknown_fields; if (include_start_and_end) { ow->StartObject(name); @@ -157,14 +203,19 @@ Status ProtoStreamObjectSource::WriteMessage(const google::protobuf::Type& type, if (tag != last_tag) { // Update field only if tag is changed. last_tag = tag; field = FindAndVerifyField(type, tag); - if (field != NULL) { - field_name = field->json_name(); + if (field != nullptr) { + if (preserve_proto_field_names_) { + field_name = field->name(); + } else { + field_name = field->json_name(); + } } } - if (field == NULL) { + if (field == nullptr) { // If we didn't find a field, skip this unknown tag. // TODO(wpoon): Check return boolean value. - WireFormat::SkipField(stream_, tag, NULL); + WireFormat::SkipField(stream_, tag, + render_unknown_fields_ ? &unknown_fields : nullptr); tag = stream_->ReadTag(); continue; } @@ -186,10 +237,12 @@ Status ProtoStreamObjectSource::WriteMessage(const google::protobuf::Type& type, tag = stream_->ReadTag(); } } + + if (include_start_and_end) { ow->EndObject(); } - return Status::OK; + return util::Status(); } StatusOr<uint32> ProtoStreamObjectSource::RenderList( @@ -229,8 +282,8 @@ StatusOr<uint32> ProtoStreamObjectSource::RenderMap( for (uint32 tag = stream_->ReadTag(); tag != 0; tag = stream_->ReadTag()) { const google::protobuf::Field* field = FindAndVerifyField(*field_type, tag); - if (field == NULL) { - WireFormat::SkipField(stream_, tag, NULL); + if (field == nullptr) { + WireFormat::SkipField(stream_, tag, nullptr); continue; } // Map field numbers are key = 1 and value = 2 @@ -238,9 +291,21 @@ StatusOr<uint32> ProtoStreamObjectSource::RenderMap( map_key = ReadFieldValueAsString(*field); } else if (field->number() == 2) { if (map_key.empty()) { - return Status(util::error::INTERNAL, "Map key must be non-empty"); + // An absent map key is treated as the default. + const google::protobuf::Field* key_field = + FindFieldByNumber(*field_type, 1); + if (key_field == nullptr) { + // The Type info for this map entry is incorrect. It should always + // have a field named "key" and with field number 1. + return Status(util::error::INTERNAL, "Invalid map entry."); + } + ASSIGN_OR_RETURN(map_key, MapKeyDefaultValueAsString(*key_field)); } RETURN_IF_ERROR(RenderField(field, map_key, ow)); + } else { + // The Type info for this map entry is incorrect. It should contain + // exactly two fields with field number 1 and 2. + return Status(util::error::INTERNAL, "Invalid map entry."); } } stream_->PopLimit(old_limit); @@ -257,16 +322,16 @@ Status ProtoStreamObjectSource::RenderPacked( RETURN_IF_ERROR(RenderField(field, StringPiece(), ow)); } stream_->PopLimit(old_limit); - return Status::OK; + return util::Status(); } Status ProtoStreamObjectSource::RenderTimestamp( const ProtoStreamObjectSource* os, const google::protobuf::Type& type, StringPiece field_name, ObjectWriter* ow) { - pair<int64, int32> p = os->ReadSecondsAndNanos(type); + std::pair<int64, int32> p = os->ReadSecondsAndNanos(type); int64 seconds = p.first; int32 nanos = p.second; - if (seconds > kMaxSeconds || seconds < kMinSeconds) { + if (seconds > kTimestampMaxSeconds || seconds < kTimestampMinSeconds) { return Status( util::error::INTERNAL, StrCat("Timestamp seconds exceeds limit for field: ", field_name)); @@ -281,16 +346,16 @@ Status ProtoStreamObjectSource::RenderTimestamp( ow->RenderString(field_name, ::google::protobuf::internal::FormatTime(seconds, nanos)); - return Status::OK; + return util::Status(); } Status ProtoStreamObjectSource::RenderDuration( const ProtoStreamObjectSource* os, const google::protobuf::Type& type, StringPiece field_name, ObjectWriter* ow) { - pair<int64, int32> p = os->ReadSecondsAndNanos(type); + std::pair<int64, int32> p = os->ReadSecondsAndNanos(type); int64 seconds = p.first; int32 nanos = p.second; - if (seconds > kMaxSeconds || seconds < kMinSeconds) { + if (seconds > kDurationMaxSeconds || seconds < kDurationMinSeconds) { return Status( util::error::INTERNAL, StrCat("Duration seconds exceeds limit for field: ", field_name)); @@ -317,10 +382,12 @@ Status ProtoStreamObjectSource::RenderDuration( sign = "-"; nanos = -nanos; } - string formatted_duration = StringPrintf("%s%lld%ss", sign.c_str(), seconds, - FormatNanos(nanos).c_str()); + string formatted_duration = StringPrintf( + "%s%lld%ss", sign.c_str(), seconds, + FormatNanos(nanos, os->add_trailing_zeros_for_timestamp_and_duration_) + .c_str()); ow->RenderString(field_name, formatted_duration); - return Status::OK; + return util::Status(); } Status ProtoStreamObjectSource::RenderDouble(const ProtoStreamObjectSource* os, @@ -334,7 +401,7 @@ Status ProtoStreamObjectSource::RenderDouble(const ProtoStreamObjectSource* os, os->stream_->ReadTag(); } ow->RenderDouble(field_name, bit_cast<double>(buffer64)); - return Status::OK; + return util::Status(); } Status ProtoStreamObjectSource::RenderFloat(const ProtoStreamObjectSource* os, @@ -348,7 +415,7 @@ Status ProtoStreamObjectSource::RenderFloat(const ProtoStreamObjectSource* os, os->stream_->ReadTag(); } ow->RenderFloat(field_name, bit_cast<float>(buffer32)); - return Status::OK; + return util::Status(); } Status ProtoStreamObjectSource::RenderInt64(const ProtoStreamObjectSource* os, @@ -362,7 +429,7 @@ Status ProtoStreamObjectSource::RenderInt64(const ProtoStreamObjectSource* os, os->stream_->ReadTag(); } ow->RenderInt64(field_name, bit_cast<int64>(buffer64)); - return Status::OK; + return util::Status(); } Status ProtoStreamObjectSource::RenderUInt64(const ProtoStreamObjectSource* os, @@ -376,7 +443,7 @@ Status ProtoStreamObjectSource::RenderUInt64(const ProtoStreamObjectSource* os, os->stream_->ReadTag(); } ow->RenderUint64(field_name, bit_cast<uint64>(buffer64)); - return Status::OK; + return util::Status(); } Status ProtoStreamObjectSource::RenderInt32(const ProtoStreamObjectSource* os, @@ -390,7 +457,7 @@ Status ProtoStreamObjectSource::RenderInt32(const ProtoStreamObjectSource* os, os->stream_->ReadTag(); } ow->RenderInt32(field_name, bit_cast<int32>(buffer32)); - return Status::OK; + return util::Status(); } Status ProtoStreamObjectSource::RenderUInt32(const ProtoStreamObjectSource* os, @@ -404,7 +471,7 @@ Status ProtoStreamObjectSource::RenderUInt32(const ProtoStreamObjectSource* os, os->stream_->ReadTag(); } ow->RenderUint32(field_name, bit_cast<uint32>(buffer32)); - return Status::OK; + return util::Status(); } Status ProtoStreamObjectSource::RenderBool(const ProtoStreamObjectSource* os, @@ -419,7 +486,7 @@ Status ProtoStreamObjectSource::RenderBool(const ProtoStreamObjectSource* os, os->stream_->ReadTag(); } ow->RenderBool(field_name, buffer64 != 0); - return Status::OK; + return util::Status(); } Status ProtoStreamObjectSource::RenderString(const ProtoStreamObjectSource* os, @@ -435,7 +502,7 @@ Status ProtoStreamObjectSource::RenderString(const ProtoStreamObjectSource* os, os->stream_->ReadTag(); } ow->RenderString(field_name, str); - return Status::OK; + return util::Status(); } Status ProtoStreamObjectSource::RenderBytes(const ProtoStreamObjectSource* os, @@ -451,14 +518,14 @@ Status ProtoStreamObjectSource::RenderBytes(const ProtoStreamObjectSource* os, os->stream_->ReadTag(); } ow->RenderBytes(field_name, str); - return Status::OK; + return util::Status(); } Status ProtoStreamObjectSource::RenderStruct(const ProtoStreamObjectSource* os, const google::protobuf::Type& type, StringPiece field_name, ObjectWriter* ow) { - const google::protobuf::Field* field = NULL; + const google::protobuf::Field* field = nullptr; uint32 tag = os->stream_->ReadTag(); ow->StartObject(field_name); while (tag != 0) { @@ -470,23 +537,23 @@ Status ProtoStreamObjectSource::RenderStruct(const ProtoStreamObjectSource* os, } } ow->EndObject(); - return Status::OK; + return util::Status(); } Status ProtoStreamObjectSource::RenderStructValue( const ProtoStreamObjectSource* os, const google::protobuf::Type& type, StringPiece field_name, ObjectWriter* ow) { - const google::protobuf::Field* field = NULL; + const google::protobuf::Field* field = nullptr; for (uint32 tag = os->stream_->ReadTag(); tag != 0; tag = os->stream_->ReadTag()) { field = os->FindAndVerifyField(type, tag); - if (field == NULL) { - WireFormat::SkipField(os->stream_, tag, NULL); + if (field == nullptr) { + WireFormat::SkipField(os->stream_, tag, nullptr); continue; } RETURN_IF_ERROR(os->RenderField(field, field_name, ow)); } - return Status::OK; + return util::Status(); } // TODO(skarvaje): Avoid code duplication of for loops and SkipField logic. @@ -499,19 +566,19 @@ Status ProtoStreamObjectSource::RenderStructListValue( if (tag == 0) { ow->StartList(field_name); ow->EndList(); - return Status::OK; + return util::Status(); } while (tag != 0) { const google::protobuf::Field* field = os->FindAndVerifyField(type, tag); - if (field == NULL) { - WireFormat::SkipField(os->stream_, tag, NULL); + if (field == nullptr) { + WireFormat::SkipField(os->stream_, tag, nullptr); tag = os->stream_->ReadTag(); continue; } ASSIGN_OR_RETURN(tag, os->RenderList(field, field_name, tag, ow)); } - return Status::OK; + return util::Status(); } Status ProtoStreamObjectSource::RenderAny(const ProtoStreamObjectSource* os, @@ -526,8 +593,8 @@ Status ProtoStreamObjectSource::RenderAny(const ProtoStreamObjectSource* os, // First read out the type_url and value from the proto stream for (tag = os->stream_->ReadTag(); tag != 0; tag = os->stream_->ReadTag()) { const google::protobuf::Field* field = os->FindAndVerifyField(type, tag); - if (field == NULL) { - WireFormat::SkipField(os->stream_, tag, NULL); + if (field == nullptr) { + WireFormat::SkipField(os->stream_, tag, nullptr); continue; } // 'type_url' has field number of 1 and 'value' has field number 2 @@ -553,7 +620,7 @@ Status ProtoStreamObjectSource::RenderAny(const ProtoStreamObjectSource* os, ow->RenderString("@type", type_url); } ow->EndObject(); - return util::Status::OK; + return util::Status(); } // If there is a value but no type, we cannot render it, so report an error. @@ -600,7 +667,7 @@ Status ProtoStreamObjectSource::RenderFieldMask( tag = os->stream_->ReadTag()) { if (paths_field_tag == 0) { const google::protobuf::Field* field = os->FindAndVerifyField(type, tag); - if (field != NULL && field->number() == 1 && + if (field != nullptr && field->number() == 1 && field->name() == "paths") { paths_field_tag = tag; } @@ -618,7 +685,7 @@ Status ProtoStreamObjectSource::RenderFieldMask( combined.append(ConvertFieldMaskPath(str, &ToCamelCase)); } ow->RenderString(field_name, combined); - return Status::OK; + return util::Status(); } @@ -687,7 +754,7 @@ Status ProtoStreamObjectSource::RenderField( // Get the nested message type for this field. const google::protobuf::Type* type = typeinfo_->GetTypeByTypeUrl(field->type_url()); - if (type == NULL) { + if (type == nullptr) { return Status(util::error::INTERNAL, StrCat("Invalid configuration. Could not find the type: ", field->type_url())); @@ -696,12 +763,14 @@ Status ProtoStreamObjectSource::RenderField( // Short-circuit any special type rendering to save call-stack space. const TypeRenderer* type_renderer = FindTypeRenderer(type->name()); - bool use_type_renderer = type_renderer != NULL; + bool use_type_renderer = type_renderer != nullptr; if (use_type_renderer) { RETURN_IF_ERROR((*type_renderer)(this, *type, field_name, ow)); } else { + RETURN_IF_ERROR(IncrementRecursionDepth(type->name(), field_name)); RETURN_IF_ERROR(WriteMessage(*type, field_name, 0, true, ow)); + --recursion_depth_; } if (!stream_->ConsumedEntireMessage()) { return Status(util::error::INVALID_ARGUMENT, @@ -712,7 +781,7 @@ Status ProtoStreamObjectSource::RenderField( // Render all other non-message types. return RenderNonMessageField(field, field_name, ow); } - return Status::OK; + return util::Status(); } Status ProtoStreamObjectSource::RenderNonMessageField( @@ -802,15 +871,25 @@ Status ProtoStreamObjectSource::RenderNonMessageField( // up. const google::protobuf::Enum* en = typeinfo_->GetEnumByTypeUrl(field->type_url()); - // Lookup the name of the enum, and render that. Skips unknown enums. - if (en != NULL) { + // Lookup the name of the enum, and render that. Unknown enum values + // are printed as integers. + if (en != nullptr) { const google::protobuf::EnumValue* enum_value = FindEnumValueByNumber(*en, buffer32); - if (enum_value != NULL) { - ow->RenderString(field_name, enum_value->name()); + if (enum_value != nullptr) { + if (use_ints_for_enums_) { + ow->RenderInt32(field_name, buffer32); + } else if (use_lower_camel_for_enums_) { + ow->RenderString(field_name, + EnumValueNameToLowerCamelCase(enum_value->name())); + } else { + ow->RenderString(field_name, enum_value->name()); + } + } else if (render_unknown_enum_values_) { + ow->RenderInt32(field_name, buffer32); } - } else { - GOOGLE_LOG(INFO) << "Unknown enum skipped: " << field->type_url(); + } else if (render_unknown_enum_values_) { + ow->RenderInt32(field_name, buffer32); } break; } @@ -829,7 +908,7 @@ Status ProtoStreamObjectSource::RenderNonMessageField( default: break; } - return Status::OK; + return util::Status(); } // TODO(skarvaje): Fix this to avoid code duplication. @@ -924,10 +1003,10 @@ const string ProtoStreamObjectSource::ReadFieldValueAsString( const google::protobuf::Enum* en = typeinfo_->GetEnumByTypeUrl(field.type_url()); // Lookup the name of the enum, and render that. Skips unknown enums. - if (en != NULL) { + if (en != nullptr) { const google::protobuf::EnumValue* enum_value = FindEnumValueByNumber(*en, buffer32); - if (enum_value != NULL) { + if (enum_value != nullptr) { result = enum_value->name(); } } @@ -957,12 +1036,8 @@ bool ProtoStreamObjectSource::IsMap( const google::protobuf::Field& field) const { const google::protobuf::Type* field_type = typeinfo_->GetTypeByTypeUrl(field.type_url()); - - // TODO(xiaofeng): Unify option names. return field.kind() == google::protobuf::Field_Kind_TYPE_MESSAGE && - (GetBoolOptionOrDefault(field_type->options(), - "google.protobuf.MessageOptions.map_entry", false) || - GetBoolOptionOrDefault(field_type->options(), "map_entry", false)); + google::protobuf::util::converter::IsMap(field, *field_type); } std::pair<int64, int32> ProtoStreamObjectSource::ReadSecondsAndNanos( @@ -975,8 +1050,8 @@ std::pair<int64, int32> ProtoStreamObjectSource::ReadSecondsAndNanos( for (tag = stream_->ReadTag(); tag != 0; tag = stream_->ReadTag()) { const google::protobuf::Field* field = FindAndVerifyField(type, tag); - if (field == NULL) { - WireFormat::SkipField(stream_, tag, NULL); + if (field == nullptr) { + WireFormat::SkipField(stream_, tag, nullptr); continue; } // 'seconds' has field number of 1 and 'nanos' has field number 2 @@ -994,6 +1069,17 @@ std::pair<int64, int32> ProtoStreamObjectSource::ReadSecondsAndNanos( return std::pair<int64, int32>(signed_seconds, signed_nanos); } +Status ProtoStreamObjectSource::IncrementRecursionDepth( + StringPiece type_name, StringPiece field_name) const { + if (++recursion_depth_ > max_recursion_depth_) { + return Status( + util::error::INVALID_ARGUMENT, + StrCat("Message too deep. Max recursion depth reached for type '", + type_name, "', field '", field_name, "'")); + } + return util::Status(); +} + namespace { // TODO(skarvaje): Speed this up by not doing a linear scan. const google::protobuf::Field* FindFieldByNumber( @@ -1003,7 +1089,7 @@ const google::protobuf::Field* FindFieldByNumber( return &type.fields(i); } } - return NULL; + return nullptr; } // TODO(skarvaje): Replace FieldDescriptor by implementing IsTypePackable() @@ -1024,12 +1110,16 @@ const google::protobuf::EnumValue* FindEnumValueByNumber( return &ev; } } - return NULL; + return nullptr; } // TODO(skarvaje): Look into optimizing this by not doing computation on // double. -const string FormatNanos(uint32 nanos) { +const string FormatNanos(uint32 nanos, bool with_trailing_zeros) { + if (nanos == 0) { + return with_trailing_zeros ? ".000" : ""; + } + const char* format = (nanos % 1000 != 0) ? "%.9f" : (nanos % 1000000 != 0) ? "%.6f" : "%.3f"; string formatted = diff --git a/src/google/protobuf/util/internal/protostream_objectsource.h b/src/google/protobuf/util/internal/protostream_objectsource.h index 78defa1d..b56efdf4 100644 --- a/src/google/protobuf/util/internal/protostream_objectsource.h +++ b/src/google/protobuf/util/internal/protostream_objectsource.h @@ -82,6 +82,51 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectSource : public ObjectSource { virtual util::Status NamedWriteTo(StringPiece name, ObjectWriter* ow) const; + // Sets whether or not to use lowerCamelCase casing for enum values. If set to + // false, enum values are output without any case conversions. + // + // For example, if we have an enum: + // enum Type { + // ACTION_AND_ADVENTURE = 1; + // } + // Type type = 20; + // + // And this option is set to true. Then the rendered "type" field will have + // the string "actionAndAdventure". + // { + // ... + // "type": "actionAndAdventure", + // ... + // } + // + // If set to false, the rendered "type" field will have the string + // "ACTION_AND_ADVENTURE". + // { + // ... + // "type": "ACTION_AND_ADVENTURE", + // ... + // } + void set_use_lower_camel_for_enums(bool value) { + use_lower_camel_for_enums_ = value; + } + + // Sets whether to always output enums as ints, by default this is off, and + // enums are rendered as strings. + void set_use_ints_for_enums(bool value) { use_ints_for_enums_ = value; } + + // Sets whether to use original proto field names + void set_preserve_proto_field_names(bool value) { + preserve_proto_field_names_ = value; + } + + // Sets the max recursion depth of proto message to be deserialized. Proto + // messages over this depth will fail to be deserialized. + // Default value is 64. + void set_max_recursion_depth(int max_depth) { + max_recursion_depth_ = max_depth; + } + + protected: // Writes a proto2 Message to the ObjectWriter. When the given end_tag is // found this method will complete, allowing it to be used for parsing both @@ -93,6 +138,28 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectSource : public ObjectSource { bool include_start_and_end, ObjectWriter* ow) const; + // Renders a repeating field (packed or unpacked). Returns the next tag after + // reading all sequential repeating elements. The caller should use this tag + // before reading more tags from the stream. + virtual util::StatusOr<uint32> RenderList( + const google::protobuf::Field* field, StringPiece name, uint32 list_tag, + ObjectWriter* ow) const; + + // Looks up a field and verify its consistency with wire type in tag. + const google::protobuf::Field* FindAndVerifyField( + const google::protobuf::Type& type, uint32 tag) const; + + // Renders a field value to the ObjectWriter. + util::Status RenderField(const google::protobuf::Field* field, + StringPiece field_name, ObjectWriter* ow) const; + + // Reads field value according to Field spec in 'field' and returns the read + // value as string. This only works for primitive datatypes (no message + // types). + const string ReadFieldValueAsString( + const google::protobuf::Field& field) const; + + private: ProtoStreamObjectSource(google::protobuf::io::CodedInputStream* stream, const TypeInfo* typeinfo, @@ -102,19 +169,9 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectSource : public ObjectSource { const google::protobuf::Type&, StringPiece, ObjectWriter*); - // Looks up a field and verify its consistency with wire type in tag. - const google::protobuf::Field* FindAndVerifyField( - const google::protobuf::Type& type, uint32 tag) const; - // TODO(skarvaje): Mark these methods as non-const as they modify internal // state (stream_). // - // Renders a repeating field (packed or unpacked). - // Returns the next tag after reading all sequential repeating elements. The - // caller should use this tag before reading more tags from the stream. - util::StatusOr<uint32> RenderList(const google::protobuf::Field* field, - StringPiece name, uint32 list_tag, - ObjectWriter* ow) const; // Renders a NWP map. // Returns the next tag after reading all map entries. The caller should use // this tag before reading more tags from the stream. @@ -198,10 +255,6 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectSource : public ObjectSource { static void DeleteRendererMap(); static TypeRenderer* FindTypeRenderer(const string& type_url); - // Renders a field value to the ObjectWriter. - util::Status RenderField(const google::protobuf::Field* field, - StringPiece field_name, ObjectWriter* ow) const; - // Same as above but renders all non-message field types. Callers don't call // this function directly. They just use RenderField. util::Status RenderNonMessageField(const google::protobuf::Field* field, @@ -209,12 +262,6 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectSource : public ObjectSource { ObjectWriter* ow) const; - // Reads field value according to Field spec in 'field' and returns the read - // value as string. This only works for primitive datatypes (no message - // types). - const string ReadFieldValueAsString( - const google::protobuf::Field& field) const; - // Utility function to detect proto maps. The 'field' MUST be repeated. bool IsMap(const google::protobuf::Field& field) const; @@ -223,12 +270,19 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectSource : public ObjectSource { std::pair<int64, int32> ReadSecondsAndNanos( const google::protobuf::Type& type) const; + // Helper function to check recursion depth and increment it. It will return + // Status::OK if the current depth is allowed. Otherwise an error is returned. + // type_name and field_name are used for error reporting. + util::Status IncrementRecursionDepth(StringPiece type_name, + StringPiece field_name) const; + // Input stream to read from. Ownership rests with the caller. google::protobuf::io::CodedInputStream* stream_; // Type information for all the types used in the descriptor. Used to find // google::protobuf::Type of nested messages/enums. const TypeInfo* typeinfo_; + // Whether this class owns the typeinfo_ object. If true the typeinfo_ object // should be deleted in the destructor. bool own_typeinfo_; @@ -237,6 +291,30 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectSource : public ObjectSource { const google::protobuf::Type& type_; + // Whether to render enums using lowerCamelCase. Defaults to false. + bool use_lower_camel_for_enums_; + + // Whether to render enums as ints always. Defaults to false. + bool use_ints_for_enums_; + + // Whether to preserve proto field names + bool preserve_proto_field_names_; + + // Tracks current recursion depth. + mutable int recursion_depth_; + + // Maximum allowed recursion depth. + int max_recursion_depth_; + + // Whether to render unknown fields. + bool render_unknown_fields_; + + // Whether to render unknown enum values. + bool render_unknown_enum_values_; + + // Whether to add trailing zeros for timestamp and duration. + bool add_trailing_zeros_for_timestamp_and_duration_; + GOOGLE_DISALLOW_IMPLICIT_CONSTRUCTORS(ProtoStreamObjectSource); }; diff --git a/src/google/protobuf/util/internal/protostream_objectsource_test.cc b/src/google/protobuf/util/internal/protostream_objectsource_test.cc index 561f6763..df790728 100644 --- a/src/google/protobuf/util/internal/protostream_objectsource_test.cc +++ b/src/google/protobuf/util/internal/protostream_objectsource_test.cc @@ -31,9 +31,6 @@ #include <google/protobuf/util/internal/protostream_objectsource.h> #include <memory> -#ifndef _SHARED_PTR_H -#include <google/protobuf/stubs/shared_ptr.h> -#endif #include <sstream> #include <google/protobuf/stubs/casts.h> @@ -42,14 +39,16 @@ #include <google/protobuf/io/zero_copy_stream_impl_lite.h> #include <google/protobuf/descriptor.h> #include <google/protobuf/util/internal/expecting_objectwriter.h> +#include <google/protobuf/util/internal/testdata/anys.pb.h> #include <google/protobuf/util/internal/testdata/books.pb.h> #include <google/protobuf/util/internal/testdata/field_mask.pb.h> +#include <google/protobuf/util/internal/testdata/maps.pb.h> +#include <google/protobuf/util/internal/testdata/proto3.pb.h> +#include <google/protobuf/util/internal/testdata/struct.pb.h> +#include <google/protobuf/util/internal/testdata/timestamp_duration.pb.h> #include <google/protobuf/util/internal/type_info_test_helper.h> #include <google/protobuf/util/internal/constants.h> #include <google/protobuf/stubs/strutil.h> -#include <google/protobuf/util/internal/testdata/anys.pb.h> -#include <google/protobuf/util/internal/testdata/maps.pb.h> -#include <google/protobuf/util/internal/testdata/struct.pb.h> #include <gtest/gtest.h> @@ -65,21 +64,24 @@ using google::protobuf::Message; using google::protobuf::io::ArrayInputStream; using google::protobuf::io::CodedInputStream; using util::Status; +using google::protobuf::testing::AnyM; +using google::protobuf::testing::AnyOut; using google::protobuf::testing::Author; using google::protobuf::testing::BadAuthor; using google::protobuf::testing::BadNestedBook; using google::protobuf::testing::Book; using google::protobuf::testing::Book_Label; +using google::protobuf::testing::Cyclic; +using google::protobuf::testing::FieldMaskTest; +using google::protobuf::testing::MapOut; +using google::protobuf::testing::MapOutWireFormat; using google::protobuf::testing::NestedBook; +using google::protobuf::testing::NestedFieldMask; using google::protobuf::testing::PackedPrimitive; using google::protobuf::testing::Primitive; -using google::protobuf::testing::more_author; -using google::protobuf::testing::maps::MapOut; -using google::protobuf::testing::anys::AnyOut; -using google::protobuf::testing::anys::AnyM; -using google::protobuf::testing::FieldMaskTest; -using google::protobuf::testing::NestedFieldMask; -using google::protobuf::testing::structs::StructType; +using google::protobuf::testing::Proto3Message; +using google::protobuf::testing::StructType; +using google::protobuf::testing::TimestampDuration; using ::testing::_; @@ -92,26 +94,36 @@ string GetTypeUrl(const Descriptor* descriptor) { class ProtostreamObjectSourceTest : public ::testing::TestWithParam<testing::TypeInfoSource> { protected: - ProtostreamObjectSourceTest() : helper_(GetParam()), mock_(), ow_(&mock_) { - helper_.ResetTypeInfo(Book::descriptor()); + ProtostreamObjectSourceTest() + : helper_(GetParam()), + mock_(), + ow_(&mock_), + use_lower_camel_for_enums_(false), + use_ints_for_enums_(false), + add_trailing_zeros_(false), + render_unknown_enum_values_(true) { + helper_.ResetTypeInfo(Book::descriptor(), Proto3Message::descriptor()); } virtual ~ProtostreamObjectSourceTest() {} void DoTest(const Message& msg, const Descriptor* descriptor) { Status status = ExecuteTest(msg, descriptor); - EXPECT_EQ(Status::OK, status); + EXPECT_EQ(util::Status(), status); } Status ExecuteTest(const Message& msg, const Descriptor* descriptor) { - ostringstream oss; + std::ostringstream oss; msg.SerializePartialToOstream(&oss); string proto = oss.str(); ArrayInputStream arr_stream(proto.data(), proto.size()); CodedInputStream in_stream(&arr_stream); - google::protobuf::scoped_ptr<ProtoStreamObjectSource> os( + std::unique_ptr<ProtoStreamObjectSource> os( helper_.NewProtoSource(&in_stream, GetTypeUrl(descriptor))); + if (use_lower_camel_for_enums_) os->set_use_lower_camel_for_enums(true); + if (use_ints_for_enums_) os->set_use_ints_for_enums(true); + os->set_max_recursion_depth(64); return os->WriteTo(&mock_); } @@ -256,10 +268,24 @@ class ProtostreamObjectSourceTest return primitive; } + void UseLowerCamelForEnums() { use_lower_camel_for_enums_ = true; } + + void UseIntsForEnums() { use_ints_for_enums_ = true; } + + void AddTrailingZeros() { add_trailing_zeros_ = true; } + + void SetRenderUnknownEnumValues(bool value) { + render_unknown_enum_values_ = value; + } + testing::TypeInfoTestHelper helper_; ::testing::NiceMock<MockObjectWriter> mock_; ExpectingObjectWriter ow_; + bool use_lower_camel_for_enums_; + bool use_ints_for_enums_; + bool add_trailing_zeros_; + bool render_unknown_enum_values_; }; INSTANTIATE_TEST_CASE_P(DifferentTypeInfoSourceTest, @@ -461,6 +487,106 @@ TEST_P(ProtostreamObjectSourceTest, DoTest(book, Book::descriptor()); } +TEST_P(ProtostreamObjectSourceTest, LowerCamelEnumOutputMacroCase) { + Book book; + book.set_type(Book::ACTION_AND_ADVENTURE); + + UseLowerCamelForEnums(); + + ow_.StartObject("")->RenderString("type", "actionAndAdventure")->EndObject(); + DoTest(book, Book::descriptor()); +} + +TEST_P(ProtostreamObjectSourceTest, LowerCamelEnumOutputSnakeCase) { + Book book; + book.set_type(Book::arts_and_photography); + + UseLowerCamelForEnums(); + + ow_.StartObject("")->RenderString("type", "artsAndPhotography")->EndObject(); + DoTest(book, Book::descriptor()); +} + +TEST_P(ProtostreamObjectSourceTest, LowerCamelEnumOutputWithNumber) { + Book book; + book.set_type(Book::I18N_Tech); + + UseLowerCamelForEnums(); + + ow_.StartObject("")->RenderString("type", "i18nTech")->EndObject(); + DoTest(book, Book::descriptor()); +} + +TEST_P(ProtostreamObjectSourceTest, EnumCaseIsUnchangedByDefault) { + Book book; + book.set_type(Book::ACTION_AND_ADVENTURE); + ow_.StartObject("") + ->RenderString("type", "ACTION_AND_ADVENTURE") + ->EndObject(); + DoTest(book, Book::descriptor()); +} + +TEST_P(ProtostreamObjectSourceTest, UseIntsForEnumsTest) { + Book book; + book.set_type(Book::ACTION_AND_ADVENTURE); + + UseIntsForEnums(); + + ow_.StartObject("")->RenderInt32("type", 3)->EndObject(); + DoTest(book, Book::descriptor()); +} + +TEST_P(ProtostreamObjectSourceTest, + UnknownEnumAreDroppedWhenRenderUnknownEnumValuesIsUnset) { + Proto3Message message; + message.set_enum_value(static_cast<Proto3Message::NestedEnum>(1234)); + + SetRenderUnknownEnumValues(false); + + // Unknown enum values are not output. + ow_.StartObject("")->EndObject(); + DoTest(message, Proto3Message::descriptor()); +} + +TEST_P(ProtostreamObjectSourceTest, + UnknownEnumAreOutputWhenRenderUnknownEnumValuesIsSet) { + Proto3Message message; + message.set_enum_value(static_cast<Proto3Message::NestedEnum>(1234)); + + SetRenderUnknownEnumValues(true); + + // Unknown enum values are output. + ow_.StartObject("")->RenderInt32("enumValue", 1234)->EndObject(); + DoTest(message, Proto3Message::descriptor()); +} + +TEST_P(ProtostreamObjectSourceTest, CyclicMessageDepthTest) { + Cyclic cyclic; + cyclic.set_m_int(123); + + Book* book = cyclic.mutable_m_book(); + book->set_title("book title"); + Cyclic* current = cyclic.mutable_m_cyclic(); + Author* current_author = cyclic.add_m_author(); + for (int i = 0; i < 63; ++i) { + Author* next = current_author->add_friend_(); + next->set_id(i); + next->set_name(StrCat("author_name_", i)); + next->set_alive(true); + current_author = next; + } + + // Recursive message with depth (65) > max (max is 64). + for (int i = 0; i < 64; ++i) { + Cyclic* next = current->mutable_m_cyclic(); + next->set_m_str(StrCat("count_", i)); + current = next; + } + + Status status = ExecuteTest(cyclic, Cyclic::descriptor()); + EXPECT_EQ(util::error::INVALID_ARGUMENT, status.error_code()); +} + class ProtostreamObjectSourceMapsTest : public ProtostreamObjectSourceTest { protected: ProtostreamObjectSourceMapsTest() { @@ -541,6 +667,67 @@ TEST_P(ProtostreamObjectSourceMapsTest, MapsTest) { DoTest(out, MapOut::descriptor()); } +TEST_P(ProtostreamObjectSourceMapsTest, MissingKeysTest) { + // MapOutWireFormat has the same wire representation with MapOut but uses + // repeated message fields to represent map fields so we can intentionally + // leave out the key field or the value field of a map entry. + MapOutWireFormat out; + // Create some map entries without keys. They will be rendered with the + // default values ("" for strings, "0" for integers, etc.). + // { + // "map1": { + // "": { + // "foo": "foovalue" + // } + // }, + // "map2": { + // "": { + // "map1": { + // "nested_key1": { + // "foo": "nested_foo" + // } + // } + // } + // }, + // "map3": { + // "0": "one one one" + // }, + // "map4": { + // "false": "bool" + // } + // } + out.add_map1()->mutable_value()->set_foo("foovalue"); + MapOut* nested = out.add_map2()->mutable_value(); + (*nested->mutable_map1())["nested_key1"].set_foo("nested_foo"); + out.add_map3()->set_value("one one one"); + out.add_map4()->set_value("bool"); + + ow_.StartObject("") + ->StartObject("map1") + ->StartObject("") + ->RenderString("foo", "foovalue") + ->EndObject() + ->EndObject() + ->StartObject("map2") + ->StartObject("") + ->StartObject("map1") + ->StartObject("nested_key1") + ->RenderString("foo", "nested_foo") + ->EndObject() + ->EndObject() + ->EndObject() + ->EndObject() + ->StartObject("map3") + ->RenderString("0", "one one one") + ->EndObject() + ->StartObject("map4") + ->RenderString("false", "bool") + ->EndObject() + ->EndObject(); + + DoTest(out, MapOut::descriptor()); +} + class ProtostreamObjectSourceAnysTest : public ProtostreamObjectSourceTest { protected: ProtostreamObjectSourceAnysTest() { @@ -559,7 +746,7 @@ INSTANTIATE_TEST_CASE_P(DifferentTypeInfoSourceTest, // This is the example expected output. // { // "any": { -// "@type": "type.googleapis.com/google.protobuf.testing.anys.AnyM" +// "@type": "type.googleapis.com/google.protobuf.testing.AnyM" // "foo": "foovalue" // } // } @@ -574,7 +761,7 @@ TEST_P(ProtostreamObjectSourceAnysTest, BasicAny) { ow_.StartObject("") ->StartObject("any") ->RenderString("@type", - "type.googleapis.com/google.protobuf.testing.anys.AnyM") + "type.googleapis.com/google.protobuf.testing.AnyM") ->RenderString("foo", "foovalue") ->EndObject() ->EndObject(); @@ -588,8 +775,7 @@ TEST_P(ProtostreamObjectSourceAnysTest, RecursiveAny) { any->set_type_url("type.googleapis.com/google.protobuf.Any"); ::google::protobuf::Any nested_any; - nested_any.set_type_url( - "type.googleapis.com/google.protobuf.testing.anys.AnyM"); + nested_any.set_type_url("type.googleapis.com/google.protobuf.testing.AnyM"); AnyM m; m.set_foo("foovalue"); @@ -602,7 +788,7 @@ TEST_P(ProtostreamObjectSourceAnysTest, RecursiveAny) { ->RenderString("@type", "type.googleapis.com/google.protobuf.Any") ->StartObject("value") ->RenderString("@type", - "type.googleapis.com/google.protobuf.testing.anys.AnyM") + "type.googleapis.com/google.protobuf.testing.AnyM") ->RenderString("foo", "foovalue") ->EndObject() ->EndObject() @@ -621,7 +807,7 @@ TEST_P(ProtostreamObjectSourceAnysTest, DoubleRecursiveAny) { ::google::protobuf::Any second_nested_any; second_nested_any.set_type_url( - "type.googleapis.com/google.protobuf.testing.anys.AnyM"); + "type.googleapis.com/google.protobuf.testing.AnyM"); AnyM m; m.set_foo("foovalue"); @@ -636,7 +822,7 @@ TEST_P(ProtostreamObjectSourceAnysTest, DoubleRecursiveAny) { ->RenderString("@type", "type.googleapis.com/google.protobuf.Any") ->StartObject("value") ->RenderString("@type", - "type.googleapis.com/google.protobuf.testing.anys.AnyM") + "type.googleapis.com/google.protobuf.testing.AnyM") ->RenderString("foo", "foovalue") ->EndObject() ->EndObject() @@ -824,6 +1010,77 @@ TEST_P(ProtostreamObjectSourceFieldMaskTest, FieldMaskRenderSuccess) { DoTest(out, FieldMaskTest::descriptor()); } +class ProtostreamObjectSourceTimestampTest + : public ProtostreamObjectSourceTest { + protected: + ProtostreamObjectSourceTimestampTest() { + helper_.ResetTypeInfo(TimestampDuration::descriptor()); + } +}; + +INSTANTIATE_TEST_CASE_P(DifferentTypeInfoSourceTest, + ProtostreamObjectSourceTimestampTest, + ::testing::Values( + testing::USE_TYPE_RESOLVER)); + +TEST_P(ProtostreamObjectSourceTimestampTest, InvalidTimestampBelowMinTest) { + TimestampDuration out; + google::protobuf::Timestamp* ts = out.mutable_ts(); + // Min allowed seconds - 1 + ts->set_seconds(kTimestampMinSeconds - 1); + ow_.StartObject(""); + + Status status = ExecuteTest(out, TimestampDuration::descriptor()); + EXPECT_EQ(util::error::INTERNAL, status.error_code()); +} + +TEST_P(ProtostreamObjectSourceTimestampTest, InvalidTimestampAboveMaxTest) { + TimestampDuration out; + google::protobuf::Timestamp* ts = out.mutable_ts(); + // Max allowed seconds + 1 + ts->set_seconds(kTimestampMaxSeconds + 1); + ow_.StartObject(""); + + Status status = ExecuteTest(out, TimestampDuration::descriptor()); + EXPECT_EQ(util::error::INTERNAL, status.error_code()); +} + +TEST_P(ProtostreamObjectSourceTimestampTest, InvalidDurationBelowMinTest) { + TimestampDuration out; + google::protobuf::Duration* dur = out.mutable_dur(); + // Min allowed seconds - 1 + dur->set_seconds(kDurationMinSeconds - 1); + ow_.StartObject(""); + + Status status = ExecuteTest(out, TimestampDuration::descriptor()); + EXPECT_EQ(util::error::INTERNAL, status.error_code()); +} + +TEST_P(ProtostreamObjectSourceTimestampTest, InvalidDurationAboveMaxTest) { + TimestampDuration out; + google::protobuf::Duration* dur = out.mutable_dur(); + // Min allowed seconds + 1 + dur->set_seconds(kDurationMaxSeconds + 1); + ow_.StartObject(""); + + Status status = ExecuteTest(out, TimestampDuration::descriptor()); + EXPECT_EQ(util::error::INTERNAL, status.error_code()); +} + +TEST_P(ProtostreamObjectSourceTimestampTest, TimestampDurationDefaultValue) { + TimestampDuration out; + out.mutable_ts()->set_seconds(0); + out.mutable_dur()->set_seconds(0); + + ow_.StartObject("") + ->RenderString("ts", "1970-01-01T00:00:00Z") + ->RenderString("dur", "0s") + ->EndObject(); + + DoTest(out, TimestampDuration::descriptor()); +} + + } // namespace converter } // namespace util } // namespace protobuf diff --git a/src/google/protobuf/util/internal/protostream_objectwriter.cc b/src/google/protobuf/util/internal/protostream_objectwriter.cc index 786bf0be..2edfd075 100644 --- a/src/google/protobuf/util/internal/protostream_objectwriter.cc +++ b/src/google/protobuf/util/internal/protostream_objectwriter.cc @@ -58,27 +58,33 @@ using util::StatusOr; ProtoStreamObjectWriter::ProtoStreamObjectWriter( TypeResolver* type_resolver, const google::protobuf::Type& type, - strings::ByteSink* output, ErrorListener* listener) + strings::ByteSink* output, ErrorListener* listener, + const ProtoStreamObjectWriter::Options& options) : ProtoWriter(type_resolver, type, output, listener), master_type_(type), - current_(NULL) {} + current_(nullptr), + options_(options) { + set_ignore_unknown_fields(options_.ignore_unknown_fields); + set_use_lower_camel_for_enums(options_.use_lower_camel_for_enums); +} ProtoStreamObjectWriter::ProtoStreamObjectWriter( const TypeInfo* typeinfo, const google::protobuf::Type& type, strings::ByteSink* output, ErrorListener* listener) : ProtoWriter(typeinfo, type, output, listener), master_type_(type), - current_(NULL) {} + current_(nullptr), + options_(ProtoStreamObjectWriter::Options::Defaults()) {} ProtoStreamObjectWriter::~ProtoStreamObjectWriter() { - if (current_ == NULL) return; + if (current_ == nullptr) return; // Cleanup explicitly in order to avoid destructor stack overflow when input // is deeply nested. // Cast to BaseElement to avoid doing additional checks (like missing fields) // during pop(). - google::protobuf::scoped_ptr<BaseElement> element( + std::unique_ptr<BaseElement> element( static_cast<BaseElement*>(current_.get())->pop<BaseElement>()); - while (element != NULL) { + while (element != nullptr) { element.reset(element->pop<BaseElement>()); } } @@ -167,7 +173,7 @@ Status GetNanosFromStringPiece(StringPiece s_nanos, *nanos = i_nanos * conversion; } - return Status::OK; + return Status(); } } // namespace @@ -179,39 +185,47 @@ ProtoStreamObjectWriter::AnyWriter::AnyWriter(ProtoStreamObjectWriter* parent) data_(), output_(&data_), depth_(0), - has_injected_value_message_(false) {} + is_well_known_type_(false), + well_known_type_render_(nullptr) {} ProtoStreamObjectWriter::AnyWriter::~AnyWriter() {} void ProtoStreamObjectWriter::AnyWriter::StartObject(StringPiece name) { ++depth_; // If an object writer is absent, that means we have not called StartAny() - // before reaching here. This is an invalid state. StartAny() gets called - // whenever we see an "@type" being rendered (see AnyWriter::RenderDataPiece). - if (ow_ == NULL) { - // Make sure we are not already in an invalid state. This avoids making - // multiple unnecessary InvalidValue calls. - if (!invalid_) { + // before reaching here, which happens when we have data before the "@type" + // field. + if (ow_ == nullptr) { + // Save data before the "@type" field for later replay. + uninterpreted_events_.push_back(Event(Event::START_OBJECT, name)); + } else if (is_well_known_type_ && depth_ == 1) { + // For well-known types, the only other field besides "@type" should be a + // "value" field. + if (name != "value" && !invalid_) { parent_->InvalidValue("Any", - StrCat("Missing or invalid @type for any field in ", - parent_->master_type_.name())); + "Expect a \"value\" field for well-known types."); invalid_ = true; } - } else if (!has_injected_value_message_ || depth_ != 1 || name != "value") { - // We don't propagate to ow_ StartObject("value") calls for nested Anys or - // Struct at depth 1 as they are nested one level deep with an injected - // "value" field. + ow_->StartObject(""); + } else { + // Forward the call to the child writer if: + // 1. the type is not a well-known type. + // 2. or, we are in a nested Any, Struct, or Value object. ow_->StartObject(name); } } bool ProtoStreamObjectWriter::AnyWriter::EndObject() { --depth_; - // As long as depth_ >= 0, we know we haven't reached the end of Any. - // Propagate these EndObject() calls to the contained ow_. If we are in a - // nested Any or Struct type, ignore the second to last EndObject call (depth_ - // == -1) - if (ow_ != NULL && (!has_injected_value_message_ || depth_ >= 0)) { + if (ow_ == nullptr) { + if (depth_ >= 0) { + // Save data before the "@type" field for later replay. + uninterpreted_events_.push_back(Event(Event::END_OBJECT)); + } + } else if (depth_ >= 0 || !is_well_known_type_) { + // As long as depth_ >= 0, we know we haven't reached the end of Any. + // Propagate these EndObject() calls to the contained ow_. For regular + // message types, we propagate the end of Any as well. ow_->EndObject(); } // A negative depth_ implies that we have reached the end of Any @@ -225,14 +239,16 @@ bool ProtoStreamObjectWriter::AnyWriter::EndObject() { void ProtoStreamObjectWriter::AnyWriter::StartList(StringPiece name) { ++depth_; - // We expect ow_ to be present as this call only makes sense inside an Any. - if (ow_ == NULL) { - if (!invalid_) { + if (ow_ == nullptr) { + // Save data before the "@type" field for later replay. + uninterpreted_events_.push_back(Event(Event::START_LIST, name)); + } else if (is_well_known_type_ && depth_ == 1) { + if (name != "value" && !invalid_) { parent_->InvalidValue("Any", - StrCat("Missing or invalid @type for any field in ", - parent_->master_type_.name())); + "Expect a \"value\" field for well-known types."); invalid_ = true; } + ow_->StartList(""); } else { ow_->StartList(name); } @@ -244,8 +260,10 @@ void ProtoStreamObjectWriter::AnyWriter::EndList() { GOOGLE_LOG(DFATAL) << "Mismatched EndList found, should not be possible"; depth_ = 0; } - // We don't write an error on the close, only on the open - if (ow_ != NULL) { + if (ow_ == nullptr) { + // Save data before the "@type" field for later replay. + uninterpreted_events_.push_back(Event(Event::END_LIST)); + } else { ow_->EndList(); } } @@ -254,26 +272,32 @@ void ProtoStreamObjectWriter::AnyWriter::RenderDataPiece( StringPiece name, const DataPiece& value) { // Start an Any only at depth_ 0. Other RenderDataPiece calls with "@type" // should go to the contained ow_ as they indicate nested Anys. - if (depth_ == 0 && ow_ == NULL && name == "@type") { + if (depth_ == 0 && ow_ == nullptr && name == "@type") { StartAny(value); - } else if (ow_ == NULL) { - if (!invalid_) { + } else if (ow_ == nullptr) { + // Save data before the "@type" field. + uninterpreted_events_.push_back(Event(name, value)); + } else if (depth_ == 0 && is_well_known_type_) { + if (name != "value" && !invalid_) { parent_->InvalidValue("Any", - StrCat("Missing or invalid @type for any field in ", - parent_->master_type_.name())); + "Expect a \"value\" field for well-known types."); invalid_ = true; } - } else { - // Check to see if the data needs to be rendered with well-known-type - // renderer. - const TypeRenderer* type_renderer = - FindTypeRenderer(GetFullTypeWithUrl(ow_->master_type_.name())); - if (type_renderer) { - Status status = (*type_renderer)(ow_.get(), value); - if (!status.ok()) ow_->InvalidValue("Any", status.error_message()); + if (well_known_type_render_ == nullptr) { + // Only Any and Struct don't have a special type render but both of + // them expect a JSON object (i.e., a StartObject() call). + if (value.type() != DataPiece::TYPE_NULL && !invalid_) { + parent_->InvalidValue("Any", "Expect a JSON object."); + invalid_ = true; + } } else { - ow_->RenderDataPiece(name, value); + ow_->ProtoWriter::StartObject(""); + Status status = (*well_known_type_render_)(ow_.get(), value); + if (!status.ok()) ow_->InvalidValue("Any", status.error_message()); + ow_->ProtoWriter::EndObject(); } + } else { + ow_->RenderDataPiece(name, value); } } @@ -302,26 +326,54 @@ void ProtoStreamObjectWriter::AnyWriter::StartAny(const DataPiece& value) { // At this point, type is never null. const google::protobuf::Type* type = resolved_type.ValueOrDie(); - // If this is the case of an Any in an Any or Struct in an Any, we need to - // expect a StartObject call with "value" while we're at depth_ 0, which we - // should ignore (not propagate to our nested object writer). We also need to - // ignore the second-to-last EndObject call, and not propagate that either. - if (type->name() == kAnyType || type->name() == kStructType) { - has_injected_value_message_ = true; + well_known_type_render_ = FindTypeRenderer(type_url_); + if (well_known_type_render_ != nullptr || + // Explicitly list Any and Struct here because they don't have a + // custom renderer. + type->name() == kAnyType || type->name() == kStructType) { + is_well_known_type_ = true; } // Create our object writer and initialize it with the first StartObject // call. ow_.reset(new ProtoStreamObjectWriter(parent_->typeinfo(), *type, &output_, parent_->listener())); - ow_->StartObject(""); + + // Don't call StartObject() for well-known types yet. Depending on the + // type of actual data, we may not need to call StartObject(). For + // example: + // { + // "@type": "type.googleapis.com/google.protobuf.Value", + // "value": [1, 2, 3], + // } + // With the above JSON representation, we will only call StartList() on the + // contained ow_. + if (!is_well_known_type_) { + ow_->StartObject(""); + } + + // Now we know the proto type and can interpret all data fields we gathered + // before the "@type" field. + for (int i = 0; i < uninterpreted_events_.size(); ++i) { + uninterpreted_events_[i].Replay(this); + } } void ProtoStreamObjectWriter::AnyWriter::WriteAny() { - if (ow_ == NULL) { - // If we had no object writer, we never got any content, so just return - // immediately, which is equivalent to writing an empty Any. - return; + if (ow_ == nullptr) { + if (uninterpreted_events_.empty()) { + // We never got any content, so just return immediately, which is + // equivalent to writing an empty Any. + return; + } else { + // There are uninterpreted data, but we never got a "@type" field. + if (!invalid_) { + parent_->InvalidValue("Any", StrCat("Missing @type for any field in ", + parent_->master_type_.name())); + invalid_ = true; + } + return; + } } // Render the type_url and value fields directly to the stream. // type_url has tag 1 and value has tag 2. @@ -331,10 +383,45 @@ void ProtoStreamObjectWriter::AnyWriter::WriteAny() { } } +void ProtoStreamObjectWriter::AnyWriter::Event::Replay( + AnyWriter* writer) const { + switch (type_) { + case START_OBJECT: + writer->StartObject(name_); + break; + case END_OBJECT: + writer->EndObject(); + break; + case START_LIST: + writer->StartList(name_); + break; + case END_LIST: + writer->EndList(); + break; + case RENDER_DATA_PIECE: + writer->RenderDataPiece(name_, value_); + break; + } +} + +void ProtoStreamObjectWriter::AnyWriter::Event::DeepCopy() { + // DataPiece only contains a string reference. To make sure the referenced + // string value stays valid, we make a copy of the string value and update + // DataPiece to reference our own copy. + if (value_.type() == DataPiece::TYPE_STRING) { + StrAppend(&value_storage_, value_.str()); + value_ = DataPiece(value_storage_, value_.use_strict_base64_decoding()); + } else if (value_.type() == DataPiece::TYPE_BYTES) { + value_storage_ = value_.ToBytes().ValueOrDie(); + value_ = + DataPiece(value_storage_, true, value_.use_strict_base64_decoding()); + } +} + ProtoStreamObjectWriter::Item::Item(ProtoStreamObjectWriter* enclosing, ItemType item_type, bool is_placeholder, bool is_list) - : BaseElement(NULL), + : BaseElement(nullptr), ow_(enclosing), any_(), item_type_(item_type), @@ -343,6 +430,9 @@ ProtoStreamObjectWriter::Item::Item(ProtoStreamObjectWriter* enclosing, if (item_type_ == ANY) { any_.reset(new AnyWriter(ow_)); } + if (item_type == MAP) { + map_keys_.reset(new hash_set<string>); + } } ProtoStreamObjectWriter::Item::Item(ProtoStreamObjectWriter::Item* parent, @@ -357,11 +447,14 @@ ProtoStreamObjectWriter::Item::Item(ProtoStreamObjectWriter::Item* parent, if (item_type == ANY) { any_.reset(new AnyWriter(ow_)); } + if (item_type == MAP) { + map_keys_.reset(new hash_set<string>); + } } bool ProtoStreamObjectWriter::Item::InsertMapKeyIfNotPresent( StringPiece map_key) { - return InsertIfNotPresent(&map_keys_, map_key.ToString()); + return InsertIfNotPresent(map_keys_.get(), map_key.ToString()); } ProtoStreamObjectWriter* ProtoStreamObjectWriter::StartObject( @@ -374,7 +467,7 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::StartObject( // Starting the root message. Create the root Item and return. // ANY message type does not need special handling, just set the ItemType // to ANY. - if (current_ == NULL) { + if (current_ == nullptr) { ProtoWriter::StartObject(name); current_.reset(new Item( this, master_type_.name() == kAnyType ? Item::ANY : Item::MESSAGE, @@ -439,7 +532,8 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::StartObject( // name): // { "key": "<name>", "value": { Push("", Item::MESSAGE, false, false); - ProtoWriter::RenderDataPiece("key", DataPiece(name)); + ProtoWriter::RenderDataPiece("key", + DataPiece(name, use_strict_base64_decoding())); Push("value", Item::MESSAGE, true, false); // Make sure we are valid so far after starting map fields. @@ -447,14 +541,14 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::StartObject( // If top of stack is g.p.Struct type, start the struct the map field within // it. - if (element() != NULL && IsStruct(*element()->parent_field())) { + if (element() != nullptr && IsStruct(*element()->parent_field())) { // Render "fields": [ Push("fields", Item::MAP, true, true); return this; } // If top of stack is g.p.Value type, start the Struct within it. - if (element() != NULL && IsStructValue(*element()->parent_field())) { + if (element() != nullptr && IsStructValue(*element()->parent_field())) { // Render // "struct_value": { // "fields": [ @@ -465,7 +559,7 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::StartObject( } const google::protobuf::Field* field = BeginNamed(name, false); - if (field == NULL) return this; + if (field == nullptr) return this; if (IsStruct(*field)) { // Start a struct object. @@ -513,7 +607,7 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::EndObject() { return this; } - if (current_ == NULL) return this; + if (current_ == nullptr) return this; if (current_->IsAny()) { if (current_->any()->EndObject()) return this; @@ -533,7 +627,7 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::StartList(StringPiece name) { // Since we cannot have a top-level repeated item in protobuf, the only way // this is valid is if we start a special type google.protobuf.ListValue or // google.protobuf.Value. - if (current_ == NULL) { + if (current_ == nullptr) { if (!name.empty()) { InvalidName(name, "Root element should not be named."); IncrementInvalidDepth(); @@ -604,14 +698,15 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::StartList(StringPiece name) { // Render // { "key": "<name>", "value": { Push("", Item::MESSAGE, false, false); - ProtoWriter::RenderDataPiece("key", DataPiece(name)); + ProtoWriter::RenderDataPiece("key", + DataPiece(name, use_strict_base64_decoding())); Push("value", Item::MESSAGE, true, false); // Make sure we are valid after pushing all above items. if (invalid_depth() > 0) return this; // case i and ii above. Start "list_value" field within g.p.Value - if (element() != NULL && element()->parent_field() != NULL) { + if (element() != nullptr && element()->parent_field() != nullptr) { // Render // "list_value": { // "values": [ // Start this list @@ -639,7 +734,7 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::StartList(StringPiece name) { // When name is empty and stack is not empty, we are rendering an item within // a list. if (name.empty()) { - if (element() != NULL && element()->parent_field() != NULL) { + if (element() != nullptr && element()->parent_field() != nullptr) { if (IsStructValue(*element()->parent_field())) { // Since it is g.p.Value, we bind directly to the list_value. // Render @@ -670,7 +765,7 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::StartList(StringPiece name) { // name is not empty const google::protobuf::Field* field = Lookup(name); - if (field == NULL) { + if (field == nullptr) { IncrementInvalidDepth(); return this; } @@ -742,7 +837,7 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::EndList() { return this; } - if (current_ == NULL) return this; + if (current_ == nullptr) return this; if (current_->IsAny()) { current_->any()->EndList(); @@ -758,9 +853,46 @@ Status ProtoStreamObjectWriter::RenderStructValue(ProtoStreamObjectWriter* ow, string struct_field_name; switch (data.type()) { // Our JSON parser parses numbers as either int64, uint64, or double. - case DataPiece::TYPE_INT64: - case DataPiece::TYPE_UINT64: + case DataPiece::TYPE_INT64: { + // If the option to treat integers as strings is set, then render them as + // strings. Otherwise, fallback to rendering them as double. + if (ow->options_.struct_integers_as_strings) { + StatusOr<int64> int_value = data.ToInt64(); + if (int_value.ok()) { + ow->ProtoWriter::RenderDataPiece( + "string_value", + DataPiece(SimpleItoa(int_value.ValueOrDie()), true)); + return Status(); + } + } + struct_field_name = "number_value"; + break; + } + case DataPiece::TYPE_UINT64: { + // If the option to treat integers as strings is set, then render them as + // strings. Otherwise, fallback to rendering them as double. + if (ow->options_.struct_integers_as_strings) { + StatusOr<uint64> int_value = data.ToUint64(); + if (int_value.ok()) { + ow->ProtoWriter::RenderDataPiece( + "string_value", + DataPiece(SimpleItoa(int_value.ValueOrDie()), true)); + return Status(); + } + } + struct_field_name = "number_value"; + break; + } case DataPiece::TYPE_DOUBLE: { + if (ow->options_.struct_integers_as_strings) { + StatusOr<double> double_value = data.ToDouble(); + if (double_value.ok()) { + ow->ProtoWriter::RenderDataPiece( + "string_value", + DataPiece(SimpleDtoa(double_value.ValueOrDie()), true)); + return Status(); + } + } struct_field_name = "number_value"; break; } @@ -783,11 +915,12 @@ Status ProtoStreamObjectWriter::RenderStructValue(ProtoStreamObjectWriter* ow, } } ow->ProtoWriter::RenderDataPiece(struct_field_name, data); - return Status::OK; + return Status(); } Status ProtoStreamObjectWriter::RenderTimestamp(ProtoStreamObjectWriter* ow, const DataPiece& data) { + if (data.type() == DataPiece::TYPE_NULL) return Status(); if (data.type() != DataPiece::TYPE_STRING) { return Status(INVALID_ARGUMENT, StrCat("Invalid data type for timestamp, value is ", @@ -806,18 +939,19 @@ Status ProtoStreamObjectWriter::RenderTimestamp(ProtoStreamObjectWriter* ow, ow->ProtoWriter::RenderDataPiece("seconds", DataPiece(seconds)); ow->ProtoWriter::RenderDataPiece("nanos", DataPiece(nanos)); - return Status::OK; + return Status(); } static inline util::Status RenderOneFieldPath(ProtoStreamObjectWriter* ow, StringPiece path) { ow->ProtoWriter::RenderDataPiece( - "paths", DataPiece(ConvertFieldMaskPath(path, &ToSnakeCase))); - return Status::OK; + "paths", DataPiece(ConvertFieldMaskPath(path, &ToSnakeCase), true)); + return Status(); } Status ProtoStreamObjectWriter::RenderFieldMask(ProtoStreamObjectWriter* ow, const DataPiece& data) { + if (data.type() == DataPiece::TYPE_NULL) return Status(); if (data.type() != DataPiece::TYPE_STRING) { return Status(INVALID_ARGUMENT, StrCat("Invalid data type for field mask, value is ", @@ -827,13 +961,14 @@ Status ProtoStreamObjectWriter::RenderFieldMask(ProtoStreamObjectWriter* ow, // TODO(tsun): figure out how to do proto descriptor based snake case // conversions as much as possible. Because ToSnakeCase sometimes returns the // wrong value. - google::protobuf::scoped_ptr<ResultCallback1<util::Status, StringPiece> > callback( - google::protobuf::internal::NewPermanentCallback(&RenderOneFieldPath, ow)); + std::unique_ptr<ResultCallback1<util::Status, StringPiece> > callback( + ::google::protobuf::NewPermanentCallback(&RenderOneFieldPath, ow)); return DecodeCompactFieldMaskPaths(data.str(), callback.get()); } Status ProtoStreamObjectWriter::RenderDuration(ProtoStreamObjectWriter* ow, const DataPiece& data) { + if (data.type() == DataPiece::TYPE_NULL) return Status(); if (data.type() != DataPiece::TYPE_STRING) { return Status(INVALID_ARGUMENT, StrCat("Invalid data type for duration, value is ", @@ -842,13 +977,13 @@ Status ProtoStreamObjectWriter::RenderDuration(ProtoStreamObjectWriter* ow, StringPiece value(data.str()); - if (!value.ends_with("s")) { + if (!StringEndsWith(value, "s")) { return Status(INVALID_ARGUMENT, "Illegal duration format; duration must end with 's'"); } value = value.substr(0, value.size() - 1); int sign = 1; - if (value.starts_with("-")) { + if (StringStartsWith(value, "-")) { sign = -1; value = value.substr(1); } @@ -871,20 +1006,21 @@ Status ProtoStreamObjectWriter::RenderDuration(ProtoStreamObjectWriter* ow, nanos = sign * nanos; int64 seconds = sign * unsigned_seconds; - if (seconds > kMaxSeconds || seconds < kMinSeconds || + if (seconds > kDurationMaxSeconds || seconds < kDurationMinSeconds || nanos <= -kNanosPerSecond || nanos >= kNanosPerSecond) { return Status(INVALID_ARGUMENT, "Duration value exceeds limits"); } ow->ProtoWriter::RenderDataPiece("seconds", DataPiece(seconds)); ow->ProtoWriter::RenderDataPiece("nanos", DataPiece(nanos)); - return Status::OK; + return Status(); } Status ProtoStreamObjectWriter::RenderWrapperType(ProtoStreamObjectWriter* ow, const DataPiece& data) { + if (data.type() == DataPiece::TYPE_NULL) return Status(); ow->ProtoWriter::RenderDataPiece("value", data); - return Status::OK; + return Status(); } ProtoStreamObjectWriter* ProtoStreamObjectWriter::RenderDataPiece( @@ -892,10 +1028,10 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::RenderDataPiece( Status status; if (invalid_depth() > 0) return this; - if (current_ == NULL) { + if (current_ == nullptr) { const TypeRenderer* type_renderer = FindTypeRenderer(GetFullTypeWithUrl(master_type_.name())); - if (type_renderer == NULL) { + if (type_renderer == nullptr) { InvalidName(name, "Root element must be a message."); return this; } @@ -918,22 +1054,24 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::RenderDataPiece( return this; } - const google::protobuf::Field* field = NULL; + const google::protobuf::Field* field = nullptr; if (current_->IsMap()) { if (!ValidMapKey(name)) return this; // Render an item in repeated map list. // { "key": "<name>", "value": Push("", Item::MESSAGE, false, false); - ProtoWriter::RenderDataPiece("key", DataPiece(name)); + ProtoWriter::RenderDataPiece("key", + DataPiece(name, use_strict_base64_decoding())); field = Lookup("value"); - if (field == NULL) { + if (field == nullptr) { + Pop(); GOOGLE_LOG(DFATAL) << "Map does not have a value field."; return this; } const TypeRenderer* type_renderer = FindTypeRenderer(field->type_url()); - if (type_renderer != NULL) { + if (type_renderer != nullptr) { // Map's value type is a special type. Render it like a message: // "value": { // ... Render special type ... @@ -952,6 +1090,7 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::RenderDataPiece( // not of the google.protobuf.NullType type, we do nothing. if (data.type() == DataPiece::TYPE_NULL && field->type_url() != kStructNullValueTypeUrl) { + Pop(); return this; } @@ -962,18 +1101,23 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::RenderDataPiece( } field = Lookup(name); - if (field == NULL) return this; + if (field == nullptr) return this; // Check if the field is of special type. Render it accordingly if so. const TypeRenderer* type_renderer = FindTypeRenderer(field->type_url()); - if (type_renderer != NULL) { - Push(name, Item::MESSAGE, false, false); - status = (*type_renderer)(this, data); - if (!status.ok()) { - InvalidValue(field->type_url(), - StrCat("Field '", name, "', ", status.error_message())); + if (type_renderer != nullptr) { + // Pass through null value only for google.protobuf.Value. For other + // types we ignore null value just like for regular field types. + if (data.type() != DataPiece::TYPE_NULL || + field->type_url() == kStructValueTypeUrl) { + Push(name, Item::MESSAGE, false, false); + status = (*type_renderer)(this, data); + if (!status.ok()) { + InvalidValue(field->type_url(), + StrCat("Field '", name, "', ", status.error_message())); + } + Pop(); } - Pop(); return this; } @@ -1055,7 +1199,7 @@ ProtoStreamObjectWriter::FindTypeRenderer(const string& type_url) { } bool ProtoStreamObjectWriter::ValidMapKey(StringPiece unnormalized_name) { - if (current_ == NULL) return true; + if (current_ == nullptr) return true; if (!current_->InsertMapKeyIfNotPresent(unnormalized_name)) { listener()->InvalidName( @@ -1080,10 +1224,10 @@ void ProtoStreamObjectWriter::Push(StringPiece name, Item::ItemType item_type, void ProtoStreamObjectWriter::Pop() { // Pop all placeholder items sending StartObject or StartList events to // ProtoWriter according to is_list value. - while (current_ != NULL && current_->is_placeholder()) { + while (current_ != nullptr && current_->is_placeholder()) { PopOneElement(); } - if (current_ != NULL) { + if (current_ != nullptr) { PopOneElement(); } } @@ -1103,10 +1247,7 @@ bool ProtoStreamObjectWriter::IsMap(const google::protobuf::Field& field) { const google::protobuf::Type* field_type = typeinfo()->GetTypeByTypeUrl(field.type_url()); - // TODO(xiaofeng): Unify option names. - return GetBoolOptionOrDefault(field_type->options(), - "google.protobuf.MessageOptions.map_entry", false) || - GetBoolOptionOrDefault(field_type->options(), "map_entry", false); + return google::protobuf::util::converter::IsMap(field, *field_type); } bool ProtoStreamObjectWriter::IsAny(const google::protobuf::Field& field) { diff --git a/src/google/protobuf/util/internal/protostream_objectwriter.h b/src/google/protobuf/util/internal/protostream_objectwriter.h index 08ac6e33..c33a4639 100644 --- a/src/google/protobuf/util/internal/protostream_objectwriter.h +++ b/src/google/protobuf/util/internal/protostream_objectwriter.h @@ -74,10 +74,44 @@ class ObjectLocationTracker; // It also supports streaming. class LIBPROTOBUF_EXPORT ProtoStreamObjectWriter : public ProtoWriter { public: + // Options that control ProtoStreamObjectWriter class's behavior. + struct Options { + // Treats numeric inputs in google.protobuf.Struct as strings. Normally, + // numeric values are returned in double field "number_value" of + // google.protobuf.Struct. However, this can cause precision loss for + // int64/uint64/double inputs. This option is provided for cases that want + // to preserve number precision. + // + // TODO(skarvaje): Rename to struct_numbers_as_strings as it covers double + // as well. + bool struct_integers_as_strings; + + // Not treat unknown fields as an error. If there is an unknown fields, + // just ignore it and continue to process the rest. + bool ignore_unknown_fields; + + // If true, check if enum name in camel case or without underscore matches + // the field name. + bool use_lower_camel_for_enums; + + Options() + : struct_integers_as_strings(false), + ignore_unknown_fields(false), + use_lower_camel_for_enums(false) {} + + // Default instance of Options with all options set to defaults. + static const Options& Defaults() { + static Options defaults; + return defaults; + } + }; + // Constructor. Does not take ownership of any parameter passed in. ProtoStreamObjectWriter(TypeResolver* type_resolver, const google::protobuf::Type& type, - strings::ByteSink* output, ErrorListener* listener); + strings::ByteSink* output, ErrorListener* listener, + const ProtoStreamObjectWriter::Options& options = + ProtoStreamObjectWriter::Options::Defaults()); virtual ~ProtoStreamObjectWriter(); // ObjectWriter methods. @@ -120,6 +154,57 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectWriter : public ProtoWriter { void RenderDataPiece(StringPiece name, const DataPiece& value); private: + // Before the "@type" field is encountered, we store all incoming data + // into this Event struct and replay them after we get the "@type" field. + class LIBPROTOBUF_EXPORT Event { + public: + enum Type { + START_OBJECT = 0, + END_OBJECT = 1, + START_LIST = 2, + END_LIST = 3, + RENDER_DATA_PIECE = 4, + }; + + // Constructor for END_OBJECT and END_LIST events. + explicit Event(Type type) : type_(type), value_(DataPiece::NullData()) {} + + // Constructor for START_OBJECT and START_LIST events. + explicit Event(Type type, StringPiece name) + : type_(type), + name_(name.ToString()), + value_(DataPiece::NullData()) {} + + // Constructor for RENDER_DATA_PIECE events. + explicit Event(StringPiece name, const DataPiece& value) + : type_(RENDER_DATA_PIECE), name_(name.ToString()), value_(value) { + DeepCopy(); + } + + Event(const Event& other) + : type_(other.type_), name_(other.name_), value_(other.value_) { + DeepCopy(); + } + + Event& operator=(const Event& other) { + type_ = other.type_; + name_ = other.name_; + value_ = other.value_; + DeepCopy(); + return *this; + } + + void Replay(AnyWriter* writer) const; + + private: + void DeepCopy(); + + Type type_; + string name_; + DataPiece value_; + string value_storage_; + }; + // Handles starting up the any once we have a type. void StartAny(const DataPiece& value); @@ -131,7 +216,7 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectWriter : public ProtoWriter { ProtoStreamObjectWriter* parent_; // The nested object writer, used to write events. - google::protobuf::scoped_ptr<ProtoStreamObjectWriter> ow_; + std::unique_ptr<ProtoStreamObjectWriter> ow_; // The type_url_ that this Any represents. string type_url_; @@ -147,9 +232,17 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectWriter : public ProtoWriter { // The depth within the Any, so we can track when we're done. int depth_; - // True if the message type contained in Any has a special "value" message - // injected. This is true for well-known message types like Any or Struct. - bool has_injected_value_message_; + // True if the type is a well-known type. Well-known types in Any + // has a special formating: + // { + // "@type": "type.googleapis.com/google.protobuf.XXX", + // "value": <JSON representation of the type>, + // } + bool is_well_known_type_; + TypeRenderer* well_known_type_render_; + + // Store data before the "@type" field. + std::vector<Event> uninterpreted_events_; }; // Represents an item in a stack of items used to keep state between @@ -199,14 +292,14 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectWriter : public ProtoWriter { ProtoStreamObjectWriter* ow_; // A writer for Any objects, handles all Any-related nonsense. - google::protobuf::scoped_ptr<AnyWriter> any_; + std::unique_ptr<AnyWriter> any_; // The type of this element, see enum for permissible types. ItemType item_type_; // Set of map keys already seen for the type_. Used to validate incoming // messages so no map key appears more than once. - hash_set<string> map_keys_; + std::unique_ptr<hash_set<string> > map_keys_; // Conveys whether this Item is a placeholder or not. Placeholder items are // pushed to stack to account for special types. @@ -224,19 +317,19 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectWriter : public ProtoWriter { strings::ByteSink* output, ErrorListener* listener); // Returns true if the field is a map. - bool IsMap(const google::protobuf::Field& field); + inline bool IsMap(const google::protobuf::Field& field); // Returns true if the field is an any. - bool IsAny(const google::protobuf::Field& field); + inline bool IsAny(const google::protobuf::Field& field); // Returns true if the field is google.protobuf.Struct. - bool IsStruct(const google::protobuf::Field& field); + inline bool IsStruct(const google::protobuf::Field& field); // Returns true if the field is google.protobuf.Value. - bool IsStructValue(const google::protobuf::Field& field); + inline bool IsStructValue(const google::protobuf::Field& field); // Returns true if the field is google.protobuf.ListValue. - bool IsStructListValue(const google::protobuf::Field& field); + inline bool IsStructListValue(const google::protobuf::Field& field); // Renders google.protobuf.Value in struct.proto. It picks the right oneof // type based on value's type. @@ -299,7 +392,10 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectWriter : public ProtoWriter { const google::protobuf::Type& master_type_; // The current element, variable for internal state processing. - google::protobuf::scoped_ptr<Item> current_; + std::unique_ptr<Item> current_; + + // Reference to the options that control this class's behavior. + const ProtoStreamObjectWriter::Options options_; GOOGLE_DISALLOW_IMPLICIT_CONSTRUCTORS(ProtoStreamObjectWriter); }; diff --git a/src/google/protobuf/util/internal/protostream_objectwriter_test.cc b/src/google/protobuf/util/internal/protostream_objectwriter_test.cc index 5f9ffb95..7f0df567 100644 --- a/src/google/protobuf/util/internal/protostream_objectwriter_test.cc +++ b/src/google/protobuf/util/internal/protostream_objectwriter_test.cc @@ -41,18 +41,20 @@ #include <google/protobuf/dynamic_message.h> #include <google/protobuf/message.h> #include <google/protobuf/util/internal/mock_error_listener.h> +#include <google/protobuf/util/internal/testdata/anys.pb.h> #include <google/protobuf/util/internal/testdata/books.pb.h> #include <google/protobuf/util/internal/testdata/field_mask.pb.h> +#include <google/protobuf/util/internal/testdata/maps.pb.h> +#include <google/protobuf/util/internal/testdata/oneofs.pb.h> +#include <google/protobuf/util/internal/testdata/proto3.pb.h> +#include <google/protobuf/util/internal/testdata/struct.pb.h> +#include <google/protobuf/util/internal/testdata/timestamp_duration.pb.h> +#include <google/protobuf/util/internal/testdata/wrappers.pb.h> #include <google/protobuf/util/internal/type_info_test_helper.h> #include <google/protobuf/util/internal/constants.h> #include <google/protobuf/util/message_differencer.h> #include <google/protobuf/stubs/bytestream.h> #include <google/protobuf/stubs/strutil.h> -#include <google/protobuf/util/internal/testdata/anys.pb.h> -#include <google/protobuf/util/internal/testdata/maps.pb.h> -#include <google/protobuf/util/internal/testdata/oneofs.pb.h> -#include <google/protobuf/util/internal/testdata/struct.pb.h> -#include <google/protobuf/util/internal/testdata/timestamp_duration.pb.h> #include <gtest/gtest.h> @@ -61,27 +63,30 @@ namespace protobuf { namespace util { namespace converter { +using google::protobuf::testing::AnyM; +using google::protobuf::testing::AnyOut; using google::protobuf::testing::Author; using google::protobuf::testing::Book; -using google::protobuf::testing::Book_Data; +using google::protobuf::testing::FieldMaskTest; +using google::protobuf::testing::Int32Wrapper; +using google::protobuf::testing::MapIn; using google::protobuf::testing::Primitive; +using google::protobuf::testing::Proto3Message; using google::protobuf::testing::Publisher; +using google::protobuf::testing::StructType; +using google::protobuf::testing::TestJsonName1; +using google::protobuf::testing::TestJsonName2; +using google::protobuf::testing::TimestampDuration; +using google::protobuf::testing::ValueWrapper; +using google::protobuf::testing::oneofs::OneOfsRequest; using google::protobuf::Descriptor; using google::protobuf::DescriptorPool; using google::protobuf::DynamicMessageFactory; using google::protobuf::FileDescriptorProto; using google::protobuf::Message; -using google::protobuf::io::ArrayInputStream; using strings::GrowingArrayByteSink; using ::testing::_; using ::testing::Args; -using google::protobuf::testing::anys::AnyM; -using google::protobuf::testing::anys::AnyOut; -using google::protobuf::testing::oneofs::OneOfsRequest; -using google::protobuf::testing::FieldMaskTest; -using google::protobuf::testing::maps::MapIn; -using google::protobuf::testing::structs::StructType; -using google::protobuf::testing::timestampduration::TimestampDuration; namespace { @@ -90,6 +95,12 @@ string GetTypeUrl(const Descriptor* descriptor) { } } // namespace +#if __cplusplus >= 201103L + using std::get; +#else + using std::tr1::get; +#endif + class BaseProtoStreamObjectWriterTest : public ::testing::TestWithParam<testing::TypeInfoSource> { protected: @@ -104,13 +115,13 @@ class BaseProtoStreamObjectWriterTest listener_(), output_(new GrowingArrayByteSink(1000)), ow_() { - vector<const Descriptor*> descriptors; + std::vector<const Descriptor*> descriptors; descriptors.push_back(descriptor); ResetTypeInfo(descriptors); } explicit BaseProtoStreamObjectWriterTest( - vector<const Descriptor*> descriptors) + std::vector<const Descriptor*> descriptors) : helper_(GetParam()), listener_(), output_(new GrowingArrayByteSink(1000)), @@ -118,18 +129,24 @@ class BaseProtoStreamObjectWriterTest ResetTypeInfo(descriptors); } - void ResetTypeInfo(vector<const Descriptor*> descriptors) { + void ResetTypeInfo(std::vector<const Descriptor*> descriptors) { GOOGLE_CHECK(!descriptors.empty()) << "Must have at least one descriptor!"; helper_.ResetTypeInfo(descriptors); ow_.reset(helper_.NewProtoWriter(GetTypeUrl(descriptors[0]), output_.get(), - &listener_)); + &listener_, options_)); + } + + void ResetTypeInfo(const Descriptor* descriptor) { + std::vector<const Descriptor*> descriptors; + descriptors.push_back(descriptor); + ResetTypeInfo(descriptors); } virtual ~BaseProtoStreamObjectWriterTest() {} void CheckOutput(const Message& expected, int expected_length) { size_t nbytes; - google::protobuf::scoped_array<char> buffer(output_->GetBuffer(&nbytes)); + std::unique_ptr<char[]> buffer(output_->GetBuffer(&nbytes)); if (expected_length >= 0) { EXPECT_EQ(expected_length, nbytes); } @@ -137,7 +154,7 @@ class BaseProtoStreamObjectWriterTest std::stringbuf str_buf(str, std::ios_base::in); std::istream istream(&str_buf); - google::protobuf::scoped_ptr<Message> message(expected.New()); + std::unique_ptr<Message> message(expected.New()); message->ParsePartialFromIstream(&istream); if (!MessageDifferencer::Equivalent(expected, *message)) { @@ -153,18 +170,14 @@ class BaseProtoStreamObjectWriterTest testing::TypeInfoTestHelper helper_; MockErrorListener listener_; - google::protobuf::scoped_ptr<GrowingArrayByteSink> output_; - google::protobuf::scoped_ptr<ProtoStreamObjectWriter> ow_; + std::unique_ptr<GrowingArrayByteSink> output_; + std::unique_ptr<ProtoStreamObjectWriter> ow_; + ProtoStreamObjectWriter::Options options_; }; MATCHER_P(HasObjectLocation, expected, "Verifies the expected object location") { - string actual; -#if __cplusplus >= 201103L - actual = std::get<0>(arg).ToString(); -#else - actual = std::tr1::get<0>(arg).ToString(); -#endif + string actual = get<0>(arg).ToString(); if (actual.compare(expected) == 0) return true; *result_listener << "actual location is: " << actual; return false; @@ -175,6 +188,10 @@ class ProtoStreamObjectWriterTest : public BaseProtoStreamObjectWriterTest { ProtoStreamObjectWriterTest() : BaseProtoStreamObjectWriterTest(Book::descriptor()) {} + void ResetProtoWriter() { + ResetTypeInfo(Book::descriptor()); + } + virtual ~ProtoStreamObjectWriterTest() {} }; @@ -256,6 +273,100 @@ TEST_P(ProtoStreamObjectWriterTest, CustomJsonName) { CheckOutput(book); } +// Test that two messages can have different fields mapped to the same JSON +// name. See: https://github.com/google/protobuf/issues/1415 +TEST_P(ProtoStreamObjectWriterTest, ConflictingJsonName) { + ResetTypeInfo(TestJsonName1::descriptor()); + TestJsonName1 message1; + message1.set_one_value(12345); + ow_->StartObject("")->RenderInt32("value", 12345)->EndObject(); + CheckOutput(message1); + + ResetTypeInfo(TestJsonName2::descriptor()); + TestJsonName2 message2; + message2.set_another_value(12345); + ow_->StartObject("")->RenderInt32("value", 12345)->EndObject(); + CheckOutput(message2); +} + +TEST_P(ProtoStreamObjectWriterTest, IntEnumValuesAreAccepted) { + Book book; + book.set_title("Some Book"); + book.set_type(google::protobuf::testing::Book_Type_KIDS); + Author* robert = book.mutable_author(); + robert->set_name("robert"); + + ow_->StartObject("") + ->RenderString("title", "Some Book") + ->RenderString("type", "2") + ->StartObject("author") + ->RenderString("name", "robert") + ->EndObject() + ->EndObject(); + CheckOutput(book); +} + +TEST_P(ProtoStreamObjectWriterTest, EnumValuesWithoutUnderscoreAreAccepted) { + Book book; + book.set_title("Some Book"); + book.set_type(google::protobuf::testing::Book_Type_ACTION_AND_ADVENTURE); + Author* robert = book.mutable_author(); + robert->set_name("robert"); + + options_.use_lower_camel_for_enums = true; + ResetProtoWriter(); + + ow_->StartObject("") + ->RenderString("title", "Some Book") + ->RenderString("type", "ACTIONANDADVENTURE") + ->StartObject("author") + ->RenderString("name", "robert") + ->EndObject() + ->EndObject(); + CheckOutput(book); +} + +TEST_P(ProtoStreamObjectWriterTest, EnumValuesInCamelCaseAreAccepted) { + Book book; + book.set_title("Some Book"); + book.set_type(google::protobuf::testing::Book_Type_ACTION_AND_ADVENTURE); + Author* robert = book.mutable_author(); + robert->set_name("robert"); + + options_.use_lower_camel_for_enums = true; + ResetProtoWriter(); + + ow_->StartObject("") + ->RenderString("title", "Some Book") + ->RenderString("type", "actionAndAdventure") + ->StartObject("author") + ->RenderString("name", "robert") + ->EndObject() + ->EndObject(); + CheckOutput(book); +} + +TEST_P(ProtoStreamObjectWriterTest, + EnumValuesInCamelCaseWithNameNotUppercaseAreAccepted) { + Book book; + book.set_title("Some Book"); + book.set_type(google::protobuf::testing::Book_Type_arts_and_photography); + Author* robert = book.mutable_author(); + robert->set_name("robert"); + + options_.use_lower_camel_for_enums = true; + ResetProtoWriter(); + + ow_->StartObject("") + ->RenderString("title", "Some Book") + ->RenderString("type", "artsAndPhotography") + ->StartObject("author") + ->RenderString("name", "robert") + ->EndObject() + ->EndObject(); + CheckOutput(book); +} + TEST_P(ProtoStreamObjectWriterTest, PrimitiveFromStringConversion) { Primitive full; full.set_fix32(101); @@ -289,8 +400,7 @@ TEST_P(ProtoStreamObjectWriterTest, PrimitiveFromStringConversion) { full.add_rep_double(-8.05L); full.add_rep_bool(false); - ow_.reset(helper_.NewProtoWriter(GetTypeUrl(Primitive::descriptor()), - output_.get(), &listener_)); + ResetTypeInfo(Primitive::descriptor()); ow_->StartObject("") ->RenderString("fix32", "101") @@ -363,8 +473,7 @@ TEST_P(ProtoStreamObjectWriterTest, InfinityInputTest) { full.set_float_(std::numeric_limits<float>::infinity()); full.set_str("-Infinity"); - ow_.reset(helper_.NewProtoWriter(GetTypeUrl(Primitive::descriptor()), - output_.get(), &listener_)); + ResetTypeInfo(Primitive::descriptor()); EXPECT_CALL(listener_, InvalidValue(_, StringPiece("TYPE_INT32"), StringPiece("\"Infinity\""))) @@ -397,8 +506,7 @@ TEST_P(ProtoStreamObjectWriterTest, NaNInputTest) { full.set_float_(std::numeric_limits<float>::quiet_NaN()); full.set_str("NaN"); - ow_.reset(helper_.NewProtoWriter(GetTypeUrl(Primitive::descriptor()), - output_.get(), &listener_)); + ResetTypeInfo(Primitive::descriptor()); EXPECT_CALL(listener_, InvalidValue(_, StringPiece("TYPE_INT32"), StringPiece("\"NaN\""))) @@ -704,6 +812,132 @@ TEST_P(ProtoStreamObjectWriterTest, UnknownListAtPublisher) { CheckOutput(expected); } +TEST_P(ProtoStreamObjectWriterTest, IgnoreUnknownFieldAtRoot) { + Book empty; + + options_.ignore_unknown_fields = true; + ResetProtoWriter(); + + EXPECT_CALL(listener_, InvalidName(_, _, _)).Times(0); + ow_->StartObject("")->RenderString("unknown", "Nope!")->EndObject(); + CheckOutput(empty, 0); +} + +TEST_P(ProtoStreamObjectWriterTest, IgnoreUnknownFieldAtAuthorFriend) { + Book expected; + Author* paul = expected.mutable_author(); + paul->set_name("Paul"); + Author* mark = paul->add_friend_(); + mark->set_name("Mark"); + Author* john = paul->add_friend_(); + john->set_name("John"); + Author* luke = paul->add_friend_(); + luke->set_name("Luke"); + + options_.ignore_unknown_fields = true; + ResetProtoWriter(); + + EXPECT_CALL(listener_, InvalidName(_, _, _)).Times(0); + ow_->StartObject("") + ->StartObject("author") + ->RenderString("name", "Paul") + ->StartList("friend") + ->StartObject("") + ->RenderString("name", "Mark") + ->EndObject() + ->StartObject("") + ->RenderString("name", "John") + ->RenderString("address", "Patmos") + ->EndObject() + ->StartObject("") + ->RenderString("name", "Luke") + ->EndObject() + ->EndList() + ->EndObject() + ->EndObject(); + CheckOutput(expected); +} + +TEST_P(ProtoStreamObjectWriterTest, IgnoreUnknownObjectAtRoot) { + Book empty; + + options_.ignore_unknown_fields = true; + ResetProtoWriter(); + + EXPECT_CALL(listener_, InvalidName(_, StringPiece("unknown"), + StringPiece("Cannot find field."))) + .Times(0); + ow_->StartObject("")->StartObject("unknown")->EndObject()->EndObject(); + CheckOutput(empty, 0); +} + +TEST_P(ProtoStreamObjectWriterTest, IgnoreUnknownObjectAtAuthor) { + Book expected; + Author* author = expected.mutable_author(); + author->set_name("William"); + author->add_pseudonym("Bill"); + + options_.ignore_unknown_fields = true; + ResetProtoWriter(); + + EXPECT_CALL(listener_, InvalidName(_, _, _)).Times(0); + ow_->StartObject("") + ->StartObject("author") + ->RenderString("name", "William") + ->StartObject("wife") + ->RenderString("name", "Hilary") + ->EndObject() + ->RenderString("pseudonym", "Bill") + ->EndObject() + ->EndObject(); + CheckOutput(expected); +} + +TEST_P(ProtoStreamObjectWriterTest, IgnoreUnknownListAtRoot) { + Book empty; + + options_.ignore_unknown_fields = true; + ResetProtoWriter(); + + EXPECT_CALL(listener_, InvalidName(_, _, _)).Times(0); + ow_->StartObject("")->StartList("unknown")->EndList()->EndObject(); + CheckOutput(empty, 0); +} + +TEST_P(ProtoStreamObjectWriterTest, IgnoreUnknownListAtPublisher) { + Book expected; + expected.set_title("Brainwashing"); + Publisher* publisher = expected.mutable_publisher(); + publisher->set_name("propaganda"); + + options_.ignore_unknown_fields = true; + ResetProtoWriter(); + + EXPECT_CALL(listener_, InvalidName(_, _, _)).Times(0); + ow_->StartObject("") + ->StartObject("publisher") + ->RenderString("name", "propaganda") + ->StartList("alliance") + ->EndList() + ->EndObject() + ->RenderString("title", "Brainwashing") + ->EndObject(); + CheckOutput(expected); +} + +TEST_P(ProtoStreamObjectWriterTest, AcceptUnknownEnumValue) { + ResetTypeInfo(Proto3Message::descriptor()); + + Proto3Message expected; + expected.set_enum_value(static_cast<Proto3Message::NestedEnum>(12345)); + + EXPECT_CALL(listener_, InvalidValue(_, _, _)).Times(0); + ow_->StartObject("") + ->RenderInt32("enumValue", 12345) + ->EndObject(); + CheckOutput(expected); +} + TEST_P(ProtoStreamObjectWriterTest, MissingRequiredField) { Book expected; expected.set_title("My Title"); @@ -862,7 +1096,7 @@ class ProtoStreamObjectWriterTimestampDurationTest : public BaseProtoStreamObjectWriterTest { protected: ProtoStreamObjectWriterTimestampDurationTest() { - vector<const Descriptor*> descriptors; + std::vector<const Descriptor*> descriptors; descriptors.push_back(TimestampDuration::descriptor()); descriptors.push_back(google::protobuf::Timestamp::descriptor()); descriptors.push_back(google::protobuf::Duration::descriptor()); @@ -887,6 +1121,124 @@ TEST_P(ProtoStreamObjectWriterTimestampDurationTest, ParseTimestamp) { CheckOutput(timestamp); } +TEST_P(ProtoStreamObjectWriterTimestampDurationTest, + ParseTimestampYearNotZeroPadded) { + TimestampDuration timestamp; + google::protobuf::Timestamp* ts = timestamp.mutable_ts(); + ts->set_seconds(-61665654145); + ts->set_nanos(33155000); + + ow_->StartObject("") + ->RenderString("ts", "15-11-23T03:37:35.033155Z") + ->EndObject(); + CheckOutput(timestamp); +} + +TEST_P(ProtoStreamObjectWriterTimestampDurationTest, + ParseTimestampYearZeroPadded) { + TimestampDuration timestamp; + google::protobuf::Timestamp* ts = timestamp.mutable_ts(); + ts->set_seconds(-61665654145); + ts->set_nanos(33155000); + + ow_->StartObject("") + ->RenderString("ts", "0015-11-23T03:37:35.033155Z") + ->EndObject(); + CheckOutput(timestamp); +} + +TEST_P(ProtoStreamObjectWriterTimestampDurationTest, + ParseTimestampWithPositiveOffset) { + TimestampDuration timestamp; + google::protobuf::Timestamp* ts = timestamp.mutable_ts(); + ts->set_seconds(1448249855); + ts->set_nanos(33155000); + + ow_->StartObject("") + ->RenderString("ts", "2015-11-23T11:47:35.033155+08:10") + ->EndObject(); + CheckOutput(timestamp); +} + +TEST_P(ProtoStreamObjectWriterTimestampDurationTest, + ParseTimestampWithNegativeOffset) { + TimestampDuration timestamp; + google::protobuf::Timestamp* ts = timestamp.mutable_ts(); + ts->set_seconds(1448249855); + ts->set_nanos(33155000); + + ow_->StartObject("") + ->RenderString("ts", "2015-11-22T19:47:35.033155-07:50") + ->EndObject(); + CheckOutput(timestamp); +} + +TEST_P(ProtoStreamObjectWriterTimestampDurationTest, + TimestampWithInvalidOffset1) { + TimestampDuration timestamp; + + EXPECT_CALL( + listener_, + InvalidValue(_, + StringPiece("type.googleapis.com/google.protobuf.Timestamp"), + StringPiece("Field 'ts', Invalid time format: " + "2016-03-07T15:14:23+"))); + + ow_->StartObject("")->RenderString("ts", "2016-03-07T15:14:23+")->EndObject(); + CheckOutput(timestamp); +} + +TEST_P(ProtoStreamObjectWriterTimestampDurationTest, + TimestampWithInvalidOffset2) { + TimestampDuration timestamp; + + EXPECT_CALL( + listener_, + InvalidValue(_, + StringPiece("type.googleapis.com/google.protobuf.Timestamp"), + StringPiece("Field 'ts', Invalid time format: " + "2016-03-07T15:14:23+08-10"))); + + ow_->StartObject("") + ->RenderString("ts", "2016-03-07T15:14:23+08-10") + ->EndObject(); + CheckOutput(timestamp); +} + +TEST_P(ProtoStreamObjectWriterTimestampDurationTest, + TimestampWithInvalidOffset3) { + TimestampDuration timestamp; + + EXPECT_CALL( + listener_, + InvalidValue(_, + StringPiece("type.googleapis.com/google.protobuf.Timestamp"), + StringPiece("Field 'ts', Invalid time format: " + "2016-03-07T15:14:23+24:10"))); + + ow_->StartObject("") + ->RenderString("ts", "2016-03-07T15:14:23+24:10") + ->EndObject(); + CheckOutput(timestamp); +} + +TEST_P(ProtoStreamObjectWriterTimestampDurationTest, + TimestampWithInvalidOffset4) { + TimestampDuration timestamp; + + EXPECT_CALL( + listener_, + InvalidValue(_, + StringPiece("type.googleapis.com/google.protobuf.Timestamp"), + StringPiece("Field 'ts', Invalid time format: " + "2016-03-07T15:14:23+04:60"))); + + ow_->StartObject("") + ->RenderString("ts", "2016-03-07T15:14:23+04:60") + ->EndObject(); + CheckOutput(timestamp); +} + TEST_P(ProtoStreamObjectWriterTimestampDurationTest, InvalidTimestampError1) { TimestampDuration timestamp; @@ -937,10 +1289,10 @@ TEST_P(ProtoStreamObjectWriterTimestampDurationTest, InvalidTimestampError4) { InvalidValue(_, StringPiece("type.googleapis.com/google.protobuf.Timestamp"), StringPiece("Field 'ts', Invalid time format: " - "-8032-10-18T00:00:00.000Z"))); + "-8031-10-18T00:00:00.000Z"))); ow_->StartObject("") - ->RenderString("ts", "-8032-10-18T00:00:00.000Z") + ->RenderString("ts", "-8031-10-18T00:00:00.000Z") ->EndObject(); CheckOutput(timestamp); } @@ -996,6 +1348,22 @@ TEST_P(ProtoStreamObjectWriterTimestampDurationTest, InvalidTimestampError7) { CheckOutput(timestamp); } +TEST_P(ProtoStreamObjectWriterTimestampDurationTest, InvalidTimestampError8) { + TimestampDuration timestamp; + + EXPECT_CALL( + listener_, + InvalidValue(_, + StringPiece("type.googleapis.com/google.protobuf.Timestamp"), + StringPiece("Field 'ts', Invalid time format: " + "0-12-31T23:59:59.000Z"))); + + ow_->StartObject("") + ->RenderString("ts", "0-12-31T23:59:59.000Z") + ->EndObject(); + CheckOutput(timestamp); +} + TEST_P(ProtoStreamObjectWriterTimestampDurationTest, ParseDuration) { TimestampDuration duration; google::protobuf::Duration* dur = duration.mutable_dur(); @@ -1082,9 +1450,9 @@ TEST_P(ProtoStreamObjectWriterTimestampDurationTest, InvalidValue( _, StringPiece("type.googleapis.com/google.protobuf.Timestamp"), StringPiece( - "Field 'ts', Invalid data type for timestamp, value is null"))) + "Field 'ts', Invalid data type for timestamp, value is 1"))) .With(Args<0>(HasObjectLocation("ts"))); - ow_->StartObject("")->RenderNull("ts")->EndObject(); + ow_->StartObject("")->RenderInt32("ts", 1)->EndObject(); CheckOutput(timestamp); } @@ -1096,8 +1464,22 @@ TEST_P(ProtoStreamObjectWriterTimestampDurationTest, InvalidValue( _, StringPiece("type.googleapis.com/google.protobuf.Duration"), StringPiece( - "Field 'dur', Invalid data type for duration, value is null"))) + "Field 'dur', Invalid data type for duration, value is 1"))) .With(Args<0>(HasObjectLocation("dur"))); + ow_->StartObject("")->RenderInt32("dur", 1)->EndObject(); + CheckOutput(duration); +} + +TEST_P(ProtoStreamObjectWriterTimestampDurationTest, TimestampAcceptsNull) { + TimestampDuration timestamp; + EXPECT_CALL(listener_, InvalidValue(_, _, _)).Times(0); + ow_->StartObject("")->RenderNull("ts")->EndObject(); + CheckOutput(timestamp); +} + +TEST_P(ProtoStreamObjectWriterTimestampDurationTest, DurationAcceptsNull) { + TimestampDuration duration; + EXPECT_CALL(listener_, InvalidValue(_, _, _)).Times(0); ow_->StartObject("")->RenderNull("dur")->EndObject(); CheckOutput(duration); } @@ -1105,8 +1487,11 @@ TEST_P(ProtoStreamObjectWriterTimestampDurationTest, class ProtoStreamObjectWriterStructTest : public BaseProtoStreamObjectWriterTest { protected: - ProtoStreamObjectWriterStructTest() { - vector<const Descriptor*> descriptors; + ProtoStreamObjectWriterStructTest() { ResetProtoWriter(); } + + // Resets ProtoWriter with current set of options and other state. + void ResetProtoWriter() { + std::vector<const Descriptor*> descriptors; descriptors.push_back(StructType::descriptor()); descriptors.push_back(google::protobuf::Struct::descriptor()); ResetTypeInfo(descriptors); @@ -1156,6 +1541,28 @@ TEST_P(ProtoStreamObjectWriterStructTest, StructInvalidInputFailure) { CheckOutput(struct_type); } +TEST_P(ProtoStreamObjectWriterStructTest, StructAcceptsNull) { + StructType struct_type; + EXPECT_CALL(listener_, InvalidValue(_, _, _)).Times(0); + + ow_->StartObject("")->RenderNull("object")->EndObject(); + CheckOutput(struct_type); +} + +TEST_P(ProtoStreamObjectWriterStructTest, StructValuePreservesNull) { + StructType struct_type; + (*struct_type.mutable_object()->mutable_fields())["key"].set_null_value( + google::protobuf::NULL_VALUE); + EXPECT_CALL(listener_, InvalidValue(_, _, _)).Times(0); + + ow_->StartObject("") + ->StartObject("object") + ->RenderNull("key") + ->EndObject() + ->EndObject(); + CheckOutput(struct_type); +} + TEST_P(ProtoStreamObjectWriterStructTest, SimpleRepeatedStructMapKeyTest) { EXPECT_CALL( listener_, @@ -1201,6 +1608,37 @@ TEST_P(ProtoStreamObjectWriterStructTest, RepeatedStructMapObjectKeyTest) { ->EndObject(); } +TEST_P(ProtoStreamObjectWriterStructTest, OptionStructIntAsStringsTest) { + StructType struct_type; + google::protobuf::Struct* s = struct_type.mutable_object(); + s->mutable_fields()->operator[]("k1").set_string_value("123"); + s->mutable_fields()->operator[]("k2").set_bool_value(true); + s->mutable_fields()->operator[]("k3").set_string_value("-222222222"); + s->mutable_fields()->operator[]("k4").set_string_value("33333333"); + + options_.struct_integers_as_strings = true; + ResetProtoWriter(); + + ow_->StartObject("") + ->StartObject("object") + ->RenderDouble("k1", 123) + ->RenderBool("k2", true) + ->RenderInt64("k3", -222222222) + ->RenderUint64("k4", 33333333) + ->EndObject() + ->EndObject(); + CheckOutput(struct_type); +} + +TEST_P(ProtoStreamObjectWriterStructTest, ValuePreservesNull) { + ValueWrapper value; + value.mutable_value()->set_null_value(google::protobuf::NULL_VALUE); + + EXPECT_CALL(listener_, InvalidValue(_, _, _)).Times(0); + ow_->StartObject("")->RenderNull("value")->EndObject(); + CheckOutput(value); +} + class ProtoStreamObjectWriterMapTest : public BaseProtoStreamObjectWriterTest { protected: ProtoStreamObjectWriterMapTest() @@ -1244,11 +1682,16 @@ TEST_P(ProtoStreamObjectWriterMapTest, RepeatedMapKeyTest) { class ProtoStreamObjectWriterAnyTest : public BaseProtoStreamObjectWriterTest { protected: ProtoStreamObjectWriterAnyTest() { - vector<const Descriptor*> descriptors; + std::vector<const Descriptor*> descriptors; descriptors.push_back(AnyOut::descriptor()); + descriptors.push_back(Book::descriptor()); + descriptors.push_back(google::protobuf::Any::descriptor()); descriptors.push_back(google::protobuf::DoubleValue::descriptor()); + descriptors.push_back(google::protobuf::FieldMask::descriptor()); + descriptors.push_back(google::protobuf::Int32Value::descriptor()); + descriptors.push_back(google::protobuf::Struct::descriptor()); descriptors.push_back(google::protobuf::Timestamp::descriptor()); - descriptors.push_back(google::protobuf::Any::descriptor()); + descriptors.push_back(google::protobuf::Value::descriptor()); ResetTypeInfo(descriptors); } }; @@ -1281,8 +1724,7 @@ TEST_P(ProtoStreamObjectWriterAnyTest, RecursiveAny) { any->set_type_url("type.googleapis.com/google.protobuf.Any"); ::google::protobuf::Any nested_any; - nested_any.set_type_url( - "type.googleapis.com/google.protobuf.testing.anys.AnyM"); + nested_any.set_type_url("type.googleapis.com/google.protobuf.testing.AnyM"); AnyM m; m.set_foo("foovalue"); @@ -1295,11 +1737,12 @@ TEST_P(ProtoStreamObjectWriterAnyTest, RecursiveAny) { ->RenderString("@type", "type.googleapis.com/google.protobuf.Any") ->StartObject("value") ->RenderString("@type", - "type.googleapis.com/google.protobuf.testing.anys.AnyM") + "type.googleapis.com/google.protobuf.testing.AnyM") ->RenderString("foo", "foovalue") ->EndObject() ->EndObject() ->EndObject(); + CheckOutput(out, 107); } TEST_P(ProtoStreamObjectWriterAnyTest, DoubleRecursiveAny) { @@ -1312,7 +1755,7 @@ TEST_P(ProtoStreamObjectWriterAnyTest, DoubleRecursiveAny) { ::google::protobuf::Any second_nested_any; second_nested_any.set_type_url( - "type.googleapis.com/google.protobuf.testing.anys.AnyM"); + "type.googleapis.com/google.protobuf.testing.AnyM"); AnyM m; m.set_foo("foovalue"); @@ -1328,18 +1771,110 @@ TEST_P(ProtoStreamObjectWriterAnyTest, DoubleRecursiveAny) { ->RenderString("@type", "type.googleapis.com/google.protobuf.Any") ->StartObject("value") ->RenderString("@type", - "type.googleapis.com/google.protobuf.testing.anys.AnyM") + "type.googleapis.com/google.protobuf.testing.AnyM") ->RenderString("foo", "foovalue") ->EndObject() ->EndObject() ->EndObject() ->EndObject(); + CheckOutput(out, 151); +} + +TEST_P(ProtoStreamObjectWriterAnyTest, TypeUrlAtEnd) { + Book book; + book.set_title("C++"); + book.set_length(1234); + book.set_content("Hello World!"); + + ::google::protobuf::Any any; + any.PackFrom(book); + + ::google::protobuf::Any outer_any; + outer_any.PackFrom(any); + + AnyOut out; + out.mutable_any()->PackFrom(outer_any); + + // Put the @type field at the end of each Any message. Parsers should + // be able to accept that. + ow_->StartObject("") + ->StartObject("any") + ->StartObject("value") + ->StartObject("value") + ->RenderString("title", "C++") + ->RenderInt32("length", 1234) + ->RenderBytes("content", "Hello World!") + ->RenderString("@type", + "type.googleapis.com/google.protobuf.testing.Book") + ->EndObject() + ->RenderString("@type", "type.googleapis.com/google.protobuf.Any") + ->EndObject() + ->RenderString("@type", "type.googleapis.com/google.protobuf.Any") + ->EndObject() + ->EndObject(); + CheckOutput(out); +} + +// Same as TypeUrlAtEnd, but use temporary string values to make sure we don't +// mistakenly store StringPiece objects pointing to invalid memory. +TEST_P(ProtoStreamObjectWriterAnyTest, TypeUrlAtEndWithTemporaryStrings) { + Book book; + book.set_title("C++"); + book.set_length(1234); + book.set_content("Hello World!"); + + ::google::protobuf::Any any; + any.PackFrom(book); + + ::google::protobuf::Any outer_any; + outer_any.PackFrom(any); + + AnyOut out; + out.mutable_any()->PackFrom(outer_any); + + string name, value; + // Put the @type field at the end of each Any message. Parsers should + // be able to accept that. + ow_->StartObject("")->StartObject("any"); + { + ow_->StartObject("value"); + { + ow_->StartObject("value"); + { + name = "title"; + value = "C++"; + ow_->RenderString(name, value); + name = "length"; + ow_->RenderInt32(name, 1234); + name = "content"; + value = "Hello World!"; + ow_->RenderBytes(name, value); + name = "@type"; + value = "type.googleapis.com/google.protobuf.testing.Book"; + ow_->RenderString(name, value); + } + ow_->EndObject(); + + name = "@type"; + value = "type.googleapis.com/google.protobuf.Any"; + ow_->RenderString(name, value); + } + ow_->EndObject(); + + name = "@type"; + value = "type.googleapis.com/google.protobuf.Any"; + ow_->RenderString(name, value); + } + ow_->EndObject()->EndObject(); + CheckOutput(out); } TEST_P(ProtoStreamObjectWriterAnyTest, EmptyAnyFromEmptyObject) { AnyOut out; out.mutable_any(); + EXPECT_CALL(listener_, InvalidValue(_, _, _)).Times(0); + ow_->StartObject("")->StartObject("any")->EndObject()->EndObject(); CheckOutput(out, 2); @@ -1348,11 +1883,10 @@ TEST_P(ProtoStreamObjectWriterAnyTest, EmptyAnyFromEmptyObject) { TEST_P(ProtoStreamObjectWriterAnyTest, AnyWithoutTypeUrlFails1) { AnyOut any; - EXPECT_CALL( - listener_, - InvalidValue(_, StringPiece("Any"), - StringPiece("Missing or invalid @type for any field in " - "google.protobuf.testing.anys.AnyOut"))); + EXPECT_CALL(listener_, + InvalidValue(_, StringPiece("Any"), + StringPiece("Missing @type for any field in " + "google.protobuf.testing.AnyOut"))); ow_->StartObject("") ->StartObject("any") @@ -1366,11 +1900,10 @@ TEST_P(ProtoStreamObjectWriterAnyTest, AnyWithoutTypeUrlFails1) { TEST_P(ProtoStreamObjectWriterAnyTest, AnyWithoutTypeUrlFails2) { AnyOut any; - EXPECT_CALL( - listener_, - InvalidValue(_, StringPiece("Any"), - StringPiece("Missing or invalid @type for any field in " - "google.protobuf.testing.anys.AnyOut"))); + EXPECT_CALL(listener_, + InvalidValue(_, StringPiece("Any"), + StringPiece("Missing @type for any field in " + "google.protobuf.testing.AnyOut"))); ow_->StartObject("") ->StartObject("any") @@ -1384,11 +1917,10 @@ TEST_P(ProtoStreamObjectWriterAnyTest, AnyWithoutTypeUrlFails2) { TEST_P(ProtoStreamObjectWriterAnyTest, AnyWithoutTypeUrlFails3) { AnyOut any; - EXPECT_CALL( - listener_, - InvalidValue(_, StringPiece("Any"), - StringPiece("Missing or invalid @type for any field in " - "google.protobuf.testing.anys.AnyOut"))); + EXPECT_CALL(listener_, + InvalidValue(_, StringPiece("Any"), + StringPiece("Missing @type for any field in " + "google.protobuf.testing.AnyOut"))); ow_->StartObject("") ->StartObject("any") @@ -1433,9 +1965,21 @@ TEST_P(ProtoStreamObjectWriterAnyTest, AnyWithUnknownTypeFails) { CheckOutput(any); } -TEST_P(ProtoStreamObjectWriterAnyTest, AnyNullInputFails) { +TEST_P(ProtoStreamObjectWriterAnyTest, AnyIncorrectInputTypeFails) { AnyOut any; + EXPECT_CALL( + listener_, + InvalidValue(_, StringPiece("type.googleapis.com/google.protobuf.Any"), + StringPiece("1"))); + ow_->StartObject("")->RenderInt32("any", 1)->EndObject(); + CheckOutput(any); +} + +TEST_P(ProtoStreamObjectWriterAnyTest, AnyAcceptsNull) { + AnyOut any; + + EXPECT_CALL(listener_, InvalidValue(_, _, _)).Times(0); ow_->StartObject("")->RenderNull("any")->EndObject(); CheckOutput(any); } @@ -1457,11 +2001,332 @@ TEST_P(ProtoStreamObjectWriterAnyTest, AnyWellKnownTypeErrorTest) { CheckOutput(any); } +// Test the following case: +// +// { +// "any": { +// "@type": "type.googleapis.com/google.protobuf.Value", +// "value": "abc" +// } +// } +TEST_P(ProtoStreamObjectWriterAnyTest, AnyWithNestedPrimitiveValue) { + AnyOut out; + ::google::protobuf::Any* any = out.mutable_any(); + + ::google::protobuf::Value value; + value.set_string_value("abc"); + any->PackFrom(value); + + ow_->StartObject("") + ->StartObject("any") + ->RenderString("@type", "type.googleapis.com/google.protobuf.Value") + ->RenderString("value", "abc") + ->EndObject() + ->EndObject(); + CheckOutput(out); +} + +// Test the following case: +// +// { +// "any": { +// "@type": "type.googleapis.com/google.protobuf.Value", +// "value": { +// "foo": "abc" +// } +// } +// } +TEST_P(ProtoStreamObjectWriterAnyTest, AnyWithNestedObjectValue) { + AnyOut out; + ::google::protobuf::Any* any = out.mutable_any(); + + ::google::protobuf::Value value; + (*value.mutable_struct_value()->mutable_fields())["foo"].set_string_value( + "abc"); + any->PackFrom(value); + + ow_->StartObject("") + ->StartObject("any") + ->RenderString("@type", "type.googleapis.com/google.protobuf.Value") + ->StartObject("value") + ->RenderString("foo", "abc") + ->EndObject() + ->EndObject() + ->EndObject(); + CheckOutput(out); +} + +// Test the following case: +// +// { +// "any": { +// "@type": "type.googleapis.com/google.protobuf.Value", +// "value": ["hello"], +// } +// } +TEST_P(ProtoStreamObjectWriterAnyTest, AnyWithNestedArrayValue) { + AnyOut out; + ::google::protobuf::Any* any = out.mutable_any(); + + ::google::protobuf::Value value; + value.mutable_list_value()->add_values()->set_string_value("hello"); + any->PackFrom(value); + + ow_->StartObject("") + ->StartObject("any") + ->RenderString("@type", "type.googleapis.com/google.protobuf.Value") + ->StartList("value") + ->RenderString("", "hello") + ->EndList() + ->EndObject() + ->EndObject() + ->EndObject(); + CheckOutput(out); +} + +// Test the following case: +// +// { +// "any": { +// "@type": "type.googleapis.com/google.protobuf.Value", +// "not_value": "" +// } +// } +TEST_P(ProtoStreamObjectWriterAnyTest, + AnyWellKnownTypesNoValueFieldForPrimitive) { + EXPECT_CALL( + listener_, + InvalidValue( + _, StringPiece("Any"), + StringPiece("Expect a \"value\" field for well-known types."))); + AnyOut any; + google::protobuf::Any* any_type = any.mutable_any(); + any_type->set_type_url("type.googleapis.com/google.protobuf.Value"); + + ow_->StartObject("") + ->StartObject("any") + ->RenderString("@type", "type.googleapis.com/google.protobuf.Value") + ->RenderString("not_value", "") + ->EndObject() + ->EndObject(); + CheckOutput(any); +} + +// Test the following case: +// +// { +// "any": { +// "@type": "type.googleapis.com/google.protobuf.Value", +// "not_value": {} +// } +// } +TEST_P(ProtoStreamObjectWriterAnyTest, AnyWellKnownTypesNoValueFieldForObject) { + EXPECT_CALL( + listener_, + InvalidValue( + _, StringPiece("Any"), + StringPiece("Expect a \"value\" field for well-known types."))); + AnyOut any; + google::protobuf::Any* any_type = any.mutable_any(); + any_type->set_type_url("type.googleapis.com/google.protobuf.Value"); + + ow_->StartObject("") + ->StartObject("any") + ->RenderString("@type", "type.googleapis.com/google.protobuf.Value") + ->StartObject("not_value") + ->EndObject() + ->EndObject() + ->EndObject(); + CheckOutput(any); +} + +// Test the following case: +// +// { +// "any": { +// "@type": "type.googleapis.com/google.protobuf.Value", +// "not_value": [], +// } +// } +TEST_P(ProtoStreamObjectWriterAnyTest, AnyWellKnownTypesNoValueFieldForArray) { + EXPECT_CALL( + listener_, + InvalidValue( + _, StringPiece("Any"), + StringPiece("Expect a \"value\" field for well-known types."))); + AnyOut any; + google::protobuf::Any* any_type = any.mutable_any(); + any_type->set_type_url("type.googleapis.com/google.protobuf.Value"); + + ow_->StartObject("") + ->StartObject("any") + ->RenderString("@type", "type.googleapis.com/google.protobuf.Value") + ->StartList("not_value") + ->EndList() + ->EndObject() + ->EndObject() + ->EndObject(); + CheckOutput(any); +} + +// Test the following case: +// +// { +// "any": { +// "@type": "type.googleapis.com/google.protobuf.Struct", +// "value": "", +// } +// } +TEST_P(ProtoStreamObjectWriterAnyTest, AnyWellKnownTypesExpectObjectForStruct) { + EXPECT_CALL(listener_, InvalidValue(_, StringPiece("Any"), + StringPiece("Expect a JSON object."))); + AnyOut any; + google::protobuf::Any* any_type = any.mutable_any(); + any_type->set_type_url("type.googleapis.com/google.protobuf.Struct"); + + ow_->StartObject("") + ->StartObject("any") + ->RenderString("@type", "type.googleapis.com/google.protobuf.Struct") + ->RenderString("value", "") + ->EndObject() + ->EndObject(); + CheckOutput(any); +} + +// Test the following case: +// +// { +// "any": { +// "@type": "type.googleapis.com/google.protobuf.Any", +// "value": "", +// } +// } +TEST_P(ProtoStreamObjectWriterAnyTest, AnyWellKnownTypesExpectObjectForAny) { + EXPECT_CALL(listener_, InvalidValue(_, StringPiece("Any"), + StringPiece("Expect a JSON object."))); + AnyOut any; + google::protobuf::Any* any_type = any.mutable_any(); + any_type->set_type_url("type.googleapis.com/google.protobuf.Any"); + + ow_->StartObject("") + ->StartObject("any") + ->RenderString("@type", "type.googleapis.com/google.protobuf.Any") + ->RenderString("value", "") + ->EndObject() + ->EndObject(); + CheckOutput(any); +} + +// { +// "any": { +// "@type": "type.googleapis.com/google.protobuf.Any", +// "value": null +// } +// } +TEST_P(ProtoStreamObjectWriterAnyTest, AnyInAnyAcceptsNull) { + AnyOut out; + google::protobuf::Any empty; + out.mutable_any()->PackFrom(empty); + + EXPECT_CALL(listener_, InvalidValue(_, _, _)).Times(0); + ow_->StartObject("") + ->StartObject("any") + ->RenderString("@type", "type.googleapis.com/google.protobuf.Any") + ->RenderNull("value") + ->EndObject() + ->EndObject(); + CheckOutput(out); +} + +// { +// "any": { +// "@type": "type.googleapis.com/google.protobuf.Timestamp", +// "value": null +// } +// } +TEST_P(ProtoStreamObjectWriterAnyTest, TimestampInAnyAcceptsNull) { + AnyOut out; + google::protobuf::Timestamp empty; + out.mutable_any()->PackFrom(empty); + + EXPECT_CALL(listener_, InvalidValue(_, _, _)).Times(0); + ow_->StartObject("") + ->StartObject("any") + ->RenderString("@type", "type.googleapis.com/google.protobuf.Timestamp") + ->RenderNull("value") + ->EndObject() + ->EndObject(); + CheckOutput(out); +} + +// { +// "any": { +// "@type": "type.googleapis.com/google.protobuf.Duration", +// "value": null +// } +// } +TEST_P(ProtoStreamObjectWriterAnyTest, DurationInAnyAcceptsNull) { + AnyOut out; + google::protobuf::Duration empty; + out.mutable_any()->PackFrom(empty); + + EXPECT_CALL(listener_, InvalidValue(_, _, _)).Times(0); + ow_->StartObject("") + ->StartObject("any") + ->RenderString("@type", "type.googleapis.com/google.protobuf.Duration") + ->RenderNull("value") + ->EndObject() + ->EndObject(); + CheckOutput(out); +} + +// { +// "any": { +// "@type": "type.googleapis.com/google.protobuf.FieldMask", +// "value": null +// } +// } +TEST_P(ProtoStreamObjectWriterAnyTest, FieldMaskInAnyAcceptsNull) { + AnyOut out; + google::protobuf::FieldMask empty; + out.mutable_any()->PackFrom(empty); + + EXPECT_CALL(listener_, InvalidValue(_, _, _)).Times(0); + ow_->StartObject("") + ->StartObject("any") + ->RenderString("@type", "type.googleapis.com/google.protobuf.FieldMask") + ->RenderNull("value") + ->EndObject() + ->EndObject(); + CheckOutput(out); +} + +// { +// "any": { +// "@type": "type.googleapis.com/google.protobuf.Int32Value", +// "value": null +// } +// } +TEST_P(ProtoStreamObjectWriterAnyTest, WrapperInAnyAcceptsNull) { + AnyOut out; + google::protobuf::Int32Value empty; + out.mutable_any()->PackFrom(empty); + + EXPECT_CALL(listener_, InvalidValue(_, _, _)).Times(0); + ow_->StartObject("") + ->StartObject("any") + ->RenderString("@type", "type.googleapis.com/google.protobuf.Int32Value") + ->RenderNull("value") + ->EndObject() + ->EndObject(); + CheckOutput(out); +} + class ProtoStreamObjectWriterFieldMaskTest : public BaseProtoStreamObjectWriterTest { protected: ProtoStreamObjectWriterFieldMaskTest() { - vector<const Descriptor*> descriptors; + std::vector<const Descriptor*> descriptors; descriptors.push_back(FieldMaskTest::descriptor()); descriptors.push_back(google::protobuf::FieldMask::descriptor()); ResetTypeInfo(descriptors); @@ -1702,11 +2567,41 @@ TEST_P(ProtoStreamObjectWriterFieldMaskTest, MapKeyCanContainAnyChars) { CheckOutput(expected); } +TEST_P(ProtoStreamObjectWriterFieldMaskTest, FieldMaskAcceptsNull) { + FieldMaskTest expected; + EXPECT_CALL(listener_, InvalidValue(_, _, _)).Times(0); + ow_->StartObject("")->RenderNull("single_mask")->EndObject(); + CheckOutput(expected); +} + +class ProtoStreamObjectWriterWrappersTest + : public BaseProtoStreamObjectWriterTest { + protected: + ProtoStreamObjectWriterWrappersTest() { + std::vector<const Descriptor*> descriptors; + descriptors.push_back(Int32Wrapper::descriptor()); + descriptors.push_back(google::protobuf::Int32Value::descriptor()); + ResetTypeInfo(descriptors); + } +}; + +INSTANTIATE_TEST_CASE_P(DifferentTypeInfoSourceTest, + ProtoStreamObjectWriterWrappersTest, + ::testing::Values( + testing::USE_TYPE_RESOLVER)); + +TEST_P(ProtoStreamObjectWriterWrappersTest, WrapperAcceptsNull) { + Int32Wrapper wrapper; + EXPECT_CALL(listener_, InvalidName(_, _, _)).Times(0); + ow_->StartObject("")->RenderNull("int32")->EndObject(); + CheckOutput(wrapper); +} + class ProtoStreamObjectWriterOneOfsTest : public BaseProtoStreamObjectWriterTest { protected: ProtoStreamObjectWriterOneOfsTest() { - vector<const Descriptor*> descriptors; + std::vector<const Descriptor*> descriptors; descriptors.push_back(OneOfsRequest::descriptor()); descriptors.push_back(google::protobuf::Struct::descriptor()); ResetTypeInfo(descriptors); @@ -1869,7 +2764,6 @@ TEST_P(ProtoStreamObjectWriterOneOfsTest, StringPiece("oneof field 'data' is already set. " "Cannot set 'intData'"))); - using google::protobuf::testing::oneofs::OneOfsRequest; // JSON: // { "anyData": // { "@type": diff --git a/src/google/protobuf/util/internal/structured_objectwriter.h b/src/google/protobuf/util/internal/structured_objectwriter.h index 3f065d6b..8e63222b 100644 --- a/src/google/protobuf/util/internal/structured_objectwriter.h +++ b/src/google/protobuf/util/internal/structured_objectwriter.h @@ -32,9 +32,6 @@ #define GOOGLE_PROTOBUF_UTIL_CONVERTER_STRUCTURED_OBJECTWRITER_H__ #include <memory> -#ifndef _SHARED_PTR_H -#include <google/protobuf/stubs/shared_ptr.h> -#endif #include <google/protobuf/stubs/casts.h> #include <google/protobuf/stubs/common.h> @@ -80,7 +77,7 @@ class LIBPROTOBUF_EXPORT StructuredObjectWriter : public ObjectWriter { } // Returns true if this element is the root. - bool is_root() const { return parent_ == NULL; } + bool is_root() const { return parent_ == nullptr; } // Returns the number of hops from this element to the root element. int level() const { return level_; } @@ -91,10 +88,10 @@ class LIBPROTOBUF_EXPORT StructuredObjectWriter : public ObjectWriter { private: // Pointer to the parent Element. - google::protobuf::scoped_ptr<BaseElement> parent_; + std::unique_ptr<BaseElement> parent_; // Number of hops to the root Element. - // The root Element has NULL parent_ and a level_ of 0. + // The root Element has nullptr parent_ and a level_ of 0. const int level_; GOOGLE_DISALLOW_IMPLICIT_CONSTRUCTORS(BaseElement); diff --git a/src/google/protobuf/util/internal/testdata/anys.proto b/src/google/protobuf/util/internal/testdata/anys.proto index 18c59cbb..a9ebca3d 100644 --- a/src/google/protobuf/util/internal/testdata/anys.proto +++ b/src/google/protobuf/util/internal/testdata/anys.proto @@ -28,16 +28,75 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// Proto to test Proto3 Any serialization. syntax = "proto3"; -package google.protobuf.testing.anys; -option java_package = "com.google.protobuf.testing.anys"; +package google.protobuf.testing; import "google/protobuf/any.proto"; +import "google/protobuf/struct.proto"; +import "google/protobuf/timestamp.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; + +// Top-level test cases proto used by MarshallingTest. See description +// at the top of the class MarshallingTest for details on how to write +// test cases. +message AnyTestCases { + AnyWrapper empty_any = 1; + AnyWrapper type_only_any = 2; + AnyWrapper wrapper_any = 3; + AnyWrapper any_with_timestamp_value = 4; + AnyWrapper any_with_duration_value = 5; + AnyWrapper any_with_struct_value = 6; + AnyWrapper recursive_any = 7; + AnyWrapper any_with_message_value = 8; + AnyWrapper any_with_nested_message = 9; + AnyWrapper any_with_message_with_wrapper_type = 10; + AnyWrapper any_with_message_with_timestamp = 11; + AnyWrapper any_with_message_containing_map = 12; + AnyWrapper any_with_message_containing_struct = 13; + AnyWrapper any_with_message_containing_repeated_message = 14; + AnyWrapper recursive_any_with_type_field_at_end = 15; + + google.protobuf.Any top_level_any = 50; + google.protobuf.Any top_level_any_with_type_field_at_end = 51; +} + +message AnyWrapper { + google.protobuf.Any any = 1; +} + +// Hack to make sure the types we put into the any are included in the types. +// Real solution is to add these types to the service config. +message Imports { + google.protobuf.DoubleValue dbl = 1; + google.protobuf.Struct struct = 2; + google.protobuf.Timestamp timestamp = 3; + google.protobuf.Duration duration = 4; + google.protobuf.Int32Value i32 = 5; + Data data = 100; +} + +message Data { + int32 attr = 1; + string str = 2; + repeated string msgs = 3; + Data nested_data = 4; + google.protobuf.Int32Value int_wrapper = 5; + google.protobuf.Timestamp time = 6; + map<string, string> map_data = 7; + google.protobuf.Struct struct_data = 8; + repeated Data repeated_data = 9; +} + +service AnyTestService { + rpc Call(AnyTestCases) returns (AnyTestCases); + rpc Call1(Imports) returns (Imports); +} message AnyIn { string something = 1; + google.protobuf.Any any = 2; } message AnyOut { @@ -47,7 +106,3 @@ message AnyOut { message AnyM { string foo = 1; } - -service TestService { - rpc Call(AnyIn) returns (AnyOut); -} diff --git a/src/google/protobuf/util/internal/testdata/books.proto b/src/google/protobuf/util/internal/testdata/books.proto index 82b81760..5630cc78 100644 --- a/src/google/protobuf/util/internal/testdata/books.proto +++ b/src/google/protobuf/util/internal/testdata/books.proto @@ -31,6 +31,10 @@ // Author: sven@google.com (Sven Mawson) // // Sample protos for testing. + +// Some of the older enums don't use CAPITALS_WITH_UNDERSCORES for testing. +// LINT: LEGACY_NAMES + syntax = "proto2"; package google.protobuf.testing; @@ -56,6 +60,15 @@ message Book { optional Publisher publisher = 9; repeated Label labels = 10; + enum Type { + FICTION = 1; + KIDS = 2; + ACTION_AND_ADVENTURE = 3; + arts_and_photography = 4; + I18N_Tech = 5; + } + optional Type type = 11; + extensions 200 to 499; } @@ -169,3 +182,21 @@ message NestedBook { message BadNestedBook { repeated uint32 book = 1 [packed=true]; // Packed to optional message. } + +// A recursively defined message. +message Cyclic { + optional int32 m_int = 1; + optional string m_str = 2; + optional Book m_book = 3; + repeated Author m_author = 5; + optional Cyclic m_cyclic = 4; +} + +// Test that two messages can have different fields mapped to the same JSON +// name. See: https://github.com/google/protobuf/issues/1415 +message TestJsonName1 { + optional int32 one_value = 1 [json_name = "value"]; +} +message TestJsonName2 { + optional int32 another_value = 1 [json_name = "value"]; +} diff --git a/src/google/protobuf/util/internal/testdata/maps.proto b/src/google/protobuf/util/internal/testdata/maps.proto index 6475ecdd..0f381b32 100644 --- a/src/google/protobuf/util/internal/testdata/maps.proto +++ b/src/google/protobuf/util/internal/testdata/maps.proto @@ -28,11 +28,76 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// Proto to test proto3 maps. syntax = "proto3"; -package google.protobuf.testing.maps; -option java_package = "com.google.protobuf.testing.maps"; +package google.protobuf.testing; + +// Top-level test cases proto used by MarshallingTest. See description +// at the top of the class MarshallingTest for details on how to write +// test cases. +message MapsTestCases { + EmptyMap empty_map = 1; + StringtoInt string_to_int = 2; + IntToString int_to_string = 3; + Mixed1 mixed1 = 4; + Mixed2 mixed2 = 5; + MapOfObjects map_of_objects = 6; + + // Empty key tests + StringtoInt empty_key_string_to_int1 = 7; + StringtoInt empty_key_string_to_int2 = 8; + StringtoInt empty_key_string_to_int3 = 9; + BoolToString empty_key_bool_to_string = 10; + IntToString empty_key_int_to_string = 11; + Mixed1 empty_key_mixed = 12; + MapOfObjects empty_key_map_objects = 13; +} + +message EmptyMap { + map<int32, int32> map = 1; +} + +message StringtoInt { + map<string, int32> map = 1; +} + +message IntToString { + map<int32, string> map = 1; +} + +message BoolToString { + map<bool, string> map = 1; +} + +message Mixed1 { + string msg = 1; + map<string, float> map = 2; +} + +message Mixed2 { + enum E { + E0 = 0; + E1 = 1; + E2 = 2; + E3 = 3; + } + map<int32, bool> map = 1; + E ee = 2; +} + +message MapOfObjects { + message M { + string inner_text = 1; + } + map<string, M> map = 1; +} + +message DummyRequest { +} + +service MapsTestService { + rpc Call(DummyRequest) returns (MapsTestCases); +} message MapIn { string other = 1; @@ -79,8 +144,3 @@ message MapOutWireFormat { message MapM { string foo = 1; } - -service TestService { - rpc Call1(MapIn) returns (MapOut); - rpc Call2(MapIn) returns (MapOut); -} diff --git a/src/google/protobuf/util/internal/testdata/oneofs.proto b/src/google/protobuf/util/internal/testdata/oneofs.proto index 5bc9fa08..c37da083 100644 --- a/src/google/protobuf/util/internal/testdata/oneofs.proto +++ b/src/google/protobuf/util/internal/testdata/oneofs.proto @@ -28,7 +28,7 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// Proto to test oneofs. +// Proto to test proto3 oneofs. syntax = "proto3"; import "google/protobuf/any.proto"; @@ -36,7 +36,6 @@ import "google/protobuf/struct.proto"; import "google/protobuf/timestamp.proto"; package google.protobuf.testing.oneofs; -option java_package = "com.google.protobuf.testing.oneofs"; message OneOfsRequest { string value = 1; @@ -45,24 +44,34 @@ message OneOfsRequest { int32 int_data = 3; // Simple message Data message_data = 4; + MoreData more_data = 5; // Well known types - google.protobuf.Struct struct_data = 5; - google.protobuf.Value value_data = 6; - google.protobuf.ListValue list_value_data = 7; - google.protobuf.Timestamp ts_data = 8; + google.protobuf.Struct struct_data = 6; + google.protobuf.Value value_data = 7; + google.protobuf.ListValue list_value_data = 8; + google.protobuf.Timestamp ts_data = 9; } google.protobuf.Any any_data = 19; } +message RequestWithSimpleOneof { + string value = 1; + oneof data { + string str_data = 2; + int32 int_data = 3; + Data message_data = 4; + MoreData more_data = 5; + } +} + message Data { int32 data_value = 1; } -message Response { - string value = 1; +message MoreData { + string str_value = 1; } -service TestService { - // Test call. - rpc Call(OneOfsRequest) returns (Response); +message Response { + string value = 1; } diff --git a/src/google/protobuf/util/internal/testdata/proto3.proto b/src/google/protobuf/util/internal/testdata/proto3.proto new file mode 100644 index 00000000..c013cee3 --- /dev/null +++ b/src/google/protobuf/util/internal/testdata/proto3.proto @@ -0,0 +1,42 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf.testing; + +message Proto3Message { + enum NestedEnum { + FOO = 0; + BAR = 1; + BAZ = 2; + } + NestedEnum enum_value = 1; +} diff --git a/src/google/protobuf/util/internal/testdata/struct.proto b/src/google/protobuf/util/internal/testdata/struct.proto index c15aba0d..7b1cc1b9 100644 --- a/src/google/protobuf/util/internal/testdata/struct.proto +++ b/src/google/protobuf/util/internal/testdata/struct.proto @@ -28,18 +28,90 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// Proto to test proto3 struct. syntax = "proto3"; -package google.protobuf.testing.structs; -option java_package = "com.google.protobuf.testing.structs"; +package google.protobuf.testing; import "google/protobuf/struct.proto"; -message StructType { - google.protobuf.Struct object = 1; +message StructTestCases { + StructWrapper empty_value = 1; + StructWrapper empty_value2 = 2; + StructWrapper null_value = 3; + StructWrapper simple_struct = 4; + StructWrapper longer_struct = 5; + StructWrapper struct_with_nested_struct = 6; + StructWrapper struct_with_nested_list = 7; + StructWrapper struct_with_list_of_nulls = 8; + StructWrapper struct_with_list_of_lists = 9; + StructWrapper struct_with_list_of_structs = 10; + StructWrapper struct_with_empty_list = 11; + StructWrapper struct_with_list_with_empty_struct = 12; + google.protobuf.Struct top_level_struct = 13; + google.protobuf.Struct top_level_struct_with_empty_list = 14; + google.protobuf.Struct top_level_struct_with_list_with_empty_struct = 15; + ValueWrapper value_wrapper_simple = 16; + ValueWrapper value_wrapper_with_struct = 17; + ValueWrapper value_wrapper_with_list = 18; + ValueWrapper value_wrapper_with_empty_list = 19; + ValueWrapper value_wrapper_with_list_with_empty_struct = 20; + ListValueWrapper list_value_wrapper = 21; + ListValueWrapper list_value_wrapper_with_empty_list = 22; + ListValueWrapper list_value_wrapper_with_list_with_empty_struct = 23; + google.protobuf.Value top_level_value_simple = 24; + google.protobuf.Value top_level_value_with_struct = 25; + google.protobuf.Value top_level_value_with_list = 26; + google.protobuf.Value top_level_value_with_empty_list = 27; + google.protobuf.Value top_level_value_with_list_with_empty_struct = 28; + google.protobuf.ListValue top_level_listvalue = 29; + google.protobuf.ListValue top_level_empty_listvalue = 30; + google.protobuf.ListValue top_level_listvalue_with_empty_struct = 31; + RepeatedValueWrapper repeated_value = 32; + RepeatedValueWrapper repeated_value_nested_list = 33; + RepeatedValueWrapper repeated_value_nested_list2 = 34; + RepeatedValueWrapper repeated_value_nested_list3 = 35; + RepeatedListValueWrapper repeated_listvalue = 36; + MapOfStruct map_of_struct = 37; + MapOfStruct map_of_struct_value = 38; + MapOfStruct map_of_listvalue = 39; +} + +message StructWrapper { + google.protobuf.Struct struct = 1; +} + +message ValueWrapper { + google.protobuf.Value value = 1; +} + +message RepeatedValueWrapper { + repeated google.protobuf.Value values = 1; +} + +message ListValueWrapper { + google.protobuf.ListValue shopping_list = 1; } -service TestService { - rpc Call(StructType) returns (StructType); +message RepeatedListValueWrapper { + repeated google.protobuf.ListValue dimensions = 1; +} + +message MapOfStruct { + map<string, google.protobuf.Struct> struct_map = 1; + map<string, google.protobuf.Value> value_map = 2; + map<string, google.protobuf.ListValue> listvalue_map = 3; +} + +// Hack to test return types with Struct as top-level message. Struct typers +// cannot be directly used in API requests. Hence using Dummy as request type. +message Dummy { + string text = 1; +} + +service StructTestService { + rpc Call(Dummy) returns (StructTestCases); +} + +message StructType { + google.protobuf.Struct object = 1; } diff --git a/src/google/protobuf/util/internal/testdata/timestamp_duration.proto b/src/google/protobuf/util/internal/testdata/timestamp_duration.proto index 56351f16..b74484ce 100644 --- a/src/google/protobuf/util/internal/testdata/timestamp_duration.proto +++ b/src/google/protobuf/util/internal/testdata/timestamp_duration.proto @@ -28,20 +28,53 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// Proto to test proto3 Timestamp and Duration. syntax = "proto3"; -package google.protobuf.testing.timestampduration; -option java_package = "com.google.protobuf.testing.timestampduration"; +package google.protobuf.testing; import "google/protobuf/timestamp.proto"; import "google/protobuf/duration.proto"; +message TimestampDurationTestCases { + // Timestamp tests + TimeStampType epoch = 1; + TimeStampType epoch2 = 2; + TimeStampType mintime = 3; + TimeStampType maxtime = 4; + TimeStampType timeval1 = 5; + TimeStampType timeval2 = 6; + TimeStampType timeval3 = 7; + TimeStampType timeval4 = 8; + TimeStampType timeval5 = 9; + TimeStampType timeval6 = 10; + TimeStampType timeval7 = 11; + google.protobuf.Timestamp timeval8 = 12; + + // Duration tests + DurationType zero_duration = 101; + DurationType min_duration = 102; + DurationType max_duration = 103; + DurationType duration1 = 104; + DurationType duration2 = 105; + DurationType duration3 = 106; + DurationType duration4 = 107; + google.protobuf.Duration duration5 = 108; +} + +message TimeStampType { + google.protobuf.Timestamp timestamp = 1; +} + +message DurationType { + google.protobuf.Duration duration = 1; +} + +service TimestampDurationTestService { + rpc Call(TimestampDurationTestCases) returns (TimestampDurationTestCases); +} + message TimestampDuration { google.protobuf.Timestamp ts = 1; google.protobuf.Duration dur = 2; -} - -service TestService { - rpc Call(TimestampDuration) returns (TimestampDuration); + repeated google.protobuf.Timestamp rep_ts = 3; } diff --git a/src/google/protobuf/util/internal/type_info.cc b/src/google/protobuf/util/internal/type_info.cc index 00a8ee7a..3847b179 100644 --- a/src/google/protobuf/util/internal/type_info.cc +++ b/src/google/protobuf/util/internal/type_info.cc @@ -60,7 +60,8 @@ class TypeInfoForTypeResolver : public TypeInfo { virtual util::StatusOr<const google::protobuf::Type*> ResolveTypeUrl( StringPiece type_url) const { - map<StringPiece, StatusOrType>::iterator it = cached_types_.find(type_url); + std::map<StringPiece, StatusOrType>::iterator it = + cached_types_.find(type_url); if (it != cached_types_.end()) { return it->second; } @@ -68,7 +69,7 @@ class TypeInfoForTypeResolver : public TypeInfo { // cached_types_ map. const string& string_type_url = *string_storage_.insert(type_url.ToString()).first; - google::protobuf::scoped_ptr<google::protobuf::Type> type(new google::protobuf::Type()); + std::unique_ptr<google::protobuf::Type> type(new google::protobuf::Type()); util::Status status = type_resolver_->ResolveMessageType(string_type_url, type.get()); StatusOrType result = @@ -85,7 +86,8 @@ class TypeInfoForTypeResolver : public TypeInfo { virtual const google::protobuf::Enum* GetEnumByTypeUrl( StringPiece type_url) const { - map<StringPiece, StatusOrEnum>::iterator it = cached_enums_.find(type_url); + std::map<StringPiece, StatusOrEnum>::iterator it = + cached_enums_.find(type_url); if (it != cached_enums_.end()) { return it->second.ok() ? it->second.ValueOrDie() : NULL; } @@ -93,7 +95,7 @@ class TypeInfoForTypeResolver : public TypeInfo { // cached_enums_ map. const string& string_type_url = *string_storage_.insert(type_url.ToString()).first; - google::protobuf::scoped_ptr<google::protobuf::Enum> enum_type( + std::unique_ptr<google::protobuf::Enum> enum_type( new google::protobuf::Enum()); util::Status status = type_resolver_->ResolveEnumType(string_type_url, enum_type.get()); @@ -105,12 +107,14 @@ class TypeInfoForTypeResolver : public TypeInfo { virtual const google::protobuf::Field* FindField( const google::protobuf::Type* type, StringPiece camel_case_name) const { - if (indexed_types_.find(type) == indexed_types_.end()) { - PopulateNameLookupTable(type); - indexed_types_.insert(type); - } + std::map<const google::protobuf::Type*, CamelCaseNameTable>::const_iterator + it = indexed_types_.find(type); + const CamelCaseNameTable& camel_case_name_table = + (it == indexed_types_.end()) + ? PopulateNameLookupTable(type, &indexed_types_[type]) + : it->second; StringPiece name = - FindWithDefault(camel_case_name_table_, camel_case_name, StringPiece()); + FindWithDefault(camel_case_name_table, camel_case_name, StringPiece()); if (name.empty()) { // Didn't find a mapping. Use whatever provided. name = camel_case_name; @@ -121,10 +125,11 @@ class TypeInfoForTypeResolver : public TypeInfo { private: typedef util::StatusOr<const google::protobuf::Type*> StatusOrType; typedef util::StatusOr<const google::protobuf::Enum*> StatusOrEnum; + typedef std::map<StringPiece, StringPiece> CamelCaseNameTable; template <typename T> - static void DeleteCachedTypes(map<StringPiece, T>* cached_types) { - for (typename map<StringPiece, T>::iterator it = cached_types->begin(); + static void DeleteCachedTypes(std::map<StringPiece, T>* cached_types) { + for (typename std::map<StringPiece, T>::iterator it = cached_types->begin(); it != cached_types->end(); ++it) { if (it->second.ok()) { delete it->second.ValueOrDie(); @@ -132,32 +137,35 @@ class TypeInfoForTypeResolver : public TypeInfo { } } - void PopulateNameLookupTable(const google::protobuf::Type* type) const { + const CamelCaseNameTable& PopulateNameLookupTable( + const google::protobuf::Type* type, + CamelCaseNameTable* camel_case_name_table) const { for (int i = 0; i < type->fields_size(); ++i) { const google::protobuf::Field& field = type->fields(i); StringPiece name = field.name(); StringPiece camel_case_name = field.json_name(); - const StringPiece* existing = InsertOrReturnExisting( - &camel_case_name_table_, camel_case_name, name); + const StringPiece* existing = + InsertOrReturnExisting(camel_case_name_table, camel_case_name, name); if (existing && *existing != name) { GOOGLE_LOG(WARNING) << "Field '" << name << "' and '" << *existing << "' map to the same camel case name '" << camel_case_name << "'."; } } + return *camel_case_name_table; } TypeResolver* type_resolver_; // Stores string values that will be referenced by StringPieces in - // cached_types_, cached_enums_ and camel_case_name_table_. - mutable set<string> string_storage_; + // cached_types_, cached_enums_. + mutable std::set<string> string_storage_; - mutable map<StringPiece, StatusOrType> cached_types_; - mutable map<StringPiece, StatusOrEnum> cached_enums_; + mutable std::map<StringPiece, StatusOrType> cached_types_; + mutable std::map<StringPiece, StatusOrEnum> cached_enums_; - mutable set<const google::protobuf::Type*> indexed_types_; - mutable map<StringPiece, StringPiece> camel_case_name_table_; + mutable std::map<const google::protobuf::Type*, CamelCaseNameTable> + indexed_types_; }; } // namespace diff --git a/src/google/protobuf/util/internal/type_info_test_helper.cc b/src/google/protobuf/util/internal/type_info_test_helper.cc index 1b9c5154..281a7f58 100644 --- a/src/google/protobuf/util/internal/type_info_test_helper.cc +++ b/src/google/protobuf/util/internal/type_info_test_helper.cc @@ -31,9 +31,6 @@ #include <google/protobuf/util/internal/type_info_test_helper.h> #include <memory> -#ifndef _SHARED_PTR_H -#include <google/protobuf/stubs/shared_ptr.h> -#endif #include <vector> #include <google/protobuf/stubs/logging.h> @@ -55,7 +52,7 @@ namespace testing { void TypeInfoTestHelper::ResetTypeInfo( - const vector<const Descriptor*>& descriptors) { + const std::vector<const Descriptor*>& descriptors) { switch (type_) { case USE_TYPE_RESOLVER: { const DescriptorPool* pool = descriptors[0]->file()->pool(); @@ -73,14 +70,14 @@ void TypeInfoTestHelper::ResetTypeInfo( } void TypeInfoTestHelper::ResetTypeInfo(const Descriptor* descriptor) { - vector<const Descriptor*> descriptors; + std::vector<const Descriptor*> descriptors; descriptors.push_back(descriptor); ResetTypeInfo(descriptors); } void TypeInfoTestHelper::ResetTypeInfo(const Descriptor* descriptor1, const Descriptor* descriptor2) { - vector<const Descriptor*> descriptors; + std::vector<const Descriptor*> descriptors; descriptors.push_back(descriptor1); descriptors.push_back(descriptor2); ResetTypeInfo(descriptors); @@ -102,13 +99,13 @@ ProtoStreamObjectSource* TypeInfoTestHelper::NewProtoSource( } ProtoStreamObjectWriter* TypeInfoTestHelper::NewProtoWriter( - const string& type_url, strings::ByteSink* output, - ErrorListener* listener) { + const string& type_url, strings::ByteSink* output, ErrorListener* listener, + const ProtoStreamObjectWriter::Options& options) { const google::protobuf::Type* type = typeinfo_->GetTypeByTypeUrl(type_url); switch (type_) { case USE_TYPE_RESOLVER: { return new ProtoStreamObjectWriter(type_resolver_.get(), *type, output, - listener); + listener, options); } } GOOGLE_LOG(FATAL) << "Can not reach here."; diff --git a/src/google/protobuf/util/internal/type_info_test_helper.h b/src/google/protobuf/util/internal/type_info_test_helper.h index 6916a73b..5a077e04 100644 --- a/src/google/protobuf/util/internal/type_info_test_helper.h +++ b/src/google/protobuf/util/internal/type_info_test_helper.h @@ -32,15 +32,12 @@ #define GOOGLE_PROTOBUF_UTIL_CONVERTER_TYPE_INFO_TEST_HELPER_H__ #include <memory> -#ifndef _SHARED_PTR_H -#include <google/protobuf/stubs/shared_ptr.h> -#endif #include <vector> #include <google/protobuf/io/coded_stream.h> #include <google/protobuf/descriptor.h> -#include <google/protobuf/util/internal/type_info.h> #include <google/protobuf/util/internal/default_value_objectwriter.h> +#include <google/protobuf/util/internal/type_info.h> #include <google/protobuf/util/internal/protostream_objectsource.h> #include <google/protobuf/util/internal/protostream_objectwriter.h> #include <google/protobuf/util/type_resolver.h> @@ -64,7 +61,7 @@ class TypeInfoTestHelper { explicit TypeInfoTestHelper(TypeInfoSource type) : type_(type) {} // Creates a TypeInfo object for the given set of descriptors. - void ResetTypeInfo(const vector<const Descriptor*>& descriptors); + void ResetTypeInfo(const std::vector<const Descriptor*>& descriptors); // Convinent overloads. void ResetTypeInfo(const Descriptor* descriptor); @@ -77,17 +74,17 @@ class TypeInfoTestHelper { ProtoStreamObjectSource* NewProtoSource(io::CodedInputStream* coded_input, const string& type_url); - ProtoStreamObjectWriter* NewProtoWriter(const string& type_url, - strings::ByteSink* output, - ErrorListener* listener); + ProtoStreamObjectWriter* NewProtoWriter( + const string& type_url, strings::ByteSink* output, + ErrorListener* listener, const ProtoStreamObjectWriter::Options& options); DefaultValueObjectWriter* NewDefaultValueWriter(const string& type_url, ObjectWriter* writer); private: TypeInfoSource type_; - google::protobuf::scoped_ptr<TypeInfo> typeinfo_; - google::protobuf::scoped_ptr<TypeResolver> type_resolver_; + std::unique_ptr<TypeInfo> typeinfo_; + std::unique_ptr<TypeResolver> type_resolver_; }; } // namespace testing } // namespace converter diff --git a/src/google/protobuf/util/internal/utility.cc b/src/google/protobuf/util/internal/utility.cc index 1ddf2487..b8d917ce 100644 --- a/src/google/protobuf/util/internal/utility.cc +++ b/src/google/protobuf/util/internal/utility.cc @@ -30,6 +30,8 @@ #include <google/protobuf/util/internal/utility.h> +#include <algorithm> + #include <google/protobuf/stubs/callback.h> #include <google/protobuf/stubs/common.h> #include <google/protobuf/stubs/logging.h> @@ -46,21 +48,11 @@ namespace protobuf { namespace util { namespace converter { -namespace { -const StringPiece SkipWhiteSpace(StringPiece str) { - StringPiece::size_type i; - for (i = 0; i < str.size() && isspace(str[i]); ++i) { - } - GOOGLE_DCHECK(i == str.size() || !isspace(str[i])); - return StringPiece(str, i); -} -} // namespace - bool GetBoolOptionOrDefault( const google::protobuf::RepeatedPtrField<google::protobuf::Option>& options, const string& option_name, bool default_value) { const google::protobuf::Option* opt = FindOptionOrNull(options, option_name); - if (opt == NULL) { + if (opt == nullptr) { return default_value; } return GetBoolFromAny(opt->value()); @@ -70,7 +62,7 @@ int64 GetInt64OptionOrDefault( const google::protobuf::RepeatedPtrField<google::protobuf::Option>& options, const string& option_name, int64 default_value) { const google::protobuf::Option* opt = FindOptionOrNull(options, option_name); - if (opt == NULL) { + if (opt == nullptr) { return default_value; } return GetInt64FromAny(opt->value()); @@ -80,7 +72,7 @@ double GetDoubleOptionOrDefault( const google::protobuf::RepeatedPtrField<google::protobuf::Option>& options, const string& option_name, double default_value) { const google::protobuf::Option* opt = FindOptionOrNull(options, option_name); - if (opt == NULL) { + if (opt == nullptr) { return default_value; } return GetDoubleFromAny(opt->value()); @@ -90,7 +82,7 @@ string GetStringOptionOrDefault( const google::protobuf::RepeatedPtrField<google::protobuf::Option>& options, const string& option_name, const string& default_value) { const google::protobuf::Option* opt = FindOptionOrNull(options, option_name); - if (opt == NULL) { + if (opt == nullptr) { return default_value; } return GetStringFromAny(opt->value()); @@ -128,8 +120,15 @@ string GetStringFromAny(const google::protobuf::Any& any) { } const StringPiece GetTypeWithoutUrl(StringPiece type_url) { - size_t idx = type_url.rfind('/'); - return type_url.substr(idx + 1); + if (type_url.size() > kTypeUrlSize && type_url[kTypeUrlSize] == '/') { + return type_url.substr(kTypeUrlSize + 1); + } else { + size_t idx = type_url.rfind('/'); + if (idx != type_url.npos) { + type_url.remove_prefix(idx + 1); + } + return type_url; + } } const string GetFullTypeWithUrl(StringPiece simple_type) { @@ -145,12 +144,12 @@ const google::protobuf::Option* FindOptionOrNull( return &opt; } } - return NULL; + return nullptr; } const google::protobuf::Field* FindFieldInTypeOrNull( const google::protobuf::Type* type, StringPiece field_name) { - if (type != NULL) { + if (type != nullptr) { for (int i = 0; i < type->fields_size(); ++i) { const google::protobuf::Field& field = type->fields(i); if (field.name() == field_name) { @@ -158,12 +157,12 @@ const google::protobuf::Field* FindFieldInTypeOrNull( } } } - return NULL; + return nullptr; } const google::protobuf::Field* FindJsonFieldInTypeOrNull( const google::protobuf::Type* type, StringPiece json_name) { - if (type != NULL) { + if (type != nullptr) { for (int i = 0; i < type->fields_size(); ++i) { const google::protobuf::Field& field = type->fields(i); if (field.json_name() == json_name) { @@ -171,12 +170,25 @@ const google::protobuf::Field* FindJsonFieldInTypeOrNull( } } } - return NULL; + return nullptr; +} + +const google::protobuf::Field* FindFieldInTypeByNumberOrNull( + const google::protobuf::Type* type, int32 number) { + if (type != nullptr) { + for (int i = 0; i < type->fields_size(); ++i) { + const google::protobuf::Field& field = type->fields(i); + if (field.number() == number) { + return &field; + } + } + } + return nullptr; } const google::protobuf::EnumValue* FindEnumValueByNameOrNull( const google::protobuf::Enum* enum_type, StringPiece enum_name) { - if (enum_type != NULL) { + if (enum_type != nullptr) { for (int i = 0; i < enum_type->enumvalue_size(); ++i) { const google::protobuf::EnumValue& enum_value = enum_type->enumvalue(i); if (enum_value.name() == enum_name) { @@ -184,12 +196,12 @@ const google::protobuf::EnumValue* FindEnumValueByNameOrNull( } } } - return NULL; + return nullptr; } const google::protobuf::EnumValue* FindEnumValueByNumberOrNull( const google::protobuf::Enum* enum_type, int32 value) { - if (enum_type != NULL) { + if (enum_type != nullptr) { for (int i = 0; i < enum_type->enumvalue_size(); ++i) { const google::protobuf::EnumValue& enum_value = enum_type->enumvalue(i); if (enum_value.number() == value) { @@ -197,7 +209,40 @@ const google::protobuf::EnumValue* FindEnumValueByNumberOrNull( } } } - return NULL; + return nullptr; +} + +const google::protobuf::EnumValue* FindEnumValueByNameWithoutUnderscoreOrNull( + const google::protobuf::Enum* enum_type, StringPiece enum_name) { + if (enum_type != nullptr) { + for (int i = 0; i < enum_type->enumvalue_size(); ++i) { + const google::protobuf::EnumValue& enum_value = enum_type->enumvalue(i); + string enum_name_without_underscore = enum_value.name(); + + // Remove underscore from the name. + enum_name_without_underscore.erase( + std::remove(enum_name_without_underscore.begin(), + enum_name_without_underscore.end(), '_'), + enum_name_without_underscore.end()); + // Make the name uppercase. + for (string::iterator it = enum_name_without_underscore.begin(); + it != enum_name_without_underscore.end(); ++it) { + *it = ascii_toupper(*it); + } + + if (enum_name_without_underscore == enum_name) { + return &enum_value; + } + } + } + return nullptr; +} + +string EnumValueNameToLowerCamelCase(const StringPiece input) { + string input_string(input); + std::transform(input_string.begin(), input_string.end(), input_string.begin(), + ::tolower); + return ToCamelCase(input_string); } string ToCamelCase(const StringPiece input) { @@ -222,6 +267,7 @@ string ToCamelCase(const StringPiece input) { if (!result.empty() && is_cap && (!was_cap || (i + 1 < input.size() && ascii_islower(input[i + 1])))) { first_word = false; + result.push_back(input[i]); } else { result.push_back(ascii_tolower(input[i])); continue; @@ -231,9 +277,13 @@ string ToCamelCase(const StringPiece input) { if (ascii_islower(input[i])) { result.push_back(ascii_toupper(input[i])); continue; + } else { + result.push_back(input[i]); + continue; } + } else { + result.push_back(ascii_tolower(input[i])); } - result.push_back(input[i]); } return result; } @@ -274,7 +324,7 @@ string ToSnakeCase(StringPiece input) { return result; } -set<string>* well_known_types_ = NULL; +std::set<string>* well_known_types_ = NULL; GOOGLE_PROTOBUF_DECLARE_ONCE(well_known_types_init_); const char* well_known_types_name_array_[] = { "google.protobuf.Timestamp", "google.protobuf.Duration", @@ -287,7 +337,7 @@ const char* well_known_types_name_array_[] = { void DeleteWellKnownTypes() { delete well_known_types_; } void InitWellKnownTypes() { - well_known_types_ = new set<string>; + well_known_types_ = new std::set<string>; for (int i = 0; i < GOOGLE_ARRAYSIZE(well_known_types_name_array_); ++i) { well_known_types_->insert(well_known_types_name_array_[i]); } @@ -306,15 +356,20 @@ bool IsValidBoolString(const string& bool_string) { bool IsMap(const google::protobuf::Field& field, const google::protobuf::Type& type) { - return (field.cardinality() == - google::protobuf::Field_Cardinality_CARDINALITY_REPEATED && + return field.cardinality() == + google::protobuf::Field_Cardinality_CARDINALITY_REPEATED && + (GetBoolOptionOrDefault(type.options(), "map_entry", false) || GetBoolOptionOrDefault(type.options(), - "google.protobuf.MessageOptions.map_entry", false)); + "google.protobuf.MessageOptions.map_entry", + false)); } bool IsMessageSetWireFormat(const google::protobuf::Type& type) { - return GetBoolOptionOrDefault( - type.options(), "google.protobuf.MessageOptions.message_set_wire_format", false); + return GetBoolOptionOrDefault(type.options(), "message_set_wire_format", + false) || + GetBoolOptionOrDefault( + type.options(), + "google.protobuf.MessageOptions.message_set_wire_format", false); } string DoubleAsString(double value) { @@ -350,6 +405,13 @@ bool SafeStrToFloat(StringPiece str, float* value) { return true; } +bool StringStartsWith(StringPiece text, StringPiece prefix) { + return text.starts_with(prefix); +} + +bool StringEndsWith(StringPiece text, StringPiece suffix) { + return text.ends_with(suffix); +} } // namespace converter } // namespace util } // namespace protobuf diff --git a/src/google/protobuf/util/internal/utility.h b/src/google/protobuf/util/internal/utility.h index 33df8eda..d8e06a97 100644 --- a/src/google/protobuf/util/internal/utility.h +++ b/src/google/protobuf/util/internal/utility.h @@ -32,9 +32,6 @@ #define GOOGLE_PROTOBUF_UTIL_CONVERTER_UTILITY_H__ #include <memory> -#ifndef _SHARED_PTR_H -#include <google/protobuf/stubs/shared_ptr.h> -#endif #include <string> #include <utility> @@ -64,6 +61,10 @@ class EnumValue; namespace protobuf { namespace util { namespace converter { + +// Size of "type.googleapis.com" +static const int64 kTypeUrlSize = 19; + // Finds the tech option identified by option_name. Parses the boolean value and // returns it. // When the option with the given name is not found, default_value is returned. @@ -117,13 +118,13 @@ LIBPROTOBUF_EXPORT const StringPiece GetTypeWithoutUrl(StringPiece type_url); LIBPROTOBUF_EXPORT const string GetFullTypeWithUrl(StringPiece simple_type); // Finds and returns option identified by name and option_name within the -// provided map. Returns NULL if none found. +// provided map. Returns nullptr if none found. const google::protobuf::Option* FindOptionOrNull( const google::protobuf::RepeatedPtrField<google::protobuf::Option>& options, const string& option_name); // Finds and returns the field identified by field_name in the passed tech Type -// object. Returns NULL if none found. +// object. Returns nullptr if none found. const google::protobuf::Field* FindFieldInTypeOrNull( const google::protobuf::Type* type, StringPiece field_name); @@ -132,19 +133,33 @@ const google::protobuf::Field* FindFieldInTypeOrNull( const google::protobuf::Field* FindJsonFieldInTypeOrNull( const google::protobuf::Type* type, StringPiece json_name); +// Similar to FindFieldInTypeOrNull, but this looks up fields by number. +const google::protobuf::Field* FindFieldInTypeByNumberOrNull( + const google::protobuf::Type* type, int32 number); + // Finds and returns the EnumValue identified by enum_name in the passed tech -// Enum object. Returns NULL if none found. +// Enum object. Returns nullptr if none found. const google::protobuf::EnumValue* FindEnumValueByNameOrNull( const google::protobuf::Enum* enum_type, StringPiece enum_name); // Finds and returns the EnumValue identified by value in the passed tech -// Enum object. Returns NULL if none found. +// Enum object. Returns nullptr if none found. const google::protobuf::EnumValue* FindEnumValueByNumberOrNull( const google::protobuf::Enum* enum_type, int32 value); +// Finds and returns the EnumValue identified by enum_name without underscore in +// the passed tech Enum object. Returns nullptr if none found. +// For Ex. if enum_name is ACTIONANDADVENTURE it can get accepted if +// EnumValue's name is action_and_adventure or ACTION_AND_ADVENTURE. +const google::protobuf::EnumValue* FindEnumValueByNameWithoutUnderscoreOrNull( + const google::protobuf::Enum* enum_type, StringPiece enum_name); + // Converts input to camel-case and returns it. LIBPROTOBUF_EXPORT string ToCamelCase(const StringPiece input); +// Converts enum name string to camel-case and returns it. +string EnumValueNameToLowerCamelCase(const StringPiece input); + // Converts input to snake_case and returns it. LIBPROTOBUF_EXPORT string ToSnakeCase(StringPiece input); @@ -185,6 +200,12 @@ inline string ValueAsString(double value) { // Converts a string to float. Unlike safe_strtof, conversion will fail if the // value fits into double but not float (e.g., DBL_MAX). LIBPROTOBUF_EXPORT bool SafeStrToFloat(StringPiece str, float* value); + +// Returns whether a StringPiece begins with the provided prefix. +bool StringStartsWith(StringPiece text, StringPiece prefix); + +// Returns whether a StringPiece ends with the provided suffix. +bool StringEndsWith(StringPiece text, StringPiece suffix); } // namespace converter } // namespace util } // namespace protobuf |