diff options
Diffstat (limited to 'src/google/protobuf/util')
58 files changed, 3478 insertions, 1182 deletions
diff --git a/src/google/protobuf/util/delimited_message_util.cc b/src/google/protobuf/util/delimited_message_util.cc new file mode 100644 index 00000000..3ba930e7 --- /dev/null +++ b/src/google/protobuf/util/delimited_message_util.cc @@ -0,0 +1,79 @@ +// Adapted from the patch of kenton@google.com (Kenton Varda) +// See https://github.com/google/protobuf/pull/710 for details. + +#include <google/protobuf/util/delimited_message_util.h> + +namespace google { +namespace protobuf { +namespace util { + +bool SerializeDelimitedToFileDescriptor(const MessageLite& message, int file_descriptor) { + io::FileOutputStream output(file_descriptor); + return SerializeDelimitedToZeroCopyStream(message, &output); +} + +bool SerializeDelimitedToOstream(const MessageLite& message, std::ostream* output) { + { + io::OstreamOutputStream zero_copy_output(output); + if (!SerializeDelimitedToZeroCopyStream(message, &zero_copy_output)) return false; + } + return output->good(); +} + +bool ParseDelimitedFromZeroCopyStream(MessageLite* message, io::ZeroCopyInputStream* input, bool* clean_eof) { + google::protobuf::io::CodedInputStream coded_input(input); + return ParseDelimitedFromCodedStream(message, &coded_input, clean_eof); +} + +bool ParseDelimitedFromCodedStream(MessageLite* message, io::CodedInputStream* input, bool* clean_eof) { + if (clean_eof != NULL) *clean_eof = false; + int start = input->CurrentPosition(); + + // Read the size. + uint32 size; + if (!input->ReadVarint32(&size)) { + if (clean_eof != NULL) *clean_eof = input->CurrentPosition() == start; + return false; + } + + // Tell the stream not to read beyond that size. + google::protobuf::io::CodedInputStream::Limit limit = input->PushLimit(size); + + // Parse the message. + if (!message->MergeFromCodedStream(input)) return false; + if (!input->ConsumedEntireMessage()) return false; + + // Release the limit. + input->PopLimit(limit); + + return true; +} + +bool SerializeDelimitedToZeroCopyStream(const MessageLite& message, io::ZeroCopyOutputStream* output) { + google::protobuf::io::CodedOutputStream coded_output(output); + return SerializeDelimitedToCodedStream(message, &coded_output); +} + +bool SerializeDelimitedToCodedStream(const MessageLite& message, io::CodedOutputStream* output) { + // Write the size. + int size = message.ByteSize(); + output->WriteVarint32(size); + + // Write the content. + uint8* buffer = output->GetDirectBufferForNBytesAndAdvance(size); + if (buffer != NULL) { + // Optimization: The message fits in one buffer, so use the faster + // direct-to-array serialization path. + message.SerializeWithCachedSizesToArray(buffer); + } else { + // Slightly-slower path when the message is multiple buffers. + message.SerializeWithCachedSizes(output); + if (output->HadError()) return false; + } + + return true; +} + +} // namespace util +} // namespace protobuf +} // namespace google diff --git a/src/google/protobuf/util/delimited_message_util.h b/src/google/protobuf/util/delimited_message_util.h new file mode 100644 index 00000000..e8a7204a --- /dev/null +++ b/src/google/protobuf/util/delimited_message_util.h @@ -0,0 +1,66 @@ +// Adapted from the patch of kenton@google.com (Kenton Varda) +// See https://github.com/google/protobuf/pull/710 for details. + +#ifndef GOOGLE_PROTOBUF_UTIL_DELIMITED_MESSAGE_UTIL_H__ +#define GOOGLE_PROTOBUF_UTIL_DELIMITED_MESSAGE_UTIL_H__ + +#include <ostream> + +#include <google/protobuf/message_lite.h> +#include <google/protobuf/io/coded_stream.h> +#include <google/protobuf/io/zero_copy_stream_impl.h> + +namespace google { +namespace protobuf { +namespace util { + +// Write a single size-delimited message from the given stream. Delimited +// format allows a single file or stream to contain multiple messages, +// whereas normally writing multiple non-delimited messages to the same +// stream would cause them to be merged. A delimited message is a varint +// encoding the message size followed by a message of exactly that size. +// +// Note that if you want to *read* a delimited message from a file descriptor +// or istream, you will need to construct an io::FileInputStream or +// io::OstreamInputStream (implementations of io::ZeroCopyStream) and use the +// utility function ParseDelimitedFromZeroCopyStream(). You must then +// continue to use the same ZeroCopyInputStream to read all further data from +// the stream until EOF. This is because these ZeroCopyInputStream +// implementations are buffered: they read a big chunk of data at a time, +// then parse it. As a result, they may read past the end of the delimited +// message. There is no way for them to push the extra data back into the +// underlying source, so instead you must keep using the same stream object. +bool LIBPROTOBUF_EXPORT SerializeDelimitedToFileDescriptor(const MessageLite& message, int file_descriptor); + +bool LIBPROTOBUF_EXPORT SerializeDelimitedToOstream(const MessageLite& message, std::ostream* output); + +// Read a single size-delimited message from the given stream. Delimited +// format allows a single file or stream to contain multiple messages, +// whereas normally parsing consumes the entire input. A delimited message +// is a varint encoding the message size followed by a message of exactly +// that size. +// +// If |clean_eof| is not NULL, then it will be set to indicate whether the +// stream ended cleanly. That is, if the stream ends without this method +// having read any data at all from it, then *clean_eof will be set true, +// otherwise it will be set false. Note that these methods return false +// on EOF, but they also return false on other errors, so |clean_eof| is +// needed to distinguish a clean end from errors. +bool LIBPROTOBUF_EXPORT ParseDelimitedFromZeroCopyStream(MessageLite* message, io::ZeroCopyInputStream* input, bool* clean_eof); + +bool LIBPROTOBUF_EXPORT ParseDelimitedFromCodedStream(MessageLite* message, io::CodedInputStream* input, bool* clean_eof); + +// Write a single size-delimited message from the given stream. Delimited +// format allows a single file or stream to contain multiple messages, +// whereas normally writing multiple non-delimited messages to the same +// stream would cause them to be merged. A delimited message is a varint +// encoding the message size followed by a message of exactly that size. +bool LIBPROTOBUF_EXPORT SerializeDelimitedToZeroCopyStream(const MessageLite& message, io::ZeroCopyOutputStream* output); + +bool LIBPROTOBUF_EXPORT SerializeDelimitedToCodedStream(const MessageLite& message, io::CodedOutputStream* output); + +} // namespace util +} // namespace protobuf +} // namespace google + +#endif // GOOGLE_PROTOBUF_UTIL_DELIMITED_MESSAGE_UTIL_H__ diff --git a/src/google/protobuf/util/delimited_message_util_test.cc b/src/google/protobuf/util/delimited_message_util_test.cc new file mode 100644 index 00000000..157a8411 --- /dev/null +++ b/src/google/protobuf/util/delimited_message_util_test.cc @@ -0,0 +1,57 @@ +// Adapted from the patch of kenton@google.com (Kenton Varda) +// See https://github.com/google/protobuf/pull/710 for details. + +#include <google/protobuf/util/delimited_message_util.h> + +#include <sstream> + +#include <google/protobuf/test_util.h> +#include <google/protobuf/unittest.pb.h> +#include <google/protobuf/testing/googletest.h> +#include <gtest/gtest.h> + +namespace google { +namespace protobuf { +namespace util { + +TEST(DelimitedMessageUtilTest, DelimitedMessages) { + std::stringstream stream; + + { + protobuf_unittest::TestAllTypes message1; + TestUtil::SetAllFields(&message1); + EXPECT_TRUE(SerializeDelimitedToOstream(message1, &stream)); + + protobuf_unittest::TestPackedTypes message2; + TestUtil::SetPackedFields(&message2); + EXPECT_TRUE(SerializeDelimitedToOstream(message2, &stream)); + } + + { + bool clean_eof; + io::IstreamInputStream zstream(&stream); + + protobuf_unittest::TestAllTypes message1; + clean_eof = true; + EXPECT_TRUE(ParseDelimitedFromZeroCopyStream(&message1, + &zstream, &clean_eof)); + EXPECT_FALSE(clean_eof); + TestUtil::ExpectAllFieldsSet(message1); + + protobuf_unittest::TestPackedTypes message2; + clean_eof = true; + EXPECT_TRUE(ParseDelimitedFromZeroCopyStream(&message2, + &zstream, &clean_eof)); + EXPECT_FALSE(clean_eof); + TestUtil::ExpectPackedFieldsSet(message2); + + clean_eof = false; + EXPECT_FALSE(ParseDelimitedFromZeroCopyStream(&message2, + &zstream, &clean_eof)); + EXPECT_TRUE(clean_eof); + } +} + +} // namespace util +} // namespace protobuf +} // namespace google diff --git a/src/google/protobuf/util/field_comparator.cc b/src/google/protobuf/util/field_comparator.cc index a1a56ee6..86ddf06a 100644 --- a/src/google/protobuf/util/field_comparator.cc +++ b/src/google/protobuf/util/field_comparator.cc @@ -36,6 +36,7 @@ #include <google/protobuf/descriptor.h> #include <google/protobuf/message.h> +#include <google/protobuf/util/message_differencer.h> #include <google/protobuf/stubs/map_util.h> #include <google/protobuf/stubs/mathlimits.h> #include <google/protobuf/stubs/mathutil.h> @@ -130,6 +131,15 @@ FieldComparator::ComparisonResult DefaultFieldComparator::Compare( } } +bool DefaultFieldComparator::Compare( + MessageDifferencer* differencer, + const Message& message1, + const Message& message2, + const google::protobuf::util::FieldContext* field_context) { + return differencer->Compare( + message1, message2, field_context->parent_fields()); +} + void DefaultFieldComparator::SetDefaultFractionAndMargin(double fraction, double margin) { default_tolerance_ = Tolerance(fraction, margin); diff --git a/src/google/protobuf/util/field_comparator.h b/src/google/protobuf/util/field_comparator.h index 715560ed..27ef4c77 100644 --- a/src/google/protobuf/util/field_comparator.h +++ b/src/google/protobuf/util/field_comparator.h @@ -30,13 +30,12 @@ // Defines classes for field comparison. -// Author: ksroka@google.com (Krzysztof Sroka) - #ifndef GOOGLE_PROTOBUF_UTIL_FIELD_COMPARATOR_H__ #define GOOGLE_PROTOBUF_UTIL_FIELD_COMPARATOR_H__ #include <map> #include <string> +#include <vector> #include <google/protobuf/stubs/common.h> @@ -50,6 +49,7 @@ class FieldDescriptor; namespace util { class FieldContext; +class MessageDifferencer; // Base class specifying the interface for comparing protocol buffer fields. // Regular users should consider using or subclassing DefaultFieldComparator @@ -155,6 +155,15 @@ class LIBPROTOBUF_EXPORT DefaultFieldComparator : public FieldComparator { // REQUIRES: float_comparison_ == APPROXIMATE void SetDefaultFractionAndMargin(double fraction, double margin); + protected: + // Compare using the provided message_differencer. For example, a subclass can + // use this method to compare some field in a certain way using the same + // message_differencer instance and the field context. + bool Compare(MessageDifferencer* differencer, + const google::protobuf::Message& message1, + const google::protobuf::Message& message2, + const google::protobuf::util::FieldContext* field_context); + private: // Defines the tolerance for floating point comparison (fraction and margin). struct Tolerance { @@ -169,7 +178,7 @@ class LIBPROTOBUF_EXPORT DefaultFieldComparator : public FieldComparator { }; // Defines the map to store the tolerances for floating point comparison. - typedef map<const FieldDescriptor*, Tolerance> ToleranceMap; + typedef std::map<const FieldDescriptor*, Tolerance> ToleranceMap; // The following methods get executed when CompareFields is called for the // basic types (instead of submessages). They return true on success. One @@ -239,7 +248,7 @@ class LIBPROTOBUF_EXPORT DefaultFieldComparator : public FieldComparator { // True iff default_tolerance_ has been explicitly set. // - // If false, then the default tolerance for flaots and doubles is that which + // If false, then the default tolerance for floats and doubles is that which // is used by MathUtil::AlmostEquals(). bool has_default_tolerance_; diff --git a/src/google/protobuf/util/field_comparator_test.cc b/src/google/protobuf/util/field_comparator_test.cc index 6fd631d8..249b8d54 100644 --- a/src/google/protobuf/util/field_comparator_test.cc +++ b/src/google/protobuf/util/field_comparator_test.cc @@ -365,10 +365,10 @@ TEST_F(DefaultFieldComparatorTest, // +inf should be equal even though they are not technically within margin or // fraction. - message_1_.set_optional_float(numeric_limits<float>::infinity()); - message_2_.set_optional_float(numeric_limits<float>::infinity()); - message_1_.set_optional_double(numeric_limits<double>::infinity()); - message_2_.set_optional_double(numeric_limits<double>::infinity()); + message_1_.set_optional_float(std::numeric_limits<float>::infinity()); + message_2_.set_optional_float(std::numeric_limits<float>::infinity()); + message_1_.set_optional_double(std::numeric_limits<double>::infinity()); + message_2_.set_optional_double(std::numeric_limits<double>::infinity()); comparator_.SetFractionAndMargin(field_float, 0.0, 0.0); comparator_.SetFractionAndMargin(field_double, 0.0, 0.0); EXPECT_EQ(FieldComparator::SAME, @@ -380,10 +380,10 @@ TEST_F(DefaultFieldComparatorTest, // -inf should be equal even though they are not technically within margin or // fraction. - message_1_.set_optional_float(-numeric_limits<float>::infinity()); - message_2_.set_optional_float(-numeric_limits<float>::infinity()); - message_1_.set_optional_double(-numeric_limits<double>::infinity()); - message_2_.set_optional_double(-numeric_limits<double>::infinity()); + message_1_.set_optional_float(-std::numeric_limits<float>::infinity()); + message_2_.set_optional_float(-std::numeric_limits<float>::infinity()); + message_1_.set_optional_double(-std::numeric_limits<double>::infinity()); + message_2_.set_optional_double(-std::numeric_limits<double>::infinity()); comparator_.SetFractionAndMargin(field_float, 0.0, 0.0); comparator_.SetFractionAndMargin(field_double, 0.0, 0.0); EXPECT_EQ(FieldComparator::SAME, diff --git a/src/google/protobuf/util/field_mask_util.cc b/src/google/protobuf/util/field_mask_util.cc index 409010a0..a2e2a388 100644 --- a/src/google/protobuf/util/field_mask_util.cc +++ b/src/google/protobuf/util/field_mask_util.cc @@ -31,6 +31,7 @@ #include <google/protobuf/util/field_mask_util.h> #include <google/protobuf/stubs/strutil.h> + #include <google/protobuf/stubs/map_util.h> namespace google { @@ -45,7 +46,7 @@ string FieldMaskUtil::ToString(const FieldMask& mask) { void FieldMaskUtil::FromString(StringPiece str, FieldMask* out) { out->Clear(); - vector<string> paths = Split(str, ","); + std::vector<string> paths = Split(str, ","); for (int i = 0; i < paths.size(); ++i) { if (paths[i].empty()) continue; out->add_paths(paths[i]); @@ -116,7 +117,7 @@ bool FieldMaskUtil::ToJsonString(const FieldMask& mask, string* out) { bool FieldMaskUtil::FromJsonString(StringPiece str, FieldMask* out) { out->Clear(); - vector<string> paths = Split(str, ","); + std::vector<string> paths = Split(str, ","); for (int i = 0; i < paths.size(); ++i) { if (paths[i].empty()) continue; string snakecase_path; @@ -128,23 +129,30 @@ bool FieldMaskUtil::FromJsonString(StringPiece str, FieldMask* out) { return true; } -bool FieldMaskUtil::InternalIsValidPath(const Descriptor* descriptor, - StringPiece path) { - vector<string> parts = Split(path, "."); +bool FieldMaskUtil::GetFieldDescriptors( + const Descriptor* descriptor, StringPiece path, + std::vector<const FieldDescriptor*>* field_descriptors) { + if (field_descriptors != nullptr) { + field_descriptors->clear(); + } + std::vector<string> parts = Split(path, "."); for (int i = 0; i < parts.size(); ++i) { const string& field_name = parts[i]; - if (descriptor == NULL) { + if (descriptor == nullptr) { return false; } const FieldDescriptor* field = descriptor->FindFieldByName(field_name); - if (field == NULL) { + if (field == nullptr) { return false; } + if (field_descriptors != nullptr) { + field_descriptors->push_back(field); + } if (!field->is_repeated() && field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { descriptor = field->message_type(); } else { - descriptor = NULL; + descriptor = nullptr; } } return true; @@ -185,6 +193,13 @@ class FieldMaskTree { // children removed because the path matches all the node's children. void AddPath(const string& path); + // Remove a path from the tree. + // If the path is a sub-path of an existing field path in the tree, it means + // we need remove the existing fied path and add all sub-paths except + // specified path. If the path matches an existing node in the tree, this node + // will be moved. + void RemovePath(const string& path, const Descriptor* descriptor); + // Calculate the intersection part of a field path with this tree and add // the intersection field path into out. void IntersectPath(const string& path, FieldMaskTree* out); @@ -200,6 +215,18 @@ class FieldMaskTree { MergeMessage(&root_, source, options, destination); } + // Add required field path of the message to this tree based on current tree + // structure. If a message is present in the tree, add the path of its + // required field to the tree. This is to make sure that after trimming a + // message with required fields are set, check IsInitialized() will not fail. + void AddRequiredFieldPath(const Descriptor* descriptor) { + // Do nothing if the tree is empty. + if (root_.children.empty()) { + return; + } + AddRequiredFieldPath(&root_, descriptor); + } + // Trims all fields not specified by this tree from the given message. void TrimMessage(Message* message) { // Do nothing if the tree is empty. @@ -216,14 +243,14 @@ class FieldMaskTree { ~Node() { ClearChildren(); } void ClearChildren() { - for (map<string, Node*>::iterator it = children.begin(); + for (std::map<string, Node*>::iterator it = children.begin(); it != children.end(); ++it) { delete it->second; } children.clear(); } - map<string, Node*> children; + std::map<string, Node*> children; private: GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(Node); @@ -242,6 +269,12 @@ class FieldMaskTree { const FieldMaskUtil::MergeOptions& options, Message* destination); + // Add required field path of the message to this tree based on current tree + // structure. If a message is present in the tree, add the path of its + // required field to the tree. This is to make sure that after trimming a + // message with required fields are set, check IsInitialized() will not fail. + void AddRequiredFieldPath(Node* node, const Descriptor* descriptor); + // Trims all fields not specified by this sub-tree from the given message. void TrimMessage(const Node* node, Message* message); @@ -274,7 +307,7 @@ void FieldMaskTree::MergeToFieldMask(const string& prefix, const Node* node, out->add_paths(prefix); return; } - for (map<string, Node*>::const_iterator it = node->children.begin(); + for (std::map<string, Node*>::const_iterator it = node->children.begin(); it != node->children.end(); ++it) { string current_path = prefix.empty() ? it->first : prefix + "." + it->first; MergeToFieldMask(current_path, it->second, out); @@ -282,7 +315,7 @@ void FieldMaskTree::MergeToFieldMask(const string& prefix, const Node* node, } void FieldMaskTree::AddPath(const string& path) { - vector<string> parts = Split(path, "."); + std::vector<string> parts = Split(path, "."); if (parts.empty()) { return; } @@ -308,8 +341,67 @@ void FieldMaskTree::AddPath(const string& path) { } } +void FieldMaskTree::RemovePath(const string& path, + const Descriptor* descriptor) { + if (root_.children.empty()) { + // Nothing to be removed from an empty tree. We shortcut it here so an empty + // tree won't be interpreted as a field mask containing all fields by the + // code below. + return; + } + std::vector<string> parts = Split(path, "."); + if (parts.empty()) { + return; + } + std::vector<Node*> nodes(parts.size()); + Node* node = &root_; + const Descriptor* current_descriptor = descriptor; + Node* new_branch_node = nullptr; + for (int i = 0; i < parts.size(); ++i) { + nodes[i] = node; + const FieldDescriptor* field_descriptor = + current_descriptor->FindFieldByName(parts[i]); + if (field_descriptor == nullptr || + (field_descriptor->cpp_type() != FieldDescriptor::CPPTYPE_MESSAGE && + i != parts.size() - 1)) { + // Invalid path. + if (new_branch_node != nullptr) { + // If add any new nodes, cleanup. + new_branch_node->ClearChildren(); + } + return; + } + + if (node->children.empty()) { + if (new_branch_node == nullptr) { + new_branch_node = node; + } + for (int i = 0; i < current_descriptor->field_count(); ++i) { + node->children[current_descriptor->field(i)->name()] = new Node(); + } + } + if (ContainsKey(node->children, parts[i])) { + node = node->children[parts[i]]; + if (field_descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + current_descriptor = field_descriptor->message_type(); + } + } else { + // Path does not exist. + return; + } + } + // Remove path. + for (int i = parts.size() - 1; i >= 0; i--) { + delete nodes[i]->children[parts[i]]; + nodes[i]->children.erase(parts[i]); + if (!nodes[i]->children.empty()) { + break; + } + } +} + void FieldMaskTree::IntersectPath(const string& path, FieldMaskTree* out) { - vector<string> parts = Split(path, "."); + std::vector<string> parts = Split(path, "."); if (parts.empty()) { return; } @@ -339,7 +431,7 @@ void FieldMaskTree::MergeLeafNodesToTree(const string& prefix, const Node* node, if (node->children.empty()) { out->AddPath(prefix); } - for (map<string, Node*>::const_iterator it = node->children.begin(); + for (std::map<string, Node*>::const_iterator it = node->children.begin(); it != node->children.end(); ++it) { string current_path = prefix.empty() ? it->first : prefix + "." + it->first; MergeLeafNodesToTree(current_path, it->second, out); @@ -353,7 +445,7 @@ void FieldMaskTree::MergeMessage(const Node* node, const Message& source, const Reflection* source_reflection = source.GetReflection(); const Reflection* destination_reflection = destination->GetReflection(); const Descriptor* descriptor = source.GetDescriptor(); - for (map<string, Node*>::const_iterator it = node->children.begin(); + for (std::map<string, Node*>::const_iterator it = node->children.begin(); it != node->children.end(); ++it) { const string& field_name = it->first; const Node* child = it->second; @@ -449,6 +541,41 @@ void FieldMaskTree::MergeMessage(const Node* node, const Message& source, } } +void FieldMaskTree::AddRequiredFieldPath( + Node* node, const Descriptor* descriptor) { + const int32 field_count = descriptor->field_count(); + for (int index = 0; index < field_count; ++index) { + const FieldDescriptor* field = descriptor->field(index); + if (field->is_required()) { + const string& node_name = field->name(); + Node*& child = node->children[node_name]; + if (child == nullptr) { + // Add required field path to the tree + child = new Node(); + } else if (child->children.empty()){ + // If the required field is in the tree and does not have any children, + // do nothing. + continue; + } + // Add required field in the children to the tree if the field is message. + if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + AddRequiredFieldPath(child, field->message_type()); + } + } else if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { + std::map<string, Node*>::const_iterator it = + node->children.find(field->name()); + if (it != node->children.end()) { + // Add required fields in the children to the + // tree if the field is a message and present in the tree. + Node* child = it->second; + if (!child->children.empty()) { + AddRequiredFieldPath(child, field->message_type()); + } + } + } + } +} + void FieldMaskTree::TrimMessage(const Node* node, Message* message) { GOOGLE_DCHECK(!node->children.empty()); const Reflection* reflection = message->GetReflection(); @@ -456,12 +583,14 @@ void FieldMaskTree::TrimMessage(const Node* node, Message* message) { const int32 field_count = descriptor->field_count(); for (int index = 0; index < field_count; ++index) { const FieldDescriptor* field = descriptor->field(index); - if (!ContainsKey(node->children, field->name())) { + std::map<string, Node*>::const_iterator it = + node->children.find(field->name()); + if (it == node->children.end()) { reflection->ClearField(message, field); } else { if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) { - Node* child = node->children.at(field->name()); - if (!child->children.empty()) { + Node* child = it->second; + if (!child->children.empty() && reflection->HasField(*message, field)) { TrimMessage(child, reflection->MutableMessage(message, field)); } } @@ -498,6 +627,22 @@ void FieldMaskUtil::Intersect(const FieldMask& mask1, const FieldMask& mask2, intersection.MergeToFieldMask(out); } +void FieldMaskUtil::InternalSubtract(const Descriptor* descriptor, + const FieldMask& mask1, + const FieldMask& mask2, FieldMask* out) { + if (mask1.paths().empty()) { + out->Clear(); + return; + } + FieldMaskTree tree; + tree.MergeFromFieldMask(mask1); + for (int i = 0; i < mask2.paths_size(); ++i) { + tree.RemovePath(mask2.paths(i), descriptor); + } + out->Clear(); + tree.MergeToFieldMask(out); +} + bool FieldMaskUtil::IsPathInFieldMask(StringPiece path, const FieldMask& mask) { for (int i = 0; i < mask.paths_size(); ++i) { const string& mask_path = mask.paths(i); @@ -533,6 +678,20 @@ void FieldMaskUtil::TrimMessage(const FieldMask& mask, Message* destination) { tree.TrimMessage(GOOGLE_CHECK_NOTNULL(destination)); } +void FieldMaskUtil::TrimMessage(const FieldMask& mask, Message* destination, + const TrimOptions& options) { + // Build a FieldMaskTree and walk through the tree to merge all specified + // fields. + FieldMaskTree tree; + tree.MergeFromFieldMask(mask); + // If keep_required_fields is true, implicitely add required fields of + // a message present in the tree to prevent from trimming. + if (options.keep_required_fields()) { + tree.AddRequiredFieldPath(GOOGLE_CHECK_NOTNULL(destination->GetDescriptor())); + } + tree.TrimMessage(GOOGLE_CHECK_NOTNULL(destination)); +} + } // namespace util } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/util/field_mask_util.h b/src/google/protobuf/util/field_mask_util.h index 396c9aea..f0299de9 100644 --- a/src/google/protobuf/util/field_mask_util.h +++ b/src/google/protobuf/util/field_mask_util.h @@ -59,17 +59,26 @@ class LIBPROTOBUF_EXPORT FieldMaskUtil { static bool ToJsonString(const FieldMask& mask, string* out); static bool FromJsonString(StringPiece str, FieldMask* out); + // Get the descriptors of the fields which the given path from the message + // descriptor traverses, if field_descriptors is not null. + // Return false if the path is not valid, and the content of field_descriptors + // is unspecified. + static bool GetFieldDescriptors( + const Descriptor* descriptor, StringPiece path, + std::vector<const FieldDescriptor*>* field_descriptors); + // Checks whether the given path is valid for type T. template <typename T> static bool IsValidPath(StringPiece path) { - return InternalIsValidPath(T::descriptor(), path); + return GetFieldDescriptors(T::descriptor(), path, nullptr); } // Checks whether the given FieldMask is valid for type T. template <typename T> static bool IsValidFieldMask(const FieldMask& mask) { for (int i = 0; i < mask.paths_size(); ++i) { - if (!InternalIsValidPath(T::descriptor(), mask.paths(i))) return false; + if (!GetFieldDescriptors(T::descriptor(), mask.paths(i), nullptr)) + return false; } return true; } @@ -85,6 +94,13 @@ class LIBPROTOBUF_EXPORT FieldMaskUtil { // Creates a FieldMask with all fields of type T. This FieldMask only // contains fields of T but not any sub-message fields. template <typename T> + static FieldMask GetFieldMaskForAllFields() { + FieldMask out; + InternalGetFieldMaskForAllFields(T::descriptor(), &out); + return out; + } + template <typename T> + PROTOBUF_RUNTIME_DEPRECATED("Use *out = GetFieldMaskForAllFields() instead") static void GetFieldMaskForAllFields(FieldMask* out) { InternalGetFieldMaskForAllFields(T::descriptor(), out); } @@ -104,8 +120,17 @@ class LIBPROTOBUF_EXPORT FieldMaskUtil { static void Intersect(const FieldMask& mask1, const FieldMask& mask2, FieldMask* out); + // Subtracts mask2 from mask1 base of type T. + template <typename T> + static void Subtract(const FieldMask& mask1, const FieldMask& mask2, + FieldMask* out) { + InternalSubtract(T::descriptor(), mask1, mask2, out); + } + // Returns true if path is covered by the given FieldMask. Note that path // "foo.bar" covers all paths like "foo.bar.baz", "foo.bar.quz.x", etc. + // Also note that parent paths are not covered by explicit child path, i.e. + // "foo.bar" does NOT cover "foo", even if "bar" is the only child. static bool IsPathInFieldMask(StringPiece path, const FieldMask& mask); class MergeOptions; @@ -115,10 +140,17 @@ class LIBPROTOBUF_EXPORT FieldMaskUtil { static void MergeMessageTo(const Message& source, const FieldMask& mask, const MergeOptions& options, Message* destination); + class TrimOptions; // Removes from 'message' any field that is not represented in the given // FieldMask. If the FieldMask is empty, does nothing. static void TrimMessage(const FieldMask& mask, Message* message); + // Removes from 'message' any field that is not represented in the given + // FieldMask with customized TrimOptions. + // If the FieldMask is empty, does nothing. + static void TrimMessage(const FieldMask& mask, Message* message, + const TrimOptions& options); + private: friend class SnakeCaseCamelCaseTest; // Converts a field name from snake_case to camelCase: @@ -149,11 +181,12 @@ class LIBPROTOBUF_EXPORT FieldMaskUtil { // successfully. static bool CamelCaseToSnakeCase(StringPiece input, string* output); - static bool InternalIsValidPath(const Descriptor* descriptor, - StringPiece path); - static void InternalGetFieldMaskForAllFields(const Descriptor* descriptor, FieldMask* out); + + static void InternalSubtract(const Descriptor* descriptor, + const FieldMask& mask1, const FieldMask& mask2, + FieldMask* out); }; // Note that for compatibility with the defined behaviour for FieldMask in @@ -188,6 +221,23 @@ class LIBPROTOBUF_EXPORT FieldMaskUtil::MergeOptions { bool replace_repeated_fields_; }; +class LIBPROTOBUF_EXPORT FieldMaskUtil::TrimOptions { + public: + TrimOptions() + : keep_required_fields_(false) {} + // When trimming message fields, the default behavior is to trim required + // fields of the present message if they are not specified in the field mask. + // If you instead want to keep required fields of the present message even + // they are not speicifed in the field mask, set this flag to true. + void set_keep_required_fields(bool value) { + keep_required_fields_ = value; + } + bool keep_required_fields() const { return keep_required_fields_; } + + private: + bool keep_required_fields_; +}; + } // namespace util } // namespace protobuf diff --git a/src/google/protobuf/util/field_mask_util_test.cc b/src/google/protobuf/util/field_mask_util_test.cc index 43fb7905..3ba30aa3 100644 --- a/src/google/protobuf/util/field_mask_util_test.cc +++ b/src/google/protobuf/util/field_mask_util_test.cc @@ -114,6 +114,8 @@ TEST_F(SnakeCaseCamelCaseTest, RoundTripTest) { } using protobuf_unittest::TestAllTypes; +using protobuf_unittest::TestRequired; +using protobuf_unittest::TestRequiredMessage; using protobuf_unittest::NestedTestAllTypes; using google::protobuf::FieldMask; @@ -159,6 +161,27 @@ TEST(FieldMaskUtilTest, JsonStringFormat) { EXPECT_EQ("baz_quz", mask.paths(1)); } +TEST(FieldMaskUtilTest, GetFieldDescriptors) { + std::vector<const FieldDescriptor*> field_descriptors; + EXPECT_TRUE(FieldMaskUtil::GetFieldDescriptors( + TestAllTypes::descriptor(), "optional_int32", &field_descriptors)); + EXPECT_EQ(1, field_descriptors.size()); + EXPECT_EQ("optional_int32", field_descriptors[0]->name()); + EXPECT_FALSE(FieldMaskUtil::GetFieldDescriptors( + TestAllTypes::descriptor(), "optional_nonexist", nullptr)); + EXPECT_TRUE(FieldMaskUtil::GetFieldDescriptors(TestAllTypes::descriptor(), + "optional_nested_message.bb", + &field_descriptors)); + EXPECT_EQ(2, field_descriptors.size()); + EXPECT_EQ("optional_nested_message", field_descriptors[0]->name()); + EXPECT_EQ("bb", field_descriptors[1]->name()); + EXPECT_FALSE(FieldMaskUtil::GetFieldDescriptors( + TestAllTypes::descriptor(), "optional_nested_message.nonexist", nullptr)); + // FieldMask cannot be used to specify sub-fields of a repeated message. + EXPECT_FALSE(FieldMaskUtil::GetFieldDescriptors( + TestAllTypes::descriptor(), "repeated_nested_message.bb", nullptr)); +} + TEST(FieldMaskUtilTest, TestIsVaildPath) { EXPECT_TRUE(FieldMaskUtil::IsValidPath<TestAllTypes>("optional_int32")); EXPECT_FALSE(FieldMaskUtil::IsValidPath<TestAllTypes>("optional_nonexist")); @@ -183,12 +206,12 @@ TEST(FieldMaskUtilTest, TestIsValidFieldMask) { TEST(FieldMaskUtilTest, TestGetFieldMaskForAllFields) { FieldMask mask; - FieldMaskUtil::GetFieldMaskForAllFields<TestAllTypes::NestedMessage>(&mask); + mask = FieldMaskUtil::GetFieldMaskForAllFields<TestAllTypes::NestedMessage>(); EXPECT_EQ(1, mask.paths_size()); EXPECT_TRUE(FieldMaskUtil::IsPathInFieldMask("bb", mask)); - FieldMaskUtil::GetFieldMaskForAllFields<TestAllTypes>(&mask); - EXPECT_EQ(76, mask.paths_size()); + mask = FieldMaskUtil::GetFieldMaskForAllFields<TestAllTypes>(); + EXPECT_EQ(75, mask.paths_size()); EXPECT_TRUE(FieldMaskUtil::IsPathInFieldMask("optional_int32", mask)); EXPECT_TRUE(FieldMaskUtil::IsPathInFieldMask("optional_int64", mask)); EXPECT_TRUE(FieldMaskUtil::IsPathInFieldMask("optional_uint32", mask)); @@ -325,6 +348,53 @@ TEST(FieldMaskUtilTest, TestIntersect) { EXPECT_EQ("foo.bar.baz", FieldMaskUtil::ToString(out)); } +TEST(FieldMaskUtilTest, TestSubtract) { + FieldMask mask1, mask2, out; + // Normal case. + FieldMaskUtil::FromString( + "optional_int32,optional_uint64,optional_nested_message,optional_foreign_" + "message,repeated_int32,repeated_foreign_message,repeated_nested_message." + "bb", + &mask1); + + FieldMaskUtil::FromString( + "optional_int32,optional_nested_message.bb,optional_foreign_message.c," + "repeated_int32,repeated_nested_message.bb,repeated_foreign_message.f," + "repeated_foreign_message.d,repeated_nested_message.bb,repeated_uint32", + &mask2); + + FieldMaskUtil::Subtract<TestAllTypes>(mask1, mask2, &out); + EXPECT_EQ( + "optional_foreign_message.d,optional_uint64,repeated_foreign_message.c", + FieldMaskUtil::ToString(out)); + + // mask1 is empty. + FieldMaskUtil::FromString("", &mask1); + FieldMaskUtil::Subtract<TestAllTypes>(mask1, mask2, &out); + EXPECT_EQ("", FieldMaskUtil::ToString(out)); + + // mask1 is "optional_nested_message" and mask2 is + // "optional_nested_message.nonexist_field". + FieldMaskUtil::FromString("optional_nested_message", &mask1); + FieldMaskUtil::FromString("optional_nested_message.nonexist_field", &mask2); + FieldMaskUtil::Subtract<TestAllTypes>(mask1, mask2, &out); + EXPECT_EQ("optional_nested_message", FieldMaskUtil::ToString(out)); + + // mask1 is "optional_nested_message" and mask2 is + // "optional_nested_message". + FieldMaskUtil::FromString("optional_nested_message", &mask1); + FieldMaskUtil::FromString("optional_nested_message", &mask2); + FieldMaskUtil::Subtract<TestAllTypes>(mask1, mask2, &out); + EXPECT_EQ("", FieldMaskUtil::ToString(out)); + + // Regression test for b/72727550 + FieldMaskUtil::FromString("optional_foreign_message.c", &mask1); + FieldMaskUtil::FromString("optional_foreign_message,optional_nested_message", + &mask2); + FieldMaskUtil::Subtract<TestAllTypes>(mask1, mask2, &out); + EXPECT_EQ("", FieldMaskUtil::ToString(out)); +} + TEST(FieldMaskUtilTest, TestIspathInFieldMask) { FieldMask mask; FieldMaskUtil::FromString("foo.bar", &mask); @@ -597,6 +667,84 @@ TEST(FieldMaskUtilTest, TrimMessage) { FieldMask empty_mask; FieldMaskUtil::TrimMessage(empty_mask, &trimmed_all_types); EXPECT_EQ(trimmed_all_types.DebugString(), all_types_msg.DebugString()); + + // Test trim required fields with keep_required_fields is set true. + FieldMaskUtil::TrimOptions options; + TestRequired required_msg_1; + required_msg_1.set_a(1234); + required_msg_1.set_b(3456); + required_msg_1.set_c(5678); + TestRequired trimmed_required_msg_1(required_msg_1); + FieldMaskUtil::FromString("dummy2", &mask); + options.set_keep_required_fields(true); + FieldMaskUtil::TrimMessage(mask, &trimmed_required_msg_1, options); + EXPECT_EQ(trimmed_required_msg_1.DebugString(), required_msg_1.DebugString()); + + // Test trim required fields with keep_required_fields is set false. + required_msg_1.clear_a(); + required_msg_1.clear_b(); + required_msg_1.clear_c(); + options.set_keep_required_fields(false); + FieldMaskUtil::TrimMessage(mask, &trimmed_required_msg_1, options); + EXPECT_EQ(trimmed_required_msg_1.DebugString(), required_msg_1.DebugString()); + + // Test trim required message with keep_required_fields is set true. + TestRequiredMessage required_msg_2; + required_msg_2.mutable_optional_message()->set_a(1234); + required_msg_2.mutable_optional_message()->set_b(3456); + required_msg_2.mutable_optional_message()->set_c(5678); + required_msg_2.mutable_required_message()->set_a(1234); + required_msg_2.mutable_required_message()->set_b(3456); + required_msg_2.mutable_required_message()->set_c(5678); + required_msg_2.mutable_required_message()->set_dummy2(7890); + TestRequired* repeated_msg = required_msg_2.add_repeated_message(); + repeated_msg->set_a(1234); + repeated_msg->set_b(3456); + repeated_msg->set_c(5678); + TestRequiredMessage trimmed_required_msg_2(required_msg_2); + FieldMaskUtil::FromString("optional_message.dummy2", &mask); + options.set_keep_required_fields(true); + required_msg_2.clear_repeated_message(); + required_msg_2.mutable_required_message()->clear_dummy2(); + FieldMaskUtil::TrimMessage(mask, &trimmed_required_msg_2, options); + EXPECT_EQ(trimmed_required_msg_2.DebugString(), + required_msg_2.DebugString()); + + FieldMaskUtil::FromString("required_message", &mask); + required_msg_2.mutable_required_message()->set_dummy2(7890); + trimmed_required_msg_2.mutable_required_message()->set_dummy2(7890); + required_msg_2.clear_optional_message(); + FieldMaskUtil::TrimMessage(mask, &trimmed_required_msg_2, options); + EXPECT_EQ(trimmed_required_msg_2.DebugString(), + required_msg_2.DebugString()); + + // Test trim required message with keep_required_fields is set false. + FieldMaskUtil::FromString("required_message.dummy2", &mask); + required_msg_2.mutable_required_message()->clear_a(); + required_msg_2.mutable_required_message()->clear_b(); + required_msg_2.mutable_required_message()->clear_c(); + options.set_keep_required_fields(false); + FieldMaskUtil::TrimMessage(mask, &trimmed_required_msg_2, options); + EXPECT_EQ(trimmed_required_msg_2.DebugString(), + required_msg_2.DebugString()); + + // Verify that trimming an empty message has no effect. In particular, fields + // mentioned in the field mask should not be created or changed. + TestAllTypes empty_msg; + FieldMaskUtil::FromString( + "optional_int32,optional_bytes,optional_nested_message.bb", &mask); + FieldMaskUtil::TrimMessage(mask, &empty_msg); + EXPECT_FALSE(empty_msg.has_optional_int32()); + EXPECT_FALSE(empty_msg.has_optional_bytes()); + EXPECT_FALSE(empty_msg.has_optional_nested_message()); + + // Verify trimming of oneof fields. This should work as expected even if + // multiple elements of the same oneof are included in the FieldMask. + TestAllTypes oneof_msg; + oneof_msg.set_oneof_uint32(11); + FieldMaskUtil::FromString("oneof_uint32,oneof_nested_message.bb", &mask); + FieldMaskUtil::TrimMessage(mask, &oneof_msg); + EXPECT_EQ(11, oneof_msg.oneof_uint32()); } diff --git a/src/google/protobuf/util/internal/constants.h b/src/google/protobuf/util/internal/constants.h index e556888c..a018a09e 100644 --- a/src/google/protobuf/util/internal/constants.h +++ b/src/google/protobuf/util/internal/constants.h @@ -50,16 +50,16 @@ const char kRfc3339TimeFormat[] = "%E4Y-%m-%dT%H:%M:%S"; const char kRfc3339TimeFormatNoPadding[] = "%Y-%m-%dT%H:%M:%S"; // Minimun seconds allowed in a google.protobuf.Timestamp value. -const int64 kTimestampMinSeconds = -62135596800; +const int64 kTimestampMinSeconds = -62135596800LL; // Maximum seconds allowed in a google.protobuf.Timestamp value. -const int64 kTimestampMaxSeconds = 253402300799; +const int64 kTimestampMaxSeconds = 253402300799LL; // Minimum seconds allowed in a google.protobuf.Duration value. -const int64 kDurationMinSeconds = -315576000000; +const int64 kDurationMinSeconds = -315576000000LL; // Maximum seconds allowed in a google.protobuf.Duration value. -const int64 kDurationMaxSeconds = 315576000000; +const int64 kDurationMaxSeconds = 315576000000LL; // Nano seconds in a second. const int32 kNanosPerSecond = 1000000000; diff --git a/src/google/protobuf/util/internal/datapiece.cc b/src/google/protobuf/util/internal/datapiece.cc index ef8da427..59bc28ae 100644 --- a/src/google/protobuf/util/internal/datapiece.cc +++ b/src/google/protobuf/util/internal/datapiece.cc @@ -64,9 +64,9 @@ StatusOr<To> ValidateNumberConversion(To after, From before) { MathUtil::Sign<From>(before) == MathUtil::Sign<To>(after)) { return after; } else { - return InvalidArgument(::google::protobuf::internal::is_integral<From>::value + return InvalidArgument(std::is_integral<From>::value ? ValueAsString(before) - : ::google::protobuf::internal::is_same<From, double>::value + : std::is_same<From, double>::value ? DoubleAsString(before) : FloatAsString(before)); } @@ -77,7 +77,7 @@ StatusOr<To> ValidateNumberConversion(To after, From before) { // except conversion between double and float. template <typename To, typename From> StatusOr<To> NumberConvertAndCheck(From before) { - if (::google::protobuf::internal::is_same<From, To>::value) return before; + if (std::is_same<From, To>::value) return before; To after = static_cast<To>(before); return ValidateNumberConversion(after, before); @@ -87,26 +87,31 @@ StatusOr<To> NumberConvertAndCheck(From before) { // point types (double, float) only. template <typename To, typename From> StatusOr<To> FloatingPointToIntConvertAndCheck(From before) { - if (::google::protobuf::internal::is_same<From, To>::value) return before; + if (std::is_same<From, To>::value) return before; To after = static_cast<To>(before); return ValidateNumberConversion(after, before); } // For conversion between double and float only. -template <typename To, typename From> -StatusOr<To> FloatingPointConvertAndCheck(From before) { - if (MathLimits<From>::IsNaN(before)) { - return std::numeric_limits<To>::quiet_NaN(); - } +StatusOr<double> FloatToDouble(float before) { + // Casting float to double should just work as double has more precision + // than float. + return static_cast<double>(before); +} - To after = static_cast<To>(before); - if (MathUtil::AlmostEquals<To>(after, before)) { - return after; +StatusOr<float> DoubleToFloat(double before) { + if (MathLimits<double>::IsNaN(before)) { + return std::numeric_limits<float>::quiet_NaN(); + } else if (!MathLimits<double>::IsFinite(before)) { + // Converting a double +inf/-inf to float should just work. + return static_cast<float>(before); + } else if (before > std::numeric_limits<float>::max() || + before < -std::numeric_limits<float>::max()) { + // Double value outside of the range of float. + return InvalidArgument(DoubleAsString(before)); } else { - return InvalidArgument(::google::protobuf::internal::is_same<From, double>::value - ? DoubleAsString(before) - : FloatAsString(before)); + return static_cast<float>(before); } } @@ -162,20 +167,27 @@ StatusOr<uint64> DataPiece::ToUint64() const { StatusOr<double> DataPiece::ToDouble() const { if (type_ == TYPE_FLOAT) { - return FloatingPointConvertAndCheck<double, float>(float_); + return FloatToDouble(float_); } if (type_ == TYPE_STRING) { if (str_ == "Infinity") return std::numeric_limits<double>::infinity(); if (str_ == "-Infinity") return -std::numeric_limits<double>::infinity(); if (str_ == "NaN") return std::numeric_limits<double>::quiet_NaN(); - return StringToNumber<double>(safe_strtod); + StatusOr<double> value = StringToNumber<double>(safe_strtod); + if (value.ok() && !MathLimits<double>::IsFinite(value.ValueOrDie())) { + // safe_strtod converts out-of-range values to +inf/-inf, but we want + // to report them as errors. + return InvalidArgument(StrCat("\"", str_, "\"")); + } else { + return value; + } } return GenericConvert<double>(); } StatusOr<float> DataPiece::ToFloat() const { if (type_ == TYPE_DOUBLE) { - return FloatingPointConvertAndCheck<float, double>(double_); + return DoubleToFloat(double_); } if (type_ == TYPE_STRING) { if (str_ == "Infinity") return std::numeric_limits<float>::infinity(); @@ -259,7 +271,9 @@ StatusOr<string> DataPiece::ToBytes() const { } } -StatusOr<int> DataPiece::ToEnum(const google::protobuf::Enum* enum_type) const { +StatusOr<int> DataPiece::ToEnum(const google::protobuf::Enum* enum_type, + bool use_lower_camel_for_enums, + bool ignore_unknown_enum_values) const { if (type_ == TYPE_NULL) return google::protobuf::NULL_VALUE; if (type_ == TYPE_STRING) { @@ -267,21 +281,39 @@ StatusOr<int> DataPiece::ToEnum(const google::protobuf::Enum* enum_type) const { string enum_name = str_.ToString(); const google::protobuf::EnumValue* value = FindEnumValueByNameOrNull(enum_type, enum_name); - if (value != NULL) return value->number(); + if (value != nullptr) return value->number(); + + // Check if int version of enum is sent as string. + StatusOr<int32> int_value = ToInt32(); + if (int_value.ok()) { + if (const google::protobuf::EnumValue* enum_value = + FindEnumValueByNumberOrNull(enum_type, int_value.ValueOrDie())) { + return enum_value->number(); + } + } + // Next try a normalized name. for (string::iterator it = enum_name.begin(); it != enum_name.end(); ++it) { *it = *it == '-' ? '_' : ascii_toupper(*it); } value = FindEnumValueByNameOrNull(enum_type, enum_name); - if (value != NULL) return value->number(); - } else { - StatusOr<int32> value = ToInt32(); - if (value.ok()) { - if (const google::protobuf::EnumValue* enum_value = - FindEnumValueByNumberOrNull(enum_type, value.ValueOrDie())) { - return enum_value->number(); - } + if (value != nullptr) return value->number(); + + // If use_lower_camel_for_enums is true try with enum name without + // underscore. This will also accept camel case names as the enum_name has + // been normalized before. + if (use_lower_camel_for_enums) { + value = FindEnumValueByNameWithoutUnderscoreOrNull(enum_type, enum_name); + if (value != nullptr) return value->number(); } + + // If ignore_unknown_enum_values is true an unknown enum value is treated + // as the default + if (ignore_unknown_enum_values) return enum_type->enumvalue(0).number(); + } else { + // We don't need to check whether the value is actually declared in the + // enum because we preserve unknown enum values as well. + return ToInt32(); } return InvalidArgument( ValueAsStringOrDefault("Cannot find enum with given value.")); @@ -330,7 +362,8 @@ bool DataPiece::DecodeBase64(StringPiece src, string* dest) const { WebSafeBase64Escape(*dest, &encoded); // Remove trailing padding '=' characters before comparison. StringPiece src_no_padding = StringPiece(src).substr( - 0, src.ends_with("=") ? src.find_last_not_of('=') + 1 : src.length()); + 0, StringEndsWith(src, "=") ? src.find_last_not_of('=') + 1 + : src.length()); return encoded == src_no_padding; } return true; @@ -343,7 +376,8 @@ bool DataPiece::DecodeBase64(StringPiece src, string* dest) const { reinterpret_cast<const unsigned char*>(dest->data()), dest->length(), &encoded, false); StringPiece src_no_padding = StringPiece(src).substr( - 0, src.ends_with("=") ? src.find_last_not_of('=') + 1 : src.length()); + 0, StringEndsWith(src, "=") ? src.find_last_not_of('=') + 1 + : src.length()); return encoded == src_no_padding; } return true; @@ -354,6 +388,7 @@ bool DataPiece::DecodeBase64(StringPiece src, string* dest) const { void DataPiece::InternalCopy(const DataPiece& other) { type_ = other.type_; + use_strict_base64_decoding_ = other.use_strict_base64_decoding_; switch (type_) { case TYPE_INT32: case TYPE_INT64: diff --git a/src/google/protobuf/util/internal/datapiece.h b/src/google/protobuf/util/internal/datapiece.h index e82cdbac..95b133da 100644 --- a/src/google/protobuf/util/internal/datapiece.h +++ b/src/google/protobuf/util/internal/datapiece.h @@ -76,13 +76,22 @@ class LIBPROTOBUF_EXPORT DataPiece { }; // Constructors and Destructor - explicit DataPiece(const int32 value) : type_(TYPE_INT32), i32_(value) {} - explicit DataPiece(const int64 value) : type_(TYPE_INT64), i64_(value) {} - explicit DataPiece(const uint32 value) : type_(TYPE_UINT32), u32_(value) {} - explicit DataPiece(const uint64 value) : type_(TYPE_UINT64), u64_(value) {} - explicit DataPiece(const double value) : type_(TYPE_DOUBLE), double_(value) {} - explicit DataPiece(const float value) : type_(TYPE_FLOAT), float_(value) {} - explicit DataPiece(const bool value) : type_(TYPE_BOOL), bool_(value) {} + explicit DataPiece(const int32 value) + : type_(TYPE_INT32), i32_(value), use_strict_base64_decoding_(false) {} + explicit DataPiece(const int64 value) + : type_(TYPE_INT64), i64_(value), use_strict_base64_decoding_(false) {} + explicit DataPiece(const uint32 value) + : type_(TYPE_UINT32), u32_(value), use_strict_base64_decoding_(false) {} + explicit DataPiece(const uint64 value) + : type_(TYPE_UINT64), u64_(value), use_strict_base64_decoding_(false) {} + explicit DataPiece(const double value) + : type_(TYPE_DOUBLE), + double_(value), + use_strict_base64_decoding_(false) {} + explicit DataPiece(const float value) + : type_(TYPE_FLOAT), float_(value), use_strict_base64_decoding_(false) {} + explicit DataPiece(const bool value) + : type_(TYPE_BOOL), bool_(value), use_strict_base64_decoding_(false) {} DataPiece(StringPiece value, bool use_strict_base64_decoding) : type_(TYPE_STRING), str_(StringPiecePod::CreateFromStringPiece(value)), @@ -108,6 +117,8 @@ class LIBPROTOBUF_EXPORT DataPiece { // Accessors Type type() const { return type_; } + bool use_strict_base64_decoding() { return use_strict_base64_decoding_; } + StringPiece str() const { GOOGLE_LOG_IF(DFATAL, type_ != TYPE_STRING) << "Not a string type."; return str_; @@ -148,16 +159,21 @@ class LIBPROTOBUF_EXPORT DataPiece { // string, first attempts conversion by name, trying names as follows: // 1) the directly provided string value. // 2) the value upper-cased and replacing '-' by '_' + // 3) if use_lower_camel_for_enums is true it also attempts by comparing + // enum name without underscore with the value upper cased above. // If the value is not a string, attempts to convert to a 32-bit integer. // If none of these succeeds, returns a conversion error status. - util::StatusOr<int> ToEnum(const google::protobuf::Enum* enum_type) const; + util::StatusOr<int> ToEnum(const google::protobuf::Enum* enum_type, + bool use_lower_camel_for_enums, + bool ignore_unknown_enum_values) const; private: // Disallow implicit constructor. DataPiece(); // Helper to create NULL or ENUM types. - DataPiece(Type type, int32 val) : type_(type), i32_(val) {} + DataPiece(Type type, int32 val) + : type_(type), i32_(val), use_strict_base64_decoding_(false) {} // For numeric conversion between // int32, int64, uint32, uint64, double, float and bool diff --git a/src/google/protobuf/util/internal/default_value_objectwriter.cc b/src/google/protobuf/util/internal/default_value_objectwriter.cc index fa31f763..b41feb7a 100644 --- a/src/google/protobuf/util/internal/default_value_objectwriter.cc +++ b/src/google/protobuf/util/internal/default_value_objectwriter.cc @@ -62,10 +62,12 @@ DefaultValueObjectWriter::DefaultValueObjectWriter( : typeinfo_(TypeInfo::NewTypeInfo(type_resolver)), own_typeinfo_(true), type_(type), - current_(NULL), - root_(NULL), + current_(nullptr), + root_(nullptr), suppress_empty_list_(false), - field_scrub_callback_(NULL), + preserve_proto_field_names_(false), + use_ints_for_enums_(false), + field_scrub_callback_(nullptr), ow_(ow) {} DefaultValueObjectWriter::~DefaultValueObjectWriter() { @@ -79,7 +81,7 @@ DefaultValueObjectWriter::~DefaultValueObjectWriter() { DefaultValueObjectWriter* DefaultValueObjectWriter::RenderBool(StringPiece name, bool value) { - if (current_ == NULL) { + if (current_ == nullptr) { ow_->RenderBool(name, value); } else { RenderDataPiece(name, DataPiece(value)); @@ -89,7 +91,7 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::RenderBool(StringPiece name, DefaultValueObjectWriter* DefaultValueObjectWriter::RenderInt32( StringPiece name, int32 value) { - if (current_ == NULL) { + if (current_ == nullptr) { ow_->RenderInt32(name, value); } else { RenderDataPiece(name, DataPiece(value)); @@ -99,7 +101,7 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::RenderInt32( DefaultValueObjectWriter* DefaultValueObjectWriter::RenderUint32( StringPiece name, uint32 value) { - if (current_ == NULL) { + if (current_ == nullptr) { ow_->RenderUint32(name, value); } else { RenderDataPiece(name, DataPiece(value)); @@ -109,7 +111,7 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::RenderUint32( DefaultValueObjectWriter* DefaultValueObjectWriter::RenderInt64( StringPiece name, int64 value) { - if (current_ == NULL) { + if (current_ == nullptr) { ow_->RenderInt64(name, value); } else { RenderDataPiece(name, DataPiece(value)); @@ -119,7 +121,7 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::RenderInt64( DefaultValueObjectWriter* DefaultValueObjectWriter::RenderUint64( StringPiece name, uint64 value) { - if (current_ == NULL) { + if (current_ == nullptr) { ow_->RenderUint64(name, value); } else { RenderDataPiece(name, DataPiece(value)); @@ -129,7 +131,7 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::RenderUint64( DefaultValueObjectWriter* DefaultValueObjectWriter::RenderDouble( StringPiece name, double value) { - if (current_ == NULL) { + if (current_ == nullptr) { ow_->RenderDouble(name, value); } else { RenderDataPiece(name, DataPiece(value)); @@ -139,7 +141,7 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::RenderDouble( DefaultValueObjectWriter* DefaultValueObjectWriter::RenderFloat( StringPiece name, float value) { - if (current_ == NULL) { + if (current_ == nullptr) { ow_->RenderBool(name, value); } else { RenderDataPiece(name, DataPiece(value)); @@ -149,7 +151,7 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::RenderFloat( DefaultValueObjectWriter* DefaultValueObjectWriter::RenderString( StringPiece name, StringPiece value) { - if (current_ == NULL) { + if (current_ == nullptr) { ow_->RenderString(name, value); } else { // Since StringPiece is essentially a pointer, takes a copy of "value" to @@ -162,7 +164,7 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::RenderString( DefaultValueObjectWriter* DefaultValueObjectWriter::RenderBytes( StringPiece name, StringPiece value) { - if (current_ == NULL) { + if (current_ == nullptr) { ow_->RenderBytes(name, value); } else { // Since StringPiece is essentially a pointer, takes a copy of "value" to @@ -175,7 +177,7 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::RenderBytes( DefaultValueObjectWriter* DefaultValueObjectWriter::RenderNull( StringPiece name) { - if (current_ == NULL) { + if (current_ == nullptr) { ow_->RenderNull(name); } else { RenderDataPiece(name, DataPiece::NullData()); @@ -188,9 +190,27 @@ void DefaultValueObjectWriter::RegisterFieldScrubCallBack( field_scrub_callback_.reset(field_scrub_callback.release()); } +DefaultValueObjectWriter::Node* DefaultValueObjectWriter::CreateNewNode( + const string& name, const google::protobuf::Type* type, NodeKind kind, + const DataPiece& data, bool is_placeholder, const std::vector<string>& path, + bool suppress_empty_list, FieldScrubCallBack* field_scrub_callback) { + return new Node(name, type, kind, data, is_placeholder, path, + suppress_empty_list, field_scrub_callback); +} + +DefaultValueObjectWriter::Node* DefaultValueObjectWriter::CreateNewNode( + const string& name, const google::protobuf::Type* type, NodeKind kind, + const DataPiece& data, bool is_placeholder, const std::vector<string>& path, + bool suppress_empty_list, bool preserve_proto_field_names, bool use_ints_for_enums, + FieldScrubCallBack* field_scrub_callback) { + return new Node(name, type, kind, data, is_placeholder, path, + suppress_empty_list, preserve_proto_field_names, use_ints_for_enums, + field_scrub_callback); +} + DefaultValueObjectWriter::Node::Node( const string& name, const google::protobuf::Type* type, NodeKind kind, - const DataPiece& data, bool is_placeholder, const vector<string>& path, + const DataPiece& data, bool is_placeholder, const std::vector<string>& path, bool suppress_empty_list, FieldScrubCallBack* field_scrub_callback) : name_(name), type_(type), @@ -200,12 +220,31 @@ DefaultValueObjectWriter::Node::Node( is_placeholder_(is_placeholder), path_(path), suppress_empty_list_(suppress_empty_list), + preserve_proto_field_names_(false), + use_ints_for_enums_(false), + field_scrub_callback_(field_scrub_callback) {} + +DefaultValueObjectWriter::Node::Node( + const string& name, const google::protobuf::Type* type, NodeKind kind, + const DataPiece& data, bool is_placeholder, const std::vector<string>& path, + bool suppress_empty_list, bool preserve_proto_field_names, bool use_ints_for_enums, + FieldScrubCallBack* field_scrub_callback) + : name_(name), + type_(type), + kind_(kind), + is_any_(false), + data_(data), + is_placeholder_(is_placeholder), + path_(path), + suppress_empty_list_(suppress_empty_list), + preserve_proto_field_names_(preserve_proto_field_names), + use_ints_for_enums_(use_ints_for_enums), field_scrub_callback_(field_scrub_callback) {} DefaultValueObjectWriter::Node* DefaultValueObjectWriter::Node::FindChild( StringPiece name) { if (name.empty() || kind_ != OBJECT) { - return NULL; + return nullptr; } for (int i = 0; i < children_.size(); ++i) { Node* child = children_[i]; @@ -213,7 +252,7 @@ DefaultValueObjectWriter::Node* DefaultValueObjectWriter::Node::FindChild( return child; } } - return NULL; + return nullptr; } void DefaultValueObjectWriter::Node::WriteTo(ObjectWriter* ow) { @@ -281,7 +320,7 @@ const google::protobuf::Type* DefaultValueObjectWriter::Node::GetMapValueType( } break; } - return NULL; + return nullptr; } void DefaultValueObjectWriter::Node::PopulateChildren( @@ -292,7 +331,7 @@ void DefaultValueObjectWriter::Node::PopulateChildren( // TODO(tsun): remove "kStructValueType" from the list. It's being checked // now because of a bug in the tool-chain that causes the "oneof_index" // of kStructValueType to not be set correctly. - if (type_ == NULL || type_->name() == kAnyType || + if (type_ == nullptr || type_->name() == kAnyType || type_->name() == kStructType || type_->name() == kTimestampType || type_->name() == kDurationType || type_->name() == kStructValueType) { return; @@ -310,12 +349,12 @@ void DefaultValueObjectWriter::Node::PopulateChildren( // This code is checking if the field to be added to the tree should be // scrubbed or not by calling the field_scrub_callback_ callback function. - vector<string> path; + std::vector<string> path; if (!path_.empty()) { path.insert(path.begin(), path_.begin(), path_.end()); } path.push_back(field.name()); - if (field_scrub_callback_ != NULL && + if (field_scrub_callback_ != nullptr && field_scrub_callback_->Run(path, &field)) { continue; } @@ -326,11 +365,11 @@ void DefaultValueObjectWriter::Node::PopulateChildren( // of children. if (found != orig_children_map.end()) { new_children.push_back(children_[found->second]); - children_[found->second] = NULL; + children_[found->second] = nullptr; continue; } - const google::protobuf::Type* field_type = NULL; + const google::protobuf::Type* field_type = nullptr; bool is_map = false; NodeKind kind = PRIMITIVE; @@ -363,25 +402,28 @@ void DefaultValueObjectWriter::Node::PopulateChildren( } // If oneof_index() != 0, the child field is part of a "oneof", which means - // the child field is optional and we shouldn't populate its default value. - if (field.oneof_index() != 0) continue; + // the child field is optional and we shouldn't populate its default + // primitive value. + if (field.oneof_index() != 0 && kind == PRIMITIVE) continue; // If the child field is of primitive type, sets its data to the default // value of its type. - google::protobuf::scoped_ptr<Node> child(new Node( - field.json_name(), field_type, kind, - kind == PRIMITIVE ? CreateDefaultDataPieceForField(field, typeinfo) + std::unique_ptr<Node> child(new Node( + preserve_proto_field_names_ ? field.name() : field.json_name(), + field_type, kind, + kind == PRIMITIVE ? CreateDefaultDataPieceForField(field, typeinfo, use_ints_for_enums_) : DataPiece::NullData(), - true, path, suppress_empty_list_, field_scrub_callback_)); + true, path, suppress_empty_list_, preserve_proto_field_names_, use_ints_for_enums_, + field_scrub_callback_)); new_children.push_back(child.release()); } // Adds all leftover nodes in children_ to the beginning of new_child. for (int i = 0; i < children_.size(); ++i) { - if (children_[i] == NULL) { + if (children_[i] == nullptr) { continue; } new_children.insert(new_children.begin(), children_[i]); - children_[i] = NULL; + children_[i] = nullptr; } children_.swap(new_children); } @@ -389,14 +431,14 @@ void DefaultValueObjectWriter::Node::PopulateChildren( void DefaultValueObjectWriter::MaybePopulateChildrenOfAny(Node* node) { // If this is an "Any" node with "@type" already given and no other children // have been added, populates its children. - if (node != NULL && node->is_any() && node->type() != NULL && + if (node != nullptr && node->is_any() && node->type() != nullptr && node->type()->name() != kAnyType && node->number_of_children() == 1) { node->PopulateChildren(typeinfo_); } } DataPiece DefaultValueObjectWriter::FindEnumDefault( - const google::protobuf::Field& field, const TypeInfo* typeinfo) { + const google::protobuf::Field& field, const TypeInfo* typeinfo, bool use_ints_for_enums) { if (!field.default_value().empty()) return DataPiece(field.default_value(), true); @@ -409,12 +451,12 @@ DataPiece DefaultValueObjectWriter::FindEnumDefault( } // We treat the first value as the default if none is specified. return enum_type->enumvalue_size() > 0 - ? DataPiece(enum_type->enumvalue(0).name(), true) + ? (use_ints_for_enums ? DataPiece(enum_type->enumvalue(0).number()) : DataPiece(enum_type->enumvalue(0).name(), true)) : DataPiece::NullData(); } DataPiece DefaultValueObjectWriter::CreateDefaultDataPieceForField( - const google::protobuf::Field& field, const TypeInfo* typeinfo) { + const google::protobuf::Field& field, const TypeInfo* typeinfo, bool use_ints_for_enums) { switch (field.kind()) { case google::protobuf::Field_Kind_TYPE_DOUBLE: { return DataPiece(ConvertTo<double>( @@ -457,7 +499,7 @@ DataPiece DefaultValueObjectWriter::CreateDefaultDataPieceForField( field.default_value(), &DataPiece::ToUint32, static_cast<uint32>(0))); } case google::protobuf::Field_Kind_TYPE_ENUM: { - return FindEnumDefault(field, typeinfo); + return FindEnumDefault(field, typeinfo, use_ints_for_enums); } default: { return DataPiece::NullData(); } } @@ -465,27 +507,30 @@ DataPiece DefaultValueObjectWriter::CreateDefaultDataPieceForField( DefaultValueObjectWriter* DefaultValueObjectWriter::StartObject( StringPiece name) { - if (current_ == NULL) { - vector<string> path; - root_.reset(new Node(name.ToString(), &type_, OBJECT, DataPiece::NullData(), - false, path, suppress_empty_list_, - field_scrub_callback_.get())); + if (current_ == nullptr) { + std::vector<string> path; + root_.reset(CreateNewNode(string(name), &type_, OBJECT, + DataPiece::NullData(), false, path, + suppress_empty_list_, preserve_proto_field_names_, use_ints_for_enums_, + field_scrub_callback_.get())); root_->PopulateChildren(typeinfo_); current_ = root_.get(); return this; } MaybePopulateChildrenOfAny(current_); Node* child = current_->FindChild(name); - if (current_->kind() == LIST || current_->kind() == MAP || child == NULL) { + if (current_->kind() == LIST || current_->kind() == MAP || child == nullptr) { // If current_ is a list or a map node, we should create a new child and use // the type of current_ as the type of the new child. - google::protobuf::scoped_ptr<Node> node(new Node( - name.ToString(), ((current_->kind() == LIST || current_->kind() == MAP) - ? current_->type() - : NULL), - OBJECT, DataPiece::NullData(), false, - child == NULL ? current_->path() : child->path(), - suppress_empty_list_, field_scrub_callback_.get())); + std::unique_ptr<Node> node( + CreateNewNode(string(name), + ((current_->kind() == LIST || current_->kind() == MAP) + ? current_->type() + : nullptr), + OBJECT, DataPiece::NullData(), false, + child == nullptr ? current_->path() : child->path(), + suppress_empty_list_, preserve_proto_field_names_, use_ints_for_enums_, + field_scrub_callback_.get())); child = node.get(); current_->AddChild(node.release()); } @@ -513,21 +558,23 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::EndObject() { DefaultValueObjectWriter* DefaultValueObjectWriter::StartList( StringPiece name) { - if (current_ == NULL) { - vector<string> path; - root_.reset(new Node(name.ToString(), &type_, LIST, DataPiece::NullData(), - false, path, suppress_empty_list_, - field_scrub_callback_.get())); + if (current_ == nullptr) { + std::vector<string> path; + root_.reset(CreateNewNode(string(name), &type_, LIST, DataPiece::NullData(), + false, path, suppress_empty_list_, + preserve_proto_field_names_, use_ints_for_enums_, + field_scrub_callback_.get())); current_ = root_.get(); return this; } MaybePopulateChildrenOfAny(current_); Node* child = current_->FindChild(name); - if (child == NULL || child->kind() != LIST) { - google::protobuf::scoped_ptr<Node> node( - new Node(name.ToString(), NULL, LIST, DataPiece::NullData(), false, - child == NULL ? current_->path() : child->path(), - suppress_empty_list_, field_scrub_callback_.get())); + if (child == nullptr || child->kind() != LIST) { + std::unique_ptr<Node> node( + CreateNewNode(string(name), nullptr, LIST, DataPiece::NullData(), false, + child == nullptr ? current_->path() : child->path(), + suppress_empty_list_, preserve_proto_field_names_, use_ints_for_enums_, + field_scrub_callback_.get())); child = node.get(); current_->AddChild(node.release()); } @@ -540,8 +587,8 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::StartList( void DefaultValueObjectWriter::WriteRoot() { root_->WriteTo(ow_); - root_.reset(NULL); - current_ = NULL; + root_.reset(nullptr); + current_ = nullptr; } DefaultValueObjectWriter* DefaultValueObjectWriter::EndList() { @@ -557,39 +604,43 @@ DefaultValueObjectWriter* DefaultValueObjectWriter::EndList() { void DefaultValueObjectWriter::RenderDataPiece(StringPiece name, const DataPiece& data) { MaybePopulateChildrenOfAny(current_); - util::StatusOr<string> data_string = data.ToString(); - if (current_->type() != NULL && current_->type()->name() == kAnyType && - name == "@type" && data_string.ok()) { - const string& string_value = data_string.ValueOrDie(); - // If the type of current_ is "Any" and its "@type" field is being set here, - // sets the type of current_ to be the type specified by the "@type". - util::StatusOr<const google::protobuf::Type*> found_type = - typeinfo_->ResolveTypeUrl(string_value); - if (!found_type.ok()) { - GOOGLE_LOG(WARNING) << "Failed to resolve type '" << string_value << "'."; - } else { - current_->set_type(found_type.ValueOrDie()); - } - current_->set_is_any(true); - // If the "@type" field is placed after other fields, we should populate - // other children of primitive type now. Otherwise, we should wait until the - // first value field is rendered before we populate the children, because - // the "value" field of a Any message could be omitted. - if (current_->number_of_children() > 1 && current_->type() != NULL) { - current_->PopulateChildren(typeinfo_); + if (current_->type() != nullptr && current_->type()->name() == kAnyType && + name == "@type") { + util::StatusOr<string> data_string = data.ToString(); + if (data_string.ok()) { + const string& string_value = data_string.ValueOrDie(); + // If the type of current_ is "Any" and its "@type" field is being set + // here, sets the type of current_ to be the type specified by the + // "@type". + util::StatusOr<const google::protobuf::Type*> found_type = + typeinfo_->ResolveTypeUrl(string_value); + if (!found_type.ok()) { + GOOGLE_LOG(WARNING) << "Failed to resolve type '" << string_value << "'."; + } else { + current_->set_type(found_type.ValueOrDie()); + } + current_->set_is_any(true); + // If the "@type" field is placed after other fields, we should populate + // other children of primitive type now. Otherwise, we should wait until + // the first value field is rendered before we populate the children, + // because the "value" field of a Any message could be omitted. + if (current_->number_of_children() > 1 && current_->type() != nullptr) { + current_->PopulateChildren(typeinfo_); + } } } Node* child = current_->FindChild(name); - if (child == NULL || child->kind() != PRIMITIVE) { + if (child == nullptr || child->kind() != PRIMITIVE) { // No children are found, creates a new child. - google::protobuf::scoped_ptr<Node> node( - new Node(name.ToString(), NULL, PRIMITIVE, data, false, - child == NULL ? current_->path() : child->path(), - suppress_empty_list_, field_scrub_callback_.get())); - child = node.get(); + std::unique_ptr<Node> node( + CreateNewNode(string(name), nullptr, PRIMITIVE, data, false, + child == nullptr ? current_->path() : child->path(), + suppress_empty_list_, preserve_proto_field_names_, use_ints_for_enums_, + field_scrub_callback_.get())); current_->AddChild(node.release()); } else { child->set_data(data); + child->set_is_placeholder(false); } } diff --git a/src/google/protobuf/util/internal/default_value_objectwriter.h b/src/google/protobuf/util/internal/default_value_objectwriter.h index 5f3b25f3..6e71f9c8 100644 --- a/src/google/protobuf/util/internal/default_value_objectwriter.h +++ b/src/google/protobuf/util/internal/default_value_objectwriter.h @@ -32,9 +32,6 @@ #define GOOGLE_PROTOBUF_UTIL_CONVERTER_DEFAULT_VALUE_OBJECTWRITER_H__ #include <memory> -#ifndef _SHARED_PTR_H -#include <google/protobuf/stubs/shared_ptr.h> -#endif #include <stack> #include <vector> @@ -77,7 +74,7 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter { FieldScrubCallBack; // A unique pointer to a DefaultValueObjectWriter::FieldScrubCallBack. - typedef google::protobuf::scoped_ptr<FieldScrubCallBack> FieldScrubCallBackPtr; + typedef std::unique_ptr<FieldScrubCallBack> FieldScrubCallBackPtr; DefaultValueObjectWriter(TypeResolver* type_resolver, const google::protobuf::Type& type, @@ -126,7 +123,18 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter { // are written. void set_suppress_empty_list(bool value) { suppress_empty_list_ = value; } - private: + // If set to true, original proto field names are used + void set_preserve_proto_field_names(bool value) { + preserve_proto_field_names_ = value; + } + + // If set to true, enums are rendered as ints from output when default values + // are written. + void set_print_enums_as_ints(bool value) { + use_ints_for_enums_ = value; + } + + protected: enum NodeKind { PRIMITIVE = 0, OBJECT = 1, @@ -139,8 +147,14 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter { class LIBPROTOBUF_EXPORT Node { public: Node(const string& name, const google::protobuf::Type* type, NodeKind kind, - const DataPiece& data, bool is_placeholder, const vector<string>& path, - bool suppress_empty_list, FieldScrubCallBack* field_scrub_callback); + const DataPiece& data, bool is_placeholder, + const std::vector<string>& path, bool suppress_empty_list, + FieldScrubCallBack* field_scrub_callback); + Node(const string& name, const google::protobuf::Type* type, NodeKind kind, + const DataPiece& data, bool is_placeholder, + const std::vector<string>& path, bool suppress_empty_list, + bool preserve_proto_field_names, bool use_ints_for_enums, + FieldScrubCallBack* field_scrub_callback); virtual ~Node() { for (int i = 0; i < children_.size(); ++i) { delete children_[i]; @@ -156,17 +170,17 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter { // Populates children of this Node based on its type. If there are already // children created, they will be merged to the result. Caller should pass // in TypeInfo for looking up types of the children. - void PopulateChildren(const TypeInfo* typeinfo); + virtual void PopulateChildren(const TypeInfo* typeinfo); // If this node is a leaf (has data), writes the current node to the // ObjectWriter; if not, then recursively writes the children to the // ObjectWriter. - void WriteTo(ObjectWriter* ow); + virtual void WriteTo(ObjectWriter* ow); // Accessors const string& name() const { return name_; } - const vector<string>& path() const { return path_; } + const std::vector<string>& path() const { return path_; } const google::protobuf::Type* type() const { return type_; } @@ -186,7 +200,7 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter { is_placeholder_ = is_placeholder; } - private: + protected: // Returns the Value Type of a map given the Type of the map entry and a // TypeInfo instance. const google::protobuf::Type* GetMapValueType( @@ -219,25 +233,55 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter { // Whether to suppress empty list output. bool suppress_empty_list_; + // Whether to preserve original proto field names + bool preserve_proto_field_names_; + + // Whether to always print enums as ints + bool use_ints_for_enums_; + // Pointer to function for determining whether a field needs to be scrubbed // or not. This callback is owned by the creator of this node. FieldScrubCallBack* field_scrub_callback_; + private: GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(Node); }; + // Creates a new Node and returns it. Caller owns memory of returned object. + virtual Node* CreateNewNode(const string& name, + const google::protobuf::Type* type, NodeKind kind, + const DataPiece& data, bool is_placeholder, + const std::vector<string>& path, + bool suppress_empty_list, + FieldScrubCallBack* field_scrub_callback); + + // Creates a new Node and returns it. Caller owns memory of returned object. + virtual Node* CreateNewNode(const string& name, + const google::protobuf::Type* type, NodeKind kind, + const DataPiece& data, bool is_placeholder, + const std::vector<string>& path, + bool suppress_empty_list, + bool preserve_proto_field_names, + bool use_ints_for_enums, + FieldScrubCallBack* field_scrub_callback); + + // Creates a DataPiece containing the default value of the type of the field. + static DataPiece CreateDefaultDataPieceForField( + const google::protobuf::Field& field, const TypeInfo* typeinfo, bool use_ints_for_enums); + + protected: + // Returns a pointer to current Node in tree. + Node* current() { return current_; } + + private: // Populates children of "node" if it is an "any" Node and its real type has // been given. void MaybePopulateChildrenOfAny(Node* node); // Writes the root_ node to ow_ and resets the root_ and current_ pointer to - // NULL. + // nullptr. void WriteRoot(); - // Creates a DataPiece containing the default value of the type of the field. - static DataPiece CreateDefaultDataPieceForField( - const google::protobuf::Field& field, const TypeInfo* typeinfo); - // Adds or replaces the data_ of a primitive child node. void RenderDataPiece(StringPiece name, const DataPiece& data); @@ -245,7 +289,8 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter { // there is no default. For proto3, where we cannot specify an explicit // default, a zero value will always be returned. static DataPiece FindEnumDefault(const google::protobuf::Field& field, - const TypeInfo* typeinfo); + const TypeInfo* typeinfo, + bool use_ints_for_enums); // Type information for all the types used in the descriptor. Used to find // google::protobuf::Type of nested messages/enums. @@ -255,18 +300,24 @@ class LIBPROTOBUF_EXPORT DefaultValueObjectWriter : public ObjectWriter { // google::protobuf::Type of the root message type. const google::protobuf::Type& type_; // Holds copies of strings passed to RenderString. - vector<string*> string_values_; + std::vector<string*> string_values_; // The current Node. Owned by its parents. Node* current_; // The root Node. - google::protobuf::scoped_ptr<Node> root_; + std::unique_ptr<Node> root_; // The stack to hold the path of Nodes from current_ to root_; std::stack<Node*> stack_; // Whether to suppress output of empty lists. bool suppress_empty_list_; + // Whether to preserve original proto field names + bool preserve_proto_field_names_; + + // Whether to always print enums as ints + bool use_ints_for_enums_; + // Unique Pointer to function for determining whether a field needs to be // scrubbed or not. FieldScrubCallBackPtr field_scrub_callback_; diff --git a/src/google/protobuf/util/internal/default_value_objectwriter_test.cc b/src/google/protobuf/util/internal/default_value_objectwriter_test.cc index e1dd697a..0c4af61b 100644 --- a/src/google/protobuf/util/internal/default_value_objectwriter_test.cc +++ b/src/google/protobuf/util/internal/default_value_objectwriter_test.cc @@ -60,7 +60,7 @@ class BaseDefaultValueObjectWriterTest TypeInfoTestHelper helper_; MockObjectWriter mock_; ExpectingObjectWriter expects_; - google::protobuf::scoped_ptr<DefaultValueObjectWriter> testing_; + std::unique_ptr<DefaultValueObjectWriter> testing_; }; // Tests to cover some basic DefaultValueObjectWriter use cases. More tests are @@ -156,7 +156,7 @@ class DefaultValueObjectWriterSuppressListTest : BaseDefaultValueObjectWriterTest(DefaultValueTest::descriptor()) { testing_->set_suppress_empty_list(true); } - ~DefaultValueObjectWriterSuppressListTest() {} + ~DefaultValueObjectWriterSuppressListTest() override {} }; INSTANTIATE_TEST_CASE_P(DifferentTypeInfoSourceTest, diff --git a/src/google/protobuf/util/internal/error_listener.h b/src/google/protobuf/util/internal/error_listener.h index 3f063936..a19bd3f7 100644 --- a/src/google/protobuf/util/internal/error_listener.h +++ b/src/google/protobuf/util/internal/error_listener.h @@ -33,9 +33,6 @@ #include <algorithm> #include <memory> -#ifndef _SHARED_PTR_H -#include <google/protobuf/stubs/shared_ptr.h> -#endif #include <string> #include <vector> @@ -57,7 +54,7 @@ class LIBPROTOBUF_EXPORT ErrorListener { // Reports an invalid name at the given location. virtual void InvalidName(const LocationTrackerInterface& loc, - StringPiece unknown_name, StringPiece message) = 0; + StringPiece invalid_name, StringPiece message) = 0; // Reports an invalid value for a field. virtual void InvalidValue(const LocationTrackerInterface& loc, @@ -82,7 +79,7 @@ class LIBPROTOBUF_EXPORT NoopErrorListener : public ErrorListener { virtual ~NoopErrorListener() {} virtual void InvalidName(const LocationTrackerInterface& loc, - StringPiece unknown_name, StringPiece message) {} + StringPiece invalid_name, StringPiece message) {} virtual void InvalidValue(const LocationTrackerInterface& loc, StringPiece type_name, StringPiece value) {} diff --git a/src/google/protobuf/util/internal/field_mask_utility.cc b/src/google/protobuf/util/internal/field_mask_utility.cc index f0e8fc88..778a4510 100644 --- a/src/google/protobuf/util/internal/field_mask_utility.cc +++ b/src/google/protobuf/util/internal/field_mask_utility.cc @@ -30,6 +30,7 @@ #include <google/protobuf/util/internal/field_mask_utility.h> +#include <google/protobuf/util/internal/utility.h> #include <google/protobuf/stubs/strutil.h> #include <google/protobuf/stubs/status_macros.h> @@ -44,11 +45,6 @@ inline util::Status CallPathSink(PathSinkCallback path_sink, return path_sink->Run(arg); } -util::Status CreatePublicError(util::error::Code code, - const string& message) { - return util::Status(code, message); -} - // Appends a FieldMask path segment to a prefix. string AppendPathSegmentToPrefix(StringPiece prefix, StringPiece segment) { if (prefix.empty()) { @@ -58,7 +54,7 @@ string AppendPathSegmentToPrefix(StringPiece prefix, StringPiece segment) { return prefix.ToString(); } // If the segment is a map key, appends it to the prefix without the ".". - if (segment.starts_with("[\"")) { + if (StringStartsWith(segment, "[\"")) { return StrCat(prefix, segment); } return StrCat(prefix, ".", segment); @@ -112,7 +108,7 @@ string ConvertFieldMaskPath(const StringPiece path, util::Status DecodeCompactFieldMaskPaths(StringPiece paths, PathSinkCallback path_sink) { - stack<string> prefix; + std::stack<string> prefix; int length = paths.length(); int previous_position = 0; bool in_map_key = false; @@ -216,7 +212,7 @@ util::Status DecodeCompactFieldMaskPaths(StringPiece paths, StrCat("Invalid FieldMask '", paths, "'. Cannot find matching ')' for all '('.")); } - return util::Status::OK; + return util::Status(); } } // namespace converter diff --git a/src/google/protobuf/util/internal/json_escaping.cc b/src/google/protobuf/util/internal/json_escaping.cc index 06d2791b..06b9a7f2 100644 --- a/src/google/protobuf/util/internal/json_escaping.cc +++ b/src/google/protobuf/util/internal/json_escaping.cc @@ -84,30 +84,6 @@ static const char kCommonEscapes[160][7] = { "\\u009c", "\\u009d", "\\u009e", "\\u009f" }; -// Determines if the given char value is a unicode high-surrogate code unit. -// Such values do not represent characters by themselves, but are used in the -// representation of supplementary characters in the utf-16 encoding. -inline bool IsHighSurrogate(uint16 c) { - // Optimized form of: - // return c >= kMinHighSurrogate && c <= kMaxHighSurrogate; - // (Reduced from 3 ALU instructions to 2 ALU instructions) - return (c & ~(JsonEscaping::kMaxHighSurrogate - - JsonEscaping::kMinHighSurrogate)) - == JsonEscaping::kMinHighSurrogate; -} - -// Determines if the given char value is a unicode low-surrogate code unit. -// Such values do not represent characters by themselves, but are used in the -// representation of supplementary characters in the utf-16 encoding. -inline bool IsLowSurrogate(uint16 c) { - // Optimized form of: - // return c >= kMinLowSurrogate && c <= kMaxLowSurrogate; - // (Reduced from 3 ALU instructions to 2 ALU instructions) - return (c & ~(JsonEscaping::kMaxLowSurrogate - - JsonEscaping::kMinLowSurrogate)) - == JsonEscaping::kMinLowSurrogate; -} - // Determines if the given char value is a unicode surrogate code unit (either // high-surrogate or low-surrogate). inline bool IsSurrogate(uint32 c) { @@ -117,36 +93,12 @@ inline bool IsSurrogate(uint32 c) { return (c & 0xfffff800) == JsonEscaping::kMinHighSurrogate; } -// Returns true if the given unicode code point cp is -// in the supplementary character range. -inline bool IsSupplementalCodePoint(uint32 cp) { - // Optimized form of: - // return kMinSupplementaryCodePoint <= cp && cp <= kMaxCodePoint; - // (Reduced from 3 ALU instructions to 2 ALU instructions) - return (cp & ~(JsonEscaping::kMinSupplementaryCodePoint - 1)) - < JsonEscaping::kMaxCodePoint; -} - // Returns true if the given unicode code point cp is a valid // unicode code point (i.e. in the range 0 <= cp <= kMaxCodePoint). inline bool IsValidCodePoint(uint32 cp) { return cp <= JsonEscaping::kMaxCodePoint; } -// Converts the specified surrogate pair to its supplementary code point value. -// It is the callers' responsibility to validate the specified surrogate pair. -inline uint32 ToCodePoint(uint16 high, uint16 low) { - // Optimized form of: - // return ((high - kMinHighSurrogate) << 10) - // + (low - kMinLowSurrogate) - // + kMinSupplementaryCodePoint; - // (Reduced from 5 ALU instructions to 3 ALU instructions) - return (high << 10) + low + - (JsonEscaping::kMinSupplementaryCodePoint - - (static_cast<unsigned>(JsonEscaping::kMinHighSurrogate) << 10) - - JsonEscaping::kMinLowSurrogate); -} - // Returns the low surrogate for the given unicode code point. The result is // meaningless if the given code point is not a supplementary character. inline uint16 ToLowSurrogate(uint32 cp) { @@ -255,7 +207,7 @@ StringPiece ToHex(uint16 cp, char* buffer) { buffer[3] = kHex[cp & 0x0f]; cp >>= 4; buffer[2] = kHex[cp & 0x0f]; - return StringPiece(buffer).substr(0, 6); + return StringPiece(buffer, 6); } // Stores the 32-bit unicode code point as its hexadecimal digits in buffer @@ -336,19 +288,19 @@ StringPiece EscapeCodePoint(uint32 cp, char* buffer, bool force_output) { cp >>= 6; if (cp <= 0x1f) { buffer[4] = cp | 0xc0; - sp.set(buffer + 4, 2); + sp = StringPiece(buffer + 4, 2); return sp; } buffer[4] = (cp & 0x3f) | 0x80; cp >>= 6; if (cp <= 0x0f) { buffer[3] = cp | 0xe0; - sp.set(buffer + 3, 3); + sp = StringPiece(buffer + 3, 3); return sp; } buffer[3] = (cp & 0x3f) | 0x80; buffer[2] = ((cp >> 6) & 0x07) | 0xf0; - sp.set(buffer + 2, 4); + sp = StringPiece(buffer + 2, 4); } return sp; } diff --git a/src/google/protobuf/util/internal/json_escaping.h b/src/google/protobuf/util/internal/json_escaping.h index e3e329fc..5495c57f 100644 --- a/src/google/protobuf/util/internal/json_escaping.h +++ b/src/google/protobuf/util/internal/json_escaping.h @@ -28,8 +28,8 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#ifndef NET_PROTO2_UTIL_CONVERTER_STRINGS_JSON_ESCAPING_H_ -#define NET_PROTO2_UTIL_CONVERTER_STRINGS_JSON_ESCAPING_H_ +#ifndef GOOGLE_PROTOBUF_UTIL_INTERNAL__JSON_ESCAPING_H__ +#define GOOGLE_PROTOBUF_UTIL_INTERNAL__JSON_ESCAPING_H__ #include <google/protobuf/stubs/common.h> #include <google/protobuf/stubs/bytestream.h> @@ -87,5 +87,5 @@ class JsonEscaping { } // namespace util } // namespace protobuf -#endif // NET_PROTO2_UTIL_CONVERTER_STRINGS_JSON_ESCAPING_H_ } // namespace google +#endif // GOOGLE_PROTOBUF_UTIL_INTERNAL__JSON_ESCAPING_H__ diff --git a/src/google/protobuf/util/internal/json_objectwriter.cc b/src/google/protobuf/util/internal/json_objectwriter.cc index b84210c1..a431177a 100644 --- a/src/google/protobuf/util/internal/json_objectwriter.cc +++ b/src/google/protobuf/util/internal/json_objectwriter.cc @@ -57,7 +57,7 @@ JsonObjectWriter::~JsonObjectWriter() { JsonObjectWriter* JsonObjectWriter::StartObject(StringPiece name) { WritePrefix(name); WriteChar('{'); - Push(); + PushObject(); return this; } @@ -71,7 +71,7 @@ JsonObjectWriter* JsonObjectWriter::EndObject() { JsonObjectWriter* JsonObjectWriter::StartList(StringPiece name) { WritePrefix(name); WriteChar('['); - Push(); + PushArray(); return this; } @@ -147,7 +147,7 @@ JsonObjectWriter* JsonObjectWriter::RenderBytes(StringPiece name, string base64; if (use_websafe_base64_for_bytes_) - WebSafeBase64Escape(value.ToString(), &base64); + WebSafeBase64EscapeWithPadding(value.ToString(), &base64); else Base64Escape(value, &base64); @@ -164,14 +164,20 @@ JsonObjectWriter* JsonObjectWriter::RenderNull(StringPiece name) { return RenderSimple(name, "null"); } +JsonObjectWriter* JsonObjectWriter::RenderNullAsEmpty(StringPiece name) { + return RenderSimple(name, ""); +} + void JsonObjectWriter::WritePrefix(StringPiece name) { bool not_first = !element()->is_first(); if (not_first) WriteChar(','); if (not_first || !element()->is_root()) NewLine(); - if (!name.empty()) { + if (!name.empty() || element()->is_json_object()) { WriteChar('"'); - ArrayByteSource source(name); - JsonEscaping::Escape(&source, &sink_); + if (!name.empty()) { + ArrayByteSource source(name); + JsonEscaping::Escape(&source, &sink_); + } stream_->WriteString("\":"); if (!indent_string_.empty()) WriteChar(' '); } diff --git a/src/google/protobuf/util/internal/json_objectwriter.h b/src/google/protobuf/util/internal/json_objectwriter.h index cb7e2fb3..81644dab 100644 --- a/src/google/protobuf/util/internal/json_objectwriter.h +++ b/src/google/protobuf/util/internal/json_objectwriter.h @@ -32,9 +32,6 @@ #define GOOGLE_PROTOBUF_UTIL_CONVERTER_JSON_OBJECTWRITER_H__ #include <memory> -#ifndef _SHARED_PTR_H -#include <google/protobuf/stubs/shared_ptr.h> -#endif #include <string> #include <google/protobuf/io/coded_stream.h> @@ -89,7 +86,7 @@ class LIBPROTOBUF_EXPORT JsonObjectWriter : public StructuredObjectWriter { public: JsonObjectWriter(StringPiece indent_string, google::protobuf::io::CodedOutputStream* out) - : element_(new Element(NULL)), + : element_(new Element(/*parent=*/nullptr, /*is_json_object=*/false)), stream_(out), sink_(out), indent_string_(indent_string.ToString()), @@ -111,6 +108,7 @@ class LIBPROTOBUF_EXPORT JsonObjectWriter : public StructuredObjectWriter { virtual JsonObjectWriter* RenderString(StringPiece name, StringPiece value); virtual JsonObjectWriter* RenderBytes(StringPiece name, StringPiece value); virtual JsonObjectWriter* RenderNull(StringPiece name); + virtual JsonObjectWriter* RenderNullAsEmpty(StringPiece name); void set_use_websafe_base64_for_bytes(bool value) { use_websafe_base64_for_bytes_ = value; @@ -119,7 +117,10 @@ class LIBPROTOBUF_EXPORT JsonObjectWriter : public StructuredObjectWriter { protected: class LIBPROTOBUF_EXPORT Element : public BaseElement { public: - explicit Element(Element* parent) : BaseElement(parent), is_first_(true) {} + Element(Element* parent, bool is_json_object) + : BaseElement(parent), + is_first_(true), + is_json_object_(is_json_object) {} // Called before each field of the Element is to be processed. // Returns true if this is the first call (processing the first field). @@ -131,8 +132,13 @@ class LIBPROTOBUF_EXPORT JsonObjectWriter : public StructuredObjectWriter { return false; } + // Whether we are currently renderring inside a JSON object (i.e., between + // StartObject() and EndObject()). + bool is_json_object() const { return is_json_object_; } + private: bool is_first_; + bool is_json_object_; GOOGLE_DISALLOW_IMPLICIT_CONSTRUCTORS(Element); }; @@ -166,8 +172,15 @@ class LIBPROTOBUF_EXPORT JsonObjectWriter : public StructuredObjectWriter { return this; } - // Pushes a new element to the stack. - void Push() { element_.reset(new Element(element_.release())); } + // Pushes a new JSON array element to the stack. + void PushArray() { + element_.reset(new Element(element_.release(), /*is_json_object=*/false)); + } + + // Pushes a new JSON object element to the stack. + void PushObject() { + element_.reset(new Element(element_.release(), /*is_json_object=*/true)); + } // Pops an element off of the stack and deletes the popped element. void Pop() { @@ -195,7 +208,7 @@ class LIBPROTOBUF_EXPORT JsonObjectWriter : public StructuredObjectWriter { // Writes an individual character to the output. void WriteChar(const char c) { stream_->WriteRaw(&c, sizeof(c)); } - google::protobuf::scoped_ptr<Element> element_; + std::unique_ptr<Element> element_; google::protobuf::io::CodedOutputStream* stream_; ByteSinkWrapper sink_; const string indent_string_; diff --git a/src/google/protobuf/util/internal/json_objectwriter_test.cc b/src/google/protobuf/util/internal/json_objectwriter_test.cc index b87b06ac..0dc710c7 100644 --- a/src/google/protobuf/util/internal/json_objectwriter_test.cc +++ b/src/google/protobuf/util/internal/json_objectwriter_test.cc @@ -47,7 +47,7 @@ class JsonObjectWriterTest : public ::testing::Test { JsonObjectWriterTest() : str_stream_(new StringOutputStream(&output_)), out_stream_(new CodedOutputStream(str_stream_)), - ow_(NULL) {} + ow_(nullptr) {} virtual ~JsonObjectWriterTest() { delete ow_; @@ -95,6 +95,12 @@ TEST_F(JsonObjectWriterTest, EmptyList) { output_.substr(0, out_stream_->ByteCount())); } +TEST_F(JsonObjectWriterTest, EmptyObjectKey) { + ow_ = new JsonObjectWriter("", out_stream_); + ow_->StartObject("")->RenderString("", "value")->EndObject(); + EXPECT_EQ("{\"\":\"value\"}", output_.substr(0, out_stream_->ByteCount())); +} + TEST_F(JsonObjectWriterTest, ObjectInObject) { ow_ = new JsonObjectWriter("", out_stream_); ow_->StartObject("") @@ -299,11 +305,11 @@ TEST_F(JsonObjectWriterTest, TestWebsafeByteEncoding) { ow_ = new JsonObjectWriter("", out_stream_); ow_->set_use_websafe_base64_for_bytes(true); ow_->StartObject("") - ->RenderBytes("bytes", "\x03\xef\xc0") + ->RenderBytes("bytes", "\x03\xef\xc0\x10") ->EndObject(); // Test that we get websafe base64 encoding when explicitly asked. - EXPECT_EQ("{\"bytes\":\"A-_A\"}", + EXPECT_EQ("{\"bytes\":\"A-_AEA==\"}", output_.substr(0, out_stream_->ByteCount())); } diff --git a/src/google/protobuf/util/internal/json_stream_parser.cc b/src/google/protobuf/util/internal/json_stream_parser.cc index 39be7b03..1c63b31d 100644 --- a/src/google/protobuf/util/internal/json_stream_parser.cc +++ b/src/google/protobuf/util/internal/json_stream_parser.cc @@ -36,15 +36,14 @@ #include <cstdlib> #include <cstring> #include <memory> -#ifndef _SHARED_PTR_H -#include <google/protobuf/stubs/shared_ptr.h> -#endif #include <google/protobuf/stubs/logging.h> #include <google/protobuf/stubs/common.h> #include <google/protobuf/util/internal/object_writer.h> #include <google/protobuf/util/internal/json_escaping.h> #include <google/protobuf/stubs/strutil.h> +#include <google/protobuf/stubs/mathlimits.h> + namespace google { namespace protobuf { @@ -54,6 +53,7 @@ namespace util { // this file. using util::Status; namespace error { +using util::error::CANCELLED; using util::error::INTERNAL; using util::error::INVALID_ARGUMENT; } // namespace error @@ -107,7 +107,9 @@ JsonStreamParser::JsonStreamParser(ObjectWriter* ow) parsed_storage_(), string_open_(0), chunk_storage_(), - coerce_to_utf8_(false) { + coerce_to_utf8_(false), + allow_empty_null_(false), + loose_float_number_conversion_(false) { // Initialize the stack with a single value to be parsed. stack_.push(VALUE); } @@ -125,7 +127,7 @@ util::Status JsonStreamParser::Parse(StringPiece json) { // Don't point chunk to leftover_ because leftover_ will be updated in // ParseChunk(chunk). chunk_storage_.swap(leftover_); - json.AppendToString(&chunk_storage_); + StrAppend(&chunk_storage_, json); chunk = StringPiece(chunk_storage_); } @@ -136,11 +138,11 @@ util::Status JsonStreamParser::Parse(StringPiece json) { // Any leftover characters are stashed in leftover_ for later parsing when // there is more data available. - chunk.substr(n).AppendToString(&leftover_); + StrAppend(&leftover_, chunk.substr(n)); return status; } else { - chunk.CopyToString(&leftover_); - return util::Status::OK; + leftover_.assign(chunk.data(), chunk.size()); + return util::Status(); } } @@ -148,11 +150,11 @@ util::Status JsonStreamParser::FinishParse() { // If we do not expect anything and there is nothing left to parse we're all // done. if (stack_.empty() && leftover_.empty()) { - return util::Status::OK; + return util::Status(); } // Storage for UTF8-coerced string. - google::protobuf::scoped_array<char> utf8; + std::unique_ptr<char[]> utf8; if (coerce_to_utf8_) { utf8.reset(new char[leftover_.size()]); char* coerced = internal::UTF8CoerceToStructurallyValid(leftover_, utf8.get(), ' '); @@ -179,7 +181,7 @@ util::Status JsonStreamParser::FinishParse() { util::Status JsonStreamParser::ParseChunk(StringPiece chunk) { // Do not do any work if the chunk is empty. - if (chunk.empty()) return util::Status::OK; + if (chunk.empty()) return util::Status(); p_ = json_ = chunk; @@ -201,7 +203,7 @@ util::Status JsonStreamParser::ParseChunk(StringPiece chunk) { // unparsed data left, we save it for later parse. leftover_ = p_.ToString(); } - return util::Status::OK; + return util::Status(); } util::Status JsonStreamParser::RunParser() { @@ -242,20 +244,20 @@ util::Status JsonStreamParser::RunParser() { } if (!result.ok()) { // If we were cancelled, save our state and try again later. - if (!finishing_ && result == util::Status::CANCELLED) { + if (!finishing_ && result == util::Status(error::CANCELLED, "")) { stack_.push(type); // If we have a key we still need to render, make sure to save off the // contents in our own storage. if (!key_.empty() && key_storage_.empty()) { - key_.AppendToString(&key_storage_); + StrAppend(&key_storage_, key_); key_ = StringPiece(key_storage_); } - result = util::Status::OK; + result = util::Status(); } return result; } } - return util::Status::OK; + return util::Status(); } util::Status JsonStreamParser::ParseValue(TokenType type) { @@ -277,12 +279,16 @@ util::Status JsonStreamParser::ParseValue(TokenType type) { case UNKNOWN: return ReportUnknown("Expected a value."); default: { + if (allow_empty_null_ && IsEmptyNullAllowed(type)) { + return ParseEmptyNull(); + } + // Special case for having been cut off while parsing, wait for more data. // This handles things like 'fals' being at the end of the string, we // don't know if the next char would be e, completing it, or something // else, making it invalid. if (!finishing_ && p_.length() < false_len) { - return util::Status::CANCELLED; + return util::Status(error::CANCELLED, ""); } return ReportFailure("Unexpected token."); } @@ -293,8 +299,8 @@ util::Status JsonStreamParser::ParseString() { util::Status result = ParseStringHelper(); if (result.ok()) { ow_->RenderString(key_, parsed_); - key_.clear(); - parsed_.clear(); + key_ = StringPiece(); + parsed_ = StringPiece(); parsed_storage_.clear(); } return result; @@ -315,13 +321,12 @@ util::Status JsonStreamParser::ParseStringHelper() { // We're about to handle an escape, copy all bytes from last to data. if (last < data) { parsed_storage_.append(last, data - last); - last = data; } // If we ran out of string after the \, cancel or report an error // depending on if we expect more data later. if (p_.length() == 1) { if (!finishing_) { - return util::Status::CANCELLED; + return util::Status(error::CANCELLED, ""); } return ReportFailure("Closing quote expected in string."); } @@ -371,7 +376,6 @@ util::Status JsonStreamParser::ParseStringHelper() { } else { if (last < data) { parsed_storage_.append(last, data - last); - last = data; } parsed_ = StringPiece(parsed_storage_); } @@ -379,7 +383,7 @@ util::Status JsonStreamParser::ParseStringHelper() { // start fresh. string_open_ = 0; Advance(); - return util::Status::OK; + return util::Status(); } // Normal character, just advance past it. Advance(); @@ -390,7 +394,7 @@ util::Status JsonStreamParser::ParseStringHelper() { } // If we didn't find the closing quote but we expect more data, cancel for now if (!finishing_) { - return util::Status::CANCELLED; + return util::Status(error::CANCELLED, ""); } // End of string reached without a closing quote, report an error. string_open_ = 0; @@ -407,7 +411,7 @@ util::Status JsonStreamParser::ParseStringHelper() { util::Status JsonStreamParser::ParseUnicodeEscape() { if (p_.length() < kUnicodeEscapedLength) { if (!finishing_) { - return util::Status::CANCELLED; + return util::Status(error::CANCELLED, ""); } return ReportFailure("Illegal hex string."); } @@ -424,7 +428,7 @@ util::Status JsonStreamParser::ParseUnicodeEscape() { code <= JsonEscaping::kMaxHighSurrogate) { if (p_.length() < 2 * kUnicodeEscapedLength) { if (!finishing_) { - return util::Status::CANCELLED; + return util::Status(error::CANCELLED, ""); } if (!coerce_to_utf8_) { return ReportFailure("Missing low surrogate."); @@ -461,7 +465,7 @@ util::Status JsonStreamParser::ParseUnicodeEscape() { // Advance past the [final] code unit escape. p_.remove_prefix(kUnicodeEscapedLength); parsed_storage_.append(buf, len); - return util::Status::OK; + return util::Status(); } util::Status JsonStreamParser::ParseNumber() { @@ -471,17 +475,17 @@ util::Status JsonStreamParser::ParseNumber() { switch (number.type) { case NumberResult::DOUBLE: ow_->RenderDouble(key_, number.double_val); - key_.clear(); + key_ = StringPiece(); break; case NumberResult::INT: ow_->RenderInt64(key_, number.int_val); - key_.clear(); + key_ = StringPiece(); break; case NumberResult::UINT: ow_->RenderUint64(key_, number.uint_val); - key_.clear(); + key_ = StringPiece(); break; default: @@ -491,6 +495,19 @@ util::Status JsonStreamParser::ParseNumber() { return result; } +util::Status JsonStreamParser::ParseDoubleHelper( + const string& number, NumberResult* result) { + if (!safe_strtod(number, &result->double_val)) { + return ReportFailure("Unable to parse number."); + } + if (!loose_float_number_conversion_ && + !MathLimits<double>::IsFinite(result->double_val)) { + return ReportFailure("Number exceeds the range of double."); + } + result->type = NumberResult::DOUBLE; + return util::Status(); +} + util::Status JsonStreamParser::ParseNumberHelper(NumberResult* result) { const char* data = p_.data(); int length = p_.length(); @@ -518,7 +535,7 @@ util::Status JsonStreamParser::ParseNumberHelper(NumberResult* result) { // If the entire input is a valid number, and we may have more content in the // future, we abort for now and resume when we know more. if (index == length && !finishing_) { - return util::Status::CANCELLED; + return util::Status(error::CANCELLED, ""); } // Create a string containing just the number, so we can use safe_strtoX @@ -526,12 +543,11 @@ util::Status JsonStreamParser::ParseNumberHelper(NumberResult* result) { // Floating point number, parse as a double. if (floating) { - if (!safe_strtod(number, &result->double_val)) { - return ReportFailure("Unable to parse number."); + util::Status status = ParseDoubleHelper(number, result); + if (status.ok()) { + p_.remove_prefix(index); } - result->type = NumberResult::DOUBLE; - p_.remove_prefix(index); - return util::Status::OK; + return status; } // Positive non-floating point number, parse as a uint64. @@ -540,12 +556,18 @@ util::Status JsonStreamParser::ParseNumberHelper(NumberResult* result) { if (number.length() >= 2 && number[0] == '0') { return ReportFailure("Octal/hex numbers are not valid JSON values."); } - if (!safe_strtou64(number, &result->uint_val)) { - return ReportFailure("Unable to parse number."); + if (safe_strtou64(number, &result->uint_val)) { + result->type = NumberResult::UINT; + p_.remove_prefix(index); + return util::Status(); + } else { + // If the value is too large, parse it as double. + util::Status status = ParseDoubleHelper(number, result); + if (status.ok()) { + p_.remove_prefix(index); + } + return status; } - result->type = NumberResult::UINT; - p_.remove_prefix(index); - return util::Status::OK; } // Octal/Hex numbers are not valid JSON values. @@ -553,21 +575,27 @@ util::Status JsonStreamParser::ParseNumberHelper(NumberResult* result) { return ReportFailure("Octal/hex numbers are not valid JSON values."); } // Negative non-floating point number, parse as an int64. - if (!safe_strto64(number, &result->int_val)) { - return ReportFailure("Unable to parse number."); + if (safe_strto64(number, &result->int_val)) { + result->type = NumberResult::INT; + p_.remove_prefix(index); + return util::Status(); + } else { + // If the value is too large, parse it as double. + util::Status status = ParseDoubleHelper(number, result); + if (status.ok()) { + p_.remove_prefix(index); + } + return status; } - result->type = NumberResult::INT; - p_.remove_prefix(index); - return util::Status::OK; } util::Status JsonStreamParser::HandleBeginObject() { GOOGLE_DCHECK_EQ('{', *p_.data()); Advance(); ow_->StartObject(key_); - key_.clear(); + key_ = StringPiece(); stack_.push(ENTRY); - return util::Status::OK; + return util::Status(); } util::Status JsonStreamParser::ParseObjectMid(TokenType type) { @@ -579,13 +607,13 @@ util::Status JsonStreamParser::ParseObjectMid(TokenType type) { if (type == END_OBJECT) { Advance(); ow_->EndObject(); - return util::Status::OK; + return util::Status(); } // Found a comma, advance past it and get ready for an entry. if (type == VALUE_SEPARATOR) { Advance(); stack_.push(ENTRY); - return util::Status::OK; + return util::Status(); } // Illegal token after key:value pair. return ReportFailure("Expected , or } after key:value pair."); @@ -600,7 +628,7 @@ util::Status JsonStreamParser::ParseEntry(TokenType type) { if (type == END_OBJECT) { ow_->EndObject(); Advance(); - return util::Status::OK; + return util::Status(); } util::Status result; @@ -615,7 +643,7 @@ util::Status JsonStreamParser::ParseEntry(TokenType type) { } else { key_ = parsed_; } - parsed_.clear(); + parsed_ = StringPiece(); } } else if (type == BEGIN_KEY) { // Key is a bare key (back compat), create a StringPiece pointing to it. @@ -639,7 +667,7 @@ util::Status JsonStreamParser::ParseEntryMid(TokenType type) { if (type == ENTRY_SEPARATOR) { Advance(); stack_.push(VALUE); - return util::Status::OK; + return util::Status(); } return ReportFailure("Expected : between key:value pair."); } @@ -648,9 +676,9 @@ util::Status JsonStreamParser::HandleBeginArray() { GOOGLE_DCHECK_EQ('[', *p_.data()); Advance(); ow_->StartList(key_); - key_.clear(); + key_ = StringPiece(); stack_.push(ARRAY_VALUE); - return util::Status::OK; + return util::Status(); } util::Status JsonStreamParser::ParseArrayValue(TokenType type) { @@ -661,14 +689,15 @@ util::Status JsonStreamParser::ParseArrayValue(TokenType type) { if (type == END_ARRAY) { ow_->EndList(); Advance(); - return util::Status::OK; + return util::Status(); } // The ParseValue call may push something onto the stack so we need to make - // sure an ARRAY_MID is after it, so we push it on now. + // sure an ARRAY_MID is after it, so we push it on now. Also, the parsing of + // empty-null array value is relying on this ARRAY_MID token. stack_.push(ARRAY_MID); util::Status result = ParseValue(type); - if (result == util::Status::CANCELLED) { + if (result == util::Status(error::CANCELLED, "")) { // If we were cancelled, pop back off the ARRAY_MID so we don't try to // push it on again when we try over. stack_.pop(); @@ -684,14 +713,14 @@ util::Status JsonStreamParser::ParseArrayMid(TokenType type) { if (type == END_ARRAY) { ow_->EndList(); Advance(); - return util::Status::OK; + return util::Status(); } // Found a comma, advance past it and expect an array value next. if (type == VALUE_SEPARATOR) { Advance(); stack_.push(ARRAY_VALUE); - return util::Status::OK; + return util::Status(); } // Illegal token after array value. return ReportFailure("Expected , or ] after array value."); @@ -699,23 +728,35 @@ util::Status JsonStreamParser::ParseArrayMid(TokenType type) { util::Status JsonStreamParser::ParseTrue() { ow_->RenderBool(key_, true); - key_.clear(); + key_ = StringPiece(); p_.remove_prefix(true_len); - return util::Status::OK; + return util::Status(); } util::Status JsonStreamParser::ParseFalse() { ow_->RenderBool(key_, false); - key_.clear(); + key_ = StringPiece(); p_.remove_prefix(false_len); - return util::Status::OK; + return util::Status(); } util::Status JsonStreamParser::ParseNull() { ow_->RenderNull(key_); - key_.clear(); + key_ = StringPiece(); p_.remove_prefix(null_len); - return util::Status::OK; + return util::Status(); +} + +util::Status JsonStreamParser::ParseEmptyNull() { + ow_->RenderNull(key_); + key_ = StringPiece(); + return util::Status(); +} + +bool JsonStreamParser::IsEmptyNullAllowed(TokenType type) { + if (stack_.empty()) return false; + return (stack_.top() == ARRAY_MID && type == VALUE_SEPARATOR) || + stack_.top() == OBJ_MID; } util::Status JsonStreamParser::ReportFailure(StringPiece message) { @@ -735,7 +776,7 @@ util::Status JsonStreamParser::ReportFailure(StringPiece message) { util::Status JsonStreamParser::ReportUnknown(StringPiece message) { // If we aren't finishing the parse, cancel parsing and try later. if (!finishing_) { - return util::Status::CANCELLED; + return util::Status(error::CANCELLED, ""); } if (p_.empty()) { return ReportFailure(StrCat("Unexpected end of string. ", message)); @@ -765,11 +806,11 @@ util::Status JsonStreamParser::ParseKey() { // we can't know if the key was complete or not. if (!finishing_ && p_.empty()) { p_ = original; - return util::Status::CANCELLED; + return util::Status(error::CANCELLED, ""); } // Since we aren't using the key storage, clear it out. key_storage_.clear(); - return util::Status::OK; + return util::Status(); } JsonStreamParser::TokenType JsonStreamParser::GetNextTokenType() { diff --git a/src/google/protobuf/util/internal/json_stream_parser.h b/src/google/protobuf/util/internal/json_stream_parser.h index 0278c28f..31933b67 100644 --- a/src/google/protobuf/util/internal/json_stream_parser.h +++ b/src/google/protobuf/util/internal/json_stream_parser.h @@ -154,6 +154,9 @@ class LIBPROTOBUF_EXPORT JsonStreamParser { // component. util::Status ParseNumberHelper(NumberResult* result); + // Parse a number as double into a NumberResult. + util::Status ParseDoubleHelper(const string& number, NumberResult* result); + // Handles a { during parsing of a value. util::Status HandleBeginObject(); @@ -179,6 +182,10 @@ class LIBPROTOBUF_EXPORT JsonStreamParser { util::Status ParseTrue(); util::Status ParseFalse(); util::Status ParseNull(); + util::Status ParseEmptyNull(); + + // Whether an empty-null is allowed in the current state. + bool IsEmptyNullAllowed(TokenType type); // Report a failure as a util::Status. util::Status ReportFailure(StringPiece message); @@ -247,6 +254,13 @@ class LIBPROTOBUF_EXPORT JsonStreamParser { // Whether to allow non UTF-8 encoded input and replace invalid code points. bool coerce_to_utf8_; + // Whether allows empty string represented null array value or object entry + // value. + bool allow_empty_null_; + + // Whether allows out-of-range floating point numbers or reject them. + bool loose_float_number_conversion_; + GOOGLE_DISALLOW_IMPLICIT_CONSTRUCTORS(JsonStreamParser); }; diff --git a/src/google/protobuf/util/internal/json_stream_parser_test.cc b/src/google/protobuf/util/internal/json_stream_parser_test.cc index 059ea6d8..a11e9be0 100644 --- a/src/google/protobuf/util/internal/json_stream_parser_test.cc +++ b/src/google/protobuf/util/internal/json_stream_parser_test.cc @@ -81,12 +81,16 @@ using util::Status; // For each test we split the input string on every possible character to ensure // the parser is able to handle arbitrarily split input for all cases. We also // do a final test of the entire test case one character at a time. +// +// It is verified that expected calls to the mocked objects are in sequence. class JsonStreamParserTest : public ::testing::Test { protected: JsonStreamParserTest() : mock_(), ow_(&mock_) {} virtual ~JsonStreamParserTest() {} - util::Status RunTest(StringPiece json, int split, bool coerce_utf8 = false) { + util::Status RunTest(StringPiece json, int split, bool coerce_utf8 = false, + bool allow_empty_null = false, + bool loose_float_number_conversion = false) { JsonStreamParser parser(&mock_); // Special case for split == length, test parsing one character at a time. @@ -116,8 +120,11 @@ class JsonStreamParserTest : public ::testing::Test { return result; } - void DoTest(StringPiece json, int split, bool coerce_utf8 = false) { - util::Status result = RunTest(json, split, coerce_utf8); + void DoTest(StringPiece json, int split, bool coerce_utf8 = false, + bool allow_empty_null = false, + bool loose_float_number_conversion = false) { + util::Status result = RunTest(json, split, coerce_utf8, allow_empty_null, + loose_float_number_conversion); if (!result.ok()) { GOOGLE_LOG(WARNING) << result; } @@ -125,14 +132,21 @@ class JsonStreamParserTest : public ::testing::Test { } void DoErrorTest(StringPiece json, int split, StringPiece error_prefix, - bool coerce_utf8 = false) { - util::Status result = RunTest(json, split, coerce_utf8); + bool coerce_utf8 = false, bool allow_empty_null = false) { + util::Status result = + RunTest(json, split, coerce_utf8, allow_empty_null); EXPECT_EQ(util::error::INVALID_ARGUMENT, result.error_code()); StringPiece error_message(result.error_message()); EXPECT_EQ(error_prefix, error_message.substr(0, error_prefix.size())); } +#ifndef _MSC_VER + // TODO(xiaofeng): We have to disable InSequence check for MSVC because it + // causes stack overflow due to its use of a linked list that is desctructed + // recursively. + ::testing::InSequence in_sequence_; +#endif // !_MSC_VER MockObjectWriter mock_; ExpectingObjectWriter ow_; }; @@ -308,18 +322,27 @@ TEST_F(JsonStreamParserTest, ObjectKeyTypes) { } } -// - array containing array, object, values (true, false, null, num, string) -TEST_F(JsonStreamParserTest, ArrayValues) { - StringPiece str = - "[true, false, null, 'a string', \"another string\", [22, -127, 45.3, " - "-1056.4, 11779497823553162765], {'key': true}]"; +// - array containing primitive values (true, false, null, num, string) +TEST_F(JsonStreamParserTest, ArrayPrimitiveValues) { + StringPiece str = "[true, false, null, 'one', \"two\"]"; for (int i = 0; i <= str.length(); ++i) { ow_.StartList("") ->RenderBool("", true) ->RenderBool("", false) ->RenderNull("") - ->RenderString("", "a string") - ->RenderString("", "another string") + ->RenderString("", "one") + ->RenderString("", "two") + ->EndList(); + DoTest(str, i); + } +} + +// - array containing array, object +TEST_F(JsonStreamParserTest, ArrayComplexValues) { + StringPiece str = + "[[22, -127, 45.3, -1056.4, 11779497823553162765], {'key': true}]"; + for (int i = 0; i <= str.length(); ++i) { + ow_.StartList("") ->StartList("") ->RenderUint64("", 22) ->RenderInt64("", -127) @@ -335,6 +358,7 @@ TEST_F(JsonStreamParserTest, ArrayValues) { } } + // - object containing array, object, value (true, false, null, num, string) TEST_F(JsonStreamParserTest, ObjectValues) { StringPiece str = @@ -670,34 +694,35 @@ TEST_F(JsonStreamParserTest, ExtraCharactersAfterObject) { } } -// numbers too large -TEST_F(JsonStreamParserTest, PositiveNumberTooBig) { - StringPiece str = "[18446744073709551616]"; // 2^64 +TEST_F(JsonStreamParserTest, PositiveNumberTooBigIsDouble) { + StringPiece str = "18446744073709551616"; // 2^64 for (int i = 0; i <= str.length(); ++i) { - ow_.StartList(""); - DoErrorTest(str, i, "Unable to parse number."); + ow_.RenderDouble("", 18446744073709552000.0); + DoTest(str, i); } } -TEST_F(JsonStreamParserTest, NegativeNumberTooBig) { - StringPiece str = "[-18446744073709551616]"; +TEST_F(JsonStreamParserTest, NegativeNumberTooBigIsDouble) { + StringPiece str = "-18446744073709551616"; for (int i = 0; i <= str.length(); ++i) { - ow_.StartList(""); - DoErrorTest(str, i, "Unable to parse number."); + ow_.RenderDouble("", -18446744073709551616.0); + DoTest(str, i); } } -/* -TODO(sven): Fail parsing when parsing a double that is too large. - TEST_F(JsonStreamParserTest, DoubleTooBig) { - StringPiece str = "[184464073709551232321616.45]"; + StringPiece str = "[1.89769e+308]"; for (int i = 0; i <= str.length(); ++i) { ow_.StartList(""); - DoErrorTest(str, i, "Unable to parse number"); + DoErrorTest(str, i, "Number exceeds the range of double."); + } + str = "[-1.89769e+308]"; + for (int i = 0; i <= str.length(); ++i) { + ow_.StartList(""); + DoErrorTest(str, i, "Number exceeds the range of double."); } } -*/ + // invalid bare backslash. TEST_F(JsonStreamParserTest, UnfinishedEscape) { diff --git a/src/google/protobuf/util/internal/proto_writer.cc b/src/google/protobuf/util/internal/proto_writer.cc index 0c38aeb9..b7a52db6 100644 --- a/src/google/protobuf/util/internal/proto_writer.cc +++ b/src/google/protobuf/util/internal/proto_writer.cc @@ -65,7 +65,8 @@ ProtoWriter::ProtoWriter(TypeResolver* type_resolver, own_typeinfo_(true), done_(false), ignore_unknown_fields_(false), - element_(NULL), + use_lower_camel_for_enums_(false), + element_(nullptr), size_insert_(), output_(output), buffer_(), @@ -83,7 +84,8 @@ ProtoWriter::ProtoWriter(const TypeInfo* typeinfo, own_typeinfo_(false), done_(false), ignore_unknown_fields_(false), - element_(NULL), + use_lower_camel_for_enums_(false), + element_(nullptr), size_insert_(), output_(output), buffer_(), @@ -97,14 +99,14 @@ ProtoWriter::~ProtoWriter() { if (own_typeinfo_) { delete typeinfo_; } - if (element_ == NULL) return; + if (element_ == nullptr) return; // Cleanup explicitly in order to avoid destructor stack overflow when input // is deeply nested. // Cast to BaseElement to avoid doing additional checks (like missing fields) // during pop(). - google::protobuf::scoped_ptr<BaseElement> element( + std::unique_ptr<BaseElement> element( static_cast<BaseElement*>(element_.get())->pop<BaseElement>()); - while (element != NULL) { + while (element != nullptr) { element.reset(element->pop<BaseElement>()); } } @@ -264,8 +266,10 @@ inline Status WriteString(int field_number, const DataPiece& data, // Writes an ENUM field, including tag, to the stream. inline Status WriteEnum(int field_number, const DataPiece& data, const google::protobuf::Enum* enum_type, - CodedOutputStream* stream) { - StatusOr<int> e = data.ToEnum(enum_type); + CodedOutputStream* stream, + bool use_lower_camel_for_enums, + bool ignore_unknown_values) { + StatusOr<int> e = data.ToEnum(enum_type, use_lower_camel_for_enums, ignore_unknown_values); if (e.ok()) { WireFormatLite::WriteEnum(field_number, e.ValueOrDie(), stream); } @@ -291,9 +295,9 @@ std::set<const google::protobuf::Field*> GetRequiredFields( ProtoWriter::ProtoElement::ProtoElement(const TypeInfo* typeinfo, const google::protobuf::Type& type, ProtoWriter* enclosing) - : BaseElement(NULL), + : BaseElement(nullptr), ow_(enclosing), - parent_field_(NULL), + parent_field_(nullptr), typeinfo_(typeinfo), proto3_(type.syntax() == google::protobuf::SYNTAX_PROTO3), type_(type), @@ -351,7 +355,7 @@ ProtoWriter::ProtoElement* ProtoWriter::ProtoElement::pop() { if (!proto3_) { // Calls the registered error listener for any required field(s) not yet // seen. - for (set<const google::protobuf::Field*>::iterator it = + for (std::set<const google::protobuf::Field*>::iterator it = required_fields_.begin(); it != required_fields_.end(); ++it) { ow_->MissingField((*it)->name()); @@ -371,7 +375,7 @@ ProtoWriter::ProtoElement* ProtoWriter::ProtoElement::pop() { // all enclosing messages. int size = ow_->size_insert_[size_index_].size; int length = CodedOutputStream::VarintSize32(size); - for (ProtoElement* e = parent(); e != NULL; e = e->parent()) { + for (ProtoElement* e = parent(); e != nullptr; e = e->parent()) { // Only nested messages have size field, lists do not have size field. if (e->size_index_ >= 0) { ow_->size_insert_[e->size_index_].size += length; @@ -391,7 +395,7 @@ void ProtoWriter::ProtoElement::RegisterField( } string ProtoWriter::ProtoElement::ToString() const { - if (parent() == NULL) return ""; + if (parent() == nullptr) return ""; string loc = parent()->ToString(); if (!ow_->IsRepeated(*parent_field_) || parent()->parent_field_ != parent_field_) { @@ -436,7 +440,7 @@ void ProtoWriter::MissingField(StringPiece missing_name) { ProtoWriter* ProtoWriter::StartObject(StringPiece name) { // Starting the root message. Create the root ProtoElement and return. - if (element_ == NULL) { + if (element_ == nullptr) { if (!name.empty()) { InvalidName(name, "Root element should not be named."); } @@ -444,9 +448,9 @@ ProtoWriter* ProtoWriter::StartObject(StringPiece name) { return this; } - const google::protobuf::Field* field = NULL; + const google::protobuf::Field* field = nullptr; field = BeginNamed(name, false); - if (field == NULL) return this; + if (field == nullptr) return this; // Check to see if this field is a oneof and that no oneof in that group has // already been set. @@ -456,7 +460,7 @@ ProtoWriter* ProtoWriter::StartObject(StringPiece name) { } const google::protobuf::Type* type = LookupType(field); - if (type == NULL) { + if (type == nullptr) { ++invalid_depth_; InvalidName(name, StrCat("Missing descriptor for field: ", field->type_url())); @@ -472,14 +476,14 @@ ProtoWriter* ProtoWriter::EndObject() { return this; } - if (element_ != NULL) { + if (element_ != nullptr) { element_.reset(element_->pop()); } // If ending the root element, // then serialize the full message with calculated sizes. - if (element_ == NULL) { + if (element_ == nullptr) { WriteRootMessage(); } return this; @@ -487,7 +491,7 @@ ProtoWriter* ProtoWriter::EndObject() { ProtoWriter* ProtoWriter::StartList(StringPiece name) { const google::protobuf::Field* field = BeginNamed(name, true); - if (field == NULL) return this; + if (field == nullptr) return this; if (!ValidOneof(*field, name)) { ++invalid_depth_; @@ -495,7 +499,7 @@ ProtoWriter* ProtoWriter::StartList(StringPiece name) { } const google::protobuf::Type* type = LookupType(field); - if (type == NULL) { + if (type == nullptr) { ++invalid_depth_; InvalidName(name, StrCat("Missing descriptor for field: ", field->type_url())); @@ -508,7 +512,7 @@ ProtoWriter* ProtoWriter::StartList(StringPiece name) { ProtoWriter* ProtoWriter::EndList() { if (invalid_depth_ > 0) { --invalid_depth_; - } else if (element_ != NULL) { + } else if (element_ != nullptr) { element_.reset(element_->pop()); } return this; @@ -520,12 +524,12 @@ ProtoWriter* ProtoWriter::RenderDataPiece(StringPiece name, if (invalid_depth_ > 0) return this; const google::protobuf::Field* field = Lookup(name); - if (field == NULL) return this; + if (field == nullptr) return this; if (!ValidOneof(*field, name)) return this; const google::protobuf::Type* type = LookupType(field); - if (type == NULL) { + if (type == nullptr) { InvalidName(name, StrCat("Missing descriptor for field: ", field->type_url())); return this; @@ -536,7 +540,7 @@ ProtoWriter* ProtoWriter::RenderDataPiece(StringPiece name, bool ProtoWriter::ValidOneof(const google::protobuf::Field& field, StringPiece unnormalized_name) { - if (element_ == NULL) return true; + if (element_ == nullptr) return true; if (field.oneof_index() > 0) { if (element_->IsOneofIndexTaken(field.oneof_index())) { @@ -662,7 +666,8 @@ ProtoWriter* ProtoWriter::RenderPrimitiveField( case google::protobuf::Field_Kind_TYPE_ENUM: { status = WriteEnum(field.number(), data, typeinfo_->GetEnumByTypeUrl(field.type_url()), - stream_.get()); + stream_.get(), use_lower_camel_for_enums_, + ignore_unknown_fields_); break; } default: // TYPE_GROUP or TYPE_MESSAGE @@ -689,18 +694,18 @@ const google::protobuf::Field* ProtoWriter::BeginNamed(StringPiece name, bool is_list) { if (invalid_depth_ > 0) { ++invalid_depth_; - return NULL; + return nullptr; } const google::protobuf::Field* field = Lookup(name); - if (field == NULL) { + if (field == nullptr) { ++invalid_depth_; // InvalidName() already called in Lookup(). - return NULL; + return nullptr; } if (is_list && !IsRepeated(*field)) { ++invalid_depth_; InvalidName(name, "Proto field is not repeating, cannot start list."); - return NULL; + return nullptr; } return field; } @@ -708,23 +713,23 @@ const google::protobuf::Field* ProtoWriter::BeginNamed(StringPiece name, const google::protobuf::Field* ProtoWriter::Lookup( StringPiece unnormalized_name) { ProtoElement* e = element(); - if (e == NULL) { + if (e == nullptr) { InvalidName(unnormalized_name, "Root element must be a message."); - return NULL; + return nullptr; } if (unnormalized_name.empty()) { // Objects in repeated field inherit the same field descriptor. - if (e->parent_field() == NULL) { + if (e->parent_field() == nullptr) { InvalidName(unnormalized_name, "Proto fields must have a name."); } else if (!IsRepeated(*e->parent_field())) { InvalidName(unnormalized_name, "Proto fields must have a name."); - return NULL; + return nullptr; } return e->parent_field(); } const google::protobuf::Field* field = typeinfo_->FindField(&e->type(), unnormalized_name); - if (field == NULL && !ignore_unknown_fields_) { + if (field == nullptr && !ignore_unknown_fields_) { InvalidName(unnormalized_name, "Cannot find field."); } return field; @@ -743,7 +748,7 @@ void ProtoWriter::WriteRootMessage() { int curr_pos = 0; // Calls the destructor of CodedOutputStream to remove any uninitialized // memory from the Cord before we read it. - stream_.reset(NULL); + stream_.reset(nullptr); const void* data; int length; google::protobuf::io::ArrayInputStream input_stream(buffer_.data(), buffer_.size()); diff --git a/src/google/protobuf/util/internal/proto_writer.h b/src/google/protobuf/util/internal/proto_writer.h index 7f1108ab..f2b4f42f 100644 --- a/src/google/protobuf/util/internal/proto_writer.h +++ b/src/google/protobuf/util/internal/proto_writer.h @@ -81,40 +81,40 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter { virtual ~ProtoWriter(); // ObjectWriter methods. - virtual ProtoWriter* StartObject(StringPiece name); - virtual ProtoWriter* EndObject(); - virtual ProtoWriter* StartList(StringPiece name); - virtual ProtoWriter* EndList(); - virtual ProtoWriter* RenderBool(StringPiece name, bool value) { + ProtoWriter* StartObject(StringPiece name) override; + ProtoWriter* EndObject() override; + ProtoWriter* StartList(StringPiece name) override; + ProtoWriter* EndList() override; + ProtoWriter* RenderBool(StringPiece name, bool value) override { return RenderDataPiece(name, DataPiece(value)); } - virtual ProtoWriter* RenderInt32(StringPiece name, int32 value) { + ProtoWriter* RenderInt32(StringPiece name, int32 value) override { return RenderDataPiece(name, DataPiece(value)); } - virtual ProtoWriter* RenderUint32(StringPiece name, uint32 value) { + ProtoWriter* RenderUint32(StringPiece name, uint32 value) override { return RenderDataPiece(name, DataPiece(value)); } - virtual ProtoWriter* RenderInt64(StringPiece name, int64 value) { + ProtoWriter* RenderInt64(StringPiece name, int64 value) override { return RenderDataPiece(name, DataPiece(value)); } - virtual ProtoWriter* RenderUint64(StringPiece name, uint64 value) { + ProtoWriter* RenderUint64(StringPiece name, uint64 value) override { return RenderDataPiece(name, DataPiece(value)); } - virtual ProtoWriter* RenderDouble(StringPiece name, double value) { + ProtoWriter* RenderDouble(StringPiece name, double value) override { return RenderDataPiece(name, DataPiece(value)); } - virtual ProtoWriter* RenderFloat(StringPiece name, float value) { + ProtoWriter* RenderFloat(StringPiece name, float value) override { return RenderDataPiece(name, DataPiece(value)); } - virtual ProtoWriter* RenderString(StringPiece name, StringPiece value) { + ProtoWriter* RenderString(StringPiece name, StringPiece value) override { return RenderDataPiece(name, DataPiece(value, use_strict_base64_decoding())); } - virtual ProtoWriter* RenderBytes(StringPiece name, StringPiece value) { + ProtoWriter* RenderBytes(StringPiece name, StringPiece value) override { return RenderDataPiece( name, DataPiece(value, false, use_strict_base64_decoding())); } - virtual ProtoWriter* RenderNull(StringPiece name) { + ProtoWriter* RenderNull(StringPiece name) override { return RenderDataPiece(name, DataPiece::NullData()); } @@ -126,11 +126,11 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter { // Returns the location tracker to use for tracking locations for errors. const LocationTrackerInterface& location() { - return element_ != NULL ? *element_ : *tracker_; + return element_ != nullptr ? *element_ : *tracker_; } // When true, we finished writing to output a complete message. - bool done() { return done_; } + bool done() override { return done_; } // Returns the proto stream object. google::protobuf::io::CodedOutputStream* stream() { return stream_.get(); } @@ -148,6 +148,10 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter { ignore_unknown_fields_ = ignore_unknown_fields; } + void set_use_lower_camel_for_enums(bool use_lower_camel_for_enums) { + use_lower_camel_for_enums_ = use_lower_camel_for_enums; + } + protected: class LIBPROTOBUF_EXPORT ProtoElement : public BaseElement, public LocationTrackerInterface { public: @@ -169,7 +173,7 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter { ProtoElement* pop(); // Accessors - // parent_field() may be NULL if we are at root. + // parent_field() may be nullptr if we are at root. const google::protobuf::Field* parent_field() const { return parent_field_; } @@ -200,7 +204,7 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter { ProtoWriter* ow_; // Describes the element as a field in the parent message. - // parent_field_ is NULL if and only if this element is the root element. + // parent_field_ is nullptr if and only if this element is the root element. const google::protobuf::Field* parent_field_; // TypeInfo to lookup types. @@ -238,7 +242,7 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter { ProtoWriter(const TypeInfo* typeinfo, const google::protobuf::Type& type, strings::ByteSink* output, ErrorListener* listener); - virtual ProtoElement* element() { return element_.get(); } + ProtoElement* element() override { return element_.get(); } // Helper methods for calling ErrorListener. See error_listener.h. void InvalidName(StringPiece unknown_name, StringPiece message); @@ -255,7 +259,7 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter { // extensions are found. const google::protobuf::Field* Lookup(StringPiece name); - // Lookup the field type in the type descriptor. Returns NULL if the type + // Lookup the field type in the type descriptor. Returns nullptr if the type // is not known. const google::protobuf::Type* LookupType( const google::protobuf::Field* field); @@ -305,15 +309,19 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter { // Indicates whether we finished writing root message completely. bool done_; - // If true, don't report unknown field names to the listener. + // If true, don't report unknown field names and enum values to the listener. bool ignore_unknown_fields_; + // If true, check if enum name in camel case or without underscore matches the + // field name. + bool use_lower_camel_for_enums_; + // Variable for internal state processing: // element_ : the current element. // size_insert_: sizes of nested messages. // pos - position to insert the size field. // size - size value to be inserted. - google::protobuf::scoped_ptr<ProtoElement> element_; + std::unique_ptr<ProtoElement> element_; std::deque<SizeInfo> size_insert_; // Variables for output generation: @@ -324,7 +332,7 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter { strings::ByteSink* output_; string buffer_; google::protobuf::io::StringOutputStream adapter_; - google::protobuf::scoped_ptr<google::protobuf::io::CodedOutputStream> stream_; + std::unique_ptr<google::protobuf::io::CodedOutputStream> stream_; // Variables for error tracking and reporting: // listener_ : a place to report any errors found. @@ -332,7 +340,7 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter { // tracker_ : the root location tracker interface. ErrorListener* listener_; int invalid_depth_; - google::protobuf::scoped_ptr<LocationTrackerInterface> tracker_; + std::unique_ptr<LocationTrackerInterface> tracker_; GOOGLE_DISALLOW_IMPLICIT_CONSTRUCTORS(ProtoWriter); }; diff --git a/src/google/protobuf/util/internal/protostream_objectsource.cc b/src/google/protobuf/util/internal/protostream_objectsource.cc index 0048d75b..56e6db12 100644 --- a/src/google/protobuf/util/internal/protostream_objectsource.cc +++ b/src/google/protobuf/util/internal/protostream_objectsource.cc @@ -73,19 +73,19 @@ namespace { static int kDefaultMaxRecursionDepth = 64; -// Finds a field with the given number. NULL if none found. +// Finds a field with the given number. nullptr if none found. const google::protobuf::Field* FindFieldByNumber( const google::protobuf::Type& type, int number); // Returns true if the field is packable. bool IsPackable(const google::protobuf::Field& field); -// Finds an enum value with the given number. NULL if none found. +// Finds an enum value with the given number. nullptr if none found. const google::protobuf::EnumValue* FindEnumValueByNumber( const google::protobuf::Enum& tech_enum, int number); // Utility function to format nanos. -const string FormatNanos(uint32 nanos); +const string FormatNanos(uint32 nanos, bool with_trailing_zeros); StatusOr<string> MapKeyDefaultValueAsString( const google::protobuf::Field& field) { @@ -120,10 +120,14 @@ ProtoStreamObjectSource::ProtoStreamObjectSource( own_typeinfo_(true), type_(type), use_lower_camel_for_enums_(false), + use_ints_for_enums_(false), + preserve_proto_field_names_(false), recursion_depth_(0), max_recursion_depth_(kDefaultMaxRecursionDepth), - render_unknown_fields_(false) { - GOOGLE_LOG_IF(DFATAL, stream == NULL) << "Input stream is NULL."; + render_unknown_fields_(false), + render_unknown_enum_values_(true), + add_trailing_zeros_for_timestamp_and_duration_(false) { + GOOGLE_LOG_IF(DFATAL, stream == nullptr) << "Input stream is nullptr."; } ProtoStreamObjectSource::ProtoStreamObjectSource( @@ -134,10 +138,14 @@ ProtoStreamObjectSource::ProtoStreamObjectSource( own_typeinfo_(false), type_(type), use_lower_camel_for_enums_(false), + use_ints_for_enums_(false), + preserve_proto_field_names_(false), recursion_depth_(0), max_recursion_depth_(kDefaultMaxRecursionDepth), - render_unknown_fields_(false) { - GOOGLE_LOG_IF(DFATAL, stream == NULL) << "Input stream is NULL."; + render_unknown_fields_(false), + render_unknown_enum_values_(true), + add_trailing_zeros_for_timestamp_and_duration_(false) { + GOOGLE_LOG_IF(DFATAL, stream == nullptr) << "Input stream is nullptr."; } ProtoStreamObjectSource::~ProtoStreamObjectSource() { @@ -157,7 +165,7 @@ const google::protobuf::Field* ProtoStreamObjectSource::FindAndVerifyField( const google::protobuf::Field* field = FindFieldByNumber(type, tag >> 3); // Verify if the field corresponds to the wire type in tag. // If there is any discrepancy, mark the field as not found. - if (field != NULL) { + if (field != nullptr) { WireFormatLite::WireType expected_type = WireFormatLite::WireTypeForFieldType( static_cast<WireFormatLite::FieldType>(field->kind())); @@ -165,7 +173,7 @@ const google::protobuf::Field* ProtoStreamObjectSource::FindAndVerifyField( if (actual_type != expected_type && (!IsPackable(*field) || actual_type != WireFormatLite::WIRETYPE_LENGTH_DELIMITED)) { - field = NULL; + field = nullptr; } } return field; @@ -178,11 +186,11 @@ Status ProtoStreamObjectSource::WriteMessage(const google::protobuf::Type& type, ObjectWriter* ow) const { const TypeRenderer* type_renderer = FindTypeRenderer(type.name()); - if (type_renderer != NULL) { + if (type_renderer != nullptr) { return (*type_renderer)(this, type, name, ow); } - const google::protobuf::Field* field = NULL; + const google::protobuf::Field* field = nullptr; string field_name; // last_tag set to dummy value that is different from tag. uint32 tag = stream_->ReadTag(), last_tag = tag + 1; @@ -195,15 +203,19 @@ Status ProtoStreamObjectSource::WriteMessage(const google::protobuf::Type& type, if (tag != last_tag) { // Update field only if tag is changed. last_tag = tag; field = FindAndVerifyField(type, tag); - if (field != NULL) { - field_name = field->json_name(); + if (field != nullptr) { + if (preserve_proto_field_names_) { + field_name = field->name(); + } else { + field_name = field->json_name(); + } } } - if (field == NULL) { + if (field == nullptr) { // If we didn't find a field, skip this unknown tag. // TODO(wpoon): Check return boolean value. WireFormat::SkipField(stream_, tag, - render_unknown_fields_ ? &unknown_fields : NULL); + render_unknown_fields_ ? &unknown_fields : nullptr); tag = stream_->ReadTag(); continue; } @@ -230,7 +242,7 @@ Status ProtoStreamObjectSource::WriteMessage(const google::protobuf::Type& type, if (include_start_and_end) { ow->EndObject(); } - return Status::OK; + return util::Status(); } StatusOr<uint32> ProtoStreamObjectSource::RenderList( @@ -270,8 +282,8 @@ StatusOr<uint32> ProtoStreamObjectSource::RenderMap( for (uint32 tag = stream_->ReadTag(); tag != 0; tag = stream_->ReadTag()) { const google::protobuf::Field* field = FindAndVerifyField(*field_type, tag); - if (field == NULL) { - WireFormat::SkipField(stream_, tag, NULL); + if (field == nullptr) { + WireFormat::SkipField(stream_, tag, nullptr); continue; } // Map field numbers are key = 1 and value = 2 @@ -282,7 +294,7 @@ StatusOr<uint32> ProtoStreamObjectSource::RenderMap( // An absent map key is treated as the default. const google::protobuf::Field* key_field = FindFieldByNumber(*field_type, 1); - if (key_field == NULL) { + if (key_field == nullptr) { // The Type info for this map entry is incorrect. It should always // have a field named "key" and with field number 1. return Status(util::error::INTERNAL, "Invalid map entry."); @@ -310,13 +322,13 @@ Status ProtoStreamObjectSource::RenderPacked( RETURN_IF_ERROR(RenderField(field, StringPiece(), ow)); } stream_->PopLimit(old_limit); - return Status::OK; + return util::Status(); } Status ProtoStreamObjectSource::RenderTimestamp( const ProtoStreamObjectSource* os, const google::protobuf::Type& type, StringPiece field_name, ObjectWriter* ow) { - pair<int64, int32> p = os->ReadSecondsAndNanos(type); + std::pair<int64, int32> p = os->ReadSecondsAndNanos(type); int64 seconds = p.first; int32 nanos = p.second; if (seconds > kTimestampMaxSeconds || seconds < kTimestampMinSeconds) { @@ -334,13 +346,13 @@ Status ProtoStreamObjectSource::RenderTimestamp( ow->RenderString(field_name, ::google::protobuf::internal::FormatTime(seconds, nanos)); - return Status::OK; + return util::Status(); } Status ProtoStreamObjectSource::RenderDuration( const ProtoStreamObjectSource* os, const google::protobuf::Type& type, StringPiece field_name, ObjectWriter* ow) { - pair<int64, int32> p = os->ReadSecondsAndNanos(type); + std::pair<int64, int32> p = os->ReadSecondsAndNanos(type); int64 seconds = p.first; int32 nanos = p.second; if (seconds > kDurationMaxSeconds || seconds < kDurationMinSeconds) { @@ -370,10 +382,12 @@ Status ProtoStreamObjectSource::RenderDuration( sign = "-"; nanos = -nanos; } - string formatted_duration = StringPrintf("%s%lld%ss", sign.c_str(), seconds, - FormatNanos(nanos).c_str()); + string formatted_duration = StringPrintf( + "%s%lld%ss", sign.c_str(), seconds, + FormatNanos(nanos, os->add_trailing_zeros_for_timestamp_and_duration_) + .c_str()); ow->RenderString(field_name, formatted_duration); - return Status::OK; + return util::Status(); } Status ProtoStreamObjectSource::RenderDouble(const ProtoStreamObjectSource* os, @@ -387,7 +401,7 @@ Status ProtoStreamObjectSource::RenderDouble(const ProtoStreamObjectSource* os, os->stream_->ReadTag(); } ow->RenderDouble(field_name, bit_cast<double>(buffer64)); - return Status::OK; + return util::Status(); } Status ProtoStreamObjectSource::RenderFloat(const ProtoStreamObjectSource* os, @@ -401,7 +415,7 @@ Status ProtoStreamObjectSource::RenderFloat(const ProtoStreamObjectSource* os, os->stream_->ReadTag(); } ow->RenderFloat(field_name, bit_cast<float>(buffer32)); - return Status::OK; + return util::Status(); } Status ProtoStreamObjectSource::RenderInt64(const ProtoStreamObjectSource* os, @@ -415,7 +429,7 @@ Status ProtoStreamObjectSource::RenderInt64(const ProtoStreamObjectSource* os, os->stream_->ReadTag(); } ow->RenderInt64(field_name, bit_cast<int64>(buffer64)); - return Status::OK; + return util::Status(); } Status ProtoStreamObjectSource::RenderUInt64(const ProtoStreamObjectSource* os, @@ -429,7 +443,7 @@ Status ProtoStreamObjectSource::RenderUInt64(const ProtoStreamObjectSource* os, os->stream_->ReadTag(); } ow->RenderUint64(field_name, bit_cast<uint64>(buffer64)); - return Status::OK; + return util::Status(); } Status ProtoStreamObjectSource::RenderInt32(const ProtoStreamObjectSource* os, @@ -443,7 +457,7 @@ Status ProtoStreamObjectSource::RenderInt32(const ProtoStreamObjectSource* os, os->stream_->ReadTag(); } ow->RenderInt32(field_name, bit_cast<int32>(buffer32)); - return Status::OK; + return util::Status(); } Status ProtoStreamObjectSource::RenderUInt32(const ProtoStreamObjectSource* os, @@ -457,7 +471,7 @@ Status ProtoStreamObjectSource::RenderUInt32(const ProtoStreamObjectSource* os, os->stream_->ReadTag(); } ow->RenderUint32(field_name, bit_cast<uint32>(buffer32)); - return Status::OK; + return util::Status(); } Status ProtoStreamObjectSource::RenderBool(const ProtoStreamObjectSource* os, @@ -472,7 +486,7 @@ Status ProtoStreamObjectSource::RenderBool(const ProtoStreamObjectSource* os, os->stream_->ReadTag(); } ow->RenderBool(field_name, buffer64 != 0); - return Status::OK; + return util::Status(); } Status ProtoStreamObjectSource::RenderString(const ProtoStreamObjectSource* os, @@ -488,7 +502,7 @@ Status ProtoStreamObjectSource::RenderString(const ProtoStreamObjectSource* os, os->stream_->ReadTag(); } ow->RenderString(field_name, str); - return Status::OK; + return util::Status(); } Status ProtoStreamObjectSource::RenderBytes(const ProtoStreamObjectSource* os, @@ -504,14 +518,14 @@ Status ProtoStreamObjectSource::RenderBytes(const ProtoStreamObjectSource* os, os->stream_->ReadTag(); } ow->RenderBytes(field_name, str); - return Status::OK; + return util::Status(); } Status ProtoStreamObjectSource::RenderStruct(const ProtoStreamObjectSource* os, const google::protobuf::Type& type, StringPiece field_name, ObjectWriter* ow) { - const google::protobuf::Field* field = NULL; + const google::protobuf::Field* field = nullptr; uint32 tag = os->stream_->ReadTag(); ow->StartObject(field_name); while (tag != 0) { @@ -523,23 +537,23 @@ Status ProtoStreamObjectSource::RenderStruct(const ProtoStreamObjectSource* os, } } ow->EndObject(); - return Status::OK; + return util::Status(); } Status ProtoStreamObjectSource::RenderStructValue( const ProtoStreamObjectSource* os, const google::protobuf::Type& type, StringPiece field_name, ObjectWriter* ow) { - const google::protobuf::Field* field = NULL; + const google::protobuf::Field* field = nullptr; for (uint32 tag = os->stream_->ReadTag(); tag != 0; tag = os->stream_->ReadTag()) { field = os->FindAndVerifyField(type, tag); - if (field == NULL) { - WireFormat::SkipField(os->stream_, tag, NULL); + if (field == nullptr) { + WireFormat::SkipField(os->stream_, tag, nullptr); continue; } RETURN_IF_ERROR(os->RenderField(field, field_name, ow)); } - return Status::OK; + return util::Status(); } // TODO(skarvaje): Avoid code duplication of for loops and SkipField logic. @@ -552,19 +566,19 @@ Status ProtoStreamObjectSource::RenderStructListValue( if (tag == 0) { ow->StartList(field_name); ow->EndList(); - return Status::OK; + return util::Status(); } while (tag != 0) { const google::protobuf::Field* field = os->FindAndVerifyField(type, tag); - if (field == NULL) { - WireFormat::SkipField(os->stream_, tag, NULL); + if (field == nullptr) { + WireFormat::SkipField(os->stream_, tag, nullptr); tag = os->stream_->ReadTag(); continue; } ASSIGN_OR_RETURN(tag, os->RenderList(field, field_name, tag, ow)); } - return Status::OK; + return util::Status(); } Status ProtoStreamObjectSource::RenderAny(const ProtoStreamObjectSource* os, @@ -579,8 +593,8 @@ Status ProtoStreamObjectSource::RenderAny(const ProtoStreamObjectSource* os, // First read out the type_url and value from the proto stream for (tag = os->stream_->ReadTag(); tag != 0; tag = os->stream_->ReadTag()) { const google::protobuf::Field* field = os->FindAndVerifyField(type, tag); - if (field == NULL) { - WireFormat::SkipField(os->stream_, tag, NULL); + if (field == nullptr) { + WireFormat::SkipField(os->stream_, tag, nullptr); continue; } // 'type_url' has field number of 1 and 'value' has field number 2 @@ -606,7 +620,7 @@ Status ProtoStreamObjectSource::RenderAny(const ProtoStreamObjectSource* os, ow->RenderString("@type", type_url); } ow->EndObject(); - return util::Status::OK; + return util::Status(); } // If there is a value but no type, we cannot render it, so report an error. @@ -653,7 +667,7 @@ Status ProtoStreamObjectSource::RenderFieldMask( tag = os->stream_->ReadTag()) { if (paths_field_tag == 0) { const google::protobuf::Field* field = os->FindAndVerifyField(type, tag); - if (field != NULL && field->number() == 1 && + if (field != nullptr && field->number() == 1 && field->name() == "paths") { paths_field_tag = tag; } @@ -671,7 +685,7 @@ Status ProtoStreamObjectSource::RenderFieldMask( combined.append(ConvertFieldMaskPath(str, &ToCamelCase)); } ow->RenderString(field_name, combined); - return Status::OK; + return util::Status(); } @@ -740,7 +754,7 @@ Status ProtoStreamObjectSource::RenderField( // Get the nested message type for this field. const google::protobuf::Type* type = typeinfo_->GetTypeByTypeUrl(field->type_url()); - if (type == NULL) { + if (type == nullptr) { return Status(util::error::INTERNAL, StrCat("Invalid configuration. Could not find the type: ", field->type_url())); @@ -749,7 +763,7 @@ Status ProtoStreamObjectSource::RenderField( // Short-circuit any special type rendering to save call-stack space. const TypeRenderer* type_renderer = FindTypeRenderer(type->name()); - bool use_type_renderer = type_renderer != NULL; + bool use_type_renderer = type_renderer != nullptr; if (use_type_renderer) { RETURN_IF_ERROR((*type_renderer)(this, *type, field_name, ow)); @@ -767,7 +781,7 @@ Status ProtoStreamObjectSource::RenderField( // Render all other non-message types. return RenderNonMessageField(field, field_name, ow); } - return Status::OK; + return util::Status(); } Status ProtoStreamObjectSource::RenderNonMessageField( @@ -857,18 +871,25 @@ Status ProtoStreamObjectSource::RenderNonMessageField( // up. const google::protobuf::Enum* en = typeinfo_->GetEnumByTypeUrl(field->type_url()); - // Lookup the name of the enum, and render that. Skips unknown enums. - if (en != NULL) { + // Lookup the name of the enum, and render that. Unknown enum values + // are printed as integers. + if (en != nullptr) { const google::protobuf::EnumValue* enum_value = FindEnumValueByNumber(*en, buffer32); - if (enum_value != NULL) { - if (use_lower_camel_for_enums_) - ow->RenderString(field_name, ToCamelCase(enum_value->name())); - else + if (enum_value != nullptr) { + if (use_ints_for_enums_) { + ow->RenderInt32(field_name, buffer32); + } else if (use_lower_camel_for_enums_) { + ow->RenderString(field_name, + EnumValueNameToLowerCamelCase(enum_value->name())); + } else { ow->RenderString(field_name, enum_value->name()); + } + } else if (render_unknown_enum_values_) { + ow->RenderInt32(field_name, buffer32); } - } else { - GOOGLE_LOG(INFO) << "Unknown enum skipped: " << field->type_url(); + } else if (render_unknown_enum_values_) { + ow->RenderInt32(field_name, buffer32); } break; } @@ -887,7 +908,7 @@ Status ProtoStreamObjectSource::RenderNonMessageField( default: break; } - return Status::OK; + return util::Status(); } // TODO(skarvaje): Fix this to avoid code duplication. @@ -982,10 +1003,10 @@ const string ProtoStreamObjectSource::ReadFieldValueAsString( const google::protobuf::Enum* en = typeinfo_->GetEnumByTypeUrl(field.type_url()); // Lookup the name of the enum, and render that. Skips unknown enums. - if (en != NULL) { + if (en != nullptr) { const google::protobuf::EnumValue* enum_value = FindEnumValueByNumber(*en, buffer32); - if (enum_value != NULL) { + if (enum_value != nullptr) { result = enum_value->name(); } } @@ -1015,12 +1036,8 @@ bool ProtoStreamObjectSource::IsMap( const google::protobuf::Field& field) const { const google::protobuf::Type* field_type = typeinfo_->GetTypeByTypeUrl(field.type_url()); - - // TODO(xiaofeng): Unify option names. return field.kind() == google::protobuf::Field_Kind_TYPE_MESSAGE && - (GetBoolOptionOrDefault(field_type->options(), - "google.protobuf.MessageOptions.map_entry", false) || - GetBoolOptionOrDefault(field_type->options(), "map_entry", false)); + google::protobuf::util::converter::IsMap(field, *field_type); } std::pair<int64, int32> ProtoStreamObjectSource::ReadSecondsAndNanos( @@ -1033,8 +1050,8 @@ std::pair<int64, int32> ProtoStreamObjectSource::ReadSecondsAndNanos( for (tag = stream_->ReadTag(); tag != 0; tag = stream_->ReadTag()) { const google::protobuf::Field* field = FindAndVerifyField(type, tag); - if (field == NULL) { - WireFormat::SkipField(stream_, tag, NULL); + if (field == nullptr) { + WireFormat::SkipField(stream_, tag, nullptr); continue; } // 'seconds' has field number of 1 and 'nanos' has field number 2 @@ -1060,7 +1077,7 @@ Status ProtoStreamObjectSource::IncrementRecursionDepth( StrCat("Message too deep. Max recursion depth reached for type '", type_name, "', field '", field_name, "'")); } - return Status::OK; + return util::Status(); } namespace { @@ -1072,7 +1089,7 @@ const google::protobuf::Field* FindFieldByNumber( return &type.fields(i); } } - return NULL; + return nullptr; } // TODO(skarvaje): Replace FieldDescriptor by implementing IsTypePackable() @@ -1093,12 +1110,16 @@ const google::protobuf::EnumValue* FindEnumValueByNumber( return &ev; } } - return NULL; + return nullptr; } // TODO(skarvaje): Look into optimizing this by not doing computation on // double. -const string FormatNanos(uint32 nanos) { +const string FormatNanos(uint32 nanos, bool with_trailing_zeros) { + if (nanos == 0) { + return with_trailing_zeros ? ".000" : ""; + } + const char* format = (nanos % 1000 != 0) ? "%.9f" : (nanos % 1000000 != 0) ? "%.6f" : "%.3f"; string formatted = diff --git a/src/google/protobuf/util/internal/protostream_objectsource.h b/src/google/protobuf/util/internal/protostream_objectsource.h index 243f85b2..b56efdf4 100644 --- a/src/google/protobuf/util/internal/protostream_objectsource.h +++ b/src/google/protobuf/util/internal/protostream_objectsource.h @@ -110,6 +110,15 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectSource : public ObjectSource { use_lower_camel_for_enums_ = value; } + // Sets whether to always output enums as ints, by default this is off, and + // enums are rendered as strings. + void set_use_ints_for_enums(bool value) { use_ints_for_enums_ = value; } + + // Sets whether to use original proto field names + void set_preserve_proto_field_names(bool value) { + preserve_proto_field_names_ = value; + } + // Sets the max recursion depth of proto message to be deserialized. Proto // messages over this depth will fail to be deserialized. // Default value is 64. @@ -129,6 +138,28 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectSource : public ObjectSource { bool include_start_and_end, ObjectWriter* ow) const; + // Renders a repeating field (packed or unpacked). Returns the next tag after + // reading all sequential repeating elements. The caller should use this tag + // before reading more tags from the stream. + virtual util::StatusOr<uint32> RenderList( + const google::protobuf::Field* field, StringPiece name, uint32 list_tag, + ObjectWriter* ow) const; + + // Looks up a field and verify its consistency with wire type in tag. + const google::protobuf::Field* FindAndVerifyField( + const google::protobuf::Type& type, uint32 tag) const; + + // Renders a field value to the ObjectWriter. + util::Status RenderField(const google::protobuf::Field* field, + StringPiece field_name, ObjectWriter* ow) const; + + // Reads field value according to Field spec in 'field' and returns the read + // value as string. This only works for primitive datatypes (no message + // types). + const string ReadFieldValueAsString( + const google::protobuf::Field& field) const; + + private: ProtoStreamObjectSource(google::protobuf::io::CodedInputStream* stream, const TypeInfo* typeinfo, @@ -138,19 +169,9 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectSource : public ObjectSource { const google::protobuf::Type&, StringPiece, ObjectWriter*); - // Looks up a field and verify its consistency with wire type in tag. - const google::protobuf::Field* FindAndVerifyField( - const google::protobuf::Type& type, uint32 tag) const; - // TODO(skarvaje): Mark these methods as non-const as they modify internal // state (stream_). // - // Renders a repeating field (packed or unpacked). - // Returns the next tag after reading all sequential repeating elements. The - // caller should use this tag before reading more tags from the stream. - util::StatusOr<uint32> RenderList(const google::protobuf::Field* field, - StringPiece name, uint32 list_tag, - ObjectWriter* ow) const; // Renders a NWP map. // Returns the next tag after reading all map entries. The caller should use // this tag before reading more tags from the stream. @@ -234,10 +255,6 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectSource : public ObjectSource { static void DeleteRendererMap(); static TypeRenderer* FindTypeRenderer(const string& type_url); - // Renders a field value to the ObjectWriter. - util::Status RenderField(const google::protobuf::Field* field, - StringPiece field_name, ObjectWriter* ow) const; - // Same as above but renders all non-message field types. Callers don't call // this function directly. They just use RenderField. util::Status RenderNonMessageField(const google::protobuf::Field* field, @@ -245,12 +262,6 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectSource : public ObjectSource { ObjectWriter* ow) const; - // Reads field value according to Field spec in 'field' and returns the read - // value as string. This only works for primitive datatypes (no message - // types). - const string ReadFieldValueAsString( - const google::protobuf::Field& field) const; - // Utility function to detect proto maps. The 'field' MUST be repeated. bool IsMap(const google::protobuf::Field& field) const; @@ -271,6 +282,7 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectSource : public ObjectSource { // Type information for all the types used in the descriptor. Used to find // google::protobuf::Type of nested messages/enums. const TypeInfo* typeinfo_; + // Whether this class owns the typeinfo_ object. If true the typeinfo_ object // should be deleted in the destructor. bool own_typeinfo_; @@ -282,6 +294,12 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectSource : public ObjectSource { // Whether to render enums using lowerCamelCase. Defaults to false. bool use_lower_camel_for_enums_; + // Whether to render enums as ints always. Defaults to false. + bool use_ints_for_enums_; + + // Whether to preserve proto field names + bool preserve_proto_field_names_; + // Tracks current recursion depth. mutable int recursion_depth_; @@ -291,6 +309,12 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectSource : public ObjectSource { // Whether to render unknown fields. bool render_unknown_fields_; + // Whether to render unknown enum values. + bool render_unknown_enum_values_; + + // Whether to add trailing zeros for timestamp and duration. + bool add_trailing_zeros_for_timestamp_and_duration_; + GOOGLE_DISALLOW_IMPLICIT_CONSTRUCTORS(ProtoStreamObjectSource); }; diff --git a/src/google/protobuf/util/internal/protostream_objectsource_test.cc b/src/google/protobuf/util/internal/protostream_objectsource_test.cc index 3f6fdf97..df790728 100644 --- a/src/google/protobuf/util/internal/protostream_objectsource_test.cc +++ b/src/google/protobuf/util/internal/protostream_objectsource_test.cc @@ -31,9 +31,6 @@ #include <google/protobuf/util/internal/protostream_objectsource.h> #include <memory> -#ifndef _SHARED_PTR_H -#include <google/protobuf/stubs/shared_ptr.h> -#endif #include <sstream> #include <google/protobuf/stubs/casts.h> @@ -42,15 +39,16 @@ #include <google/protobuf/io/zero_copy_stream_impl_lite.h> #include <google/protobuf/descriptor.h> #include <google/protobuf/util/internal/expecting_objectwriter.h> +#include <google/protobuf/util/internal/testdata/anys.pb.h> #include <google/protobuf/util/internal/testdata/books.pb.h> #include <google/protobuf/util/internal/testdata/field_mask.pb.h> -#include <google/protobuf/util/internal/type_info_test_helper.h> -#include <google/protobuf/util/internal/constants.h> -#include <google/protobuf/stubs/strutil.h> -#include <google/protobuf/util/internal/testdata/anys.pb.h> #include <google/protobuf/util/internal/testdata/maps.pb.h> +#include <google/protobuf/util/internal/testdata/proto3.pb.h> #include <google/protobuf/util/internal/testdata/struct.pb.h> #include <google/protobuf/util/internal/testdata/timestamp_duration.pb.h> +#include <google/protobuf/util/internal/type_info_test_helper.h> +#include <google/protobuf/util/internal/constants.h> +#include <google/protobuf/stubs/strutil.h> #include <gtest/gtest.h> @@ -66,24 +64,24 @@ using google::protobuf::Message; using google::protobuf::io::ArrayInputStream; using google::protobuf::io::CodedInputStream; using util::Status; +using google::protobuf::testing::AnyM; +using google::protobuf::testing::AnyOut; using google::protobuf::testing::Author; using google::protobuf::testing::BadAuthor; using google::protobuf::testing::BadNestedBook; using google::protobuf::testing::Book; -using google::protobuf::testing::Cyclic; using google::protobuf::testing::Book_Label; +using google::protobuf::testing::Cyclic; +using google::protobuf::testing::FieldMaskTest; +using google::protobuf::testing::MapOut; +using google::protobuf::testing::MapOutWireFormat; using google::protobuf::testing::NestedBook; +using google::protobuf::testing::NestedFieldMask; using google::protobuf::testing::PackedPrimitive; using google::protobuf::testing::Primitive; -using google::protobuf::testing::more_author; -using google::protobuf::testing::maps::MapOut; -using google::protobuf::testing::maps::MapOutWireFormat; -using google::protobuf::testing::timestampduration::TimestampDuration; -using google::protobuf::testing::anys::AnyOut; -using google::protobuf::testing::anys::AnyM; -using google::protobuf::testing::FieldMaskTest; -using google::protobuf::testing::NestedFieldMask; -using google::protobuf::testing::structs::StructType; +using google::protobuf::testing::Proto3Message; +using google::protobuf::testing::StructType; +using google::protobuf::testing::TimestampDuration; using ::testing::_; @@ -100,27 +98,31 @@ class ProtostreamObjectSourceTest : helper_(GetParam()), mock_(), ow_(&mock_), - use_lower_camel_for_enums_(false) { - helper_.ResetTypeInfo(Book::descriptor()); + use_lower_camel_for_enums_(false), + use_ints_for_enums_(false), + add_trailing_zeros_(false), + render_unknown_enum_values_(true) { + helper_.ResetTypeInfo(Book::descriptor(), Proto3Message::descriptor()); } virtual ~ProtostreamObjectSourceTest() {} void DoTest(const Message& msg, const Descriptor* descriptor) { Status status = ExecuteTest(msg, descriptor); - EXPECT_EQ(Status::OK, status); + EXPECT_EQ(util::Status(), status); } Status ExecuteTest(const Message& msg, const Descriptor* descriptor) { - ostringstream oss; + std::ostringstream oss; msg.SerializePartialToOstream(&oss); string proto = oss.str(); ArrayInputStream arr_stream(proto.data(), proto.size()); CodedInputStream in_stream(&arr_stream); - google::protobuf::scoped_ptr<ProtoStreamObjectSource> os( + std::unique_ptr<ProtoStreamObjectSource> os( helper_.NewProtoSource(&in_stream, GetTypeUrl(descriptor))); if (use_lower_camel_for_enums_) os->set_use_lower_camel_for_enums(true); + if (use_ints_for_enums_) os->set_use_ints_for_enums(true); os->set_max_recursion_depth(64); return os->WriteTo(&mock_); } @@ -268,11 +270,22 @@ class ProtostreamObjectSourceTest void UseLowerCamelForEnums() { use_lower_camel_for_enums_ = true; } + void UseIntsForEnums() { use_ints_for_enums_ = true; } + + void AddTrailingZeros() { add_trailing_zeros_ = true; } + + void SetRenderUnknownEnumValues(bool value) { + render_unknown_enum_values_ = value; + } + testing::TypeInfoTestHelper helper_; ::testing::NiceMock<MockObjectWriter> mock_; ExpectingObjectWriter ow_; bool use_lower_camel_for_enums_; + bool use_ints_for_enums_; + bool add_trailing_zeros_; + bool render_unknown_enum_values_; }; INSTANTIATE_TEST_CASE_P(DifferentTypeInfoSourceTest, @@ -474,7 +487,7 @@ TEST_P(ProtostreamObjectSourceTest, DoTest(book, Book::descriptor()); } -TEST_P(ProtostreamObjectSourceTest, LowerCamelEnumOutputTest) { +TEST_P(ProtostreamObjectSourceTest, LowerCamelEnumOutputMacroCase) { Book book; book.set_type(Book::ACTION_AND_ADVENTURE); @@ -484,6 +497,26 @@ TEST_P(ProtostreamObjectSourceTest, LowerCamelEnumOutputTest) { DoTest(book, Book::descriptor()); } +TEST_P(ProtostreamObjectSourceTest, LowerCamelEnumOutputSnakeCase) { + Book book; + book.set_type(Book::arts_and_photography); + + UseLowerCamelForEnums(); + + ow_.StartObject("")->RenderString("type", "artsAndPhotography")->EndObject(); + DoTest(book, Book::descriptor()); +} + +TEST_P(ProtostreamObjectSourceTest, LowerCamelEnumOutputWithNumber) { + Book book; + book.set_type(Book::I18N_Tech); + + UseLowerCamelForEnums(); + + ow_.StartObject("")->RenderString("type", "i18nTech")->EndObject(); + DoTest(book, Book::descriptor()); +} + TEST_P(ProtostreamObjectSourceTest, EnumCaseIsUnchangedByDefault) { Book book; book.set_type(Book::ACTION_AND_ADVENTURE); @@ -493,6 +526,40 @@ TEST_P(ProtostreamObjectSourceTest, EnumCaseIsUnchangedByDefault) { DoTest(book, Book::descriptor()); } +TEST_P(ProtostreamObjectSourceTest, UseIntsForEnumsTest) { + Book book; + book.set_type(Book::ACTION_AND_ADVENTURE); + + UseIntsForEnums(); + + ow_.StartObject("")->RenderInt32("type", 3)->EndObject(); + DoTest(book, Book::descriptor()); +} + +TEST_P(ProtostreamObjectSourceTest, + UnknownEnumAreDroppedWhenRenderUnknownEnumValuesIsUnset) { + Proto3Message message; + message.set_enum_value(static_cast<Proto3Message::NestedEnum>(1234)); + + SetRenderUnknownEnumValues(false); + + // Unknown enum values are not output. + ow_.StartObject("")->EndObject(); + DoTest(message, Proto3Message::descriptor()); +} + +TEST_P(ProtostreamObjectSourceTest, + UnknownEnumAreOutputWhenRenderUnknownEnumValuesIsSet) { + Proto3Message message; + message.set_enum_value(static_cast<Proto3Message::NestedEnum>(1234)); + + SetRenderUnknownEnumValues(true); + + // Unknown enum values are output. + ow_.StartObject("")->RenderInt32("enumValue", 1234)->EndObject(); + DoTest(message, Proto3Message::descriptor()); +} + TEST_P(ProtostreamObjectSourceTest, CyclicMessageDepthTest) { Cyclic cyclic; cyclic.set_m_int(123); @@ -679,7 +746,7 @@ INSTANTIATE_TEST_CASE_P(DifferentTypeInfoSourceTest, // This is the example expected output. // { // "any": { -// "@type": "type.googleapis.com/google.protobuf.testing.anys.AnyM" +// "@type": "type.googleapis.com/google.protobuf.testing.AnyM" // "foo": "foovalue" // } // } @@ -694,7 +761,7 @@ TEST_P(ProtostreamObjectSourceAnysTest, BasicAny) { ow_.StartObject("") ->StartObject("any") ->RenderString("@type", - "type.googleapis.com/google.protobuf.testing.anys.AnyM") + "type.googleapis.com/google.protobuf.testing.AnyM") ->RenderString("foo", "foovalue") ->EndObject() ->EndObject(); @@ -708,8 +775,7 @@ TEST_P(ProtostreamObjectSourceAnysTest, RecursiveAny) { any->set_type_url("type.googleapis.com/google.protobuf.Any"); ::google::protobuf::Any nested_any; - nested_any.set_type_url( - "type.googleapis.com/google.protobuf.testing.anys.AnyM"); + nested_any.set_type_url("type.googleapis.com/google.protobuf.testing.AnyM"); AnyM m; m.set_foo("foovalue"); @@ -722,7 +788,7 @@ TEST_P(ProtostreamObjectSourceAnysTest, RecursiveAny) { ->RenderString("@type", "type.googleapis.com/google.protobuf.Any") ->StartObject("value") ->RenderString("@type", - "type.googleapis.com/google.protobuf.testing.anys.AnyM") + "type.googleapis.com/google.protobuf.testing.AnyM") ->RenderString("foo", "foovalue") ->EndObject() ->EndObject() @@ -741,7 +807,7 @@ TEST_P(ProtostreamObjectSourceAnysTest, DoubleRecursiveAny) { ::google::protobuf::Any second_nested_any; second_nested_any.set_type_url( - "type.googleapis.com/google.protobuf.testing.anys.AnyM"); + "type.googleapis.com/google.protobuf.testing.AnyM"); AnyM m; m.set_foo("foovalue"); @@ -756,7 +822,7 @@ TEST_P(ProtostreamObjectSourceAnysTest, DoubleRecursiveAny) { ->RenderString("@type", "type.googleapis.com/google.protobuf.Any") ->StartObject("value") ->RenderString("@type", - "type.googleapis.com/google.protobuf.testing.anys.AnyM") + "type.googleapis.com/google.protobuf.testing.AnyM") ->RenderString("foo", "foovalue") ->EndObject() ->EndObject() @@ -1001,6 +1067,20 @@ TEST_P(ProtostreamObjectSourceTimestampTest, InvalidDurationAboveMaxTest) { EXPECT_EQ(util::error::INTERNAL, status.error_code()); } +TEST_P(ProtostreamObjectSourceTimestampTest, TimestampDurationDefaultValue) { + TimestampDuration out; + out.mutable_ts()->set_seconds(0); + out.mutable_dur()->set_seconds(0); + + ow_.StartObject("") + ->RenderString("ts", "1970-01-01T00:00:00Z") + ->RenderString("dur", "0s") + ->EndObject(); + + DoTest(out, TimestampDuration::descriptor()); +} + + } // namespace converter } // namespace util } // namespace protobuf diff --git a/src/google/protobuf/util/internal/protostream_objectwriter.cc b/src/google/protobuf/util/internal/protostream_objectwriter.cc index 73e05cfe..2edfd075 100644 --- a/src/google/protobuf/util/internal/protostream_objectwriter.cc +++ b/src/google/protobuf/util/internal/protostream_objectwriter.cc @@ -62,9 +62,10 @@ ProtoStreamObjectWriter::ProtoStreamObjectWriter( const ProtoStreamObjectWriter::Options& options) : ProtoWriter(type_resolver, type, output, listener), master_type_(type), - current_(NULL), + current_(nullptr), options_(options) { set_ignore_unknown_fields(options_.ignore_unknown_fields); + set_use_lower_camel_for_enums(options_.use_lower_camel_for_enums); } ProtoStreamObjectWriter::ProtoStreamObjectWriter( @@ -72,18 +73,18 @@ ProtoStreamObjectWriter::ProtoStreamObjectWriter( strings::ByteSink* output, ErrorListener* listener) : ProtoWriter(typeinfo, type, output, listener), master_type_(type), - current_(NULL), + current_(nullptr), options_(ProtoStreamObjectWriter::Options::Defaults()) {} ProtoStreamObjectWriter::~ProtoStreamObjectWriter() { - if (current_ == NULL) return; + if (current_ == nullptr) return; // Cleanup explicitly in order to avoid destructor stack overflow when input // is deeply nested. // Cast to BaseElement to avoid doing additional checks (like missing fields) // during pop(). - google::protobuf::scoped_ptr<BaseElement> element( + std::unique_ptr<BaseElement> element( static_cast<BaseElement*>(current_.get())->pop<BaseElement>()); - while (element != NULL) { + while (element != nullptr) { element.reset(element->pop<BaseElement>()); } } @@ -172,7 +173,7 @@ Status GetNanosFromStringPiece(StringPiece s_nanos, *nanos = i_nanos * conversion; } - return Status::OK; + return Status(); } } // namespace @@ -185,24 +186,18 @@ ProtoStreamObjectWriter::AnyWriter::AnyWriter(ProtoStreamObjectWriter* parent) output_(&data_), depth_(0), is_well_known_type_(false), - well_known_type_render_(NULL) {} + well_known_type_render_(nullptr) {} ProtoStreamObjectWriter::AnyWriter::~AnyWriter() {} void ProtoStreamObjectWriter::AnyWriter::StartObject(StringPiece name) { ++depth_; // If an object writer is absent, that means we have not called StartAny() - // before reaching here. This is an invalid state. StartAny() gets called - // whenever we see an "@type" being rendered (see AnyWriter::RenderDataPiece). - if (ow_ == NULL) { - // Make sure we are not already in an invalid state. This avoids making - // multiple unnecessary InvalidValue calls. - if (!invalid_) { - parent_->InvalidValue("Any", - StrCat("Missing or invalid @type for any field in ", - parent_->master_type_.name())); - invalid_ = true; - } + // before reaching here, which happens when we have data before the "@type" + // field. + if (ow_ == nullptr) { + // Save data before the "@type" field for later replay. + uninterpreted_events_.push_back(Event(Event::START_OBJECT, name)); } else if (is_well_known_type_ && depth_ == 1) { // For well-known types, the only other field besides "@type" should be a // "value" field. @@ -222,10 +217,15 @@ void ProtoStreamObjectWriter::AnyWriter::StartObject(StringPiece name) { bool ProtoStreamObjectWriter::AnyWriter::EndObject() { --depth_; - // As long as depth_ >= 0, we know we haven't reached the end of Any. - // Propagate these EndObject() calls to the contained ow_. For regular - // message types, we propagate the end of Any as well. - if (ow_ != NULL && (depth_ >= 0 || !is_well_known_type_)) { + if (ow_ == nullptr) { + if (depth_ >= 0) { + // Save data before the "@type" field for later replay. + uninterpreted_events_.push_back(Event(Event::END_OBJECT)); + } + } else if (depth_ >= 0 || !is_well_known_type_) { + // As long as depth_ >= 0, we know we haven't reached the end of Any. + // Propagate these EndObject() calls to the contained ow_. For regular + // message types, we propagate the end of Any as well. ow_->EndObject(); } // A negative depth_ implies that we have reached the end of Any @@ -239,14 +239,9 @@ bool ProtoStreamObjectWriter::AnyWriter::EndObject() { void ProtoStreamObjectWriter::AnyWriter::StartList(StringPiece name) { ++depth_; - // We expect ow_ to be present as this call only makes sense inside an Any. - if (ow_ == NULL) { - if (!invalid_) { - parent_->InvalidValue("Any", - StrCat("Missing or invalid @type for any field in ", - parent_->master_type_.name())); - invalid_ = true; - } + if (ow_ == nullptr) { + // Save data before the "@type" field for later replay. + uninterpreted_events_.push_back(Event(Event::START_LIST, name)); } else if (is_well_known_type_ && depth_ == 1) { if (name != "value" && !invalid_) { parent_->InvalidValue("Any", @@ -265,8 +260,10 @@ void ProtoStreamObjectWriter::AnyWriter::EndList() { GOOGLE_LOG(DFATAL) << "Mismatched EndList found, should not be possible"; depth_ = 0; } - // We don't write an error on the close, only on the open - if (ow_ != NULL) { + if (ow_ == nullptr) { + // Save data before the "@type" field for later replay. + uninterpreted_events_.push_back(Event(Event::END_LIST)); + } else { ow_->EndList(); } } @@ -275,25 +272,21 @@ void ProtoStreamObjectWriter::AnyWriter::RenderDataPiece( StringPiece name, const DataPiece& value) { // Start an Any only at depth_ 0. Other RenderDataPiece calls with "@type" // should go to the contained ow_ as they indicate nested Anys. - if (depth_ == 0 && ow_ == NULL && name == "@type") { + if (depth_ == 0 && ow_ == nullptr && name == "@type") { StartAny(value); - } else if (ow_ == NULL) { - if (!invalid_) { - parent_->InvalidValue("Any", - StrCat("Missing or invalid @type for any field in ", - parent_->master_type_.name())); - invalid_ = true; - } + } else if (ow_ == nullptr) { + // Save data before the "@type" field. + uninterpreted_events_.push_back(Event(name, value)); } else if (depth_ == 0 && is_well_known_type_) { if (name != "value" && !invalid_) { parent_->InvalidValue("Any", "Expect a \"value\" field for well-known types."); invalid_ = true; } - if (well_known_type_render_ == NULL) { + if (well_known_type_render_ == nullptr) { // Only Any and Struct don't have a special type render but both of // them expect a JSON object (i.e., a StartObject() call). - if (!invalid_) { + if (value.type() != DataPiece::TYPE_NULL && !invalid_) { parent_->InvalidValue("Any", "Expect a JSON object."); invalid_ = true; } @@ -334,7 +327,7 @@ void ProtoStreamObjectWriter::AnyWriter::StartAny(const DataPiece& value) { const google::protobuf::Type* type = resolved_type.ValueOrDie(); well_known_type_render_ = FindTypeRenderer(type_url_); - if (well_known_type_render_ != NULL || + if (well_known_type_render_ != nullptr || // Explicitly list Any and Struct here because they don't have a // custom renderer. type->name() == kAnyType || type->name() == kStructType) { @@ -358,13 +351,29 @@ void ProtoStreamObjectWriter::AnyWriter::StartAny(const DataPiece& value) { if (!is_well_known_type_) { ow_->StartObject(""); } + + // Now we know the proto type and can interpret all data fields we gathered + // before the "@type" field. + for (int i = 0; i < uninterpreted_events_.size(); ++i) { + uninterpreted_events_[i].Replay(this); + } } void ProtoStreamObjectWriter::AnyWriter::WriteAny() { - if (ow_ == NULL) { - // If we had no object writer, we never got any content, so just return - // immediately, which is equivalent to writing an empty Any. - return; + if (ow_ == nullptr) { + if (uninterpreted_events_.empty()) { + // We never got any content, so just return immediately, which is + // equivalent to writing an empty Any. + return; + } else { + // There are uninterpreted data, but we never got a "@type" field. + if (!invalid_) { + parent_->InvalidValue("Any", StrCat("Missing @type for any field in ", + parent_->master_type_.name())); + invalid_ = true; + } + return; + } } // Render the type_url and value fields directly to the stream. // type_url has tag 1 and value has tag 2. @@ -374,10 +383,45 @@ void ProtoStreamObjectWriter::AnyWriter::WriteAny() { } } +void ProtoStreamObjectWriter::AnyWriter::Event::Replay( + AnyWriter* writer) const { + switch (type_) { + case START_OBJECT: + writer->StartObject(name_); + break; + case END_OBJECT: + writer->EndObject(); + break; + case START_LIST: + writer->StartList(name_); + break; + case END_LIST: + writer->EndList(); + break; + case RENDER_DATA_PIECE: + writer->RenderDataPiece(name_, value_); + break; + } +} + +void ProtoStreamObjectWriter::AnyWriter::Event::DeepCopy() { + // DataPiece only contains a string reference. To make sure the referenced + // string value stays valid, we make a copy of the string value and update + // DataPiece to reference our own copy. + if (value_.type() == DataPiece::TYPE_STRING) { + StrAppend(&value_storage_, value_.str()); + value_ = DataPiece(value_storage_, value_.use_strict_base64_decoding()); + } else if (value_.type() == DataPiece::TYPE_BYTES) { + value_storage_ = value_.ToBytes().ValueOrDie(); + value_ = + DataPiece(value_storage_, true, value_.use_strict_base64_decoding()); + } +} + ProtoStreamObjectWriter::Item::Item(ProtoStreamObjectWriter* enclosing, ItemType item_type, bool is_placeholder, bool is_list) - : BaseElement(NULL), + : BaseElement(nullptr), ow_(enclosing), any_(), item_type_(item_type), @@ -423,7 +467,7 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::StartObject( // Starting the root message. Create the root Item and return. // ANY message type does not need special handling, just set the ItemType // to ANY. - if (current_ == NULL) { + if (current_ == nullptr) { ProtoWriter::StartObject(name); current_.reset(new Item( this, master_type_.name() == kAnyType ? Item::ANY : Item::MESSAGE, @@ -497,14 +541,14 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::StartObject( // If top of stack is g.p.Struct type, start the struct the map field within // it. - if (element() != NULL && IsStruct(*element()->parent_field())) { + if (element() != nullptr && IsStruct(*element()->parent_field())) { // Render "fields": [ Push("fields", Item::MAP, true, true); return this; } // If top of stack is g.p.Value type, start the Struct within it. - if (element() != NULL && IsStructValue(*element()->parent_field())) { + if (element() != nullptr && IsStructValue(*element()->parent_field())) { // Render // "struct_value": { // "fields": [ @@ -515,7 +559,7 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::StartObject( } const google::protobuf::Field* field = BeginNamed(name, false); - if (field == NULL) return this; + if (field == nullptr) return this; if (IsStruct(*field)) { // Start a struct object. @@ -563,7 +607,7 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::EndObject() { return this; } - if (current_ == NULL) return this; + if (current_ == nullptr) return this; if (current_->IsAny()) { if (current_->any()->EndObject()) return this; @@ -583,7 +627,7 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::StartList(StringPiece name) { // Since we cannot have a top-level repeated item in protobuf, the only way // this is valid is if we start a special type google.protobuf.ListValue or // google.protobuf.Value. - if (current_ == NULL) { + if (current_ == nullptr) { if (!name.empty()) { InvalidName(name, "Root element should not be named."); IncrementInvalidDepth(); @@ -662,7 +706,7 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::StartList(StringPiece name) { if (invalid_depth() > 0) return this; // case i and ii above. Start "list_value" field within g.p.Value - if (element() != NULL && element()->parent_field() != NULL) { + if (element() != nullptr && element()->parent_field() != nullptr) { // Render // "list_value": { // "values": [ // Start this list @@ -690,7 +734,7 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::StartList(StringPiece name) { // When name is empty and stack is not empty, we are rendering an item within // a list. if (name.empty()) { - if (element() != NULL && element()->parent_field() != NULL) { + if (element() != nullptr && element()->parent_field() != nullptr) { if (IsStructValue(*element()->parent_field())) { // Since it is g.p.Value, we bind directly to the list_value. // Render @@ -721,7 +765,7 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::StartList(StringPiece name) { // name is not empty const google::protobuf::Field* field = Lookup(name); - if (field == NULL) { + if (field == nullptr) { IncrementInvalidDepth(); return this; } @@ -793,7 +837,7 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::EndList() { return this; } - if (current_ == NULL) return this; + if (current_ == nullptr) return this; if (current_->IsAny()) { current_->any()->EndList(); @@ -818,7 +862,7 @@ Status ProtoStreamObjectWriter::RenderStructValue(ProtoStreamObjectWriter* ow, ow->ProtoWriter::RenderDataPiece( "string_value", DataPiece(SimpleItoa(int_value.ValueOrDie()), true)); - return Status::OK; + return Status(); } } struct_field_name = "number_value"; @@ -833,13 +877,22 @@ Status ProtoStreamObjectWriter::RenderStructValue(ProtoStreamObjectWriter* ow, ow->ProtoWriter::RenderDataPiece( "string_value", DataPiece(SimpleItoa(int_value.ValueOrDie()), true)); - return Status::OK; + return Status(); } } struct_field_name = "number_value"; break; } case DataPiece::TYPE_DOUBLE: { + if (ow->options_.struct_integers_as_strings) { + StatusOr<double> double_value = data.ToDouble(); + if (double_value.ok()) { + ow->ProtoWriter::RenderDataPiece( + "string_value", + DataPiece(SimpleDtoa(double_value.ValueOrDie()), true)); + return Status(); + } + } struct_field_name = "number_value"; break; } @@ -862,11 +915,12 @@ Status ProtoStreamObjectWriter::RenderStructValue(ProtoStreamObjectWriter* ow, } } ow->ProtoWriter::RenderDataPiece(struct_field_name, data); - return Status::OK; + return Status(); } Status ProtoStreamObjectWriter::RenderTimestamp(ProtoStreamObjectWriter* ow, const DataPiece& data) { + if (data.type() == DataPiece::TYPE_NULL) return Status(); if (data.type() != DataPiece::TYPE_STRING) { return Status(INVALID_ARGUMENT, StrCat("Invalid data type for timestamp, value is ", @@ -885,18 +939,19 @@ Status ProtoStreamObjectWriter::RenderTimestamp(ProtoStreamObjectWriter* ow, ow->ProtoWriter::RenderDataPiece("seconds", DataPiece(seconds)); ow->ProtoWriter::RenderDataPiece("nanos", DataPiece(nanos)); - return Status::OK; + return Status(); } static inline util::Status RenderOneFieldPath(ProtoStreamObjectWriter* ow, StringPiece path) { ow->ProtoWriter::RenderDataPiece( "paths", DataPiece(ConvertFieldMaskPath(path, &ToSnakeCase), true)); - return Status::OK; + return Status(); } Status ProtoStreamObjectWriter::RenderFieldMask(ProtoStreamObjectWriter* ow, const DataPiece& data) { + if (data.type() == DataPiece::TYPE_NULL) return Status(); if (data.type() != DataPiece::TYPE_STRING) { return Status(INVALID_ARGUMENT, StrCat("Invalid data type for field mask, value is ", @@ -906,13 +961,14 @@ Status ProtoStreamObjectWriter::RenderFieldMask(ProtoStreamObjectWriter* ow, // TODO(tsun): figure out how to do proto descriptor based snake case // conversions as much as possible. Because ToSnakeCase sometimes returns the // wrong value. - google::protobuf::scoped_ptr<ResultCallback1<util::Status, StringPiece> > callback( - NewPermanentCallback(&RenderOneFieldPath, ow)); + std::unique_ptr<ResultCallback1<util::Status, StringPiece> > callback( + ::google::protobuf::NewPermanentCallback(&RenderOneFieldPath, ow)); return DecodeCompactFieldMaskPaths(data.str(), callback.get()); } Status ProtoStreamObjectWriter::RenderDuration(ProtoStreamObjectWriter* ow, const DataPiece& data) { + if (data.type() == DataPiece::TYPE_NULL) return Status(); if (data.type() != DataPiece::TYPE_STRING) { return Status(INVALID_ARGUMENT, StrCat("Invalid data type for duration, value is ", @@ -921,13 +977,13 @@ Status ProtoStreamObjectWriter::RenderDuration(ProtoStreamObjectWriter* ow, StringPiece value(data.str()); - if (!value.ends_with("s")) { + if (!StringEndsWith(value, "s")) { return Status(INVALID_ARGUMENT, "Illegal duration format; duration must end with 's'"); } value = value.substr(0, value.size() - 1); int sign = 1; - if (value.starts_with("-")) { + if (StringStartsWith(value, "-")) { sign = -1; value = value.substr(1); } @@ -957,13 +1013,14 @@ Status ProtoStreamObjectWriter::RenderDuration(ProtoStreamObjectWriter* ow, ow->ProtoWriter::RenderDataPiece("seconds", DataPiece(seconds)); ow->ProtoWriter::RenderDataPiece("nanos", DataPiece(nanos)); - return Status::OK; + return Status(); } Status ProtoStreamObjectWriter::RenderWrapperType(ProtoStreamObjectWriter* ow, const DataPiece& data) { + if (data.type() == DataPiece::TYPE_NULL) return Status(); ow->ProtoWriter::RenderDataPiece("value", data); - return Status::OK; + return Status(); } ProtoStreamObjectWriter* ProtoStreamObjectWriter::RenderDataPiece( @@ -971,10 +1028,10 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::RenderDataPiece( Status status; if (invalid_depth() > 0) return this; - if (current_ == NULL) { + if (current_ == nullptr) { const TypeRenderer* type_renderer = FindTypeRenderer(GetFullTypeWithUrl(master_type_.name())); - if (type_renderer == NULL) { + if (type_renderer == nullptr) { InvalidName(name, "Root element must be a message."); return this; } @@ -997,7 +1054,7 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::RenderDataPiece( return this; } - const google::protobuf::Field* field = NULL; + const google::protobuf::Field* field = nullptr; if (current_->IsMap()) { if (!ValidMapKey(name)) return this; @@ -1007,14 +1064,14 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::RenderDataPiece( ProtoWriter::RenderDataPiece("key", DataPiece(name, use_strict_base64_decoding())); field = Lookup("value"); - if (field == NULL) { + if (field == nullptr) { Pop(); GOOGLE_LOG(DFATAL) << "Map does not have a value field."; return this; } const TypeRenderer* type_renderer = FindTypeRenderer(field->type_url()); - if (type_renderer != NULL) { + if (type_renderer != nullptr) { // Map's value type is a special type. Render it like a message: // "value": { // ... Render special type ... @@ -1044,18 +1101,23 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::RenderDataPiece( } field = Lookup(name); - if (field == NULL) return this; + if (field == nullptr) return this; // Check if the field is of special type. Render it accordingly if so. const TypeRenderer* type_renderer = FindTypeRenderer(field->type_url()); - if (type_renderer != NULL) { - Push(name, Item::MESSAGE, false, false); - status = (*type_renderer)(this, data); - if (!status.ok()) { - InvalidValue(field->type_url(), - StrCat("Field '", name, "', ", status.error_message())); + if (type_renderer != nullptr) { + // Pass through null value only for google.protobuf.Value. For other + // types we ignore null value just like for regular field types. + if (data.type() != DataPiece::TYPE_NULL || + field->type_url() == kStructValueTypeUrl) { + Push(name, Item::MESSAGE, false, false); + status = (*type_renderer)(this, data); + if (!status.ok()) { + InvalidValue(field->type_url(), + StrCat("Field '", name, "', ", status.error_message())); + } + Pop(); } - Pop(); return this; } @@ -1137,7 +1199,7 @@ ProtoStreamObjectWriter::FindTypeRenderer(const string& type_url) { } bool ProtoStreamObjectWriter::ValidMapKey(StringPiece unnormalized_name) { - if (current_ == NULL) return true; + if (current_ == nullptr) return true; if (!current_->InsertMapKeyIfNotPresent(unnormalized_name)) { listener()->InvalidName( @@ -1162,10 +1224,10 @@ void ProtoStreamObjectWriter::Push(StringPiece name, Item::ItemType item_type, void ProtoStreamObjectWriter::Pop() { // Pop all placeholder items sending StartObject or StartList events to // ProtoWriter according to is_list value. - while (current_ != NULL && current_->is_placeholder()) { + while (current_ != nullptr && current_->is_placeholder()) { PopOneElement(); } - if (current_ != NULL) { + if (current_ != nullptr) { PopOneElement(); } } @@ -1185,10 +1247,7 @@ bool ProtoStreamObjectWriter::IsMap(const google::protobuf::Field& field) { const google::protobuf::Type* field_type = typeinfo()->GetTypeByTypeUrl(field.type_url()); - // TODO(xiaofeng): Unify option names. - return GetBoolOptionOrDefault(field_type->options(), - "google.protobuf.MessageOptions.map_entry", false) || - GetBoolOptionOrDefault(field_type->options(), "map_entry", false); + return google::protobuf::util::converter::IsMap(field, *field_type); } bool ProtoStreamObjectWriter::IsAny(const google::protobuf::Field& field) { diff --git a/src/google/protobuf/util/internal/protostream_objectwriter.h b/src/google/protobuf/util/internal/protostream_objectwriter.h index 2e4d14d1..c33a4639 100644 --- a/src/google/protobuf/util/internal/protostream_objectwriter.h +++ b/src/google/protobuf/util/internal/protostream_objectwriter.h @@ -76,19 +76,28 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectWriter : public ProtoWriter { public: // Options that control ProtoStreamObjectWriter class's behavior. struct Options { - // Treats integer inputs in google.protobuf.Struct as strings. Normally, - // integer values are returned in double field "number_value" of + // Treats numeric inputs in google.protobuf.Struct as strings. Normally, + // numeric values are returned in double field "number_value" of // google.protobuf.Struct. However, this can cause precision loss for - // int64/uint64 inputs. This option is provided for cases that want to - // preserve integer precision. + // int64/uint64/double inputs. This option is provided for cases that want + // to preserve number precision. + // + // TODO(skarvaje): Rename to struct_numbers_as_strings as it covers double + // as well. bool struct_integers_as_strings; // Not treat unknown fields as an error. If there is an unknown fields, // just ignore it and continue to process the rest. bool ignore_unknown_fields; + // If true, check if enum name in camel case or without underscore matches + // the field name. + bool use_lower_camel_for_enums; + Options() - : struct_integers_as_strings(false), ignore_unknown_fields(false) {} + : struct_integers_as_strings(false), + ignore_unknown_fields(false), + use_lower_camel_for_enums(false) {} // Default instance of Options with all options set to defaults. static const Options& Defaults() { @@ -145,6 +154,57 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectWriter : public ProtoWriter { void RenderDataPiece(StringPiece name, const DataPiece& value); private: + // Before the "@type" field is encountered, we store all incoming data + // into this Event struct and replay them after we get the "@type" field. + class LIBPROTOBUF_EXPORT Event { + public: + enum Type { + START_OBJECT = 0, + END_OBJECT = 1, + START_LIST = 2, + END_LIST = 3, + RENDER_DATA_PIECE = 4, + }; + + // Constructor for END_OBJECT and END_LIST events. + explicit Event(Type type) : type_(type), value_(DataPiece::NullData()) {} + + // Constructor for START_OBJECT and START_LIST events. + explicit Event(Type type, StringPiece name) + : type_(type), + name_(name.ToString()), + value_(DataPiece::NullData()) {} + + // Constructor for RENDER_DATA_PIECE events. + explicit Event(StringPiece name, const DataPiece& value) + : type_(RENDER_DATA_PIECE), name_(name.ToString()), value_(value) { + DeepCopy(); + } + + Event(const Event& other) + : type_(other.type_), name_(other.name_), value_(other.value_) { + DeepCopy(); + } + + Event& operator=(const Event& other) { + type_ = other.type_; + name_ = other.name_; + value_ = other.value_; + DeepCopy(); + return *this; + } + + void Replay(AnyWriter* writer) const; + + private: + void DeepCopy(); + + Type type_; + string name_; + DataPiece value_; + string value_storage_; + }; + // Handles starting up the any once we have a type. void StartAny(const DataPiece& value); @@ -156,7 +216,7 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectWriter : public ProtoWriter { ProtoStreamObjectWriter* parent_; // The nested object writer, used to write events. - google::protobuf::scoped_ptr<ProtoStreamObjectWriter> ow_; + std::unique_ptr<ProtoStreamObjectWriter> ow_; // The type_url_ that this Any represents. string type_url_; @@ -180,6 +240,9 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectWriter : public ProtoWriter { // } bool is_well_known_type_; TypeRenderer* well_known_type_render_; + + // Store data before the "@type" field. + std::vector<Event> uninterpreted_events_; }; // Represents an item in a stack of items used to keep state between @@ -229,14 +292,14 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectWriter : public ProtoWriter { ProtoStreamObjectWriter* ow_; // A writer for Any objects, handles all Any-related nonsense. - google::protobuf::scoped_ptr<AnyWriter> any_; + std::unique_ptr<AnyWriter> any_; // The type of this element, see enum for permissible types. ItemType item_type_; // Set of map keys already seen for the type_. Used to validate incoming // messages so no map key appears more than once. - google::protobuf::scoped_ptr<hash_set<string> > map_keys_; + std::unique_ptr<hash_set<string> > map_keys_; // Conveys whether this Item is a placeholder or not. Placeholder items are // pushed to stack to account for special types. @@ -329,7 +392,7 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectWriter : public ProtoWriter { const google::protobuf::Type& master_type_; // The current element, variable for internal state processing. - google::protobuf::scoped_ptr<Item> current_; + std::unique_ptr<Item> current_; // Reference to the options that control this class's behavior. const ProtoStreamObjectWriter::Options options_; diff --git a/src/google/protobuf/util/internal/protostream_objectwriter_test.cc b/src/google/protobuf/util/internal/protostream_objectwriter_test.cc index 30e58e62..7f0df567 100644 --- a/src/google/protobuf/util/internal/protostream_objectwriter_test.cc +++ b/src/google/protobuf/util/internal/protostream_objectwriter_test.cc @@ -41,18 +41,20 @@ #include <google/protobuf/dynamic_message.h> #include <google/protobuf/message.h> #include <google/protobuf/util/internal/mock_error_listener.h> +#include <google/protobuf/util/internal/testdata/anys.pb.h> #include <google/protobuf/util/internal/testdata/books.pb.h> #include <google/protobuf/util/internal/testdata/field_mask.pb.h> +#include <google/protobuf/util/internal/testdata/maps.pb.h> +#include <google/protobuf/util/internal/testdata/oneofs.pb.h> +#include <google/protobuf/util/internal/testdata/proto3.pb.h> +#include <google/protobuf/util/internal/testdata/struct.pb.h> +#include <google/protobuf/util/internal/testdata/timestamp_duration.pb.h> +#include <google/protobuf/util/internal/testdata/wrappers.pb.h> #include <google/protobuf/util/internal/type_info_test_helper.h> #include <google/protobuf/util/internal/constants.h> #include <google/protobuf/util/message_differencer.h> #include <google/protobuf/stubs/bytestream.h> #include <google/protobuf/stubs/strutil.h> -#include <google/protobuf/util/internal/testdata/anys.pb.h> -#include <google/protobuf/util/internal/testdata/maps.pb.h> -#include <google/protobuf/util/internal/testdata/oneofs.pb.h> -#include <google/protobuf/util/internal/testdata/struct.pb.h> -#include <google/protobuf/util/internal/testdata/timestamp_duration.pb.h> #include <gtest/gtest.h> @@ -61,27 +63,30 @@ namespace protobuf { namespace util { namespace converter { +using google::protobuf::testing::AnyM; +using google::protobuf::testing::AnyOut; using google::protobuf::testing::Author; using google::protobuf::testing::Book; -using google::protobuf::testing::Book_Data; +using google::protobuf::testing::FieldMaskTest; +using google::protobuf::testing::Int32Wrapper; +using google::protobuf::testing::MapIn; using google::protobuf::testing::Primitive; +using google::protobuf::testing::Proto3Message; using google::protobuf::testing::Publisher; +using google::protobuf::testing::StructType; +using google::protobuf::testing::TestJsonName1; +using google::protobuf::testing::TestJsonName2; +using google::protobuf::testing::TimestampDuration; +using google::protobuf::testing::ValueWrapper; +using google::protobuf::testing::oneofs::OneOfsRequest; using google::protobuf::Descriptor; using google::protobuf::DescriptorPool; using google::protobuf::DynamicMessageFactory; using google::protobuf::FileDescriptorProto; using google::protobuf::Message; -using google::protobuf::io::ArrayInputStream; using strings::GrowingArrayByteSink; using ::testing::_; using ::testing::Args; -using google::protobuf::testing::anys::AnyM; -using google::protobuf::testing::anys::AnyOut; -using google::protobuf::testing::oneofs::OneOfsRequest; -using google::protobuf::testing::FieldMaskTest; -using google::protobuf::testing::maps::MapIn; -using google::protobuf::testing::structs::StructType; -using google::protobuf::testing::timestampduration::TimestampDuration; namespace { @@ -110,13 +115,13 @@ class BaseProtoStreamObjectWriterTest listener_(), output_(new GrowingArrayByteSink(1000)), ow_() { - vector<const Descriptor*> descriptors; + std::vector<const Descriptor*> descriptors; descriptors.push_back(descriptor); ResetTypeInfo(descriptors); } explicit BaseProtoStreamObjectWriterTest( - vector<const Descriptor*> descriptors) + std::vector<const Descriptor*> descriptors) : helper_(GetParam()), listener_(), output_(new GrowingArrayByteSink(1000)), @@ -124,7 +129,7 @@ class BaseProtoStreamObjectWriterTest ResetTypeInfo(descriptors); } - void ResetTypeInfo(vector<const Descriptor*> descriptors) { + void ResetTypeInfo(std::vector<const Descriptor*> descriptors) { GOOGLE_CHECK(!descriptors.empty()) << "Must have at least one descriptor!"; helper_.ResetTypeInfo(descriptors); ow_.reset(helper_.NewProtoWriter(GetTypeUrl(descriptors[0]), output_.get(), @@ -132,7 +137,7 @@ class BaseProtoStreamObjectWriterTest } void ResetTypeInfo(const Descriptor* descriptor) { - vector<const Descriptor*> descriptors; + std::vector<const Descriptor*> descriptors; descriptors.push_back(descriptor); ResetTypeInfo(descriptors); } @@ -141,7 +146,7 @@ class BaseProtoStreamObjectWriterTest void CheckOutput(const Message& expected, int expected_length) { size_t nbytes; - google::protobuf::scoped_array<char> buffer(output_->GetBuffer(&nbytes)); + std::unique_ptr<char[]> buffer(output_->GetBuffer(&nbytes)); if (expected_length >= 0) { EXPECT_EQ(expected_length, nbytes); } @@ -149,7 +154,7 @@ class BaseProtoStreamObjectWriterTest std::stringbuf str_buf(str, std::ios_base::in); std::istream istream(&str_buf); - google::protobuf::scoped_ptr<Message> message(expected.New()); + std::unique_ptr<Message> message(expected.New()); message->ParsePartialFromIstream(&istream); if (!MessageDifferencer::Equivalent(expected, *message)) { @@ -165,8 +170,8 @@ class BaseProtoStreamObjectWriterTest testing::TypeInfoTestHelper helper_; MockErrorListener listener_; - google::protobuf::scoped_ptr<GrowingArrayByteSink> output_; - google::protobuf::scoped_ptr<ProtoStreamObjectWriter> ow_; + std::unique_ptr<GrowingArrayByteSink> output_; + std::unique_ptr<ProtoStreamObjectWriter> ow_; ProtoStreamObjectWriter::Options options_; }; @@ -268,6 +273,100 @@ TEST_P(ProtoStreamObjectWriterTest, CustomJsonName) { CheckOutput(book); } +// Test that two messages can have different fields mapped to the same JSON +// name. See: https://github.com/google/protobuf/issues/1415 +TEST_P(ProtoStreamObjectWriterTest, ConflictingJsonName) { + ResetTypeInfo(TestJsonName1::descriptor()); + TestJsonName1 message1; + message1.set_one_value(12345); + ow_->StartObject("")->RenderInt32("value", 12345)->EndObject(); + CheckOutput(message1); + + ResetTypeInfo(TestJsonName2::descriptor()); + TestJsonName2 message2; + message2.set_another_value(12345); + ow_->StartObject("")->RenderInt32("value", 12345)->EndObject(); + CheckOutput(message2); +} + +TEST_P(ProtoStreamObjectWriterTest, IntEnumValuesAreAccepted) { + Book book; + book.set_title("Some Book"); + book.set_type(google::protobuf::testing::Book_Type_KIDS); + Author* robert = book.mutable_author(); + robert->set_name("robert"); + + ow_->StartObject("") + ->RenderString("title", "Some Book") + ->RenderString("type", "2") + ->StartObject("author") + ->RenderString("name", "robert") + ->EndObject() + ->EndObject(); + CheckOutput(book); +} + +TEST_P(ProtoStreamObjectWriterTest, EnumValuesWithoutUnderscoreAreAccepted) { + Book book; + book.set_title("Some Book"); + book.set_type(google::protobuf::testing::Book_Type_ACTION_AND_ADVENTURE); + Author* robert = book.mutable_author(); + robert->set_name("robert"); + + options_.use_lower_camel_for_enums = true; + ResetProtoWriter(); + + ow_->StartObject("") + ->RenderString("title", "Some Book") + ->RenderString("type", "ACTIONANDADVENTURE") + ->StartObject("author") + ->RenderString("name", "robert") + ->EndObject() + ->EndObject(); + CheckOutput(book); +} + +TEST_P(ProtoStreamObjectWriterTest, EnumValuesInCamelCaseAreAccepted) { + Book book; + book.set_title("Some Book"); + book.set_type(google::protobuf::testing::Book_Type_ACTION_AND_ADVENTURE); + Author* robert = book.mutable_author(); + robert->set_name("robert"); + + options_.use_lower_camel_for_enums = true; + ResetProtoWriter(); + + ow_->StartObject("") + ->RenderString("title", "Some Book") + ->RenderString("type", "actionAndAdventure") + ->StartObject("author") + ->RenderString("name", "robert") + ->EndObject() + ->EndObject(); + CheckOutput(book); +} + +TEST_P(ProtoStreamObjectWriterTest, + EnumValuesInCamelCaseWithNameNotUppercaseAreAccepted) { + Book book; + book.set_title("Some Book"); + book.set_type(google::protobuf::testing::Book_Type_arts_and_photography); + Author* robert = book.mutable_author(); + robert->set_name("robert"); + + options_.use_lower_camel_for_enums = true; + ResetProtoWriter(); + + ow_->StartObject("") + ->RenderString("title", "Some Book") + ->RenderString("type", "artsAndPhotography") + ->StartObject("author") + ->RenderString("name", "robert") + ->EndObject() + ->EndObject(); + CheckOutput(book); +} + TEST_P(ProtoStreamObjectWriterTest, PrimitiveFromStringConversion) { Primitive full; full.set_fix32(101); @@ -826,6 +925,19 @@ TEST_P(ProtoStreamObjectWriterTest, IgnoreUnknownListAtPublisher) { CheckOutput(expected); } +TEST_P(ProtoStreamObjectWriterTest, AcceptUnknownEnumValue) { + ResetTypeInfo(Proto3Message::descriptor()); + + Proto3Message expected; + expected.set_enum_value(static_cast<Proto3Message::NestedEnum>(12345)); + + EXPECT_CALL(listener_, InvalidValue(_, _, _)).Times(0); + ow_->StartObject("") + ->RenderInt32("enumValue", 12345) + ->EndObject(); + CheckOutput(expected); +} + TEST_P(ProtoStreamObjectWriterTest, MissingRequiredField) { Book expected; expected.set_title("My Title"); @@ -984,7 +1096,7 @@ class ProtoStreamObjectWriterTimestampDurationTest : public BaseProtoStreamObjectWriterTest { protected: ProtoStreamObjectWriterTimestampDurationTest() { - vector<const Descriptor*> descriptors; + std::vector<const Descriptor*> descriptors; descriptors.push_back(TimestampDuration::descriptor()); descriptors.push_back(google::protobuf::Timestamp::descriptor()); descriptors.push_back(google::protobuf::Duration::descriptor()); @@ -1338,9 +1450,9 @@ TEST_P(ProtoStreamObjectWriterTimestampDurationTest, InvalidValue( _, StringPiece("type.googleapis.com/google.protobuf.Timestamp"), StringPiece( - "Field 'ts', Invalid data type for timestamp, value is null"))) + "Field 'ts', Invalid data type for timestamp, value is 1"))) .With(Args<0>(HasObjectLocation("ts"))); - ow_->StartObject("")->RenderNull("ts")->EndObject(); + ow_->StartObject("")->RenderInt32("ts", 1)->EndObject(); CheckOutput(timestamp); } @@ -1352,8 +1464,22 @@ TEST_P(ProtoStreamObjectWriterTimestampDurationTest, InvalidValue( _, StringPiece("type.googleapis.com/google.protobuf.Duration"), StringPiece( - "Field 'dur', Invalid data type for duration, value is null"))) + "Field 'dur', Invalid data type for duration, value is 1"))) .With(Args<0>(HasObjectLocation("dur"))); + ow_->StartObject("")->RenderInt32("dur", 1)->EndObject(); + CheckOutput(duration); +} + +TEST_P(ProtoStreamObjectWriterTimestampDurationTest, TimestampAcceptsNull) { + TimestampDuration timestamp; + EXPECT_CALL(listener_, InvalidValue(_, _, _)).Times(0); + ow_->StartObject("")->RenderNull("ts")->EndObject(); + CheckOutput(timestamp); +} + +TEST_P(ProtoStreamObjectWriterTimestampDurationTest, DurationAcceptsNull) { + TimestampDuration duration; + EXPECT_CALL(listener_, InvalidValue(_, _, _)).Times(0); ow_->StartObject("")->RenderNull("dur")->EndObject(); CheckOutput(duration); } @@ -1365,7 +1491,7 @@ class ProtoStreamObjectWriterStructTest // Resets ProtoWriter with current set of options and other state. void ResetProtoWriter() { - vector<const Descriptor*> descriptors; + std::vector<const Descriptor*> descriptors; descriptors.push_back(StructType::descriptor()); descriptors.push_back(google::protobuf::Struct::descriptor()); ResetTypeInfo(descriptors); @@ -1415,6 +1541,28 @@ TEST_P(ProtoStreamObjectWriterStructTest, StructInvalidInputFailure) { CheckOutput(struct_type); } +TEST_P(ProtoStreamObjectWriterStructTest, StructAcceptsNull) { + StructType struct_type; + EXPECT_CALL(listener_, InvalidValue(_, _, _)).Times(0); + + ow_->StartObject("")->RenderNull("object")->EndObject(); + CheckOutput(struct_type); +} + +TEST_P(ProtoStreamObjectWriterStructTest, StructValuePreservesNull) { + StructType struct_type; + (*struct_type.mutable_object()->mutable_fields())["key"].set_null_value( + google::protobuf::NULL_VALUE); + EXPECT_CALL(listener_, InvalidValue(_, _, _)).Times(0); + + ow_->StartObject("") + ->StartObject("object") + ->RenderNull("key") + ->EndObject() + ->EndObject(); + CheckOutput(struct_type); +} + TEST_P(ProtoStreamObjectWriterStructTest, SimpleRepeatedStructMapKeyTest) { EXPECT_CALL( listener_, @@ -1463,7 +1611,7 @@ TEST_P(ProtoStreamObjectWriterStructTest, RepeatedStructMapObjectKeyTest) { TEST_P(ProtoStreamObjectWriterStructTest, OptionStructIntAsStringsTest) { StructType struct_type; google::protobuf::Struct* s = struct_type.mutable_object(); - s->mutable_fields()->operator[]("k1").set_number_value(123); + s->mutable_fields()->operator[]("k1").set_string_value("123"); s->mutable_fields()->operator[]("k2").set_bool_value(true); s->mutable_fields()->operator[]("k3").set_string_value("-222222222"); s->mutable_fields()->operator[]("k4").set_string_value("33333333"); @@ -1482,6 +1630,15 @@ TEST_P(ProtoStreamObjectWriterStructTest, OptionStructIntAsStringsTest) { CheckOutput(struct_type); } +TEST_P(ProtoStreamObjectWriterStructTest, ValuePreservesNull) { + ValueWrapper value; + value.mutable_value()->set_null_value(google::protobuf::NULL_VALUE); + + EXPECT_CALL(listener_, InvalidValue(_, _, _)).Times(0); + ow_->StartObject("")->RenderNull("value")->EndObject(); + CheckOutput(value); +} + class ProtoStreamObjectWriterMapTest : public BaseProtoStreamObjectWriterTest { protected: ProtoStreamObjectWriterMapTest() @@ -1525,13 +1682,16 @@ TEST_P(ProtoStreamObjectWriterMapTest, RepeatedMapKeyTest) { class ProtoStreamObjectWriterAnyTest : public BaseProtoStreamObjectWriterTest { protected: ProtoStreamObjectWriterAnyTest() { - vector<const Descriptor*> descriptors; + std::vector<const Descriptor*> descriptors; descriptors.push_back(AnyOut::descriptor()); + descriptors.push_back(Book::descriptor()); + descriptors.push_back(google::protobuf::Any::descriptor()); descriptors.push_back(google::protobuf::DoubleValue::descriptor()); + descriptors.push_back(google::protobuf::FieldMask::descriptor()); + descriptors.push_back(google::protobuf::Int32Value::descriptor()); + descriptors.push_back(google::protobuf::Struct::descriptor()); descriptors.push_back(google::protobuf::Timestamp::descriptor()); - descriptors.push_back(google::protobuf::Any::descriptor()); descriptors.push_back(google::protobuf::Value::descriptor()); - descriptors.push_back(google::protobuf::Struct::descriptor()); ResetTypeInfo(descriptors); } }; @@ -1564,8 +1724,7 @@ TEST_P(ProtoStreamObjectWriterAnyTest, RecursiveAny) { any->set_type_url("type.googleapis.com/google.protobuf.Any"); ::google::protobuf::Any nested_any; - nested_any.set_type_url( - "type.googleapis.com/google.protobuf.testing.anys.AnyM"); + nested_any.set_type_url("type.googleapis.com/google.protobuf.testing.AnyM"); AnyM m; m.set_foo("foovalue"); @@ -1578,11 +1737,12 @@ TEST_P(ProtoStreamObjectWriterAnyTest, RecursiveAny) { ->RenderString("@type", "type.googleapis.com/google.protobuf.Any") ->StartObject("value") ->RenderString("@type", - "type.googleapis.com/google.protobuf.testing.anys.AnyM") + "type.googleapis.com/google.protobuf.testing.AnyM") ->RenderString("foo", "foovalue") ->EndObject() ->EndObject() ->EndObject(); + CheckOutput(out, 107); } TEST_P(ProtoStreamObjectWriterAnyTest, DoubleRecursiveAny) { @@ -1595,7 +1755,7 @@ TEST_P(ProtoStreamObjectWriterAnyTest, DoubleRecursiveAny) { ::google::protobuf::Any second_nested_any; second_nested_any.set_type_url( - "type.googleapis.com/google.protobuf.testing.anys.AnyM"); + "type.googleapis.com/google.protobuf.testing.AnyM"); AnyM m; m.set_foo("foovalue"); @@ -1611,18 +1771,110 @@ TEST_P(ProtoStreamObjectWriterAnyTest, DoubleRecursiveAny) { ->RenderString("@type", "type.googleapis.com/google.protobuf.Any") ->StartObject("value") ->RenderString("@type", - "type.googleapis.com/google.protobuf.testing.anys.AnyM") + "type.googleapis.com/google.protobuf.testing.AnyM") ->RenderString("foo", "foovalue") ->EndObject() ->EndObject() ->EndObject() ->EndObject(); + CheckOutput(out, 151); +} + +TEST_P(ProtoStreamObjectWriterAnyTest, TypeUrlAtEnd) { + Book book; + book.set_title("C++"); + book.set_length(1234); + book.set_content("Hello World!"); + + ::google::protobuf::Any any; + any.PackFrom(book); + + ::google::protobuf::Any outer_any; + outer_any.PackFrom(any); + + AnyOut out; + out.mutable_any()->PackFrom(outer_any); + + // Put the @type field at the end of each Any message. Parsers should + // be able to accept that. + ow_->StartObject("") + ->StartObject("any") + ->StartObject("value") + ->StartObject("value") + ->RenderString("title", "C++") + ->RenderInt32("length", 1234) + ->RenderBytes("content", "Hello World!") + ->RenderString("@type", + "type.googleapis.com/google.protobuf.testing.Book") + ->EndObject() + ->RenderString("@type", "type.googleapis.com/google.protobuf.Any") + ->EndObject() + ->RenderString("@type", "type.googleapis.com/google.protobuf.Any") + ->EndObject() + ->EndObject(); + CheckOutput(out); +} + +// Same as TypeUrlAtEnd, but use temporary string values to make sure we don't +// mistakenly store StringPiece objects pointing to invalid memory. +TEST_P(ProtoStreamObjectWriterAnyTest, TypeUrlAtEndWithTemporaryStrings) { + Book book; + book.set_title("C++"); + book.set_length(1234); + book.set_content("Hello World!"); + + ::google::protobuf::Any any; + any.PackFrom(book); + + ::google::protobuf::Any outer_any; + outer_any.PackFrom(any); + + AnyOut out; + out.mutable_any()->PackFrom(outer_any); + + string name, value; + // Put the @type field at the end of each Any message. Parsers should + // be able to accept that. + ow_->StartObject("")->StartObject("any"); + { + ow_->StartObject("value"); + { + ow_->StartObject("value"); + { + name = "title"; + value = "C++"; + ow_->RenderString(name, value); + name = "length"; + ow_->RenderInt32(name, 1234); + name = "content"; + value = "Hello World!"; + ow_->RenderBytes(name, value); + name = "@type"; + value = "type.googleapis.com/google.protobuf.testing.Book"; + ow_->RenderString(name, value); + } + ow_->EndObject(); + + name = "@type"; + value = "type.googleapis.com/google.protobuf.Any"; + ow_->RenderString(name, value); + } + ow_->EndObject(); + + name = "@type"; + value = "type.googleapis.com/google.protobuf.Any"; + ow_->RenderString(name, value); + } + ow_->EndObject()->EndObject(); + CheckOutput(out); } TEST_P(ProtoStreamObjectWriterAnyTest, EmptyAnyFromEmptyObject) { AnyOut out; out.mutable_any(); + EXPECT_CALL(listener_, InvalidValue(_, _, _)).Times(0); + ow_->StartObject("")->StartObject("any")->EndObject()->EndObject(); CheckOutput(out, 2); @@ -1631,11 +1883,10 @@ TEST_P(ProtoStreamObjectWriterAnyTest, EmptyAnyFromEmptyObject) { TEST_P(ProtoStreamObjectWriterAnyTest, AnyWithoutTypeUrlFails1) { AnyOut any; - EXPECT_CALL( - listener_, - InvalidValue(_, StringPiece("Any"), - StringPiece("Missing or invalid @type for any field in " - "google.protobuf.testing.anys.AnyOut"))); + EXPECT_CALL(listener_, + InvalidValue(_, StringPiece("Any"), + StringPiece("Missing @type for any field in " + "google.protobuf.testing.AnyOut"))); ow_->StartObject("") ->StartObject("any") @@ -1649,11 +1900,10 @@ TEST_P(ProtoStreamObjectWriterAnyTest, AnyWithoutTypeUrlFails1) { TEST_P(ProtoStreamObjectWriterAnyTest, AnyWithoutTypeUrlFails2) { AnyOut any; - EXPECT_CALL( - listener_, - InvalidValue(_, StringPiece("Any"), - StringPiece("Missing or invalid @type for any field in " - "google.protobuf.testing.anys.AnyOut"))); + EXPECT_CALL(listener_, + InvalidValue(_, StringPiece("Any"), + StringPiece("Missing @type for any field in " + "google.protobuf.testing.AnyOut"))); ow_->StartObject("") ->StartObject("any") @@ -1667,11 +1917,10 @@ TEST_P(ProtoStreamObjectWriterAnyTest, AnyWithoutTypeUrlFails2) { TEST_P(ProtoStreamObjectWriterAnyTest, AnyWithoutTypeUrlFails3) { AnyOut any; - EXPECT_CALL( - listener_, - InvalidValue(_, StringPiece("Any"), - StringPiece("Missing or invalid @type for any field in " - "google.protobuf.testing.anys.AnyOut"))); + EXPECT_CALL(listener_, + InvalidValue(_, StringPiece("Any"), + StringPiece("Missing @type for any field in " + "google.protobuf.testing.AnyOut"))); ow_->StartObject("") ->StartObject("any") @@ -1716,9 +1965,21 @@ TEST_P(ProtoStreamObjectWriterAnyTest, AnyWithUnknownTypeFails) { CheckOutput(any); } -TEST_P(ProtoStreamObjectWriterAnyTest, AnyNullInputFails) { +TEST_P(ProtoStreamObjectWriterAnyTest, AnyIncorrectInputTypeFails) { + AnyOut any; + + EXPECT_CALL( + listener_, + InvalidValue(_, StringPiece("type.googleapis.com/google.protobuf.Any"), + StringPiece("1"))); + ow_->StartObject("")->RenderInt32("any", 1)->EndObject(); + CheckOutput(any); +} + +TEST_P(ProtoStreamObjectWriterAnyTest, AnyAcceptsNull) { AnyOut any; + EXPECT_CALL(listener_, InvalidValue(_, _, _)).Times(0); ow_->StartObject("")->RenderNull("any")->EndObject(); CheckOutput(any); } @@ -1956,11 +2217,116 @@ TEST_P(ProtoStreamObjectWriterAnyTest, AnyWellKnownTypesExpectObjectForAny) { CheckOutput(any); } +// { +// "any": { +// "@type": "type.googleapis.com/google.protobuf.Any", +// "value": null +// } +// } +TEST_P(ProtoStreamObjectWriterAnyTest, AnyInAnyAcceptsNull) { + AnyOut out; + google::protobuf::Any empty; + out.mutable_any()->PackFrom(empty); + + EXPECT_CALL(listener_, InvalidValue(_, _, _)).Times(0); + ow_->StartObject("") + ->StartObject("any") + ->RenderString("@type", "type.googleapis.com/google.protobuf.Any") + ->RenderNull("value") + ->EndObject() + ->EndObject(); + CheckOutput(out); +} + +// { +// "any": { +// "@type": "type.googleapis.com/google.protobuf.Timestamp", +// "value": null +// } +// } +TEST_P(ProtoStreamObjectWriterAnyTest, TimestampInAnyAcceptsNull) { + AnyOut out; + google::protobuf::Timestamp empty; + out.mutable_any()->PackFrom(empty); + + EXPECT_CALL(listener_, InvalidValue(_, _, _)).Times(0); + ow_->StartObject("") + ->StartObject("any") + ->RenderString("@type", "type.googleapis.com/google.protobuf.Timestamp") + ->RenderNull("value") + ->EndObject() + ->EndObject(); + CheckOutput(out); +} + +// { +// "any": { +// "@type": "type.googleapis.com/google.protobuf.Duration", +// "value": null +// } +// } +TEST_P(ProtoStreamObjectWriterAnyTest, DurationInAnyAcceptsNull) { + AnyOut out; + google::protobuf::Duration empty; + out.mutable_any()->PackFrom(empty); + + EXPECT_CALL(listener_, InvalidValue(_, _, _)).Times(0); + ow_->StartObject("") + ->StartObject("any") + ->RenderString("@type", "type.googleapis.com/google.protobuf.Duration") + ->RenderNull("value") + ->EndObject() + ->EndObject(); + CheckOutput(out); +} + +// { +// "any": { +// "@type": "type.googleapis.com/google.protobuf.FieldMask", +// "value": null +// } +// } +TEST_P(ProtoStreamObjectWriterAnyTest, FieldMaskInAnyAcceptsNull) { + AnyOut out; + google::protobuf::FieldMask empty; + out.mutable_any()->PackFrom(empty); + + EXPECT_CALL(listener_, InvalidValue(_, _, _)).Times(0); + ow_->StartObject("") + ->StartObject("any") + ->RenderString("@type", "type.googleapis.com/google.protobuf.FieldMask") + ->RenderNull("value") + ->EndObject() + ->EndObject(); + CheckOutput(out); +} + +// { +// "any": { +// "@type": "type.googleapis.com/google.protobuf.Int32Value", +// "value": null +// } +// } +TEST_P(ProtoStreamObjectWriterAnyTest, WrapperInAnyAcceptsNull) { + AnyOut out; + google::protobuf::Int32Value empty; + out.mutable_any()->PackFrom(empty); + + EXPECT_CALL(listener_, InvalidValue(_, _, _)).Times(0); + ow_->StartObject("") + ->StartObject("any") + ->RenderString("@type", "type.googleapis.com/google.protobuf.Int32Value") + ->RenderNull("value") + ->EndObject() + ->EndObject(); + CheckOutput(out); +} + class ProtoStreamObjectWriterFieldMaskTest : public BaseProtoStreamObjectWriterTest { protected: ProtoStreamObjectWriterFieldMaskTest() { - vector<const Descriptor*> descriptors; + std::vector<const Descriptor*> descriptors; descriptors.push_back(FieldMaskTest::descriptor()); descriptors.push_back(google::protobuf::FieldMask::descriptor()); ResetTypeInfo(descriptors); @@ -2201,11 +2567,41 @@ TEST_P(ProtoStreamObjectWriterFieldMaskTest, MapKeyCanContainAnyChars) { CheckOutput(expected); } +TEST_P(ProtoStreamObjectWriterFieldMaskTest, FieldMaskAcceptsNull) { + FieldMaskTest expected; + EXPECT_CALL(listener_, InvalidValue(_, _, _)).Times(0); + ow_->StartObject("")->RenderNull("single_mask")->EndObject(); + CheckOutput(expected); +} + +class ProtoStreamObjectWriterWrappersTest + : public BaseProtoStreamObjectWriterTest { + protected: + ProtoStreamObjectWriterWrappersTest() { + std::vector<const Descriptor*> descriptors; + descriptors.push_back(Int32Wrapper::descriptor()); + descriptors.push_back(google::protobuf::Int32Value::descriptor()); + ResetTypeInfo(descriptors); + } +}; + +INSTANTIATE_TEST_CASE_P(DifferentTypeInfoSourceTest, + ProtoStreamObjectWriterWrappersTest, + ::testing::Values( + testing::USE_TYPE_RESOLVER)); + +TEST_P(ProtoStreamObjectWriterWrappersTest, WrapperAcceptsNull) { + Int32Wrapper wrapper; + EXPECT_CALL(listener_, InvalidName(_, _, _)).Times(0); + ow_->StartObject("")->RenderNull("int32")->EndObject(); + CheckOutput(wrapper); +} + class ProtoStreamObjectWriterOneOfsTest : public BaseProtoStreamObjectWriterTest { protected: ProtoStreamObjectWriterOneOfsTest() { - vector<const Descriptor*> descriptors; + std::vector<const Descriptor*> descriptors; descriptors.push_back(OneOfsRequest::descriptor()); descriptors.push_back(google::protobuf::Struct::descriptor()); ResetTypeInfo(descriptors); @@ -2368,7 +2764,6 @@ TEST_P(ProtoStreamObjectWriterOneOfsTest, StringPiece("oneof field 'data' is already set. " "Cannot set 'intData'"))); - using google::protobuf::testing::oneofs::OneOfsRequest; // JSON: // { "anyData": // { "@type": diff --git a/src/google/protobuf/util/internal/structured_objectwriter.h b/src/google/protobuf/util/internal/structured_objectwriter.h index 3f065d6b..8e63222b 100644 --- a/src/google/protobuf/util/internal/structured_objectwriter.h +++ b/src/google/protobuf/util/internal/structured_objectwriter.h @@ -32,9 +32,6 @@ #define GOOGLE_PROTOBUF_UTIL_CONVERTER_STRUCTURED_OBJECTWRITER_H__ #include <memory> -#ifndef _SHARED_PTR_H -#include <google/protobuf/stubs/shared_ptr.h> -#endif #include <google/protobuf/stubs/casts.h> #include <google/protobuf/stubs/common.h> @@ -80,7 +77,7 @@ class LIBPROTOBUF_EXPORT StructuredObjectWriter : public ObjectWriter { } // Returns true if this element is the root. - bool is_root() const { return parent_ == NULL; } + bool is_root() const { return parent_ == nullptr; } // Returns the number of hops from this element to the root element. int level() const { return level_; } @@ -91,10 +88,10 @@ class LIBPROTOBUF_EXPORT StructuredObjectWriter : public ObjectWriter { private: // Pointer to the parent Element. - google::protobuf::scoped_ptr<BaseElement> parent_; + std::unique_ptr<BaseElement> parent_; // Number of hops to the root Element. - // The root Element has NULL parent_ and a level_ of 0. + // The root Element has nullptr parent_ and a level_ of 0. const int level_; GOOGLE_DISALLOW_IMPLICIT_CONSTRUCTORS(BaseElement); diff --git a/src/google/protobuf/util/internal/testdata/anys.proto b/src/google/protobuf/util/internal/testdata/anys.proto index 18c59cbb..a9ebca3d 100644 --- a/src/google/protobuf/util/internal/testdata/anys.proto +++ b/src/google/protobuf/util/internal/testdata/anys.proto @@ -28,16 +28,75 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// Proto to test Proto3 Any serialization. syntax = "proto3"; -package google.protobuf.testing.anys; -option java_package = "com.google.protobuf.testing.anys"; +package google.protobuf.testing; import "google/protobuf/any.proto"; +import "google/protobuf/struct.proto"; +import "google/protobuf/timestamp.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; + +// Top-level test cases proto used by MarshallingTest. See description +// at the top of the class MarshallingTest for details on how to write +// test cases. +message AnyTestCases { + AnyWrapper empty_any = 1; + AnyWrapper type_only_any = 2; + AnyWrapper wrapper_any = 3; + AnyWrapper any_with_timestamp_value = 4; + AnyWrapper any_with_duration_value = 5; + AnyWrapper any_with_struct_value = 6; + AnyWrapper recursive_any = 7; + AnyWrapper any_with_message_value = 8; + AnyWrapper any_with_nested_message = 9; + AnyWrapper any_with_message_with_wrapper_type = 10; + AnyWrapper any_with_message_with_timestamp = 11; + AnyWrapper any_with_message_containing_map = 12; + AnyWrapper any_with_message_containing_struct = 13; + AnyWrapper any_with_message_containing_repeated_message = 14; + AnyWrapper recursive_any_with_type_field_at_end = 15; + + google.protobuf.Any top_level_any = 50; + google.protobuf.Any top_level_any_with_type_field_at_end = 51; +} + +message AnyWrapper { + google.protobuf.Any any = 1; +} + +// Hack to make sure the types we put into the any are included in the types. +// Real solution is to add these types to the service config. +message Imports { + google.protobuf.DoubleValue dbl = 1; + google.protobuf.Struct struct = 2; + google.protobuf.Timestamp timestamp = 3; + google.protobuf.Duration duration = 4; + google.protobuf.Int32Value i32 = 5; + Data data = 100; +} + +message Data { + int32 attr = 1; + string str = 2; + repeated string msgs = 3; + Data nested_data = 4; + google.protobuf.Int32Value int_wrapper = 5; + google.protobuf.Timestamp time = 6; + map<string, string> map_data = 7; + google.protobuf.Struct struct_data = 8; + repeated Data repeated_data = 9; +} + +service AnyTestService { + rpc Call(AnyTestCases) returns (AnyTestCases); + rpc Call1(Imports) returns (Imports); +} message AnyIn { string something = 1; + google.protobuf.Any any = 2; } message AnyOut { @@ -47,7 +106,3 @@ message AnyOut { message AnyM { string foo = 1; } - -service TestService { - rpc Call(AnyIn) returns (AnyOut); -} diff --git a/src/google/protobuf/util/internal/testdata/books.proto b/src/google/protobuf/util/internal/testdata/books.proto index 1cbbba47..5630cc78 100644 --- a/src/google/protobuf/util/internal/testdata/books.proto +++ b/src/google/protobuf/util/internal/testdata/books.proto @@ -31,6 +31,10 @@ // Author: sven@google.com (Sven Mawson) // // Sample protos for testing. + +// Some of the older enums don't use CAPITALS_WITH_UNDERSCORES for testing. +// LINT: LEGACY_NAMES + syntax = "proto2"; package google.protobuf.testing; @@ -60,6 +64,8 @@ message Book { FICTION = 1; KIDS = 2; ACTION_AND_ADVENTURE = 3; + arts_and_photography = 4; + I18N_Tech = 5; } optional Type type = 11; @@ -185,3 +191,12 @@ message Cyclic { repeated Author m_author = 5; optional Cyclic m_cyclic = 4; } + +// Test that two messages can have different fields mapped to the same JSON +// name. See: https://github.com/google/protobuf/issues/1415 +message TestJsonName1 { + optional int32 one_value = 1 [json_name = "value"]; +} +message TestJsonName2 { + optional int32 another_value = 1 [json_name = "value"]; +} diff --git a/src/google/protobuf/util/internal/testdata/maps.proto b/src/google/protobuf/util/internal/testdata/maps.proto index 6475ecdd..0f381b32 100644 --- a/src/google/protobuf/util/internal/testdata/maps.proto +++ b/src/google/protobuf/util/internal/testdata/maps.proto @@ -28,11 +28,76 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// Proto to test proto3 maps. syntax = "proto3"; -package google.protobuf.testing.maps; -option java_package = "com.google.protobuf.testing.maps"; +package google.protobuf.testing; + +// Top-level test cases proto used by MarshallingTest. See description +// at the top of the class MarshallingTest for details on how to write +// test cases. +message MapsTestCases { + EmptyMap empty_map = 1; + StringtoInt string_to_int = 2; + IntToString int_to_string = 3; + Mixed1 mixed1 = 4; + Mixed2 mixed2 = 5; + MapOfObjects map_of_objects = 6; + + // Empty key tests + StringtoInt empty_key_string_to_int1 = 7; + StringtoInt empty_key_string_to_int2 = 8; + StringtoInt empty_key_string_to_int3 = 9; + BoolToString empty_key_bool_to_string = 10; + IntToString empty_key_int_to_string = 11; + Mixed1 empty_key_mixed = 12; + MapOfObjects empty_key_map_objects = 13; +} + +message EmptyMap { + map<int32, int32> map = 1; +} + +message StringtoInt { + map<string, int32> map = 1; +} + +message IntToString { + map<int32, string> map = 1; +} + +message BoolToString { + map<bool, string> map = 1; +} + +message Mixed1 { + string msg = 1; + map<string, float> map = 2; +} + +message Mixed2 { + enum E { + E0 = 0; + E1 = 1; + E2 = 2; + E3 = 3; + } + map<int32, bool> map = 1; + E ee = 2; +} + +message MapOfObjects { + message M { + string inner_text = 1; + } + map<string, M> map = 1; +} + +message DummyRequest { +} + +service MapsTestService { + rpc Call(DummyRequest) returns (MapsTestCases); +} message MapIn { string other = 1; @@ -79,8 +144,3 @@ message MapOutWireFormat { message MapM { string foo = 1; } - -service TestService { - rpc Call1(MapIn) returns (MapOut); - rpc Call2(MapIn) returns (MapOut); -} diff --git a/src/google/protobuf/util/internal/testdata/oneofs.proto b/src/google/protobuf/util/internal/testdata/oneofs.proto index 5bc9fa08..c37da083 100644 --- a/src/google/protobuf/util/internal/testdata/oneofs.proto +++ b/src/google/protobuf/util/internal/testdata/oneofs.proto @@ -28,7 +28,7 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// Proto to test oneofs. +// Proto to test proto3 oneofs. syntax = "proto3"; import "google/protobuf/any.proto"; @@ -36,7 +36,6 @@ import "google/protobuf/struct.proto"; import "google/protobuf/timestamp.proto"; package google.protobuf.testing.oneofs; -option java_package = "com.google.protobuf.testing.oneofs"; message OneOfsRequest { string value = 1; @@ -45,24 +44,34 @@ message OneOfsRequest { int32 int_data = 3; // Simple message Data message_data = 4; + MoreData more_data = 5; // Well known types - google.protobuf.Struct struct_data = 5; - google.protobuf.Value value_data = 6; - google.protobuf.ListValue list_value_data = 7; - google.protobuf.Timestamp ts_data = 8; + google.protobuf.Struct struct_data = 6; + google.protobuf.Value value_data = 7; + google.protobuf.ListValue list_value_data = 8; + google.protobuf.Timestamp ts_data = 9; } google.protobuf.Any any_data = 19; } +message RequestWithSimpleOneof { + string value = 1; + oneof data { + string str_data = 2; + int32 int_data = 3; + Data message_data = 4; + MoreData more_data = 5; + } +} + message Data { int32 data_value = 1; } -message Response { - string value = 1; +message MoreData { + string str_value = 1; } -service TestService { - // Test call. - rpc Call(OneOfsRequest) returns (Response); +message Response { + string value = 1; } diff --git a/src/google/protobuf/util/internal/testdata/proto3.proto b/src/google/protobuf/util/internal/testdata/proto3.proto new file mode 100644 index 00000000..c013cee3 --- /dev/null +++ b/src/google/protobuf/util/internal/testdata/proto3.proto @@ -0,0 +1,42 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf.testing; + +message Proto3Message { + enum NestedEnum { + FOO = 0; + BAR = 1; + BAZ = 2; + } + NestedEnum enum_value = 1; +} diff --git a/src/google/protobuf/util/internal/testdata/struct.proto b/src/google/protobuf/util/internal/testdata/struct.proto index c15aba0d..7b1cc1b9 100644 --- a/src/google/protobuf/util/internal/testdata/struct.proto +++ b/src/google/protobuf/util/internal/testdata/struct.proto @@ -28,18 +28,90 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// Proto to test proto3 struct. syntax = "proto3"; -package google.protobuf.testing.structs; -option java_package = "com.google.protobuf.testing.structs"; +package google.protobuf.testing; import "google/protobuf/struct.proto"; -message StructType { - google.protobuf.Struct object = 1; +message StructTestCases { + StructWrapper empty_value = 1; + StructWrapper empty_value2 = 2; + StructWrapper null_value = 3; + StructWrapper simple_struct = 4; + StructWrapper longer_struct = 5; + StructWrapper struct_with_nested_struct = 6; + StructWrapper struct_with_nested_list = 7; + StructWrapper struct_with_list_of_nulls = 8; + StructWrapper struct_with_list_of_lists = 9; + StructWrapper struct_with_list_of_structs = 10; + StructWrapper struct_with_empty_list = 11; + StructWrapper struct_with_list_with_empty_struct = 12; + google.protobuf.Struct top_level_struct = 13; + google.protobuf.Struct top_level_struct_with_empty_list = 14; + google.protobuf.Struct top_level_struct_with_list_with_empty_struct = 15; + ValueWrapper value_wrapper_simple = 16; + ValueWrapper value_wrapper_with_struct = 17; + ValueWrapper value_wrapper_with_list = 18; + ValueWrapper value_wrapper_with_empty_list = 19; + ValueWrapper value_wrapper_with_list_with_empty_struct = 20; + ListValueWrapper list_value_wrapper = 21; + ListValueWrapper list_value_wrapper_with_empty_list = 22; + ListValueWrapper list_value_wrapper_with_list_with_empty_struct = 23; + google.protobuf.Value top_level_value_simple = 24; + google.protobuf.Value top_level_value_with_struct = 25; + google.protobuf.Value top_level_value_with_list = 26; + google.protobuf.Value top_level_value_with_empty_list = 27; + google.protobuf.Value top_level_value_with_list_with_empty_struct = 28; + google.protobuf.ListValue top_level_listvalue = 29; + google.protobuf.ListValue top_level_empty_listvalue = 30; + google.protobuf.ListValue top_level_listvalue_with_empty_struct = 31; + RepeatedValueWrapper repeated_value = 32; + RepeatedValueWrapper repeated_value_nested_list = 33; + RepeatedValueWrapper repeated_value_nested_list2 = 34; + RepeatedValueWrapper repeated_value_nested_list3 = 35; + RepeatedListValueWrapper repeated_listvalue = 36; + MapOfStruct map_of_struct = 37; + MapOfStruct map_of_struct_value = 38; + MapOfStruct map_of_listvalue = 39; +} + +message StructWrapper { + google.protobuf.Struct struct = 1; +} + +message ValueWrapper { + google.protobuf.Value value = 1; +} + +message RepeatedValueWrapper { + repeated google.protobuf.Value values = 1; +} + +message ListValueWrapper { + google.protobuf.ListValue shopping_list = 1; } -service TestService { - rpc Call(StructType) returns (StructType); +message RepeatedListValueWrapper { + repeated google.protobuf.ListValue dimensions = 1; +} + +message MapOfStruct { + map<string, google.protobuf.Struct> struct_map = 1; + map<string, google.protobuf.Value> value_map = 2; + map<string, google.protobuf.ListValue> listvalue_map = 3; +} + +// Hack to test return types with Struct as top-level message. Struct typers +// cannot be directly used in API requests. Hence using Dummy as request type. +message Dummy { + string text = 1; +} + +service StructTestService { + rpc Call(Dummy) returns (StructTestCases); +} + +message StructType { + google.protobuf.Struct object = 1; } diff --git a/src/google/protobuf/util/internal/testdata/timestamp_duration.proto b/src/google/protobuf/util/internal/testdata/timestamp_duration.proto index 56351f16..b74484ce 100644 --- a/src/google/protobuf/util/internal/testdata/timestamp_duration.proto +++ b/src/google/protobuf/util/internal/testdata/timestamp_duration.proto @@ -28,20 +28,53 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// Proto to test proto3 Timestamp and Duration. syntax = "proto3"; -package google.protobuf.testing.timestampduration; -option java_package = "com.google.protobuf.testing.timestampduration"; +package google.protobuf.testing; import "google/protobuf/timestamp.proto"; import "google/protobuf/duration.proto"; +message TimestampDurationTestCases { + // Timestamp tests + TimeStampType epoch = 1; + TimeStampType epoch2 = 2; + TimeStampType mintime = 3; + TimeStampType maxtime = 4; + TimeStampType timeval1 = 5; + TimeStampType timeval2 = 6; + TimeStampType timeval3 = 7; + TimeStampType timeval4 = 8; + TimeStampType timeval5 = 9; + TimeStampType timeval6 = 10; + TimeStampType timeval7 = 11; + google.protobuf.Timestamp timeval8 = 12; + + // Duration tests + DurationType zero_duration = 101; + DurationType min_duration = 102; + DurationType max_duration = 103; + DurationType duration1 = 104; + DurationType duration2 = 105; + DurationType duration3 = 106; + DurationType duration4 = 107; + google.protobuf.Duration duration5 = 108; +} + +message TimeStampType { + google.protobuf.Timestamp timestamp = 1; +} + +message DurationType { + google.protobuf.Duration duration = 1; +} + +service TimestampDurationTestService { + rpc Call(TimestampDurationTestCases) returns (TimestampDurationTestCases); +} + message TimestampDuration { google.protobuf.Timestamp ts = 1; google.protobuf.Duration dur = 2; -} - -service TestService { - rpc Call(TimestampDuration) returns (TimestampDuration); + repeated google.protobuf.Timestamp rep_ts = 3; } diff --git a/src/google/protobuf/util/internal/type_info.cc b/src/google/protobuf/util/internal/type_info.cc index 00a8ee7a..3847b179 100644 --- a/src/google/protobuf/util/internal/type_info.cc +++ b/src/google/protobuf/util/internal/type_info.cc @@ -60,7 +60,8 @@ class TypeInfoForTypeResolver : public TypeInfo { virtual util::StatusOr<const google::protobuf::Type*> ResolveTypeUrl( StringPiece type_url) const { - map<StringPiece, StatusOrType>::iterator it = cached_types_.find(type_url); + std::map<StringPiece, StatusOrType>::iterator it = + cached_types_.find(type_url); if (it != cached_types_.end()) { return it->second; } @@ -68,7 +69,7 @@ class TypeInfoForTypeResolver : public TypeInfo { // cached_types_ map. const string& string_type_url = *string_storage_.insert(type_url.ToString()).first; - google::protobuf::scoped_ptr<google::protobuf::Type> type(new google::protobuf::Type()); + std::unique_ptr<google::protobuf::Type> type(new google::protobuf::Type()); util::Status status = type_resolver_->ResolveMessageType(string_type_url, type.get()); StatusOrType result = @@ -85,7 +86,8 @@ class TypeInfoForTypeResolver : public TypeInfo { virtual const google::protobuf::Enum* GetEnumByTypeUrl( StringPiece type_url) const { - map<StringPiece, StatusOrEnum>::iterator it = cached_enums_.find(type_url); + std::map<StringPiece, StatusOrEnum>::iterator it = + cached_enums_.find(type_url); if (it != cached_enums_.end()) { return it->second.ok() ? it->second.ValueOrDie() : NULL; } @@ -93,7 +95,7 @@ class TypeInfoForTypeResolver : public TypeInfo { // cached_enums_ map. const string& string_type_url = *string_storage_.insert(type_url.ToString()).first; - google::protobuf::scoped_ptr<google::protobuf::Enum> enum_type( + std::unique_ptr<google::protobuf::Enum> enum_type( new google::protobuf::Enum()); util::Status status = type_resolver_->ResolveEnumType(string_type_url, enum_type.get()); @@ -105,12 +107,14 @@ class TypeInfoForTypeResolver : public TypeInfo { virtual const google::protobuf::Field* FindField( const google::protobuf::Type* type, StringPiece camel_case_name) const { - if (indexed_types_.find(type) == indexed_types_.end()) { - PopulateNameLookupTable(type); - indexed_types_.insert(type); - } + std::map<const google::protobuf::Type*, CamelCaseNameTable>::const_iterator + it = indexed_types_.find(type); + const CamelCaseNameTable& camel_case_name_table = + (it == indexed_types_.end()) + ? PopulateNameLookupTable(type, &indexed_types_[type]) + : it->second; StringPiece name = - FindWithDefault(camel_case_name_table_, camel_case_name, StringPiece()); + FindWithDefault(camel_case_name_table, camel_case_name, StringPiece()); if (name.empty()) { // Didn't find a mapping. Use whatever provided. name = camel_case_name; @@ -121,10 +125,11 @@ class TypeInfoForTypeResolver : public TypeInfo { private: typedef util::StatusOr<const google::protobuf::Type*> StatusOrType; typedef util::StatusOr<const google::protobuf::Enum*> StatusOrEnum; + typedef std::map<StringPiece, StringPiece> CamelCaseNameTable; template <typename T> - static void DeleteCachedTypes(map<StringPiece, T>* cached_types) { - for (typename map<StringPiece, T>::iterator it = cached_types->begin(); + static void DeleteCachedTypes(std::map<StringPiece, T>* cached_types) { + for (typename std::map<StringPiece, T>::iterator it = cached_types->begin(); it != cached_types->end(); ++it) { if (it->second.ok()) { delete it->second.ValueOrDie(); @@ -132,32 +137,35 @@ class TypeInfoForTypeResolver : public TypeInfo { } } - void PopulateNameLookupTable(const google::protobuf::Type* type) const { + const CamelCaseNameTable& PopulateNameLookupTable( + const google::protobuf::Type* type, + CamelCaseNameTable* camel_case_name_table) const { for (int i = 0; i < type->fields_size(); ++i) { const google::protobuf::Field& field = type->fields(i); StringPiece name = field.name(); StringPiece camel_case_name = field.json_name(); - const StringPiece* existing = InsertOrReturnExisting( - &camel_case_name_table_, camel_case_name, name); + const StringPiece* existing = + InsertOrReturnExisting(camel_case_name_table, camel_case_name, name); if (existing && *existing != name) { GOOGLE_LOG(WARNING) << "Field '" << name << "' and '" << *existing << "' map to the same camel case name '" << camel_case_name << "'."; } } + return *camel_case_name_table; } TypeResolver* type_resolver_; // Stores string values that will be referenced by StringPieces in - // cached_types_, cached_enums_ and camel_case_name_table_. - mutable set<string> string_storage_; + // cached_types_, cached_enums_. + mutable std::set<string> string_storage_; - mutable map<StringPiece, StatusOrType> cached_types_; - mutable map<StringPiece, StatusOrEnum> cached_enums_; + mutable std::map<StringPiece, StatusOrType> cached_types_; + mutable std::map<StringPiece, StatusOrEnum> cached_enums_; - mutable set<const google::protobuf::Type*> indexed_types_; - mutable map<StringPiece, StringPiece> camel_case_name_table_; + mutable std::map<const google::protobuf::Type*, CamelCaseNameTable> + indexed_types_; }; } // namespace diff --git a/src/google/protobuf/util/internal/type_info_test_helper.cc b/src/google/protobuf/util/internal/type_info_test_helper.cc index 49e18ed0..281a7f58 100644 --- a/src/google/protobuf/util/internal/type_info_test_helper.cc +++ b/src/google/protobuf/util/internal/type_info_test_helper.cc @@ -31,9 +31,6 @@ #include <google/protobuf/util/internal/type_info_test_helper.h> #include <memory> -#ifndef _SHARED_PTR_H -#include <google/protobuf/stubs/shared_ptr.h> -#endif #include <vector> #include <google/protobuf/stubs/logging.h> @@ -55,7 +52,7 @@ namespace testing { void TypeInfoTestHelper::ResetTypeInfo( - const vector<const Descriptor*>& descriptors) { + const std::vector<const Descriptor*>& descriptors) { switch (type_) { case USE_TYPE_RESOLVER: { const DescriptorPool* pool = descriptors[0]->file()->pool(); @@ -73,14 +70,14 @@ void TypeInfoTestHelper::ResetTypeInfo( } void TypeInfoTestHelper::ResetTypeInfo(const Descriptor* descriptor) { - vector<const Descriptor*> descriptors; + std::vector<const Descriptor*> descriptors; descriptors.push_back(descriptor); ResetTypeInfo(descriptors); } void TypeInfoTestHelper::ResetTypeInfo(const Descriptor* descriptor1, const Descriptor* descriptor2) { - vector<const Descriptor*> descriptors; + std::vector<const Descriptor*> descriptors; descriptors.push_back(descriptor1); descriptors.push_back(descriptor2); ResetTypeInfo(descriptors); diff --git a/src/google/protobuf/util/internal/type_info_test_helper.h b/src/google/protobuf/util/internal/type_info_test_helper.h index 1a279849..5a077e04 100644 --- a/src/google/protobuf/util/internal/type_info_test_helper.h +++ b/src/google/protobuf/util/internal/type_info_test_helper.h @@ -32,9 +32,6 @@ #define GOOGLE_PROTOBUF_UTIL_CONVERTER_TYPE_INFO_TEST_HELPER_H__ #include <memory> -#ifndef _SHARED_PTR_H -#include <google/protobuf/stubs/shared_ptr.h> -#endif #include <vector> #include <google/protobuf/io/coded_stream.h> @@ -64,7 +61,7 @@ class TypeInfoTestHelper { explicit TypeInfoTestHelper(TypeInfoSource type) : type_(type) {} // Creates a TypeInfo object for the given set of descriptors. - void ResetTypeInfo(const vector<const Descriptor*>& descriptors); + void ResetTypeInfo(const std::vector<const Descriptor*>& descriptors); // Convinent overloads. void ResetTypeInfo(const Descriptor* descriptor); @@ -86,8 +83,8 @@ class TypeInfoTestHelper { private: TypeInfoSource type_; - google::protobuf::scoped_ptr<TypeInfo> typeinfo_; - google::protobuf::scoped_ptr<TypeResolver> type_resolver_; + std::unique_ptr<TypeInfo> typeinfo_; + std::unique_ptr<TypeResolver> type_resolver_; }; } // namespace testing } // namespace converter diff --git a/src/google/protobuf/util/internal/utility.cc b/src/google/protobuf/util/internal/utility.cc index 5f613e77..b8d917ce 100644 --- a/src/google/protobuf/util/internal/utility.cc +++ b/src/google/protobuf/util/internal/utility.cc @@ -30,6 +30,8 @@ #include <google/protobuf/util/internal/utility.h> +#include <algorithm> + #include <google/protobuf/stubs/callback.h> #include <google/protobuf/stubs/common.h> #include <google/protobuf/stubs/logging.h> @@ -46,21 +48,11 @@ namespace protobuf { namespace util { namespace converter { -namespace { -const StringPiece SkipWhiteSpace(StringPiece str) { - StringPiece::size_type i; - for (i = 0; i < str.size() && isspace(str[i]); ++i) { - } - GOOGLE_DCHECK(i == str.size() || !isspace(str[i])); - return str.substr(i); -} -} // namespace - bool GetBoolOptionOrDefault( const google::protobuf::RepeatedPtrField<google::protobuf::Option>& options, const string& option_name, bool default_value) { const google::protobuf::Option* opt = FindOptionOrNull(options, option_name); - if (opt == NULL) { + if (opt == nullptr) { return default_value; } return GetBoolFromAny(opt->value()); @@ -70,7 +62,7 @@ int64 GetInt64OptionOrDefault( const google::protobuf::RepeatedPtrField<google::protobuf::Option>& options, const string& option_name, int64 default_value) { const google::protobuf::Option* opt = FindOptionOrNull(options, option_name); - if (opt == NULL) { + if (opt == nullptr) { return default_value; } return GetInt64FromAny(opt->value()); @@ -80,7 +72,7 @@ double GetDoubleOptionOrDefault( const google::protobuf::RepeatedPtrField<google::protobuf::Option>& options, const string& option_name, double default_value) { const google::protobuf::Option* opt = FindOptionOrNull(options, option_name); - if (opt == NULL) { + if (opt == nullptr) { return default_value; } return GetDoubleFromAny(opt->value()); @@ -90,7 +82,7 @@ string GetStringOptionOrDefault( const google::protobuf::RepeatedPtrField<google::protobuf::Option>& options, const string& option_name, const string& default_value) { const google::protobuf::Option* opt = FindOptionOrNull(options, option_name); - if (opt == NULL) { + if (opt == nullptr) { return default_value; } return GetStringFromAny(opt->value()); @@ -132,7 +124,10 @@ const StringPiece GetTypeWithoutUrl(StringPiece type_url) { return type_url.substr(kTypeUrlSize + 1); } else { size_t idx = type_url.rfind('/'); - return type_url.substr(idx + 1); + if (idx != type_url.npos) { + type_url.remove_prefix(idx + 1); + } + return type_url; } } @@ -149,12 +144,12 @@ const google::protobuf::Option* FindOptionOrNull( return &opt; } } - return NULL; + return nullptr; } const google::protobuf::Field* FindFieldInTypeOrNull( const google::protobuf::Type* type, StringPiece field_name) { - if (type != NULL) { + if (type != nullptr) { for (int i = 0; i < type->fields_size(); ++i) { const google::protobuf::Field& field = type->fields(i); if (field.name() == field_name) { @@ -162,12 +157,12 @@ const google::protobuf::Field* FindFieldInTypeOrNull( } } } - return NULL; + return nullptr; } const google::protobuf::Field* FindJsonFieldInTypeOrNull( const google::protobuf::Type* type, StringPiece json_name) { - if (type != NULL) { + if (type != nullptr) { for (int i = 0; i < type->fields_size(); ++i) { const google::protobuf::Field& field = type->fields(i); if (field.json_name() == json_name) { @@ -175,12 +170,25 @@ const google::protobuf::Field* FindJsonFieldInTypeOrNull( } } } - return NULL; + return nullptr; +} + +const google::protobuf::Field* FindFieldInTypeByNumberOrNull( + const google::protobuf::Type* type, int32 number) { + if (type != nullptr) { + for (int i = 0; i < type->fields_size(); ++i) { + const google::protobuf::Field& field = type->fields(i); + if (field.number() == number) { + return &field; + } + } + } + return nullptr; } const google::protobuf::EnumValue* FindEnumValueByNameOrNull( const google::protobuf::Enum* enum_type, StringPiece enum_name) { - if (enum_type != NULL) { + if (enum_type != nullptr) { for (int i = 0; i < enum_type->enumvalue_size(); ++i) { const google::protobuf::EnumValue& enum_value = enum_type->enumvalue(i); if (enum_value.name() == enum_name) { @@ -188,12 +196,12 @@ const google::protobuf::EnumValue* FindEnumValueByNameOrNull( } } } - return NULL; + return nullptr; } const google::protobuf::EnumValue* FindEnumValueByNumberOrNull( const google::protobuf::Enum* enum_type, int32 value) { - if (enum_type != NULL) { + if (enum_type != nullptr) { for (int i = 0; i < enum_type->enumvalue_size(); ++i) { const google::protobuf::EnumValue& enum_value = enum_type->enumvalue(i); if (enum_value.number() == value) { @@ -201,7 +209,40 @@ const google::protobuf::EnumValue* FindEnumValueByNumberOrNull( } } } - return NULL; + return nullptr; +} + +const google::protobuf::EnumValue* FindEnumValueByNameWithoutUnderscoreOrNull( + const google::protobuf::Enum* enum_type, StringPiece enum_name) { + if (enum_type != nullptr) { + for (int i = 0; i < enum_type->enumvalue_size(); ++i) { + const google::protobuf::EnumValue& enum_value = enum_type->enumvalue(i); + string enum_name_without_underscore = enum_value.name(); + + // Remove underscore from the name. + enum_name_without_underscore.erase( + std::remove(enum_name_without_underscore.begin(), + enum_name_without_underscore.end(), '_'), + enum_name_without_underscore.end()); + // Make the name uppercase. + for (string::iterator it = enum_name_without_underscore.begin(); + it != enum_name_without_underscore.end(); ++it) { + *it = ascii_toupper(*it); + } + + if (enum_name_without_underscore == enum_name) { + return &enum_value; + } + } + } + return nullptr; +} + +string EnumValueNameToLowerCamelCase(const StringPiece input) { + string input_string(input); + std::transform(input_string.begin(), input_string.end(), input_string.begin(), + ::tolower); + return ToCamelCase(input_string); } string ToCamelCase(const StringPiece input) { @@ -283,7 +324,7 @@ string ToSnakeCase(StringPiece input) { return result; } -set<string>* well_known_types_ = NULL; +std::set<string>* well_known_types_ = NULL; GOOGLE_PROTOBUF_DECLARE_ONCE(well_known_types_init_); const char* well_known_types_name_array_[] = { "google.protobuf.Timestamp", "google.protobuf.Duration", @@ -296,7 +337,7 @@ const char* well_known_types_name_array_[] = { void DeleteWellKnownTypes() { delete well_known_types_; } void InitWellKnownTypes() { - well_known_types_ = new set<string>; + well_known_types_ = new std::set<string>; for (int i = 0; i < GOOGLE_ARRAYSIZE(well_known_types_name_array_); ++i) { well_known_types_->insert(well_known_types_name_array_[i]); } @@ -315,15 +356,20 @@ bool IsValidBoolString(const string& bool_string) { bool IsMap(const google::protobuf::Field& field, const google::protobuf::Type& type) { - return (field.cardinality() == - google::protobuf::Field_Cardinality_CARDINALITY_REPEATED && + return field.cardinality() == + google::protobuf::Field_Cardinality_CARDINALITY_REPEATED && + (GetBoolOptionOrDefault(type.options(), "map_entry", false) || GetBoolOptionOrDefault(type.options(), - "google.protobuf.MessageOptions.map_entry", false)); + "google.protobuf.MessageOptions.map_entry", + false)); } bool IsMessageSetWireFormat(const google::protobuf::Type& type) { - return GetBoolOptionOrDefault( - type.options(), "google.protobuf.MessageOptions.message_set_wire_format", false); + return GetBoolOptionOrDefault(type.options(), "message_set_wire_format", + false) || + GetBoolOptionOrDefault( + type.options(), + "google.protobuf.MessageOptions.message_set_wire_format", false); } string DoubleAsString(double value) { @@ -359,6 +405,13 @@ bool SafeStrToFloat(StringPiece str, float* value) { return true; } +bool StringStartsWith(StringPiece text, StringPiece prefix) { + return text.starts_with(prefix); +} + +bool StringEndsWith(StringPiece text, StringPiece suffix) { + return text.ends_with(suffix); +} } // namespace converter } // namespace util } // namespace protobuf diff --git a/src/google/protobuf/util/internal/utility.h b/src/google/protobuf/util/internal/utility.h index 26fed444..d8e06a97 100644 --- a/src/google/protobuf/util/internal/utility.h +++ b/src/google/protobuf/util/internal/utility.h @@ -32,9 +32,6 @@ #define GOOGLE_PROTOBUF_UTIL_CONVERTER_UTILITY_H__ #include <memory> -#ifndef _SHARED_PTR_H -#include <google/protobuf/stubs/shared_ptr.h> -#endif #include <string> #include <utility> @@ -121,13 +118,13 @@ LIBPROTOBUF_EXPORT const StringPiece GetTypeWithoutUrl(StringPiece type_url); LIBPROTOBUF_EXPORT const string GetFullTypeWithUrl(StringPiece simple_type); // Finds and returns option identified by name and option_name within the -// provided map. Returns NULL if none found. +// provided map. Returns nullptr if none found. const google::protobuf::Option* FindOptionOrNull( const google::protobuf::RepeatedPtrField<google::protobuf::Option>& options, const string& option_name); // Finds and returns the field identified by field_name in the passed tech Type -// object. Returns NULL if none found. +// object. Returns nullptr if none found. const google::protobuf::Field* FindFieldInTypeOrNull( const google::protobuf::Type* type, StringPiece field_name); @@ -136,19 +133,33 @@ const google::protobuf::Field* FindFieldInTypeOrNull( const google::protobuf::Field* FindJsonFieldInTypeOrNull( const google::protobuf::Type* type, StringPiece json_name); +// Similar to FindFieldInTypeOrNull, but this looks up fields by number. +const google::protobuf::Field* FindFieldInTypeByNumberOrNull( + const google::protobuf::Type* type, int32 number); + // Finds and returns the EnumValue identified by enum_name in the passed tech -// Enum object. Returns NULL if none found. +// Enum object. Returns nullptr if none found. const google::protobuf::EnumValue* FindEnumValueByNameOrNull( const google::protobuf::Enum* enum_type, StringPiece enum_name); // Finds and returns the EnumValue identified by value in the passed tech -// Enum object. Returns NULL if none found. +// Enum object. Returns nullptr if none found. const google::protobuf::EnumValue* FindEnumValueByNumberOrNull( const google::protobuf::Enum* enum_type, int32 value); +// Finds and returns the EnumValue identified by enum_name without underscore in +// the passed tech Enum object. Returns nullptr if none found. +// For Ex. if enum_name is ACTIONANDADVENTURE it can get accepted if +// EnumValue's name is action_and_adventure or ACTION_AND_ADVENTURE. +const google::protobuf::EnumValue* FindEnumValueByNameWithoutUnderscoreOrNull( + const google::protobuf::Enum* enum_type, StringPiece enum_name); + // Converts input to camel-case and returns it. LIBPROTOBUF_EXPORT string ToCamelCase(const StringPiece input); +// Converts enum name string to camel-case and returns it. +string EnumValueNameToLowerCamelCase(const StringPiece input); + // Converts input to snake_case and returns it. LIBPROTOBUF_EXPORT string ToSnakeCase(StringPiece input); @@ -189,6 +200,12 @@ inline string ValueAsString(double value) { // Converts a string to float. Unlike safe_strtof, conversion will fail if the // value fits into double but not float (e.g., DBL_MAX). LIBPROTOBUF_EXPORT bool SafeStrToFloat(StringPiece str, float* value); + +// Returns whether a StringPiece begins with the provided prefix. +bool StringStartsWith(StringPiece text, StringPiece prefix); + +// Returns whether a StringPiece ends with the provided suffix. +bool StringEndsWith(StringPiece text, StringPiece suffix); } // namespace converter } // namespace util } // namespace protobuf diff --git a/src/google/protobuf/util/json_format_proto3.proto b/src/google/protobuf/util/json_format_proto3.proto index 3835b30e..cbc3f81f 100644 --- a/src/google/protobuf/util/json_format_proto3.proto +++ b/src/google/protobuf/util/json_format_proto3.proto @@ -32,6 +32,8 @@ syntax = "proto3"; package proto3; +option java_package = "com.google.protobuf.util"; +option java_outer_classname = "JsonFormatProto3"; import "google/protobuf/duration.proto"; import "google/protobuf/timestamp.proto"; @@ -179,3 +181,9 @@ message TestCustomJsonName { message TestExtensions { .protobuf_unittest.TestAllExtensions extensions = 1; } + +message TestEnumValue{ + EnumType enum_value1 = 1; + EnumType enum_value2 = 2; + EnumType enum_value3 = 3; +}
\ No newline at end of file diff --git a/src/google/protobuf/util/json_util.cc b/src/google/protobuf/util/json_util.cc index d7ac2dba..f81a7a30 100644 --- a/src/google/protobuf/util/json_util.cc +++ b/src/google/protobuf/util/json_util.cc @@ -49,22 +49,29 @@ namespace protobuf { namespace util { namespace internal { +ZeroCopyStreamByteSink::~ZeroCopyStreamByteSink() { + if (buffer_size_ > 0) { + stream_->BackUp(buffer_size_); + } +} + void ZeroCopyStreamByteSink::Append(const char* bytes, size_t len) { - while (len > 0) { - void* buffer; - int length; - if (!stream_->Next(&buffer, &length)) { - // There isn't a way for ByteSink to report errors. + while (true) { + if (len <= buffer_size_) { + memcpy(buffer_, bytes, len); + buffer_ = static_cast<char*>(buffer_) + len; + buffer_size_ -= len; return; } - if (len < length) { - memcpy(buffer, bytes, len); - stream_->BackUp(length - len); - break; - } else { - memcpy(buffer, bytes, length); - bytes += length; - len -= length; + if (buffer_size_ > 0) { + memcpy(buffer_, bytes, buffer_size_); + bytes += buffer_size_; + len -= buffer_size_; + } + if (!stream_->Next(&buffer_, &buffer_size_)) { + // There isn't a way for ByteSink to report errors. + buffer_size_ = 0; + return; } } } @@ -79,12 +86,19 @@ util::Status BinaryToJsonStream(TypeResolver* resolver, google::protobuf::Type type; RETURN_IF_ERROR(resolver->ResolveMessageType(type_url, &type)); converter::ProtoStreamObjectSource proto_source(&in_stream, resolver, type); + proto_source.set_use_ints_for_enums(options.always_print_enums_as_ints); + proto_source.set_preserve_proto_field_names( + options.preserve_proto_field_names); io::CodedOutputStream out_stream(json_output); converter::JsonObjectWriter json_writer(options.add_whitespace ? " " : "", &out_stream); if (options.always_print_primitive_fields) { converter::DefaultValueObjectWriter default_value_writer( resolver, type, &json_writer); + default_value_writer.set_preserve_proto_field_names( + options.preserve_proto_field_names); + default_value_writer.set_print_enums_as_ints( + options.always_print_enums_as_ints); return proto_source.WriteTo(&default_value_writer); } else { return proto_source.WriteTo(&json_writer); @@ -105,7 +119,7 @@ util::Status BinaryToJsonString(TypeResolver* resolver, namespace { class StatusErrorListener : public converter::ErrorListener { public: - StatusErrorListener() : status_(util::Status::OK) {} + StatusErrorListener() {} virtual ~StatusErrorListener() {} util::Status GetStatus() { return status_; } @@ -113,22 +127,22 @@ class StatusErrorListener : public converter::ErrorListener { virtual void InvalidName(const converter::LocationTrackerInterface& loc, StringPiece unknown_name, StringPiece message) { status_ = util::Status(util::error::INVALID_ARGUMENT, - loc.ToString() + ": " + message.ToString()); + loc.ToString() + ": " + string(message)); } virtual void InvalidValue(const converter::LocationTrackerInterface& loc, StringPiece type_name, StringPiece value) { status_ = util::Status(util::error::INVALID_ARGUMENT, - loc.ToString() + ": invalid value " + value.ToString() + - " for type " + type_name.ToString()); + loc.ToString() + ": invalid value " + string(value) + + " for type " + string(type_name)); } virtual void MissingField(const converter::LocationTrackerInterface& loc, StringPiece missing_name) { status_ = util::Status( util::error::INVALID_ARGUMENT, - loc.ToString() + ": missing field " + missing_name.ToString()); + loc.ToString() + ": missing field " + string(missing_name)); } private: diff --git a/src/google/protobuf/util/json_util.h b/src/google/protobuf/util/json_util.h index 6d3cee52..b1c69813 100644 --- a/src/google/protobuf/util/json_util.h +++ b/src/google/protobuf/util/json_util.h @@ -56,15 +56,22 @@ struct JsonPrintOptions { // Whether to add spaces, line breaks and indentation to make the JSON output // easy to read. bool add_whitespace; - // Whether to always print primitive fields. By default primitive fields with - // default values will be omitted in JSON joutput. For example, an int32 field - // set to 0 will be omitted. Set this flag to true will override the default - // behavior and print primitive fields regardless of their values. + // Whether to always print primitive fields. By default proto3 primitive + // fields with default values will be omitted in JSON output. For example, an + // int32 field set to 0 will be omitted. Set this flag to true will override + // the default behavior and print primitive fields regardless of their values. bool always_print_primitive_fields; - - JsonPrintOptions() : add_whitespace(false), - always_print_primitive_fields(false) { - } + // Whether to always print enums as ints. By default they are rendered as + // strings. + bool always_print_enums_as_ints; + // Whether to preserve proto field names + bool preserve_proto_field_names; + + JsonPrintOptions() + : add_whitespace(false), + always_print_primitive_fields(false), + always_print_enums_as_ints(false), + preserve_proto_field_names(false) {} }; // DEPRECATED. Use JsonPrintOptions instead. @@ -172,12 +179,15 @@ namespace internal { class LIBPROTOBUF_EXPORT ZeroCopyStreamByteSink : public strings::ByteSink { public: explicit ZeroCopyStreamByteSink(io::ZeroCopyOutputStream* stream) - : stream_(stream) {} + : stream_(stream), buffer_(NULL), buffer_size_(0) {} + ~ZeroCopyStreamByteSink(); virtual void Append(const char* bytes, size_t len); private: io::ZeroCopyOutputStream* stream_; + void* buffer_; + int buffer_size_; GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ZeroCopyStreamByteSink); }; diff --git a/src/google/protobuf/util/json_util_test.cc b/src/google/protobuf/util/json_util_test.cc index 24ff5fd6..ed9092df 100644 --- a/src/google/protobuf/util/json_util_test.cc +++ b/src/google/protobuf/util/json_util_test.cc @@ -34,8 +34,10 @@ #include <string> #include <google/protobuf/io/zero_copy_stream.h> +#include <google/protobuf/io/zero_copy_stream_impl.h> #include <google/protobuf/descriptor_database.h> #include <google/protobuf/dynamic_message.h> +#include <google/protobuf/util/internal/testdata/maps.pb.h> #include <google/protobuf/util/json_format_proto3.pb.h> #include <google/protobuf/util/type_resolver.h> #include <google/protobuf/util/type_resolver_util.h> @@ -46,23 +48,22 @@ namespace protobuf { namespace util { namespace { +using google::protobuf::testing::MapIn; using proto3::FOO; using proto3::BAR; using proto3::TestMessage; using proto3::TestMap; +using proto3::TestOneof; +using proto3::TestEnumValue; static const char kTypeUrlPrefix[] = "type.googleapis.com"; -static string GetTypeUrl(const Descriptor* message) { - return string(kTypeUrlPrefix) + "/" + message->full_name(); -} - // As functions defined in json_util.h are just thin wrappers around the // JSON conversion code in //net/proto2/util/converter, in this test we // only cover some very basic cases to make sure the wrappers have forwarded // parameters to the underlying implementation correctly. More detailed // tests are contained in the //net/proto2/util/converter directory. -class JsonUtilTest : public testing::Test { +class JsonUtilTest : public ::testing::Test { protected: JsonUtilTest() { } @@ -82,7 +83,7 @@ class JsonUtilTest : public testing::Test { return FromJson(json, message, JsonParseOptions()); } - google::protobuf::scoped_ptr<TypeResolver> resolver_; + std::unique_ptr<TypeResolver> resolver_; }; TEST_F(JsonUtilTest, TestWhitespaces) { @@ -156,6 +157,88 @@ TEST_F(JsonUtilTest, TestDefaultValues) { "\"repeatedMessageValue\":[]" "}", ToJson(m, options)); + + options.preserve_proto_field_names = true; + m.set_string_value("i am a test string value"); + m.set_bytes_value("i am a test bytes value"); + EXPECT_EQ( + "{\"bool_value\":false," + "\"int32_value\":0," + "\"int64_value\":\"0\"," + "\"uint32_value\":0," + "\"uint64_value\":\"0\"," + "\"float_value\":0," + "\"double_value\":0," + "\"string_value\":\"i am a test string value\"," + "\"bytes_value\":\"aSBhbSBhIHRlc3QgYnl0ZXMgdmFsdWU=\"," + "\"enum_value\":\"FOO\"," + "\"repeated_bool_value\":[]," + "\"repeated_int32_value\":[]," + "\"repeated_int64_value\":[]," + "\"repeated_uint32_value\":[]," + "\"repeated_uint64_value\":[]," + "\"repeated_float_value\":[]," + "\"repeated_double_value\":[]," + "\"repeated_string_value\":[]," + "\"repeated_bytes_value\":[]," + "\"repeated_enum_value\":[]," + "\"repeated_message_value\":[]" + "}", + ToJson(m, options)); +} + +TEST_F(JsonUtilTest, TestPreserveProtoFieldNames) { + TestMessage m; + m.mutable_message_value(); + + JsonPrintOptions options; + options.preserve_proto_field_names = true; + EXPECT_EQ("{\"message_value\":{}}", ToJson(m, options)); +} + +TEST_F(JsonUtilTest, TestAlwaysPrintEnumsAsInts) { + TestMessage orig; + orig.set_enum_value(proto3::BAR); + orig.add_repeated_enum_value(proto3::FOO); + orig.add_repeated_enum_value(proto3::BAR); + + JsonPrintOptions print_options; + print_options.always_print_enums_as_ints = true; + + string expected_json = "{\"enumValue\":1,\"repeatedEnumValue\":[0,1]}"; + EXPECT_EQ(expected_json, ToJson(orig, print_options)); + + TestMessage parsed; + JsonParseOptions parse_options; + ASSERT_TRUE(FromJson(expected_json, &parsed, parse_options)); + + EXPECT_EQ(proto3::BAR, parsed.enum_value()); + EXPECT_EQ(2, parsed.repeated_enum_value_size()); + EXPECT_EQ(proto3::FOO, parsed.repeated_enum_value(0)); + EXPECT_EQ(proto3::BAR, parsed.repeated_enum_value(1)); +} + +TEST_F(JsonUtilTest, TestPrintEnumsAsIntsWithDefaultValue) { + TestEnumValue orig; + //orig.set_enum_value1(proto3::FOO) + orig.set_enum_value2(proto3::FOO); + orig.set_enum_value3(proto3::BAR); + + JsonPrintOptions print_options; + print_options.always_print_enums_as_ints = true; + print_options.always_print_primitive_fields = true; + + string expected_json = "{\"enumValue1\":0,\"enumValue2\":0,\"enumValue3\":1}"; + EXPECT_EQ(expected_json, ToJson(orig, print_options)); + + TestEnumValue parsed; + JsonParseOptions parse_options; + ASSERT_TRUE(FromJson(expected_json, &parsed, parse_options)); + + EXPECT_EQ(proto3::FOO, parsed.enum_value1()); + EXPECT_EQ(proto3::FOO, parsed.enum_value2()); + EXPECT_EQ(proto3::BAR, parsed.enum_value3()); + } TEST_F(JsonUtilTest, ParseMessage) { @@ -196,6 +279,29 @@ TEST_F(JsonUtilTest, ParseMap) { EXPECT_EQ(message.DebugString(), other.DebugString()); } +TEST_F(JsonUtilTest, ParsePrimitiveMapIn) { + MapIn message; + JsonPrintOptions print_options; + print_options.always_print_primitive_fields = true; + JsonParseOptions parse_options; + EXPECT_EQ("{\"other\":\"\",\"things\":[],\"mapInput\":{}}", + ToJson(message, print_options)); + MapIn other; + ASSERT_TRUE(FromJson(ToJson(message, print_options), &other, parse_options)); + EXPECT_EQ(message.DebugString(), other.DebugString()); +} + +TEST_F(JsonUtilTest, PrintPrimitiveOneof) { + TestOneof message; + JsonPrintOptions options; + options.always_print_primitive_fields = true; + message.mutable_oneof_message_value(); + EXPECT_EQ("{\"oneofMessageValue\":{\"value\":0}}", ToJson(message, options)); + + message.set_oneof_int32_value(1); + EXPECT_EQ("{\"oneofInt32Value\":1}", ToJson(message, options)); +} + TEST_F(JsonUtilTest, TestParseIgnoreUnknownFields) { TestMessage m; JsonParseOptions options; @@ -231,7 +337,7 @@ TEST_F(JsonUtilTest, TestDynamicMessage) { DescriptorPool pool(&database); // A dynamic version of the test proto. DynamicMessageFactory factory; - google::protobuf::scoped_ptr<Message> message(factory.GetPrototype( + std::unique_ptr<Message> message(factory.GetPrototype( pool.FindMessageTypeByName("proto3.TestMessage"))->New()); EXPECT_TRUE(FromJson(input, message.get())); @@ -251,11 +357,69 @@ TEST_F(JsonUtilTest, TestDynamicMessage) { EXPECT_EQ(ToJson(generated, options), ToJson(*message, options)); } -typedef pair<char*, int> Segment; +TEST_F(JsonUtilTest, TestParsingUnknownEnumsAs0) { + TestMessage m; + { + JsonParseOptions options; + ASSERT_FALSE(options.ignore_unknown_fields); + string input = + "{\n" + " \"enum_value\":\"UNKNOWN_VALUE\"\n" + "}"; + m.set_enum_value(proto3::BAR); + EXPECT_FALSE(FromJson(input, &m, options)); + ASSERT_EQ(proto3::BAR, m.enum_value()); // Keep previous value + + options.ignore_unknown_fields = true; + EXPECT_TRUE(FromJson(input, &m, options)); + EXPECT_EQ(0, m.enum_value()); // Unknown enum value must be decoded as 0 + } + // Integer values are read as usual + { + JsonParseOptions options; + string input = + "{\n" + " \"enum_value\":12345\n" + "}"; + m.set_enum_value(proto3::BAR); + EXPECT_TRUE(FromJson(input, &m, options)); + ASSERT_EQ(12345, m.enum_value()); + + options.ignore_unknown_fields = true; + EXPECT_TRUE(FromJson(input, &m, options)); + EXPECT_EQ(12345, m.enum_value()); + } + + // Trying to pass an object as an enum field value is always treated as an error + { + JsonParseOptions options; + string input = + "{\n" + " \"enum_value\":{}\n" + "}"; + options.ignore_unknown_fields = true; + EXPECT_FALSE(FromJson(input, &m, options)); + options.ignore_unknown_fields = false; + EXPECT_FALSE(FromJson(input, &m, options)); + } + // Trying to pass an array as an enum field value is always treated as an error + { + JsonParseOptions options; + string input = + "{\n" + " \"enum_value\":[]\n" + "}"; + EXPECT_FALSE(FromJson(input, &m, options)); + options.ignore_unknown_fields = true; + EXPECT_FALSE(FromJson(input, &m, options)); + } +} + +typedef std::pair<char*, int> Segment; // A ZeroCopyOutputStream that writes to multiple buffers. class SegmentedZeroCopyOutputStream : public io::ZeroCopyOutputStream { public: - explicit SegmentedZeroCopyOutputStream(list<Segment> segments) + explicit SegmentedZeroCopyOutputStream(std::list<Segment> segments) : segments_(segments), last_segment_(static_cast<char*>(NULL), 0), byte_count_(0) {} virtual bool Next(void** buffer, int* length) { @@ -281,7 +445,7 @@ class SegmentedZeroCopyOutputStream : public io::ZeroCopyOutputStream { virtual int64 ByteCount() const { return byte_count_; } private: - list<Segment> segments_; + std::list<Segment> segments_; Segment last_segment_; int64 byte_count_; }; @@ -299,7 +463,7 @@ TEST(ZeroCopyStreamByteSinkTest, TestAllInputOutputPatterns) { for (int split_pattern = 0; split_pattern < (1 << (kOutputBufferLength - 1)); split_pattern += kSkippedPatternCount) { // Split the buffer into small segments according to the split_pattern. - list<Segment> segments; + std::list<Segment> segments; int segment_start = 0; for (int i = 0; i < kOutputBufferLength - 1; ++i) { if (split_pattern & (1 << i)) { @@ -376,6 +540,25 @@ TEST(ZeroCopyStreamByteSinkTest, TestAllInputOutputPatterns) { } } +TEST_F(JsonUtilTest, TestWrongJsonInput) { + const char json[] = "{\"unknown_field\":\"some_value\"}"; + io::ArrayInputStream input_stream(json, strlen(json)); + char proto_buffer[10000]; + io::ArrayOutputStream output_stream(proto_buffer, sizeof(proto_buffer)); + std::string message_type = "type.googleapis.com/proto3.TestMessage"; + TypeResolver* resolver = NewTypeResolverForDescriptorPool( + "type.googleapis.com", DescriptorPool::generated_pool()); + + auto result_status = util::JsonToBinaryStream( + resolver, message_type, &input_stream, &output_stream); + + delete resolver; + + EXPECT_FALSE(result_status.ok()); + EXPECT_EQ(result_status.error_code(), + util::error::INVALID_ARGUMENT); +} + } // namespace } // namespace util } // namespace protobuf diff --git a/src/google/protobuf/util/message_differencer.cc b/src/google/protobuf/util/message_differencer.cc index 03a334b6..9842f64c 100644 --- a/src/google/protobuf/util/message_differencer.cc +++ b/src/google/protobuf/util/message_differencer.cc @@ -40,9 +40,6 @@ #include <algorithm> #include <memory> -#ifndef _SHARED_PTR_H -#include <google/protobuf/stubs/shared_ptr.h> -#endif #include <utility> #include <google/protobuf/stubs/callback.h> @@ -53,6 +50,7 @@ #include <google/protobuf/io/printer.h> #include <google/protobuf/io/zero_copy_stream.h> #include <google/protobuf/io/zero_copy_stream_impl.h> +#include <google/protobuf/descriptor.pb.h> #include <google/protobuf/dynamic_message.h> #include <google/protobuf/text_format.h> #include <google/protobuf/util/field_comparator.h> @@ -73,7 +71,7 @@ class MessageDifferencer::MultipleFieldsMapKeyComparator public: MultipleFieldsMapKeyComparator( MessageDifferencer* message_differencer, - const vector<vector<const FieldDescriptor*> >& key_field_paths) + const std::vector<std::vector<const FieldDescriptor*> >& key_field_paths) : message_differencer_(message_differencer), key_field_paths_(key_field_paths) { GOOGLE_CHECK(!key_field_paths_.empty()); @@ -85,14 +83,14 @@ class MessageDifferencer::MultipleFieldsMapKeyComparator MessageDifferencer* message_differencer, const FieldDescriptor* key) : message_differencer_(message_differencer) { - vector<const FieldDescriptor*> key_field_path; + std::vector<const FieldDescriptor*> key_field_path; key_field_path.push_back(key); key_field_paths_.push_back(key_field_path); } virtual bool IsMatch( const Message& message1, const Message& message2, - const vector<SpecificField>& parent_fields) const { + const std::vector<SpecificField>& parent_fields) const { for (int i = 0; i < key_field_paths_.size(); ++i) { if (!IsMatchInternal(message1, message2, parent_fields, key_field_paths_[i], 0)) { @@ -105,11 +103,11 @@ class MessageDifferencer::MultipleFieldsMapKeyComparator bool IsMatchInternal( const Message& message1, const Message& message2, - const vector<SpecificField>& parent_fields, - const vector<const FieldDescriptor*>& key_field_path, + const std::vector<SpecificField>& parent_fields, + const std::vector<const FieldDescriptor*>& key_field_path, int path_index) const { const FieldDescriptor* field = key_field_path[path_index]; - vector<SpecificField> current_parent_fields(parent_fields); + std::vector<SpecificField> current_parent_fields(parent_fields); if (path_index == key_field_path.size() - 1) { if (field->is_repeated()) { if (!message_differencer_->CompareRepeatedField( @@ -146,10 +144,36 @@ class MessageDifferencer::MultipleFieldsMapKeyComparator } } MessageDifferencer* message_differencer_; - vector<vector<const FieldDescriptor*> > key_field_paths_; + std::vector<std::vector<const FieldDescriptor*> > key_field_paths_; GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(MultipleFieldsMapKeyComparator); }; +MessageDifferencer::MapEntryKeyComparator::MapEntryKeyComparator( + MessageDifferencer* message_differencer) + : message_differencer_(message_differencer) {} + +bool MessageDifferencer::MapEntryKeyComparator::IsMatch( + const Message& message1, const Message& message2, + const std::vector<SpecificField>& parent_fields) const { + // Map entry has its key in the field with tag 1. See the comment for + // map_entry in MessageOptions. + const FieldDescriptor* key = message1.GetDescriptor()->FindFieldByNumber(1); + // If key is not present in message1 and we're doing partial comparison or if + // map key is explicitly ignored treat the field as set instead, + const bool treat_as_set = + (message_differencer_->scope() == PARTIAL && + !message1.GetReflection()->HasField(message1, key)) || + message_differencer_->IsIgnored(message1, message2, key, parent_fields); + + std::vector<SpecificField> current_parent_fields(parent_fields); + if (treat_as_set) { + return message_differencer_->Compare(message1, message2, + ¤t_parent_fields); + } + return message_differencer_->CompareFieldValueUsingParentFields( + message1, message2, key, -1, -1, ¤t_parent_fields); +} + bool MessageDifferencer::Equals(const Message& message1, const Message& message2) { MessageDifferencer differencer; @@ -191,8 +215,10 @@ MessageDifferencer::MessageDifferencer() message_field_comparison_(EQUAL), scope_(FULL), repeated_field_comparison_(AS_LIST), + map_entry_key_comparator_(this), report_matches_(false), - output_string_(NULL) { } + report_moves_(true), + output_string_(NULL) {} MessageDifferencer::~MessageDifferencer() { for (int i = 0; i < owned_key_comparators_.size(); ++i) { @@ -283,10 +309,10 @@ void MessageDifferencer::TreatAsMap(const FieldDescriptor* field, void MessageDifferencer::TreatAsMapWithMultipleFieldsAsKey( const FieldDescriptor* field, - const vector<const FieldDescriptor*>& key_fields) { - vector<vector<const FieldDescriptor*> > key_field_paths; + const std::vector<const FieldDescriptor*>& key_fields) { + std::vector<std::vector<const FieldDescriptor*> > key_field_paths; for (int i = 0; i < key_fields.size(); ++i) { - vector<const FieldDescriptor*> key_field_path; + std::vector<const FieldDescriptor*> key_field_path; key_field_path.push_back(key_fields[i]); key_field_paths.push_back(key_field_path); } @@ -295,14 +321,15 @@ void MessageDifferencer::TreatAsMapWithMultipleFieldsAsKey( void MessageDifferencer::TreatAsMapWithMultipleFieldPathsAsKey( const FieldDescriptor* field, - const vector<vector<const FieldDescriptor*> >& key_field_paths) { + const std::vector<std::vector<const FieldDescriptor*> >& key_field_paths) { GOOGLE_CHECK(field->is_repeated()) << "Field must be repeated: " << field->full_name(); GOOGLE_CHECK_EQ(FieldDescriptor::CPPTYPE_MESSAGE, field->cpp_type()) << "Field has to be message type. Field name is: " << field->full_name(); for (int i = 0; i < key_field_paths.size(); ++i) { - const vector<const FieldDescriptor*>& key_field_path = key_field_paths[i]; + const std::vector<const FieldDescriptor*>& key_field_path = + key_field_paths[i]; for (int j = 0; j < key_field_path.size(); ++j) { const FieldDescriptor* parent_field = j == 0 ? field : key_field_path[j - 1]; @@ -333,9 +360,6 @@ void MessageDifferencer::TreatAsMapUsingKeyComparator( const MapKeyComparator* key_comparator) { GOOGLE_CHECK(field->is_repeated()) << "Field must be repeated: " << field->full_name(); - GOOGLE_CHECK_EQ(FieldDescriptor::CPPTYPE_MESSAGE, field->cpp_type()) - << "Field has to be message type. Field name is: " - << field->full_name(); GOOGLE_CHECK(set_fields_.find(field) == set_fields_.end()) << "Cannot treat this repeated field as both Map and Set for " << "comparison."; @@ -390,7 +414,7 @@ bool MessageDifferencer::FieldBefore(const FieldDescriptor* field1, bool MessageDifferencer::Compare(const Message& message1, const Message& message2) { - vector<SpecificField> parent_fields; + std::vector<SpecificField> parent_fields; bool result = false; @@ -411,20 +435,20 @@ bool MessageDifferencer::Compare(const Message& message1, bool MessageDifferencer::CompareWithFields( const Message& message1, const Message& message2, - const vector<const FieldDescriptor*>& message1_fields_arg, - const vector<const FieldDescriptor*>& message2_fields_arg) { + const std::vector<const FieldDescriptor*>& message1_fields_arg, + const std::vector<const FieldDescriptor*>& message2_fields_arg) { if (message1.GetDescriptor() != message2.GetDescriptor()) { GOOGLE_LOG(DFATAL) << "Comparison between two messages with different " << "descriptors."; return false; } - vector<SpecificField> parent_fields; + std::vector<SpecificField> parent_fields; bool result = false; - vector<const FieldDescriptor*> message1_fields(message1_fields_arg); - vector<const FieldDescriptor*> message2_fields(message2_fields_arg); + std::vector<const FieldDescriptor*> message1_fields(message1_fields_arg); + std::vector<const FieldDescriptor*> message2_fields(message2_fields_arg); std::sort(message1_fields.begin(), message1_fields.end(), FieldBefore); std::sort(message2_fields.begin(), message2_fields.end(), FieldBefore); @@ -451,7 +475,7 @@ bool MessageDifferencer::CompareWithFields( bool MessageDifferencer::Compare( const Message& message1, const Message& message2, - vector<SpecificField>* parent_fields) { + std::vector<SpecificField>* parent_fields) { const Descriptor* descriptor1 = message1.GetDescriptor(); const Descriptor* descriptor2 = message2.GetDescriptor(); if (descriptor1 != descriptor2) { @@ -463,9 +487,13 @@ bool MessageDifferencer::Compare( } // Expand google.protobuf.Any payload if possible. if (descriptor1->full_name() == internal::kAnyFullTypeName) { - google::protobuf::scoped_ptr<Message> data1; - google::protobuf::scoped_ptr<Message> data2; + std::unique_ptr<Message> data1; + std::unique_ptr<Message> data2; if (UnpackAny(message1, &data1) && UnpackAny(message2, &data2)) { + // Avoid DFATAL for different descriptors in google.protobuf.Any payloads. + if (data1->GetDescriptor() != data2->GetDescriptor()) { + return false; + } return Compare(*data1, *data2, parent_fields); } } @@ -473,14 +501,28 @@ bool MessageDifferencer::Compare( const Reflection* reflection2 = message2.GetReflection(); // Retrieve all the set fields, including extensions. - vector<const FieldDescriptor*> message1_fields; + std::vector<const FieldDescriptor*> message1_fields; message1_fields.reserve(1 + message1.GetDescriptor()->field_count()); - vector<const FieldDescriptor*> message2_fields; + std::vector<const FieldDescriptor*> message2_fields; message2_fields.reserve(1 + message2.GetDescriptor()->field_count()); - reflection1->ListFields(message1, &message1_fields); - reflection2->ListFields(message2, &message2_fields); + if (descriptor1->options().map_entry()) { + if (scope_ == PARTIAL) { + reflection1->ListFields(message1, &message1_fields); + } else { + // Map entry fields are always considered present. + for (int i = 0; i < descriptor1->field_count(); i++) { + message1_fields.push_back(descriptor1->field(i)); + } + } + for (int i = 0; i < descriptor1->field_count(); i++) { + message2_fields.push_back(descriptor1->field(i)); + } + } else { + reflection1->ListFields(message1, &message1_fields); + reflection2->ListFields(message2, &message2_fields); + } // Add sentinel values to deal with the // case where the number of the fields in @@ -514,15 +556,15 @@ bool MessageDifferencer::Compare( bool MessageDifferencer::CompareRequestedFieldsUsingSettings( const Message& message1, const Message& message2, - const vector<const FieldDescriptor*>& message1_fields, - const vector<const FieldDescriptor*>& message2_fields, - vector<SpecificField>* parent_fields) { + const std::vector<const FieldDescriptor*>& message1_fields, + const std::vector<const FieldDescriptor*>& message2_fields, + std::vector<SpecificField>* parent_fields) { if (scope_ == FULL) { if (message_field_comparison_ == EQUIVALENT) { // We need to merge the field lists of both messages (i.e. // we are merely checking for a difference in field values, // rather than the addition or deletion of fields). - vector<const FieldDescriptor*> fields_union; + std::vector<const FieldDescriptor*> fields_union; CombineFields(message1_fields, FULL, message2_fields, FULL, &fields_union); return CompareWithFieldsInternal(message1, message2, fields_union, @@ -544,7 +586,7 @@ bool MessageDifferencer::CompareRequestedFieldsUsingSettings( // but only the intersection for message2. This way, any fields // only present in message2 will be ignored, but any fields only // present in message1 will be marked as a difference. - vector<const FieldDescriptor*> fields_intersection; + std::vector<const FieldDescriptor*> fields_intersection; CombineFields(message1_fields, PARTIAL, message2_fields, PARTIAL, &fields_intersection); return CompareWithFieldsInternal(message1, message2, message1_fields, @@ -554,11 +596,11 @@ bool MessageDifferencer::CompareRequestedFieldsUsingSettings( } void MessageDifferencer::CombineFields( - const vector<const FieldDescriptor*>& fields1, + const std::vector<const FieldDescriptor*>& fields1, Scope fields1_scope, - const vector<const FieldDescriptor*>& fields2, + const std::vector<const FieldDescriptor*>& fields2, Scope fields2_scope, - vector<const FieldDescriptor*>* combined_fields) { + std::vector<const FieldDescriptor*>* combined_fields) { int index1 = 0; int index2 = 0; @@ -588,9 +630,9 @@ void MessageDifferencer::CombineFields( bool MessageDifferencer::CompareWithFieldsInternal( const Message& message1, const Message& message2, - const vector<const FieldDescriptor*>& message1_fields, - const vector<const FieldDescriptor*>& message2_fields, - vector<SpecificField>* parent_fields) { + const std::vector<const FieldDescriptor*>& message1_fields, + const std::vector<const FieldDescriptor*>& message2_fields, + std::vector<SpecificField>* parent_fields) { bool isDifferent = false; int field_index1 = 0; int field_index2 = 0; @@ -626,6 +668,7 @@ bool MessageDifferencer::CompareWithFieldsInternal( } if (reporter_ != NULL) { + assert(field1 != NULL); int count = field1->is_repeated() ? reflection1->FieldSize(message1, field1) : 1; @@ -706,6 +749,7 @@ bool MessageDifferencer::CompareWithFieldsInternal( } bool fieldDifferent = false; + assert(field1 != NULL); if (field1->is_repeated()) { fieldDifferent = !CompareRepeatedField(message1, message2, field1, parent_fields); @@ -744,13 +788,12 @@ bool MessageDifferencer::CompareWithFieldsInternal( return !isDifferent; } -bool MessageDifferencer::IsMatch(const FieldDescriptor* repeated_field, - const MapKeyComparator* key_comparator, - const Message* message1, - const Message* message2, - const vector<SpecificField>& parent_fields, - int index1, int index2) { - vector<SpecificField> current_parent_fields(parent_fields); +bool MessageDifferencer::IsMatch( + const FieldDescriptor* repeated_field, + const MapKeyComparator* key_comparator, const Message* message1, + const Message* message2, const std::vector<SpecificField>& parent_fields, + int index1, int index2) { + std::vector<SpecificField> current_parent_fields(parent_fields); if (repeated_field->cpp_type() != FieldDescriptor::CPPTYPE_MESSAGE) { return CompareFieldValueUsingParentFields( *message1, *message2, repeated_field, index1, index2, @@ -777,6 +820,8 @@ bool MessageDifferencer::IsMatch(const FieldDescriptor* repeated_field, reflection2->GetRepeatedMessage(*message2, repeated_field, index2); SpecificField specific_field; specific_field.field = repeated_field; + specific_field.index = index1; + specific_field.new_index = index2; current_parent_fields.push_back(specific_field); match = key_comparator->IsMatch(m1, m2, current_parent_fields); } @@ -790,7 +835,7 @@ bool MessageDifferencer::CompareRepeatedField( const Message& message1, const Message& message2, const FieldDescriptor* repeated_field, - vector<SpecificField>* parent_fields) { + std::vector<SpecificField>* parent_fields) { // the input FieldDescriptor is guaranteed to be repeated field. const Reflection* reflection1 = message1.GetReflection(); const Reflection* reflection2 = message2.GetReflection(); @@ -811,8 +856,8 @@ bool MessageDifferencer::CompareRepeatedField( // These two list are used for store the index of the correspondent // element in peer repeated field. - vector<int> match_list1; - vector<int> match_list2; + std::vector<int> match_list1; + std::vector<int> match_list2; // Try to match indices of the repeated fields. Return false if match fails // and there's no detailed report needed. @@ -847,7 +892,8 @@ bool MessageDifferencer::CompareRepeatedField( parent_fields->pop_back(); fieldDifferent = true; } else if (reporter_ != NULL && - specific_field.index != specific_field.new_index) { + specific_field.index != specific_field.new_index && + !specific_field.field->is_map() && report_moves_) { parent_fields->push_back(specific_field); reporter_->ReportMoved(message1, message2, *parent_fields); parent_fields->pop_back(); @@ -875,6 +921,7 @@ bool MessageDifferencer::CompareRepeatedField( for (int i = 0; i < count1; ++i) { if (match_list1[i] != -1) continue; + assert(reporter_ != NULL); specific_field.index = i; parent_fields->push_back(specific_field); reporter_->ReportDeleted(message1, message2, *parent_fields); @@ -896,7 +943,7 @@ bool MessageDifferencer::CompareFieldValue(const Message& message1, bool MessageDifferencer::CompareFieldValueUsingParentFields( const Message& message1, const Message& message2, const FieldDescriptor* field, int index1, int index2, - vector<SpecificField>* parent_fields) { + std::vector<SpecificField>* parent_fields) { FieldContext field_context(parent_fields); FieldComparator::ComparisonResult result = GetFieldComparisonResult( message1, message2, field, index1, index2, &field_context); @@ -935,8 +982,10 @@ bool MessageDifferencer::CompareFieldValueUsingParentFields( } bool MessageDifferencer::CheckPathChanged( - const vector<SpecificField>& field_path) { + const std::vector<SpecificField>& field_path) { for (int i = 0; i < field_path.size(); ++i) { + // Don't check indexes for map entries -- maps are unordered. + if (field_path[i].field != NULL && field_path[i].field->is_map()) continue; if (field_path[i].index != field_path[i].new_index) return true; } return false; @@ -944,7 +993,6 @@ bool MessageDifferencer::CheckPathChanged( bool MessageDifferencer::IsTreatedAsSet(const FieldDescriptor* field) { if (!field->is_repeated()) return false; - if (field->is_map()) return true; if (repeated_field_comparison_ == AS_SET) return list_fields_.find(field) == list_fields_.end(); return (set_fields_.find(field) != set_fields_.end()); @@ -959,7 +1007,7 @@ bool MessageDifferencer::IsIgnored( const Message& message1, const Message& message2, const FieldDescriptor* field, - const vector<SpecificField>& parent_fields) { + const std::vector<SpecificField>& parent_fields) { if (ignored_fields_.find(field) != ignored_fields_.end()) { return true; } @@ -974,7 +1022,8 @@ bool MessageDifferencer::IsIgnored( bool MessageDifferencer::IsUnknownFieldIgnored( const Message& message1, const Message& message2, - const SpecificField& field, const vector<SpecificField>& parent_fields) { + const SpecificField& field, + const std::vector<SpecificField>& parent_fields) { for (int i = 0; i < ignore_criteria_.size(); ++i) { if (ignore_criteria_[i]->IsUnknownFieldIgnored(message1, message2, field, parent_fields)) { @@ -984,19 +1033,25 @@ bool MessageDifferencer::IsUnknownFieldIgnored( return false; } -const MessageDifferencer::MapKeyComparator* MessageDifferencer - ::GetMapKeyComparator(const FieldDescriptor* field) { +const MessageDifferencer::MapKeyComparator* +MessageDifferencer ::GetMapKeyComparator(const FieldDescriptor* field) const { if (!field->is_repeated()) return NULL; - if (map_field_key_comparator_.find(field) != - map_field_key_comparator_.end()) { - return map_field_key_comparator_[field]; + FieldKeyComparatorMap::const_iterator it = + map_field_key_comparator_.find(field); + if (it != map_field_key_comparator_.end()) { + return it->second; + } + if (field->is_map()) { + // field cannot already be treated as list or set since TreatAsList() and + // TreatAsSet() call GetMapKeyComparator() and fail if it returns non-NULL. + return &map_entry_key_comparator_; } return NULL; } namespace { -typedef pair<int, const UnknownField*> IndexUnknownFieldPair; +typedef std::pair<int, const UnknownField*> IndexUnknownFieldPair; struct UnknownFieldOrdering { inline bool operator()(const IndexUnknownFieldPair& a, @@ -1010,7 +1065,7 @@ struct UnknownFieldOrdering { } // namespace bool MessageDifferencer::UnpackAny(const Message& any, - google::protobuf::scoped_ptr<Message>* data) { + std::unique_ptr<Message>* data) { const Reflection* reflection = any.GetReflection(); const FieldDescriptor* type_url_field; const FieldDescriptor* value_field; @@ -1047,7 +1102,7 @@ bool MessageDifferencer::CompareUnknownFields( const Message& message1, const Message& message2, const google::protobuf::UnknownFieldSet& unknown_field_set1, const google::protobuf::UnknownFieldSet& unknown_field_set2, - vector<SpecificField>* parent_field) { + std::vector<SpecificField>* parent_field) { // Ignore unknown fields in EQUIVALENT mode. if (message_field_comparison_ == EQUIVALENT) return true; @@ -1063,8 +1118,8 @@ bool MessageDifferencer::CompareUnknownFields( // two sets -- that is, differing values for the same tag. We use // IndexUnknownFieldPairs to keep track of the field's original index for // reporting purposes. - vector<IndexUnknownFieldPair> fields1; // unknown_field_set1, sorted - vector<IndexUnknownFieldPair> fields2; // unknown_field_set2, sorted + std::vector<IndexUnknownFieldPair> fields1; // unknown_field_set1, sorted + std::vector<IndexUnknownFieldPair> fields2; // unknown_field_set2, sorted fields1.reserve(unknown_field_set1.field_count()); fields2.reserve(unknown_field_set2.field_count()); @@ -1267,7 +1322,7 @@ class MaximumMatcher { // the x-th node on the right side is matched to y-th node on the left side. // match_list1[i] == -1 means the node is not matched. Same with match_list2. MaximumMatcher(int count1, int count2, NodeMatchCallback* callback, - vector<int>* match_list1, vector<int>* match_list2); + std::vector<int>* match_list1, std::vector<int>* match_list2); // Find a maximum match and return the number of matched node pairs. // If early_return is true, this method will return 0 immediately when it // finds that not all nodes on the left side can be matched. @@ -1279,21 +1334,21 @@ class MaximumMatcher { // Find an argumenting path starting from the node v on the left side. If a // path can be found, update match_list2_ to reflect the path and return // true. - bool FindArgumentPathDFS(int v, vector<bool>* visited); + bool FindArgumentPathDFS(int v, std::vector<bool>* visited); int count1_; int count2_; - google::protobuf::scoped_ptr<NodeMatchCallback> match_callback_; - map<pair<int, int>, bool> cached_match_results_; - vector<int>* match_list1_; - vector<int>* match_list2_; + std::unique_ptr<NodeMatchCallback> match_callback_; + std::map<std::pair<int, int>, bool> cached_match_results_; + std::vector<int>* match_list1_; + std::vector<int>* match_list2_; GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(MaximumMatcher); }; MaximumMatcher::MaximumMatcher(int count1, int count2, NodeMatchCallback* callback, - vector<int>* match_list1, - vector<int>* match_list2) + std::vector<int>* match_list1, + std::vector<int>* match_list2) : count1_(count1), count2_(count2), match_callback_(callback), match_list1_(match_list1), match_list2_(match_list2) { match_list1_->assign(count1, -1); @@ -1303,7 +1358,7 @@ MaximumMatcher::MaximumMatcher(int count1, int count2, int MaximumMatcher::FindMaximumMatch(bool early_return) { int result = 0; for (int i = 0; i < count1_; ++i) { - vector<bool> visited(count1_); + std::vector<bool> visited(count1_); if (FindArgumentPathDFS(i, &visited)) { ++result; } else if (early_return) { @@ -1321,8 +1376,9 @@ int MaximumMatcher::FindMaximumMatch(bool early_return) { } bool MaximumMatcher::Match(int left, int right) { - pair<int, int> p(left, right); - map<pair<int, int>, bool>::iterator it = cached_match_results_.find(p); + std::pair<int, int> p(left, right); + std::map<std::pair<int, int>, bool>::iterator it = + cached_match_results_.find(p); if (it != cached_match_results_.end()) { return it->second; } @@ -1330,7 +1386,7 @@ bool MaximumMatcher::Match(int left, int right) { return cached_match_results_[p]; } -bool MaximumMatcher::FindArgumentPathDFS(int v, vector<bool>* visited) { +bool MaximumMatcher::FindArgumentPathDFS(int v, std::vector<bool>* visited) { (*visited)[v] = true; // We try to match those un-matched nodes on the right side first. This is // the step that the navie greedy matching algorithm uses. In the best cases @@ -1366,9 +1422,9 @@ bool MessageDifferencer::MatchRepeatedFieldIndices( const Message& message1, const Message& message2, const FieldDescriptor* repeated_field, - const vector<SpecificField>& parent_fields, - vector<int>* match_list1, - vector<int>* match_list2) { + const std::vector<SpecificField>& parent_fields, + std::vector<int>* match_list1, + std::vector<int>* match_list2) { const int count1 = message1.GetReflection()->FieldSize(message1, repeated_field); const int count2 = @@ -1378,9 +1434,6 @@ bool MessageDifferencer::MatchRepeatedFieldIndices( match_list1->assign(count1, -1); match_list2->assign(count2, -1); - SpecificField specific_field; - specific_field.field = repeated_field; - bool success = true; // Find potential match if this is a special repeated field. if (key_comparator != NULL || IsTreatedAsSet(repeated_field)) { @@ -1389,7 +1442,8 @@ bool MessageDifferencer::MatchRepeatedFieldIndices( // doesn't necessarily imply Compare(b, c). Therefore a naive greedy // algorithm will fail to find a maximum matching. // Here we use the argumenting path algorithm. - MaximumMatcher::NodeMatchCallback* callback = NewPermanentCallback( + MaximumMatcher::NodeMatchCallback* callback = + ::google::protobuf::NewPermanentCallback( this, &MessageDifferencer::IsMatch, repeated_field, key_comparator, &message1, &message2, parent_fields); @@ -1402,24 +1456,35 @@ bool MessageDifferencer::MatchRepeatedFieldIndices( if (match_count != count1 && reporter_ == NULL) return false; success = success && (match_count == count1); } else { - for (int i = 0; i < count1; ++i) { + int start_offset = 0; + // If the two repeated fields are treated as sets, optimize for the case + // where both start with same items stored in the same order. + if (IsTreatedAsSet(repeated_field)) { + start_offset = std::min(count1, count2); + for (int i = 0; i < count1 && i < count2; i++) { + if (IsMatch(repeated_field, key_comparator, &message1, &message2, + parent_fields, i, i)) { + match_list1->at(i) = i; + match_list2->at(i) = i; + } else { + start_offset = i; + break; + } + } + } + for (int i = start_offset; i < count1; ++i) { // Indicates any matched elements for this repeated field. bool match = false; - specific_field.index = i; - specific_field.new_index = i; - - for (int j = 0; j < count2; j++) { + for (int j = start_offset; j < count2; j++) { if (match_list2->at(j) != -1) continue; - specific_field.index = i; - specific_field.new_index = j; match = IsMatch(repeated_field, key_comparator, &message1, &message2, parent_fields, i, j); if (match) { - match_list1->at(specific_field.index) = specific_field.new_index; - match_list2->at(specific_field.new_index) = specific_field.index; + match_list1->at(i) = j; + match_list2->at(j) = i; break; } } @@ -1482,7 +1547,7 @@ MessageDifferencer::StreamReporter::~StreamReporter() { } void MessageDifferencer::StreamReporter::PrintPath( - const vector<SpecificField>& field_path, bool left_side) { + const std::vector<SpecificField>& field_path, bool left_side) { for (int i = 0; i < field_path.size(); ++i) { if (i > 0) { printer_->Print("."); @@ -1497,6 +1562,10 @@ void MessageDifferencer::StreamReporter::PrintPath( } else { printer_->PrintRaw(specific_field.field->name()); } + if (specific_field.field->is_map()) { + // Don't print index in a map field; they are semantically unordered. + continue; + } } else { printer_->PrintRaw(SimpleItoa(specific_field.unknown_field_number)); } @@ -1509,9 +1578,15 @@ void MessageDifferencer::StreamReporter::PrintPath( } } +void MessageDifferencer::StreamReporter::PrintPath( + const std::vector<SpecificField>& field_path, bool left_side, + const Message& message) { + PrintPath(field_path, left_side); +} + void MessageDifferencer:: StreamReporter::PrintValue(const Message& message, - const vector<SpecificField>& field_path, + const std::vector<SpecificField>& field_path, bool left_side) { const SpecificField& specific_field = field_path.back(); const FieldDescriptor* field = specific_field.field; @@ -1584,9 +1659,9 @@ void MessageDifferencer::StreamReporter::Print(const string& str) { void MessageDifferencer::StreamReporter::ReportAdded( const Message& message1, const Message& message2, - const vector<SpecificField>& field_path) { + const std::vector<SpecificField>& field_path) { printer_->Print("added: "); - PrintPath(field_path, false); + PrintPath(field_path, false, message2); printer_->Print(": "); PrintValue(message2, field_path, false); printer_->Print("\n"); // Print for newlines. @@ -1595,9 +1670,9 @@ void MessageDifferencer::StreamReporter::ReportAdded( void MessageDifferencer::StreamReporter::ReportDeleted( const Message& message1, const Message& message2, - const vector<SpecificField>& field_path) { + const std::vector<SpecificField>& field_path) { printer_->Print("deleted: "); - PrintPath(field_path, true); + PrintPath(field_path, true, message1); printer_->Print(": "); PrintValue(message1, field_path, true); printer_->Print("\n"); // Print for newlines @@ -1606,7 +1681,7 @@ void MessageDifferencer::StreamReporter::ReportDeleted( void MessageDifferencer::StreamReporter::ReportModified( const Message& message1, const Message& message2, - const vector<SpecificField>& field_path) { + const std::vector<SpecificField>& field_path) { if (!report_modified_aggregates_ && field_path.back().field == NULL) { if (field_path.back().unknown_field_type == UnknownField::TYPE_GROUP) { // Any changes to the subfields have already been printed. @@ -1621,10 +1696,10 @@ void MessageDifferencer::StreamReporter::ReportModified( } printer_->Print("modified: "); - PrintPath(field_path, true); + PrintPath(field_path, true, message1); if (CheckPathChanged(field_path)) { printer_->Print(" -> "); - PrintPath(field_path, false); + PrintPath(field_path, false, message2); } printer_->Print(": "); PrintValue(message1, field_path, true); @@ -1636,11 +1711,11 @@ void MessageDifferencer::StreamReporter::ReportModified( void MessageDifferencer::StreamReporter::ReportMoved( const Message& message1, const Message& message2, - const vector<SpecificField>& field_path) { + const std::vector<SpecificField>& field_path) { printer_->Print("moved: "); - PrintPath(field_path, true); + PrintPath(field_path, true, message1); printer_->Print(" -> "); - PrintPath(field_path, false); + PrintPath(field_path, false, message2); printer_->Print(" : "); PrintValue(message1, field_path, true); printer_->Print("\n"); // Print for newlines. @@ -1649,12 +1724,12 @@ void MessageDifferencer::StreamReporter::ReportMoved( void MessageDifferencer::StreamReporter::ReportMatched( const Message& message1, const Message& message2, - const vector<SpecificField>& field_path) { + const std::vector<SpecificField>& field_path) { printer_->Print("matched: "); - PrintPath(field_path, true); + PrintPath(field_path, true, message1); if (CheckPathChanged(field_path)) { printer_->Print(" -> "); - PrintPath(field_path, false); + PrintPath(field_path, false, message2); } printer_->Print(" : "); PrintValue(message1, field_path, true); @@ -1664,28 +1739,35 @@ void MessageDifferencer::StreamReporter::ReportMatched( void MessageDifferencer::StreamReporter::ReportIgnored( const Message& message1, const Message& message2, - const vector<SpecificField>& field_path) { + const std::vector<SpecificField>& field_path) { printer_->Print("ignored: "); - PrintPath(field_path, true); + PrintPath(field_path, true, message1); if (CheckPathChanged(field_path)) { printer_->Print(" -> "); - PrintPath(field_path, false); + PrintPath(field_path, false, message2); } printer_->Print("\n"); // Print for newlines. } void MessageDifferencer::StreamReporter::ReportUnknownFieldIgnored( const Message& message1, const Message& message2, - const vector<SpecificField>& field_path) { + const std::vector<SpecificField>& field_path) { printer_->Print("ignored: "); - PrintPath(field_path, true); + PrintPath(field_path, true, message1); if (CheckPathChanged(field_path)) { printer_->Print(" -> "); - PrintPath(field_path, false); + PrintPath(field_path, false, message2); } printer_->Print("\n"); // Print for newlines. } +MessageDifferencer::MapKeyComparator* +MessageDifferencer::CreateMultipleFieldsMapKeyComparator( + const std::vector<std::vector<const FieldDescriptor*> >& key_field_paths) { + return new MultipleFieldsMapKeyComparator(this, key_field_paths); +} + + } // namespace util } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/util/message_differencer.h b/src/google/protobuf/util/message_differencer.h index 654d1a67..bbcf8498 100644 --- a/src/google/protobuf/util/message_differencer.h +++ b/src/google/protobuf/util/message_differencer.h @@ -64,6 +64,7 @@ class Printer; namespace util { +class DefaultFieldComparator; class FieldContext; // declared below MessageDifferencer // A basic differencer that can be used to determine @@ -174,10 +175,8 @@ class LIBPROTOBUF_EXPORT MessageDifferencer { // If "field" is a repeated field which is being treated as a map or // a set (see TreatAsMap() and TreatAsSet(), below), new_index indicates - // the index the position to which the element has moved. This only - // applies to ReportMoved() and (in the case of TreatAsMap()) - // ReportModified(). In all other cases, "new_index" will have the same - // value as "index". + // the index the position to which the element has moved. If the element + // has not moved, "new_index" will have the same value as "index". int new_index; // For unknown fields, these are the pointers to the UnknownFieldSet @@ -221,19 +220,19 @@ class LIBPROTOBUF_EXPORT MessageDifferencer { // Reports that a field has been added into Message2. virtual void ReportAdded( const Message& message1, const Message& message2, - const vector<SpecificField>& field_path) = 0; + const std::vector<SpecificField>& field_path) = 0; // Reports that a field has been deleted from Message1. virtual void ReportDeleted( const Message& message1, const Message& message2, - const vector<SpecificField>& field_path) = 0; + const std::vector<SpecificField>& field_path) = 0; // Reports that the value of a field has been modified. virtual void ReportModified( const Message& message1, const Message& message2, - const vector<SpecificField>& field_path) = 0; + const std::vector<SpecificField>& field_path) = 0; // Reports that a repeated field has been moved to another location. This // only applies when using TreatAsSet or TreatAsMap() -- see below. Also @@ -241,18 +240,18 @@ class LIBPROTOBUF_EXPORT MessageDifferencer { // mutually exclusive. If a field has been both moved and modified, then // only ReportModified will be called. virtual void ReportMoved( - const Message& message1, - const Message& message2, - const vector<SpecificField>& field_path) { } + const Message& /* message1 */, + const Message& /* message2 */, + const std::vector<SpecificField>& /* field_path */) { } // Reports that two fields match. Useful for doing side-by-side diffs. // This function is mutually exclusive with ReportModified and ReportMoved. // Note that you must call set_report_matches(true) before calling Compare // to make use of this function. virtual void ReportMatched( - const Message& message1, - const Message& message2, - const vector<SpecificField>& field_path) { } + const Message& /* message1 */, + const Message& /* message2 */, + const std::vector<SpecificField>& /* field_path */) { } // Reports that two fields would have been compared, but the // comparison has been skipped because the field was marked as @@ -274,16 +273,16 @@ class LIBPROTOBUF_EXPORT MessageDifferencer { // the fields are equal or not (perhaps with a second call to // Compare()), if it cares. virtual void ReportIgnored( - const Message& message1, - const Message& message2, - const vector<SpecificField>& field_path) { } + const Message& /* message1 */, + const Message& /* message2 */, + const std::vector<SpecificField>& /* field_path */) { } // Report that an unknown field is ignored. (see comment above). // Note this is a different function since the last SpecificField in field // path has a null field. This could break existing Reporter. virtual void ReportUnknownFieldIgnored( - const Message& message1, const Message& message2, - const vector<SpecificField>& field_path) {} + const Message& /* message1 */, const Message& /* message2 */, + const std::vector<SpecificField>& /* field_path */) {} private: GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(Reporter); @@ -296,9 +295,10 @@ class LIBPROTOBUF_EXPORT MessageDifferencer { MapKeyComparator(); virtual ~MapKeyComparator(); - virtual bool IsMatch(const Message& message1, - const Message& message2, - const vector<SpecificField>& parent_fields) const { + virtual bool IsMatch( + const Message& /* message1 */, + const Message& /* message2 */, + const std::vector<SpecificField>& /* parent_fields */) const { GOOGLE_CHECK(false) << "IsMatch() is not implemented."; return false; } @@ -320,18 +320,18 @@ class LIBPROTOBUF_EXPORT MessageDifferencer { // Returns true if the field should be ignored. virtual bool IsIgnored( - const Message& message1, - const Message& message2, - const FieldDescriptor* field, - const vector<SpecificField>& parent_fields) = 0; + const Message& /* message1 */, + const Message& /* message2 */, + const FieldDescriptor* /* field */, + const std::vector<SpecificField>& /* parent_fields */) = 0; // Returns true if the unknown field should be ignored. // Note: This will be called for unknown fields as well in which case // field.field will be null. virtual bool IsUnknownFieldIgnored( - const Message& message1, const Message& message2, - const SpecificField& field, - const vector<SpecificField>& parent_fields) { + const Message& /* message1 */, const Message& /* message2 */, + const SpecificField& /* field */, + const std::vector<SpecificField>& /* parent_fields */) { return false; } }; @@ -371,7 +371,7 @@ class LIBPROTOBUF_EXPORT MessageDifferencer { // repeated fields have different numbers of elements, the // unpaired elements are reported using ReportAdded() or // ReportDeleted(). - AS_SET, // Treat all the repeated fields as sets by default. + AS_SET, // Treat all the repeated fields as sets. // See TreatAsSet(), as below. }; @@ -385,6 +385,11 @@ class LIBPROTOBUF_EXPORT MessageDifferencer { // the only differences between the compared messages is that some fields // have been moved, then the comparison returns true. // + // Note that despite the name of this method, this is really + // comparison as multisets: if one side of the comparison has a duplicate + // in the repeated field but the other side doesn't, this will count as + // a mismatch. + // // If the scope of comparison is set to PARTIAL, then in addition to what's // above, extra values added to repeated fields of the second message will // not cause the comparison to fail. @@ -440,7 +445,7 @@ class LIBPROTOBUF_EXPORT MessageDifferencer { // size of each element. void TreatAsMapWithMultipleFieldsAsKey( const FieldDescriptor* field, - const vector<const FieldDescriptor*>& key_fields); + const std::vector<const FieldDescriptor*>& key_fields); // Same as TreatAsMapWithMultipleFieldsAsKey, except that each of the field // do not necessarily need to be a direct subfield. Each element in // key_field_paths indicate a path from the message being compared, listing @@ -456,7 +461,7 @@ class LIBPROTOBUF_EXPORT MessageDifferencer { // !key_field_path[i]->is_repeated() void TreatAsMapWithMultipleFieldPathsAsKey( const FieldDescriptor* field, - const vector<vector<const FieldDescriptor*> >& key_field_paths); + const std::vector<std::vector<const FieldDescriptor*> >& key_field_paths); // Uses a custom MapKeyComparator to determine if two elements have the same // key when comparing a repeated field as a map. @@ -469,6 +474,10 @@ class LIBPROTOBUF_EXPORT MessageDifferencer { const FieldDescriptor* field, const MapKeyComparator* key_comparator); + // Initiates and returns a new instance of MultipleFieldsMapKeyComparator. + MapKeyComparator* CreateMultipleFieldsMapKeyComparator( + const std::vector<std::vector<const FieldDescriptor*> >& key_field_paths); + // Add a custom ignore criteria that is evaluated in addition to the // ignored fields added with IgnoreField. // Takes ownership of ignore_criteria. @@ -517,6 +526,13 @@ class LIBPROTOBUF_EXPORT MessageDifferencer { report_matches_ = report_matches; } + // Tells the differencer whether or not to report moves (in a set or map + // repeated field). This method must be called before Compare. The default for + // a new differencer is true. + void set_report_moves(bool report_moves) { + report_moves_ = report_moves; + } + // Sets the scope of the comparison (as defined in the Scope enumeration // above) that is used by this differencer when determining which fields to // compare between the messages. @@ -549,9 +565,10 @@ class LIBPROTOBUF_EXPORT MessageDifferencer { // Same as above, except comparing only the list of fields specified by the // two vectors of FieldDescriptors. - bool CompareWithFields(const Message& message1, const Message& message2, - const vector<const FieldDescriptor*>& message1_fields, - const vector<const FieldDescriptor*>& message2_fields); + bool CompareWithFields( + const Message& message1, const Message& message2, + const std::vector<const FieldDescriptor*>& message1_fields, + const std::vector<const FieldDescriptor*>& message2_fields); // Automatically creates a reporter that will output the differences // found (if any) to the specified output string pointer. Note that this @@ -591,35 +608,40 @@ class LIBPROTOBUF_EXPORT MessageDifferencer { // The following are implementations of the methods described above. virtual void ReportAdded(const Message& message1, const Message& message2, - const vector<SpecificField>& field_path); + const std::vector<SpecificField>& field_path); virtual void ReportDeleted(const Message& message1, const Message& message2, - const vector<SpecificField>& field_path); + const std::vector<SpecificField>& field_path); virtual void ReportModified(const Message& message1, const Message& message2, - const vector<SpecificField>& field_path); + const std::vector<SpecificField>& field_path); virtual void ReportMoved(const Message& message1, const Message& message2, - const vector<SpecificField>& field_path); + const std::vector<SpecificField>& field_path); virtual void ReportMatched(const Message& message1, const Message& message2, - const vector<SpecificField>& field_path); + const std::vector<SpecificField>& field_path); virtual void ReportIgnored(const Message& message1, const Message& message2, - const vector<SpecificField>& field_path); + const std::vector<SpecificField>& field_path); virtual void ReportUnknownFieldIgnored( const Message& message1, const Message& message2, - const vector<SpecificField>& field_path); + const std::vector<SpecificField>& field_path); protected: + // Prints the specified path of fields to the buffer. message is used to + // print map keys. + virtual void PrintPath(const std::vector<SpecificField>& field_path, + bool left_side, const Message& message); + // Prints the specified path of fields to the buffer. - virtual void PrintPath(const vector<SpecificField>& field_path, + virtual void PrintPath(const std::vector<SpecificField>& field_path, bool left_side); // Prints the value of fields to the buffer. left_side is true if the @@ -628,7 +650,7 @@ class LIBPROTOBUF_EXPORT MessageDifferencer { // unknown_field_index1 or unknown_field_index2 when an unknown field // is encountered in field_path. virtual void PrintValue(const Message& message, - const vector<SpecificField>& field_path, + const std::vector<SpecificField>& field_path, bool left_side); // Prints the specified path of unknown fields to the buffer. @@ -646,11 +668,25 @@ class LIBPROTOBUF_EXPORT MessageDifferencer { }; private: + friend class DefaultFieldComparator; + // A MapKeyComparator to be used in TreatAsMapUsingKeyComparator. // Implementation of this class needs to do field value comparison which // relies on some private methods of MessageDifferencer. That's why this // class is declared as a nested class of MessageDifferencer. class MultipleFieldsMapKeyComparator; + + // A MapKeyComparator for use with map_entries. + class LIBPROTOBUF_EXPORT MapEntryKeyComparator : public MapKeyComparator { + public: + explicit MapEntryKeyComparator(MessageDifferencer* message_differencer); + virtual bool IsMatch(const Message& message1, const Message& message2, + const std::vector<SpecificField>& parent_fields) const; + + private: + MessageDifferencer* message_differencer_; + }; + // Returns true if field1's number() is less than field2's. static bool FieldBefore(const FieldDescriptor* field1, const FieldDescriptor* field2); @@ -659,11 +695,11 @@ class LIBPROTOBUF_EXPORT MessageDifferencer { // All fields present in both lists will always be included in the combined // list. Fields only present in one of the lists will only appear in the // combined list if the corresponding fields_scope option is set to FULL. - void CombineFields(const vector<const FieldDescriptor*>& fields1, + void CombineFields(const std::vector<const FieldDescriptor*>& fields1, Scope fields1_scope, - const vector<const FieldDescriptor*>& fields2, + const std::vector<const FieldDescriptor*>& fields2, Scope fields2_scope, - vector<const FieldDescriptor*>* combined_fields); + std::vector<const FieldDescriptor*>* combined_fields); // Internal version of the Compare method which performs the actual // comparison. The parent_fields vector is a vector containing field @@ -671,34 +707,34 @@ class LIBPROTOBUF_EXPORT MessageDifferencer { // (i.e. if the current message is an embedded message, the parent_fields // vector will contain the field that has this embedded message). bool Compare(const Message& message1, const Message& message2, - vector<SpecificField>* parent_fields); + std::vector<SpecificField>* parent_fields); // Compares all the unknown fields in two messages. bool CompareUnknownFields(const Message& message1, const Message& message2, const google::protobuf::UnknownFieldSet&, const google::protobuf::UnknownFieldSet&, - vector<SpecificField>* parent_fields); + std::vector<SpecificField>* parent_fields); // Compares the specified messages for the requested field lists. The field // lists are modified depending on comparison settings, and then passed to // CompareWithFieldsInternal. bool CompareRequestedFieldsUsingSettings( const Message& message1, const Message& message2, - const vector<const FieldDescriptor*>& message1_fields, - const vector<const FieldDescriptor*>& message2_fields, - vector<SpecificField>* parent_fields); + const std::vector<const FieldDescriptor*>& message1_fields, + const std::vector<const FieldDescriptor*>& message2_fields, + std::vector<SpecificField>* parent_fields); // Compares the specified messages with the specified field lists. bool CompareWithFieldsInternal( const Message& message1, const Message& message2, - const vector<const FieldDescriptor*>& message1_fields, - const vector<const FieldDescriptor*>& message2_fields, - vector<SpecificField>* parent_fields); + const std::vector<const FieldDescriptor*>& message1_fields, + const std::vector<const FieldDescriptor*>& message2_fields, + std::vector<SpecificField>* parent_fields); // Compares the repeated fields, and report the error. bool CompareRepeatedField(const Message& message1, const Message& message2, const FieldDescriptor* field, - vector<SpecificField>* parent_fields); + std::vector<SpecificField>* parent_fields); // Shorthand for CompareFieldValueUsingParentFields with NULL parent_fields. bool CompareFieldValue(const Message& message1, @@ -716,12 +752,13 @@ class LIBPROTOBUF_EXPORT MessageDifferencer { // list of parent messages if it needs to recursively compare the given field. // To avoid confusing users you should not set it to NULL unless you modified // Reporter to handle the change of parent_fields correctly. - bool CompareFieldValueUsingParentFields(const Message& message1, - const Message& message2, - const FieldDescriptor* field, - int index1, - int index2, - vector<SpecificField>* parent_fields); + bool CompareFieldValueUsingParentFields( + const Message& message1, + const Message& message2, + const FieldDescriptor* field, + int index1, + int index2, + std::vector<SpecificField>* parent_fields); // Compares the specified field on the two messages, returning comparison // result, as returned by appropriate FieldComparator. @@ -736,7 +773,7 @@ class LIBPROTOBUF_EXPORT MessageDifferencer { bool IsMatch(const FieldDescriptor* repeated_field, const MapKeyComparator* key_comparator, const Message* message1, const Message* message2, - const vector<SpecificField>& parent_fields, + const std::vector<SpecificField>& parent_fields, int index1, int index2); // Returns true when this repeated field has been configured to be treated @@ -754,17 +791,18 @@ class LIBPROTOBUF_EXPORT MessageDifferencer { const Message& message1, const Message& message2, const FieldDescriptor* field, - const vector<SpecificField>& parent_fields); + const std::vector<SpecificField>& parent_fields); // Returns true if this unknown field is to be ignored when this // MessageDifferencer compares messages. bool IsUnknownFieldIgnored(const Message& message1, const Message& message2, const SpecificField& field, - const vector<SpecificField>& parent_fields); + const std::vector<SpecificField>& parent_fields); - // Returns MapKeyComparator* when this field has been configured to - // be treated as a map. If not, returns NULL. - const MapKeyComparator* GetMapKeyComparator(const FieldDescriptor* field); + // Returns MapKeyComparator* when this field has been configured to be treated + // as a map or its is_map() return true. If not, returns NULL. + const MapKeyComparator* GetMapKeyComparator( + const FieldDescriptor* field) const; // Attempts to match indices of a repeated field, so that the contained values // match. Clears output vectors and sets their values to indices of paired @@ -773,28 +811,29 @@ class LIBPROTOBUF_EXPORT MessageDifferencer { // This method returns false if the match failed. However, it doesn't mean // that the comparison succeeds when this method returns true (you need to // double-check in this case). - bool MatchRepeatedFieldIndices(const Message& message1, - const Message& message2, - const FieldDescriptor* repeated_field, - const vector<SpecificField>& parent_fields, - vector<int>* match_list1, - vector<int>* match_list2); + bool MatchRepeatedFieldIndices( + const Message& message1, + const Message& message2, + const FieldDescriptor* repeated_field, + const std::vector<SpecificField>& parent_fields, + std::vector<int>* match_list1, + std::vector<int>* match_list2); // If "any" is of type google.protobuf.Any, extract its payload using // DynamicMessageFactory and store in "data". - bool UnpackAny(const Message& any, google::protobuf::scoped_ptr<Message>* data); + bool UnpackAny(const Message& any, std::unique_ptr<Message>* data); // Checks if index is equal to new_index in all the specific fields. - static bool CheckPathChanged(const vector<SpecificField>& parent_fields); + static bool CheckPathChanged(const std::vector<SpecificField>& parent_fields); // Defines a map between field descriptors and their MapKeyComparators. // Used for repeated fields when they are configured as TreatAsMap. - typedef map<const FieldDescriptor*, + typedef std::map<const FieldDescriptor*, const MapKeyComparator*> FieldKeyComparatorMap; // Defines a set to store field descriptors. Used for repeated fields when // they are configured as TreatAsSet. - typedef set<const FieldDescriptor*> FieldSet; + typedef std::set<const FieldDescriptor*> FieldSet; Reporter* reporter_; DefaultFieldComparator default_field_comparator_; @@ -811,18 +850,19 @@ class LIBPROTOBUF_EXPORT MessageDifferencer { // When TreatAsMap or TreatAsMapWithMultipleFieldsAsKey is called, we don't // store the supplied FieldDescriptors directly. Instead, a new // MapKeyComparator is created for comparison purpose. - vector<MapKeyComparator*> owned_key_comparators_; + std::vector<MapKeyComparator*> owned_key_comparators_; FieldKeyComparatorMap map_field_key_comparator_; - vector<IgnoreCriteria*> ignore_criteria_; + MapEntryKeyComparator map_entry_key_comparator_; + std::vector<IgnoreCriteria*> ignore_criteria_; FieldSet ignored_fields_; - bool compare_unknown_fields_; bool report_matches_; + bool report_moves_; string* output_string_; - google::protobuf::scoped_ptr<DynamicMessageFactory> dynamic_message_factory_; + std::unique_ptr<DynamicMessageFactory> dynamic_message_factory_; GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(MessageDifferencer); }; @@ -831,15 +871,15 @@ class LIBPROTOBUF_EXPORT MessageDifferencer { class LIBPROTOBUF_EXPORT FieldContext { public: explicit FieldContext( - vector<MessageDifferencer::SpecificField>* parent_fields) + std::vector<MessageDifferencer::SpecificField>* parent_fields) : parent_fields_(parent_fields) {} - vector<MessageDifferencer::SpecificField>* parent_fields() const { + std::vector<MessageDifferencer::SpecificField>* parent_fields() const { return parent_fields_; } private: - vector<MessageDifferencer::SpecificField>* parent_fields_; + std::vector<MessageDifferencer::SpecificField>* parent_fields_; }; } diff --git a/src/google/protobuf/util/message_differencer_unittest.cc b/src/google/protobuf/util/message_differencer_unittest.cc index ed43c512..a263d983 100755 --- a/src/google/protobuf/util/message_differencer_unittest.cc +++ b/src/google/protobuf/util/message_differencer_unittest.cc @@ -38,6 +38,7 @@ #include <string> #include <vector> +#include <google/protobuf/stubs/strutil.h> #include <google/protobuf/util/field_comparator.h> #include <google/protobuf/util/message_differencer.h> @@ -54,7 +55,6 @@ #include <google/protobuf/stubs/logging.h> #include <google/protobuf/stubs/common.h> -#include <google/protobuf/stubs/strutil.h> #include <google/protobuf/testing/googletest.h> #include <gtest/gtest.h> @@ -66,7 +66,7 @@ namespace { const FieldDescriptor* GetFieldDescriptor( const Message& message, const string& field_name) { - vector<string> field_path = + std::vector<string> field_path = Split(field_name, ".", true); const Descriptor* descriptor = message.GetDescriptor(); const FieldDescriptor* field = NULL; @@ -127,6 +127,27 @@ TEST(MessageDifferencerTest, RepeatedFieldInequalityTest) { EXPECT_FALSE(util::MessageDifferencer::Equals(msg1, msg2)); } +TEST(MessageDifferencerTest, RepeatedFieldSetOptimizationTest) { + util::MessageDifferencer differencer; + protobuf_unittest::TestDiffMessage msg1; + protobuf_unittest::TestDiffMessage msg2; + protobuf_unittest::TestDiffMessage::Item* item1 = msg1.add_item(); + protobuf_unittest::TestDiffMessage::Item* item2 = msg2.add_item(); + differencer.TreatAsSet(item1->GetDescriptor()->FindFieldByName("ra")); + differencer.TreatAsSet(item2->GetDescriptor()->FindFieldByName("ra")); + for (int i = 0; i < 1000; i++) { + item1->add_ra(i); + item2->add_ra(i); + } + EXPECT_TRUE(differencer.Compare(msg1, msg2)); + item2->add_ra(1001); + EXPECT_FALSE(differencer.Compare(msg1, msg2)); + item1->add_ra(1001); + EXPECT_TRUE(differencer.Compare(msg1, msg2)); + item1->add_ra(1002); + EXPECT_FALSE(differencer.Compare(msg1, msg2)); +} + TEST(MessageDifferencerTest, MapFieldEqualityTest) { // Create the testing protos unittest::TestMap msg1; @@ -790,8 +811,8 @@ TEST(MessageDifferencerTest, SpecifiedFieldsEqualityAllTest) { TestUtil::SetAllFields(&msg1); TestUtil::SetAllFields(&msg2); - vector<const FieldDescriptor*> fields1; - vector<const FieldDescriptor*> fields2; + std::vector<const FieldDescriptor*> fields1; + std::vector<const FieldDescriptor*> fields2; msg1.GetReflection()->ListFields(msg1, &fields1); msg2.GetReflection()->ListFields(msg2, &fields2); @@ -805,8 +826,8 @@ TEST(MessageDifferencerTest, SpecifiedFieldsInequalityAllTest) { TestUtil::SetAllFields(&msg1); - vector<const FieldDescriptor*> fields1; - vector<const FieldDescriptor*> fields2; + std::vector<const FieldDescriptor*> fields1; + std::vector<const FieldDescriptor*> fields2; msg1.GetReflection()->ListFields(msg1, &fields1); msg2.GetReflection()->ListFields(msg2, &fields2); @@ -820,7 +841,7 @@ TEST(MessageDifferencerTest, SpecifiedFieldsEmptyListAlwaysSucceeds) { TestUtil::SetAllFields(&msg1); - vector<const FieldDescriptor*> empty_fields; + std::vector<const FieldDescriptor*> empty_fields; util::MessageDifferencer differencer; EXPECT_TRUE(differencer.CompareWithFields(msg1, msg2, @@ -835,7 +856,7 @@ TEST(MessageDifferencerTest, SpecifiedFieldsCompareWithSelf) { unittest::TestAllTypes msg1; TestUtil::SetAllFields(&msg1); - vector<const FieldDescriptor*> fields; + std::vector<const FieldDescriptor*> fields; msg1.GetReflection()->ListFields(msg1, &fields); util::MessageDifferencer differencer; @@ -843,7 +864,7 @@ TEST(MessageDifferencerTest, SpecifiedFieldsCompareWithSelf) { { // Compare with a subset of fields. - vector<const FieldDescriptor*> compare_fields; + std::vector<const FieldDescriptor*> compare_fields; for (int i = 0; i < fields.size(); ++i) { if (i % 2 == 0) { compare_fields.push_back(fields[i]); @@ -856,8 +877,8 @@ TEST(MessageDifferencerTest, SpecifiedFieldsCompareWithSelf) { // Specify a different set of fields to compare, even though we're using the // same message. This should fail, since we are explicitly saying that the // set of fields are different. - vector<const FieldDescriptor*> compare_fields1; - vector<const FieldDescriptor*> compare_fields2; + std::vector<const FieldDescriptor*> compare_fields1; + std::vector<const FieldDescriptor*> compare_fields2; for (int i = 0; i < fields.size(); ++i) { if (i % 2 == 0) { compare_fields1.push_back(fields[i]); @@ -880,8 +901,8 @@ TEST(MessageDifferencerTest, SpecifiedFieldsEqualityAllShuffledTest) { TestUtil::SetAllFields(&msg1); TestUtil::SetAllFields(&msg2); - vector<const FieldDescriptor*> fields1; - vector<const FieldDescriptor*> fields2; + std::vector<const FieldDescriptor*> fields1; + std::vector<const FieldDescriptor*> fields2; msg1.GetReflection()->ListFields(msg1, &fields1); msg2.GetReflection()->ListFields(msg2, &fields2); @@ -899,10 +920,10 @@ TEST(MessageDifferencerTest, SpecifiedFieldsSubsetEqualityTest) { TestUtil::SetAllFields(&msg1); TestUtil::SetAllFields(&msg2); - vector<const FieldDescriptor*> fields1; + std::vector<const FieldDescriptor*> fields1; msg1.GetReflection()->ListFields(msg1, &fields1); - vector<const FieldDescriptor*> compare_fields; + std::vector<const FieldDescriptor*> compare_fields; // Only compare the field descriptors with even indices. for (int i = 0; i < fields1.size(); ++i) { if (i % 2 == 0) { @@ -925,11 +946,11 @@ TEST(MessageDifferencerTest, TestUtil::SetAllFields(&msg1); TestUtil::SetAllFields(&msg2); - vector<const FieldDescriptor*> fields1; + std::vector<const FieldDescriptor*> fields1; const Reflection* reflection = msg1.GetReflection(); reflection->ListFields(msg1, &fields1); - vector<const FieldDescriptor*> compare_fields; + std::vector<const FieldDescriptor*> compare_fields; // Only compare the field descriptors with even indices. for (int i = 0; i < fields1.size(); ++i) { if (i % 2 == 0) { @@ -954,10 +975,10 @@ TEST(MessageDifferencerTest, SpecifiedFieldsDetectsDifferencesTest) { TestUtil::SetAllFields(&msg2); TestUtil::ModifyRepeatedFields(&msg2); - vector<const FieldDescriptor*> fields1; + std::vector<const FieldDescriptor*> fields1; msg1.GetReflection()->ListFields(msg1, &fields1); - vector<const FieldDescriptor*> compare_fields; + std::vector<const FieldDescriptor*> compare_fields; // Only compare the repeated field descriptors. for (int i = 0; i < fields1.size(); ++i) { if (fields1[i]->is_repeated()) { @@ -977,8 +998,8 @@ TEST(MessageDifferencerTest, SpecifiedFieldsEquivalenceAllTest) { TestUtil::SetAllFields(&msg1); TestUtil::SetAllFields(&msg2); - vector<const FieldDescriptor*> fields1; - vector<const FieldDescriptor*> fields2; + std::vector<const FieldDescriptor*> fields1; + std::vector<const FieldDescriptor*> fields2; msg1.GetReflection()->ListFields(msg1, &fields1); msg2.GetReflection()->ListFields(msg2, &fields2); @@ -1011,8 +1032,8 @@ TEST(MessageDifferencerTest, // actually doing something. msg2.set_optional_uint64(23); - vector<const FieldDescriptor*> fields1; - vector<const FieldDescriptor*> fields2; + std::vector<const FieldDescriptor*> fields1; + std::vector<const FieldDescriptor*> fields2; fields1.push_back(optional_int32_desc); fields1.push_back(default_int64_desc); @@ -1286,7 +1307,7 @@ TEST(MessageDifferencerTest, RepeatedFieldMapTest_MultipleFieldsAsKey) { util::MessageDifferencer differencer; differencer.TreatAsSet(GetFieldDescriptor(msg1, "item.ra")); EXPECT_FALSE(differencer.Compare(msg1, msg2)); - vector<const FieldDescriptor*> key_fields; + std::vector<const FieldDescriptor*> key_fields; key_fields.push_back(GetFieldDescriptor(msg1, "item.a")); key_fields.push_back(GetFieldDescriptor(msg1, "item.ra")); differencer.TreatAsMapWithMultipleFieldsAsKey( @@ -1363,11 +1384,11 @@ TEST(MessageDifferencerTest, RepeatedFieldMapTest_MultipleFieldPathsAsKey) { util::MessageDifferencer differencer; differencer.TreatAsSet(GetFieldDescriptor(msg1, "item.m.rc")); EXPECT_FALSE(differencer.Compare(msg1, msg2)); - vector<vector<const FieldDescriptor*> > key_field_paths; - vector<const FieldDescriptor*> key_field_path1; + std::vector<std::vector<const FieldDescriptor*> > key_field_paths; + std::vector<const FieldDescriptor*> key_field_path1; key_field_path1.push_back(GetFieldDescriptor(msg1, "item.m")); key_field_path1.push_back(GetFieldDescriptor(msg1, "item.m.a")); - vector<const FieldDescriptor*> key_field_path2; + std::vector<const FieldDescriptor*> key_field_path2; key_field_path2.push_back(GetFieldDescriptor(msg1, "item.m")); key_field_path2.push_back(GetFieldDescriptor(msg1, "item.m.rc")); key_field_paths.push_back(key_field_path1); @@ -1413,7 +1434,7 @@ TEST(MessageDifferencerTest, RepeatedFieldMapTest_IgnoredKeyFields) { item->set_b("world"); // Compare util::MessageDifferencer differencer; - vector<const FieldDescriptor*> key_fields; + std::vector<const FieldDescriptor*> key_fields; key_fields.push_back(GetFieldDescriptor(msg1, "item.a")); key_fields.push_back(GetFieldDescriptor(msg1, "item.ra")); differencer.TreatAsMapWithMultipleFieldsAsKey( @@ -1457,7 +1478,8 @@ class TestIgnorer : public util::MessageDifferencer::IgnoreCriteria { virtual bool IsIgnored( const Message& message1, const Message& message2, const FieldDescriptor* field, - const vector<util::MessageDifferencer::SpecificField>& parent_fields) { + const std::vector<util::MessageDifferencer::SpecificField>& + parent_fields) { string name = ""; for (int i = 0; i < parent_fields.size(); ++i) { name += parent_fields[i].field->name() + "."; @@ -1502,7 +1524,7 @@ class ValueProductMapKeyComparator typedef util::MessageDifferencer::SpecificField SpecificField; virtual bool IsMatch( const Message &message1, const Message &message2, - const vector<SpecificField>& parent_fields) const { + const std::vector<SpecificField>& parent_fields) const { const Reflection* reflection1 = message1.GetReflection(); const Reflection* reflection2 = message2.GetReflection(); // FieldDescriptor for item.ra @@ -1557,6 +1579,43 @@ TEST(MessageDifferencerTest, RepeatedFieldMapTest_CustomMapKeyComparator) { EXPECT_EQ("ignored: item[0].ra\n", output); } +// Compares fields by their index offset by one, so index 0 matches with 1, etc. +class OffsetByOneMapKeyComparator + : public util::MessageDifferencer::MapKeyComparator { + public: + typedef util::MessageDifferencer::SpecificField SpecificField; + virtual bool IsMatch(const Message& message1, const Message& message2, + const std::vector<SpecificField>& parent_fields) const { + return parent_fields.back().index + 1 == parent_fields.back().new_index; + } +}; + +TEST(MessageDifferencerTest, RepeatedFieldMapTest_CustomIndexMapKeyComparator) { + protobuf_unittest::TestDiffMessage msg1; + protobuf_unittest::TestDiffMessage msg2; + // Treat "item" as Map, using custom key comparator to determine if two + // elements have the same key. + protobuf_unittest::TestDiffMessage::Item* item = msg1.add_item(); + item->set_b("one"); + item = msg2.add_item(); + item->set_b("zero"); + item = msg2.add_item(); + item->set_b("one"); + util::MessageDifferencer differencer; + OffsetByOneMapKeyComparator key_comparator; + differencer.TreatAsMapUsingKeyComparator(GetFieldDescriptor(msg1, "item"), + &key_comparator); + string output; + differencer.ReportDifferencesToString(&output); + // With the offset by one comparator msg1.item[0] should be compared to + // msg2.item[1] and thus be moved, msg2.item[0] should be marked as added. + EXPECT_FALSE(differencer.Compare(msg1, msg2)); + EXPECT_EQ( + "moved: item[0] -> item[1] : { b: \"one\" }\n" + "added: item[0]: { b: \"zero\" }\n", + output); +} + TEST(MessageDifferencerTest, RepeatedFieldSetTest_Subset) { protobuf_unittest::TestDiffMessage msg1; protobuf_unittest::TestDiffMessage msg2; @@ -1863,7 +1922,7 @@ TEST(MessageDifferencerTest, IgnoreField_TrumpsCompareWithFields) { const FieldDescriptor* c = GetFieldDescriptor(msg1, "c"); const FieldDescriptor* rc = GetFieldDescriptor(msg1, "rc"); - vector<const FieldDescriptor*> fields; + std::vector<const FieldDescriptor*> fields; fields.push_back(c); fields.push_back(rc); @@ -1899,12 +1958,12 @@ class ParentSavingFieldComparator : public util::FieldComparator { } } - vector<google::protobuf::util::MessageDifferencer::SpecificField> parent_fields() { + std::vector<google::protobuf::util::MessageDifferencer::SpecificField> parent_fields() { return parent_fields_; } private: - vector<google::protobuf::util::MessageDifferencer::SpecificField> parent_fields_; + std::vector<google::protobuf::util::MessageDifferencer::SpecificField> parent_fields_; }; // Tests if MessageDifferencer sends the parent fields in the FieldContext @@ -2041,6 +2100,9 @@ class ComparisonTest : public testing::Test { unittest::TestEmptyMessage empty1_; unittest::TestEmptyMessage empty2_; + unittest::TestMap map_proto1_; + unittest::TestMap map_proto2_; + UnknownFieldSet* unknown1_; UnknownFieldSet* unknown2_; @@ -2801,6 +2863,133 @@ TEST_F(ComparisonTest, EquivalentIgnoresUnknown) { EXPECT_TRUE(util::MessageDifferencer::Equivalent(message1, message2)); } +TEST_F(ComparisonTest, MapTest) { + Map<string, string>& map1 = *map_proto1_.mutable_map_string_string(); + map1["key1"] = "1"; + map1["key2"] = "2"; + map1["key3"] = "3"; + Map<string, string>& map2 = *map_proto2_.mutable_map_string_string(); + map2["key3"] = "0"; + map2["key2"] = "2"; + map2["key1"] = "1"; + + EXPECT_EQ("modified: map_string_string.value: \"3\" -> \"0\"\n", + Run(map_proto1_, map_proto2_)); +} + +TEST_F(ComparisonTest, MapIgnoreKeyTest) { + Map<string, string>& map1 = *map_proto1_.mutable_map_string_string(); + map1["key1"] = "1"; + map1["key2"] = "2"; + map1["key3"] = "3"; + Map<string, string>& map2 = *map_proto2_.mutable_map_string_string(); + map2["key4"] = "2"; + map2["key5"] = "3"; + map2["key6"] = "1"; + + util::MessageDifferencer differencer; + differencer.IgnoreField( + GetFieldDescriptor(map_proto1_, "map_string_string.key")); + EXPECT_TRUE(differencer.Compare(map_proto1_, map_proto2_)); +} + +TEST_F(ComparisonTest, MapRoundTripSyncTest) { + google::protobuf::TextFormat::Parser parser; + unittest::TestMap map_reflection1; + + // By setting via reflection, data exists in repeated field. + ASSERT_TRUE(parser.ParseFromString( + "map_int32_foreign_message { key: 1 }", &map_reflection1)); + + // During copy, data is synced from repeated field to map. + unittest::TestMap map_reflection2 = map_reflection1; + + // During comparison, data is synced from map to repeated field. + EXPECT_TRUE( + util::MessageDifferencer::Equals(map_reflection1, map_reflection2)); +} + +TEST_F(ComparisonTest, MapEntryPartialTest) { + google::protobuf::TextFormat::Parser parser; + unittest::TestMap map1; + unittest::TestMap map2; + + string output; + util::MessageDifferencer differencer; + differencer.set_scope(util::MessageDifferencer::PARTIAL); + differencer.ReportDifferencesToString(&output); + + ASSERT_TRUE(parser.ParseFromString( + "map_int32_foreign_message { key: 1 value { c: 1 } }", &map1)); + ASSERT_TRUE(parser.ParseFromString( + "map_int32_foreign_message { key: 1 value { c: 2 }}", &map2)); + EXPECT_FALSE(differencer.Compare(map1, map2)); + EXPECT_EQ("modified: map_int32_foreign_message.value.c: 1 -> 2\n", output); + + ASSERT_TRUE( + parser.ParseFromString("map_int32_foreign_message { key: 1 }", &map1)); + EXPECT_TRUE(differencer.Compare(map1, map2)); +} + +TEST_F(ComparisonTest, MapEntryPartialEmptyKeyTest) { + google::protobuf::TextFormat::Parser parser; + unittest::TestMap map1; + unittest::TestMap map2; + ASSERT_TRUE(parser.ParseFromString("map_int32_foreign_message {}", &map1)); + ASSERT_TRUE( + parser.ParseFromString("map_int32_foreign_message { key: 1 }", &map2)); + + util::MessageDifferencer differencer; + differencer.set_scope(util::MessageDifferencer::PARTIAL); + EXPECT_TRUE(differencer.Compare(map1, map2)); +} + +// Considers strings keys as equal if they have equal lengths. +class LengthMapKeyComparator + : public util::MessageDifferencer::MapKeyComparator { + public: + typedef util::MessageDifferencer::SpecificField SpecificField; + virtual bool IsMatch(const Message& message1, const Message& message2, + const std::vector<SpecificField>& parent_fields) const { + const Reflection* reflection1 = message1.GetReflection(); + const Reflection* reflection2 = message2.GetReflection(); + const FieldDescriptor* key_field = + message1.GetDescriptor()->FindFieldByName("key"); + return reflection1->GetString(message1, key_field).size() == + reflection2->GetString(message2, key_field).size(); + } +}; + +TEST_F(ComparisonTest, MapEntryCustomMapKeyComparator) { + google::protobuf::TextFormat::Parser parser; + protobuf_unittest::TestMap msg1; + protobuf_unittest::TestMap msg2; + + ASSERT_TRUE(parser.ParseFromString( + "map_string_foreign_message { key: 'key1' value { c: 1 }}", &msg1)); + ASSERT_TRUE(parser.ParseFromString( + "map_string_foreign_message { key: 'key2' value { c: 1 }}", &msg2)); + + util::MessageDifferencer differencer; + LengthMapKeyComparator key_comparator; + differencer.TreatAsMapUsingKeyComparator( + GetFieldDescriptor(msg1, "map_string_foreign_message"), &key_comparator); + string output; + differencer.ReportDifferencesToString(&output); + // Though the above two messages have different keys for their map entries, + // they are considered the same by key_comparator because their lengths are + // equal. However, in value comparison, all fields of the message are taken + // into consideration, so they are reported as different. + EXPECT_FALSE(differencer.Compare(msg1, msg2)); + EXPECT_EQ("modified: map_string_foreign_message.key: \"key1\" -> \"key2\"\n", + output); + differencer.IgnoreField( + GetFieldDescriptor(msg1, "map_string_foreign_message.key")); + output.clear(); + EXPECT_TRUE(differencer.Compare(msg1, msg2)); + EXPECT_EQ("ignored: map_string_foreign_message.key\n", output); +} + class MatchingTest : public testing::Test { public: typedef util::MessageDifferencer MessageDifferencer; @@ -3145,6 +3334,24 @@ TEST(Anytest, TreatAsSet) { EXPECT_TRUE(message_differencer.Compare(m1, m2)); } +TEST(Anytest, TreatAsSet_DifferentType) { + protobuf_unittest::TestField value1; + value1.set_a(20); + value1.set_b(30); + protobuf_unittest::TestDiffMessage value2; + value2.add_rv(40); + + protobuf_unittest::TestAny m1, m2; + m1.add_repeated_any_value()->PackFrom(value1); + m1.add_repeated_any_value()->PackFrom(value2); + m2.add_repeated_any_value()->PackFrom(value2); + m2.add_repeated_any_value()->PackFrom(value1); + + util::MessageDifferencer message_differencer; + message_differencer.TreatAsSet(GetFieldDescriptor(m1, "repeated_any_value")); + EXPECT_TRUE(message_differencer.Compare(m1, m2)); +} + } // namespace } // namespace protobuf diff --git a/src/google/protobuf/util/package_info.h b/src/google/protobuf/util/package_info.h index e37e6dc0..96019203 100644 --- a/src/google/protobuf/util/package_info.h +++ b/src/google/protobuf/util/package_info.h @@ -38,7 +38,7 @@ namespace protobuf { // Utility classes. // -// This package contains various utilities for message comprasion, JSON +// This package contains various utilities for message comparison, JSON // conversion, well known types, etc. namespace util {} diff --git a/src/google/protobuf/util/time_util.cc b/src/google/protobuf/util/time_util.cc index 031d019a..d4912837 100644 --- a/src/google/protobuf/util/time_util.cc +++ b/src/google/protobuf/util/time_util.cc @@ -30,13 +30,14 @@ #include <google/protobuf/util/time_util.h> -#include <google/protobuf/stubs/time.h> #include <google/protobuf/stubs/int128.h> -#include <google/protobuf/stubs/strutil.h> #include <google/protobuf/stubs/stringprintf.h> +#include <google/protobuf/stubs/strutil.h> +#include <google/protobuf/stubs/time.h> #include <google/protobuf/duration.pb.h> #include <google/protobuf/timestamp.pb.h> + namespace google { namespace protobuf { namespace util { @@ -49,11 +50,9 @@ static const int kNanosPerSecond = 1000000000; static const int kMicrosPerSecond = 1000000; static const int kMillisPerSecond = 1000; static const int kNanosPerMillisecond = 1000000; -static const int kMicrosPerMillisecond = 1000; static const int kNanosPerMicrosecond = 1000; static const int kSecondsPerMinute = 60; // Note that we ignore leap seconds. static const int kSecondsPerHour = 3600; -static const char kTimestampFormat[] = "%E4Y-%m-%dT%H:%M:%S"; template <typename T> T CreateNormalized(int64 seconds, int64 nanos); @@ -143,11 +142,13 @@ int64 RoundTowardZero(int64 value, int64 divider) { } // namespace // Actually define these static const integers. Required by C++ standard (but -// omitting them may still work with some compilers). +// some compilers don't like it). +#ifndef _MSC_VER const int64 TimeUtil::kTimestampMinSeconds; const int64 TimeUtil::kTimestampMaxSeconds; const int64 TimeUtil::kDurationMaxSeconds; const int64 TimeUtil::kDurationMinSeconds; +#endif // !_MSC_VER string TimeUtil::ToString(const Timestamp& timestamp) { return FormatTime(timestamp.seconds(), timestamp.nanos()); @@ -374,19 +375,6 @@ namespace { using google::protobuf::util::kNanosPerSecond; using google::protobuf::util::CreateNormalized; -// Convert a Timestamp to uint128. -void ToUint128(const Timestamp& value, uint128* result, bool* negative) { - if (value.seconds() < 0) { - *negative = true; - *result = static_cast<uint64>(-value.seconds()); - *result = *result * kNanosPerSecond - static_cast<uint32>(value.nanos()); - } else { - *negative = false; - *result = static_cast<uint64>(value.seconds()); - *result = *result * kNanosPerSecond + static_cast<uint32>(value.nanos()); - } -} - // Convert a Duration to uint128. void ToUint128(const Duration& value, uint128* result, bool* negative) { if (value.seconds() < 0 || value.nanos() < 0) { @@ -400,21 +388,6 @@ void ToUint128(const Duration& value, uint128* result, bool* negative) { } } -void ToTimestamp(const uint128& value, bool negative, Timestamp* timestamp) { - int64 seconds = static_cast<int64>(Uint128Low64(value / kNanosPerSecond)); - int32 nanos = static_cast<int32>(Uint128Low64(value % kNanosPerSecond)); - if (negative) { - seconds = -seconds; - nanos = -nanos; - if (nanos < 0) { - nanos += kNanosPerSecond; - seconds -= 1; - } - } - timestamp->set_seconds(seconds); - timestamp->set_nanos(nanos); -} - void ToDuration(const uint128& value, bool negative, Duration* duration) { int64 seconds = static_cast<int64>(Uint128Low64(value / kNanosPerSecond)); int32 nanos = static_cast<int32>(Uint128Low64(value % kNanosPerSecond)); diff --git a/src/google/protobuf/util/time_util.h b/src/google/protobuf/util/time_util.h index 432c9f0a..b8846935 100644 --- a/src/google/protobuf/util/time_util.h +++ b/src/google/protobuf/util/time_util.h @@ -237,7 +237,7 @@ inline Duration operator%(const Duration& d1, const Duration& d2) { return result %= d2; } -inline ostream& operator<<(ostream& out, const Duration& d) { +inline std::ostream& operator<<(std::ostream& out, const Duration& d) { out << google::protobuf::util::TimeUtil::ToString(d); return out; } @@ -284,7 +284,7 @@ inline Timestamp operator-(const Timestamp& t, const Duration& d) { } LIBPROTOBUF_EXPORT Duration operator-(const Timestamp& t1, const Timestamp& t2); -inline ostream& operator<<(ostream& out, const Timestamp& t) { +inline std::ostream& operator<<(std::ostream& out, const Timestamp& t) { out << google::protobuf::util::TimeUtil::ToString(t); return out; } diff --git a/src/google/protobuf/util/type_resolver_util.cc b/src/google/protobuf/util/type_resolver_util.cc index 96393903..a69ed58c 100644 --- a/src/google/protobuf/util/type_resolver_util.cc +++ b/src/google/protobuf/util/type_resolver_util.cc @@ -54,17 +54,6 @@ using util::Status; using util::error::INVALID_ARGUMENT; using util::error::NOT_FOUND; -bool SplitTypeUrl(const string& type_url, string* url_prefix, - string* message_name) { - size_t pos = type_url.find_last_of("/"); - if (pos == string::npos) { - return false; - } - *url_prefix = type_url.substr(0, pos); - *message_name = type_url.substr(pos + 1); - return true; -} - class DescriptorPoolTypeResolver : public TypeResolver { public: DescriptorPoolTypeResolver(const string& url_prefix, @@ -72,38 +61,27 @@ class DescriptorPoolTypeResolver : public TypeResolver { : url_prefix_(url_prefix), pool_(pool) {} Status ResolveMessageType(const string& type_url, Type* type) { - string url_prefix, message_name; - if (!SplitTypeUrl(type_url, &url_prefix, &message_name) || - url_prefix != url_prefix_) { - return Status(INVALID_ARGUMENT, - StrCat("Invalid type URL, type URLs must be of the form '", - url_prefix_, "/<typename>', got: ", type_url)); + string type_name; + Status status = ParseTypeUrl(type_url, &type_name); + if (!status.ok()) { + return status; } - if (url_prefix != url_prefix_) { - return Status(INVALID_ARGUMENT, - "Cannot resolve types from URL: " + url_prefix); - } - const Descriptor* descriptor = pool_->FindMessageTypeByName(message_name); + + const Descriptor* descriptor = pool_->FindMessageTypeByName(type_name); if (descriptor == NULL) { - return Status(NOT_FOUND, - "Invalid type URL, unknown type: " + message_name); + return Status(NOT_FOUND, "Invalid type URL, unknown type: " + type_name); } ConvertDescriptor(descriptor, type); return Status(); } Status ResolveEnumType(const string& type_url, Enum* enum_type) { - string url_prefix, type_name; - if (!SplitTypeUrl(type_url, &url_prefix, &type_name) || - url_prefix != url_prefix_) { - return Status(INVALID_ARGUMENT, - StrCat("Invalid type URL, type URLs must be of the form '", - url_prefix_, "/<typename>', got: ", type_url)); - } - if (url_prefix != url_prefix_) { - return Status(INVALID_ARGUMENT, - "Cannot resolve types from URL: " + url_prefix); + string type_name; + Status status = ParseTypeUrl(type_url, &type_name); + if (!status.ok()) { + return status; } + const EnumDescriptor* descriptor = pool_->FindEnumTypeByName(type_name); if (descriptor == NULL) { return Status(NOT_FOUND, "Invalid type URL, unknown type: " + type_name); @@ -117,11 +95,6 @@ class DescriptorPoolTypeResolver : public TypeResolver { type->Clear(); type->set_name(descriptor->full_name()); for (int i = 0; i < descriptor->field_count(); ++i) { - const FieldDescriptor* field = descriptor->field(i); - if (field->type() == FieldDescriptor::TYPE_GROUP) { - // Group fields cannot be represented with Type. We discard them. - continue; - } ConvertFieldDescriptor(descriptor->field(i), type->add_fields()); } for (int i = 0; i < descriptor->oneof_decl_count(); ++i) { @@ -163,7 +136,8 @@ class DescriptorPoolTypeResolver : public TypeResolver { if (descriptor->has_default_value()) { field->set_default_value(DefaultValueAsString(descriptor)); } - if (descriptor->type() == FieldDescriptor::TYPE_MESSAGE) { + if (descriptor->type() == FieldDescriptor::TYPE_MESSAGE || + descriptor->type() == FieldDescriptor::TYPE_GROUP) { field->set_type_url(GetTypeUrl(descriptor->message_type())); } else if (descriptor->type() == FieldDescriptor::TYPE_ENUM) { field->set_type_url(GetTypeUrl(descriptor->enum_type())); @@ -203,6 +177,16 @@ class DescriptorPoolTypeResolver : public TypeResolver { return url_prefix_ + "/" + descriptor->full_name(); } + Status ParseTypeUrl(const string& type_url, string* type_name) { + if (type_url.substr(0, url_prefix_.size() + 1) != url_prefix_ + "/") { + return Status(INVALID_ARGUMENT, + StrCat("Invalid type URL, type URLs must be of the form '", + url_prefix_, "/<typename>', got: ", type_url)); + } + *type_name = type_url.substr(url_prefix_.size() + 1); + return Status(); + } + string DefaultValueAsString(const FieldDescriptor* descriptor) { switch (descriptor->cpp_type()) { case FieldDescriptor::CPPTYPE_INT32: diff --git a/src/google/protobuf/util/type_resolver_util_test.cc b/src/google/protobuf/util/type_resolver_util_test.cc index 8a0bf652..9dea17ae 100644 --- a/src/google/protobuf/util/type_resolver_util_test.cc +++ b/src/google/protobuf/util/type_resolver_util_test.cc @@ -32,9 +32,6 @@ #include <limits> #include <memory> -#ifndef _SHARED_PTR_H -#include <google/protobuf/stubs/shared_ptr.h> -#endif #include <string> #include <vector> @@ -153,7 +150,7 @@ class DescriptorPoolTypeResolverTest : public testing::Test { } protected: - google::protobuf::scoped_ptr<TypeResolver> resolver_; + std::unique_ptr<TypeResolver> resolver_; }; TEST_F(DescriptorPoolTypeResolverTest, TestAllTypes) { @@ -193,6 +190,13 @@ TEST_F(DescriptorPoolTypeResolverTest, TestAllTypes) { Field::TYPE_BYTES, "optional_bytes", 15)); EXPECT_TRUE(HasField(type, Field::CARDINALITY_OPTIONAL, + Field::TYPE_GROUP, "optionalgroup", 16)); + + EXPECT_TRUE(CheckFieldTypeUrl( + type, "optionalgroup", + GetTypeUrl<protobuf_unittest::TestAllTypes::OptionalGroup>())); + + EXPECT_TRUE(HasField(type, Field::CARDINALITY_OPTIONAL, Field::TYPE_MESSAGE, "optional_nested_message", 18)); EXPECT_TRUE(HasField(type, Field::CARDINALITY_OPTIONAL, Field::TYPE_MESSAGE, "optional_foreign_message", 19)); @@ -249,6 +253,13 @@ TEST_F(DescriptorPoolTypeResolverTest, TestAllTypes) { Field::TYPE_BYTES, "repeated_bytes", 45)); EXPECT_TRUE(HasField(type, Field::CARDINALITY_REPEATED, + Field::TYPE_GROUP, "repeatedgroup", 46)); + + EXPECT_TRUE(CheckFieldTypeUrl( + type, "repeatedgroup", + GetTypeUrl<protobuf_unittest::TestAllTypes::RepeatedGroup>())); + + EXPECT_TRUE(HasField(type, Field::CARDINALITY_REPEATED, Field::TYPE_MESSAGE, "repeated_nested_message", 48)); EXPECT_TRUE(HasField(type, Field::CARDINALITY_REPEATED, Field::TYPE_MESSAGE, "repeated_foreign_message", 49)); @@ -271,13 +282,6 @@ TEST_F(DescriptorPoolTypeResolverTest, TestAllTypes) { EXPECT_TRUE(CheckFieldTypeUrl( type, "repeated_foreign_enum", GetTypeUrl("protobuf_unittest.ForeignEnum"))); - - // Groups are discarded when converting to Type. - const Descriptor* descriptor = protobuf_unittest::TestAllTypes::descriptor(); - EXPECT_TRUE(descriptor->FindFieldByName("optionalgroup") != NULL); - EXPECT_TRUE(descriptor->FindFieldByName("repeatedgroup") != NULL); - ASSERT_FALSE(HasField(type, "optionalgroup")); - ASSERT_FALSE(HasField(type, "repeatedgroup")); } TEST_F(DescriptorPoolTypeResolverTest, TestPackedField) { |