Integrated internal changes from Google

This includes all internal changes from around May 20 to now.

12 files changed, 95 insertions, 33 deletions

diff --git a/src/google/protobuf/util/internal/datapiece.cc b/src/google/protobuf/util/internal/datapiece.cc
index 72c0aca6..ef8da427 100644
--- a/src/google/protobuf/util/internal/datapiece.cc
+++ b/src/google/protobuf/util/internal/datapiece.cc
@@ -329,9 +329,8 @@ bool DataPiece::DecodeBase64(StringPiece src, string* dest) const {
       // WebSafeBase64Escape does no padding by default.
       WebSafeBase64Escape(*dest, &encoded);
       // Remove trailing padding '=' characters before comparison.
-      StringPiece src_no_padding(src, 0, src.ends_with("=")
-                                             ? src.find_last_not_of('=') + 1
-                                             : src.length());
+      StringPiece src_no_padding = StringPiece(src).substr(
+          0, src.ends_with("=") ? src.find_last_not_of('=') + 1 : src.length());
       return encoded == src_no_padding;
     }
     return true;
@@ -343,9 +342,8 @@ bool DataPiece::DecodeBase64(StringPiece src, string* dest) const {
       Base64Escape(
           reinterpret_cast<const unsigned char*>(dest->data()), dest->length(),
           &encoded, false);
-      StringPiece src_no_padding(src, 0, src.ends_with("=")
-                                             ? src.find_last_not_of('=') + 1
-                                             : src.length());
+      StringPiece src_no_padding = StringPiece(src).substr(
+          0, src.ends_with("=") ? src.find_last_not_of('=') + 1 : src.length());
       return encoded == src_no_padding;
     }
     return true;
@@ -354,6 +352,26 @@ bool DataPiece::DecodeBase64(StringPiece src, string* dest) const {
   return false;
 }
 
+void DataPiece::InternalCopy(const DataPiece& other) {
+  type_ = other.type_;
+  switch (type_) {
+    case TYPE_INT32:
+    case TYPE_INT64:
+    case TYPE_UINT32:
+    case TYPE_UINT64:
+    case TYPE_DOUBLE:
+    case TYPE_FLOAT:
+    case TYPE_BOOL:
+    case TYPE_ENUM:
+    case TYPE_NULL:
+    case TYPE_BYTES:
+    case TYPE_STRING: {
+      str_ = other.str_;
+      break;
+    }
+  }
+}
+
 }  // namespace converter
 }  // namespace util
 }  // namespace protobuf
diff --git a/src/google/protobuf/util/internal/datapiece.h b/src/google/protobuf/util/internal/datapiece.h
index 8b2e35d3..e82cdbac 100644
--- a/src/google/protobuf/util/internal/datapiece.h
+++ b/src/google/protobuf/util/internal/datapiece.h
@@ -92,10 +92,11 @@ class LIBPROTOBUF_EXPORT DataPiece {
       : type_(TYPE_BYTES),
         str_(StringPiecePod::CreateFromStringPiece(value)),
         use_strict_base64_decoding_(use_strict_base64_decoding) {}
-  DataPiece(const DataPiece& r) : type_(r.type_), str_(r.str_) {}
+
+  DataPiece(const DataPiece& r) : type_(r.type_) { InternalCopy(r); }
+
   DataPiece& operator=(const DataPiece& x) {
-    type_ = x.type_;
-    str_ = x.str_;
+    InternalCopy(x);
     return *this;
   }
 
@@ -171,6 +172,9 @@ class LIBPROTOBUF_EXPORT DataPiece {
   // Decodes a base64 string. Returns true on success.
   bool DecodeBase64(StringPiece src, string* dest) const;
 
+  // Helper function to initialize this DataPiece with 'other'.
+  void InternalCopy(const DataPiece& other);
+
   // Data type for this piece of data.
   Type type_;
 
diff --git a/src/google/protobuf/util/internal/json_escaping.cc b/src/google/protobuf/util/internal/json_escaping.cc
index 24bd554e..06d2791b 100644
--- a/src/google/protobuf/util/internal/json_escaping.cc
+++ b/src/google/protobuf/util/internal/json_escaping.cc
@@ -255,7 +255,7 @@ StringPiece ToHex(uint16 cp, char* buffer) {
   buffer[3] = kHex[cp & 0x0f];
   cp >>= 4;
   buffer[2] = kHex[cp & 0x0f];
-  return StringPiece(buffer, 0, 6);
+  return StringPiece(buffer).substr(0, 6);
 }
 
 // Stores the 32-bit unicode code point as its hexadecimal digits in buffer
diff --git a/src/google/protobuf/util/internal/object_writer.h b/src/google/protobuf/util/internal/object_writer.h
index 9f07363d..5781aa1e 100644
--- a/src/google/protobuf/util/internal/object_writer.h
+++ b/src/google/protobuf/util/internal/object_writer.h
@@ -101,6 +101,7 @@ class LIBPROTOBUF_EXPORT ObjectWriter {
   // Renders a Null value.
   virtual ObjectWriter* RenderNull(StringPiece name) = 0;
 
+
   // Renders a DataPiece object to a ObjectWriter.
   static void RenderDataPieceTo(const DataPiece& data, StringPiece name,
                                 ObjectWriter* ow);
diff --git a/src/google/protobuf/util/internal/proto_writer.cc b/src/google/protobuf/util/internal/proto_writer.cc
index 36b79410..18cc1233 100644
--- a/src/google/protobuf/util/internal/proto_writer.cc
+++ b/src/google/protobuf/util/internal/proto_writer.cc
@@ -293,10 +293,14 @@ ProtoWriter::ProtoElement::ProtoElement(const TypeInfo* typeinfo,
       ow_(enclosing),
       parent_field_(NULL),
       typeinfo_(typeinfo),
+      proto3_(type.syntax() == google::protobuf::SYNTAX_PROTO3),
       type_(type),
-      required_fields_(GetRequiredFields(type)),
       size_index_(-1),
-      array_index_(-1) {}
+      array_index_(-1) {
+  if (!proto3_) {
+    required_fields_ = GetRequiredFields(type_);
+  }
+}
 
 ProtoWriter::ProtoElement::ProtoElement(ProtoWriter::ProtoElement* parent,
                                         const google::protobuf::Field* field,
@@ -306,6 +310,7 @@ ProtoWriter::ProtoElement::ProtoElement(ProtoWriter::ProtoElement* parent,
       ow_(this->parent()->ow_),
       parent_field_(field),
       typeinfo_(this->parent()->typeinfo_),
+      proto3_(this->parent()->proto3_),
       type_(type),
       size_index_(
           !is_list && field->kind() == google::protobuf::Field_Kind_TYPE_MESSAGE
@@ -316,12 +321,15 @@ ProtoWriter::ProtoElement::ProtoElement(ProtoWriter::ProtoElement* parent,
     if (ow_->IsRepeated(*field)) {
       // Update array_index_ if it is an explicit list.
       if (this->parent()->array_index_ >= 0) this->parent()->array_index_++;
-    } else {
+    } else if (!proto3_) {
+      // For required fields tracking.
       this->parent()->RegisterField(field);
     }
 
     if (field->kind() == google::protobuf::Field_Kind_TYPE_MESSAGE) {
-      required_fields_ = GetRequiredFields(type_);
+      if (!proto3_) {
+        required_fields_ = GetRequiredFields(type_);
+      }
       int start_pos = ow_->stream_->ByteCount();
       // length of serialized message is the final buffer position minus
       // starting buffer position, plus length adjustments for size fields
@@ -334,12 +342,14 @@ ProtoWriter::ProtoElement::ProtoElement(ProtoWriter::ProtoElement* parent,
 }
 
 ProtoWriter::ProtoElement* ProtoWriter::ProtoElement::pop() {
-  // Calls the registered error listener for any required field(s) not yet
-  // seen.
-  for (set<const google::protobuf::Field*>::iterator it =
-           required_fields_.begin();
-       it != required_fields_.end(); ++it) {
-    ow_->MissingField((*it)->name());
+  if (!proto3_) {
+    // Calls the registered error listener for any required field(s) not yet
+    // seen.
+    for (set<const google::protobuf::Field*>::iterator it =
+             required_fields_.begin();
+         it != required_fields_.end(); ++it) {
+      ow_->MissingField((*it)->name());
+    }
   }
   // Computes the total number of proto bytes used by a message, also adjusts
   // the size of all parent messages by the length of this size field.
diff --git a/src/google/protobuf/util/internal/proto_writer.h b/src/google/protobuf/util/internal/proto_writer.h
index 957565e7..ffb8f60e 100644
--- a/src/google/protobuf/util/internal/proto_writer.h
+++ b/src/google/protobuf/util/internal/proto_writer.h
@@ -117,6 +117,7 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter {
     return RenderDataPiece(name, DataPiece::NullData());
   }
 
+
   // Renders a DataPiece 'value' into a field whose wire type is determined
   // from the given field 'name'.
   virtual ProtoWriter* RenderDataPiece(StringPiece name,
@@ -198,6 +199,9 @@ class LIBPROTOBUF_EXPORT ProtoWriter : public StructuredObjectWriter {
     // TypeInfo to lookup types.
     const TypeInfo* typeinfo_;
 
+    // Whether the root type is a proto3 or not.
+    bool proto3_;
+
     // Additional variables if this element is a message:
     // (Root element is always a message).
     // type_             : the type of this element.
diff --git a/src/google/protobuf/util/internal/protostream_objectsource.cc b/src/google/protobuf/util/internal/protostream_objectsource.cc
index 1f3781a4..0048d75b 100644
--- a/src/google/protobuf/util/internal/protostream_objectsource.cc
+++ b/src/google/protobuf/util/internal/protostream_objectsource.cc
@@ -121,7 +121,8 @@ ProtoStreamObjectSource::ProtoStreamObjectSource(
       type_(type),
       use_lower_camel_for_enums_(false),
       recursion_depth_(0),
-      max_recursion_depth_(kDefaultMaxRecursionDepth) {
+      max_recursion_depth_(kDefaultMaxRecursionDepth),
+      render_unknown_fields_(false) {
   GOOGLE_LOG_IF(DFATAL, stream == NULL) << "Input stream is NULL.";
 }
 
@@ -134,7 +135,8 @@ ProtoStreamObjectSource::ProtoStreamObjectSource(
       type_(type),
       use_lower_camel_for_enums_(false),
       recursion_depth_(0),
-      max_recursion_depth_(kDefaultMaxRecursionDepth) {
+      max_recursion_depth_(kDefaultMaxRecursionDepth),
+      render_unknown_fields_(false) {
   GOOGLE_LOG_IF(DFATAL, stream == NULL) << "Input stream is NULL.";
 }
 
@@ -184,6 +186,7 @@ Status ProtoStreamObjectSource::WriteMessage(const google::protobuf::Type& type,
   string field_name;
   // last_tag set to dummy value that is different from tag.
   uint32 tag = stream_->ReadTag(), last_tag = tag + 1;
+  google::protobuf::UnknownFieldSet unknown_fields;
 
   if (include_start_and_end) {
     ow->StartObject(name);
@@ -199,7 +202,8 @@ Status ProtoStreamObjectSource::WriteMessage(const google::protobuf::Type& type,
     if (field == NULL) {
       // If we didn't find a field, skip this unknown tag.
       // TODO(wpoon): Check return boolean value.
-      WireFormat::SkipField(stream_, tag, NULL);
+      WireFormat::SkipField(stream_, tag,
+                            render_unknown_fields_ ? &unknown_fields : NULL);
       tag = stream_->ReadTag();
       continue;
     }
@@ -221,6 +225,8 @@ Status ProtoStreamObjectSource::WriteMessage(const google::protobuf::Type& type,
       tag = stream_->ReadTag();
     }
   }
+
+
   if (include_start_and_end) {
     ow->EndObject();
   }
diff --git a/src/google/protobuf/util/internal/protostream_objectsource.h b/src/google/protobuf/util/internal/protostream_objectsource.h
index d7d4347b..243f85b2 100644
--- a/src/google/protobuf/util/internal/protostream_objectsource.h
+++ b/src/google/protobuf/util/internal/protostream_objectsource.h
@@ -117,6 +117,7 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectSource : public ObjectSource {
     max_recursion_depth_ = max_depth;
   }
 
+
  protected:
   // Writes a proto2 Message to the ObjectWriter. When the given end_tag is
   // found this method will complete, allowing it to be used for parsing both
@@ -287,6 +288,9 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectSource : public ObjectSource {
   // Maximum allowed recursion depth.
   int max_recursion_depth_;
 
+  // Whether to render unknown fields.
+  bool render_unknown_fields_;
+
   GOOGLE_DISALLOW_IMPLICIT_CONSTRUCTORS(ProtoStreamObjectSource);
 };
 
diff --git a/src/google/protobuf/util/internal/protostream_objectwriter.cc b/src/google/protobuf/util/internal/protostream_objectwriter.cc
index 97a7909a..8fa58a6f 100644
--- a/src/google/protobuf/util/internal/protostream_objectwriter.cc
+++ b/src/google/protobuf/util/internal/protostream_objectwriter.cc
@@ -384,6 +384,9 @@ ProtoStreamObjectWriter::Item::Item(ProtoStreamObjectWriter* enclosing,
   if (item_type_ == ANY) {
     any_.reset(new AnyWriter(ow_));
   }
+  if (item_type == MAP) {
+    map_keys_.reset(new hash_set<string>);
+  }
 }
 
 ProtoStreamObjectWriter::Item::Item(ProtoStreamObjectWriter::Item* parent,
@@ -398,11 +401,14 @@ ProtoStreamObjectWriter::Item::Item(ProtoStreamObjectWriter::Item* parent,
   if (item_type == ANY) {
     any_.reset(new AnyWriter(ow_));
   }
+  if (item_type == MAP) {
+    map_keys_.reset(new hash_set<string>);
+  }
 }
 
 bool ProtoStreamObjectWriter::Item::InsertMapKeyIfNotPresent(
     StringPiece map_key) {
-  return InsertIfNotPresent(&map_keys_, map_key.ToString());
+  return InsertIfNotPresent(map_keys_.get(), map_key.ToString());
 }
 
 ProtoStreamObjectWriter* ProtoStreamObjectWriter::StartObject(
@@ -1000,6 +1006,7 @@ ProtoStreamObjectWriter* ProtoStreamObjectWriter::RenderDataPiece(
                                  DataPiece(name, use_strict_base64_decoding()));
     field = Lookup("value");
     if (field == NULL) {
+      Pop();
       GOOGLE_LOG(DFATAL) << "Map does not have a value field.";
       return this;
     }
diff --git a/src/google/protobuf/util/internal/protostream_objectwriter.h b/src/google/protobuf/util/internal/protostream_objectwriter.h
index e1162d43..75e3d67d 100644
--- a/src/google/protobuf/util/internal/protostream_objectwriter.h
+++ b/src/google/protobuf/util/internal/protostream_objectwriter.h
@@ -231,7 +231,7 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectWriter : public ProtoWriter {
 
     // Set of map keys already seen for the type_. Used to validate incoming
     // messages so no map key appears more than once.
-    hash_set<string> map_keys_;
+    google::protobuf::scoped_ptr<hash_set<string> > map_keys_;
 
     // Conveys whether this Item is a placeholder or not. Placeholder items are
     // pushed to stack to account for special types.
@@ -249,19 +249,19 @@ class LIBPROTOBUF_EXPORT ProtoStreamObjectWriter : public ProtoWriter {
                           strings::ByteSink* output, ErrorListener* listener);
 
   // Returns true if the field is a map.
-  bool IsMap(const google::protobuf::Field& field);
+  inline bool IsMap(const google::protobuf::Field& field);
 
   // Returns true if the field is an any.
-  bool IsAny(const google::protobuf::Field& field);
+  inline bool IsAny(const google::protobuf::Field& field);
 
   // Returns true if the field is google.protobuf.Struct.
-  bool IsStruct(const google::protobuf::Field& field);
+  inline bool IsStruct(const google::protobuf::Field& field);
 
   // Returns true if the field is google.protobuf.Value.
-  bool IsStructValue(const google::protobuf::Field& field);
+  inline bool IsStructValue(const google::protobuf::Field& field);
 
   // Returns true if the field is google.protobuf.ListValue.
-  bool IsStructListValue(const google::protobuf::Field& field);
+  inline bool IsStructListValue(const google::protobuf::Field& field);
 
   // Renders google.protobuf.Value in struct.proto. It picks the right oneof
   // type based on value's type.
diff --git a/src/google/protobuf/util/internal/utility.cc b/src/google/protobuf/util/internal/utility.cc
index ee7a51fc..5f613e77 100644
--- a/src/google/protobuf/util/internal/utility.cc
+++ b/src/google/protobuf/util/internal/utility.cc
@@ -52,7 +52,7 @@ const StringPiece SkipWhiteSpace(StringPiece str) {
   for (i = 0; i < str.size() && isspace(str[i]); ++i) {
   }
   GOOGLE_DCHECK(i == str.size() || !isspace(str[i]));
-  return StringPiece(str, i);
+  return str.substr(i);
 }
 }  // namespace
 
@@ -128,8 +128,12 @@ string GetStringFromAny(const google::protobuf::Any& any) {
 }
 
 const StringPiece GetTypeWithoutUrl(StringPiece type_url) {
-  size_t idx = type_url.rfind('/');
-  return type_url.substr(idx + 1);
+  if (type_url.size() > kTypeUrlSize && type_url[kTypeUrlSize] == '/') {
+    return type_url.substr(kTypeUrlSize + 1);
+  } else {
+    size_t idx = type_url.rfind('/');
+    return type_url.substr(idx + 1);
+  }
 }
 
 const string GetFullTypeWithUrl(StringPiece simple_type) {
diff --git a/src/google/protobuf/util/internal/utility.h b/src/google/protobuf/util/internal/utility.h
index 33df8eda..26fed444 100644
--- a/src/google/protobuf/util/internal/utility.h
+++ b/src/google/protobuf/util/internal/utility.h
@@ -64,6 +64,10 @@ class EnumValue;
 namespace protobuf {
 namespace util {
 namespace converter {
+
+// Size of "type.googleapis.com"
+static const int64 kTypeUrlSize = 19;
+
 // Finds the tech option identified by option_name. Parses the boolean value and
 // returns it.
 // When the option with the given name is not found, default_value is returned.