Push out changes from internal codebase.

All Languages
* Repeated fields of primitive types (types other that string, group, and
  nested messages) may now use the option [packed = true] to get a more
  efficient encoding.  In the new encoding, the entire list is written
  as a single byte blob using the "length-delimited" wire type.  Within
  this blob, the individual values are encoded the same way they would
  be normally except without a tag before each value (thus, they are
  tightly "packed").

C++
* UnknownFieldSet now supports STL-like iteration.
* Message interface has method ParseFromBoundedZeroCopyStream() which parses
  a limited number of bytes from an input stream rather than parsing until
  EOF.

Java
* Fixed bug where Message.mergeFrom(Message) failed to merge extensions.
* Message interface has new method toBuilder() which is equivalent to
  newBuilderForType().mergeFrom(this).
* All enums now implement the ProtocolMessageEnum interface.
* Setting a field to null now throws NullPointerException.
* Fixed tendency for TextFormat's parsing to overflow the stack when
  parsing large string values.  The underlying problem is with Java's
  regex implementation (which unfortunately uses recursive backtracking
  rather than building an NFA).  Worked around by making use of possesive
  quantifiers.

Python
* Updated RPC interfaces to allow for blocking operation.  A client may
  now pass None for a callback when making an RPC, in which case the
  call will block until the response is received, and the response
  object will be returned directly to the caller.  This interface change
  cannot be used in practice until RPC implementations are updated to
  implement it.

7 files changed, 185 insertions, 49 deletions

diff --git a/src/google/protobuf/compiler/cpp/cpp_enum_field.cc b/src/google/protobuf/compiler/cpp/cpp_enum_field.cc
index c998f20b..19779a8a 100644
--- a/src/google/protobuf/compiler/cpp/cpp_enum_field.cc
+++ b/src/google/protobuf/compiler/cpp/cpp_enum_field.cc
@@ -158,7 +158,13 @@ RepeatedEnumFieldGenerator::~RepeatedEnumFieldGenerator() {}
 
 void RepeatedEnumFieldGenerator::
 GeneratePrivateMembers(io::Printer* printer) const {
-  printer->Print(variables_, "::google::protobuf::RepeatedField<int> $name$_;\n");
+  printer->Print(variables_,
+    "::google::protobuf::RepeatedField<int> $name$_;\n");
+  if (descriptor_->options().packed() &&
+      descriptor_->file()->options().optimize_for() == FileOptions::SPEED) {
+    printer->Print(variables_,
+      "mutable int _$name$_cached_byte_size_;\n");
+  }
 }
 
 void RepeatedEnumFieldGenerator::
@@ -217,31 +223,84 @@ GenerateInitializer(io::Printer* printer) const {
 
 void RepeatedEnumFieldGenerator::
 GenerateMergeFromCodedStream(io::Printer* printer) const {
-  printer->Print(variables_,
-    "int value;\n"
-    "DO_(::google::protobuf::internal::WireFormat::ReadEnum(input, &value));\n"
-    "if ($type$_IsValid(value)) {\n"
-    "  add_$name$(static_cast< $type$ >(value));\n"
-    "} else {\n"
-    "  mutable_unknown_fields()->AddField($number$)->add_varint(value);\n"
-    "}\n");
+  if (descriptor_->options().packed()) {
+    printer->Print(variables_,
+      "::google::protobuf::uint32 length;\n"
+      "DO_(input->ReadVarint32(&length));\n"
+      "::google::protobuf::io::CodedInputStream::Limit limit = "
+          "input->PushLimit(length);\n"
+      "while (input->BytesUntilLimit() > 0) {\n"
+      "  int value;\n"
+      "  DO_(::google::protobuf::internal::WireFormat::ReadEnum(input, &value));\n"
+      "  if ($type$_IsValid(value)) {\n"
+      "    add_$name$(static_cast< $type$ >(value));\n"
+      "  }\n"
+      "}\n"
+      "input->PopLimit(limit);\n");
+  } else {
+    printer->Print(variables_,
+      "int value;\n"
+      "DO_(::google::protobuf::internal::WireFormat::ReadEnum(input, &value));\n"
+      "if ($type$_IsValid(value)) {\n"
+      "  add_$name$(static_cast< $type$ >(value));\n"
+      "} else {\n"
+      "  mutable_unknown_fields()->AddField($number$)->add_varint(value);\n"
+      "}\n");
+  }
 }
 
 void RepeatedEnumFieldGenerator::
 GenerateSerializeWithCachedSizes(io::Printer* printer) const {
+  if (descriptor_->options().packed()) {
+    // Write the tag and the size.
+    printer->Print(variables_,
+      "if (this->$name$_size() > 0) {\n"
+      "  DO_(::google::protobuf::internal::WireFormat::WriteTag("
+          "$number$, ::google::protobuf::internal::WireFormat::WIRETYPE_LENGTH_DELIMITED,"
+          "output));\n"
+      "  DO_(output->WriteVarint32(_$name$_cached_byte_size_));\n"
+      "}\n");
+  }
   printer->Print(variables_,
-    "DO_(::google::protobuf::internal::WireFormat::WriteEnum("
-      "$number$, this->$name$(i), output));\n");
+      "for (int i = 0; i < this->$name$_size(); i++) {\n");
+  if (descriptor_->options().packed()) {
+    printer->Print(variables_,
+      "  DO_(::google::protobuf::internal::WireFormat::WriteEnumNoTag("
+          "this->$name$(i), output));\n");
+  } else {
+    printer->Print(variables_,
+      "  DO_(::google::protobuf::internal::WireFormat::WriteEnum("
+          "$number$, this->$name$(i), output));\n");
+  }
+  printer->Print("}\n");
 }
 
 void RepeatedEnumFieldGenerator::
 GenerateByteSize(io::Printer* printer) const {
   printer->Print(variables_,
-    "total_size += $tag_size$ * $name$_size();\n"
-    "for (int i = 0; i < $name$_size(); i++) {\n"
-    "  total_size += ::google::protobuf::internal::WireFormat::EnumSize(\n"
-    "    this->$name$(i));\n"
-    "}\n");
+    "{\n"
+    "  int data_size = 0;\n");
+  printer->Indent();
+  printer->Print(variables_,
+      "for (int i = 0; i < this->$name$_size(); i++) {\n"
+      "  data_size += ::google::protobuf::internal::WireFormat::EnumSize(\n"
+      "    this->$name$(i));\n"
+      "}\n");
+
+  if (descriptor_->options().packed()) {
+    printer->Print(variables_,
+      "if (data_size > 0) {\n"
+      "  total_size += $tag_size$ + "
+        "::google::protobuf::internal::WireFormat::Int32Size(data_size);\n"
+      "}\n"
+      "_$name$_cached_byte_size_ = data_size;\n"
+      "total_size += data_size;\n");
+  } else {
+    printer->Print(variables_,
+      "total_size += $tag_size$ * this->$name$_size() + data_size;\n");
+  }
+  printer->Outdent();
+  printer->Print("}\n");
 }
 
 }  // namespace cpp
diff --git a/src/google/protobuf/compiler/cpp/cpp_message.cc b/src/google/protobuf/compiler/cpp/cpp_message.cc
index eacceeaf..c6843e93 100644
--- a/src/google/protobuf/compiler/cpp/cpp_message.cc
+++ b/src/google/protobuf/compiler/cpp/cpp_message.cc
@@ -41,7 +41,7 @@
 #include <google/protobuf/stubs/strutil.h>
 #include <google/protobuf/io/printer.h>
 #include <google/protobuf/io/coded_stream.h>
-#include <google/protobuf/wire_format.h>
+#include <google/protobuf/wire_format_inl.h>
 #include <google/protobuf/descriptor.pb.h>
 
 namespace google {
@@ -1169,10 +1169,9 @@ GenerateMergeFromCodedStream(io::Printer* printer) {
         "    goto handle_uninterpreted;\n"
         "  }\n",
         "number", SimpleItoa(field->number()),
-        "wiretype", kWireTypeNames[
-          WireFormat::WireTypeForFieldType(field->type())]);
+        "wiretype", kWireTypeNames[WireFormat::WireTypeForField(field)]);
 
-      if (i > 0 || field->is_repeated()) {
+      if (i > 0 || (field->is_repeated() && !field->options().packed())) {
         printer->Print(
           " parse_$name$:\n",
           "name", field->name());
@@ -1184,7 +1183,7 @@ GenerateMergeFromCodedStream(io::Printer* printer) {
 
       // switch() is slow since it can't be predicted well.  Insert some if()s
       // here that attempt to predict the next tag.
-      if (field->is_repeated()) {
+      if (field->is_repeated() && !field->options().packed()) {
         // Expect repeats of this field.
         printer->Print(
           "if (input->ExpectTag($tag$)) goto parse_$name$;\n",
@@ -1283,22 +1282,20 @@ void MessageGenerator::GenerateSerializeOneField(
     io::Printer* printer, const FieldDescriptor* field) {
   PrintFieldComment(printer, field);
 
-  if (field->is_repeated()) {
-    printer->Print(
-      "for (int i = 0; i < $name$_.size(); i++) {\n",
-      "name", FieldName(field));
-  } else {
+  if (!field->is_repeated()) {
     printer->Print(
       "if (_has_bit($index$)) {\n",
       "index", SimpleItoa(field->index()));
+    printer->Indent();
   }
 
-  printer->Indent();
-
   field_generators_.get(field).GenerateSerializeWithCachedSizes(printer);
 
-  printer->Outdent();
-  printer->Print("}\n\n");
+  if (!field->is_repeated()) {
+    printer->Outdent();
+    printer->Print("}\n");
+  }
+  printer->Print("\n");
 }
 
 void MessageGenerator::GenerateSerializeOneExtensionRange(
diff --git a/src/google/protobuf/compiler/cpp/cpp_message_field.cc b/src/google/protobuf/compiler/cpp/cpp_message_field.cc
index d1c31067..7d57a6df 100644
--- a/src/google/protobuf/compiler/cpp/cpp_message_field.cc
+++ b/src/google/protobuf/compiler/cpp/cpp_message_field.cc
@@ -232,15 +232,17 @@ GenerateMergeFromCodedStream(io::Printer* printer) const {
 void RepeatedMessageFieldGenerator::
 GenerateSerializeWithCachedSizes(io::Printer* printer) const {
   printer->Print(variables_,
-    "DO_(::google::protobuf::internal::WireFormat::Write$declared_type$NoVirtual("
-      "$number$, this->$name$(i), output));\n");
+    "for (int i = 0; i < this->$name$_size(); i++) {\n"
+    "  DO_(::google::protobuf::internal::WireFormat::Write$declared_type$NoVirtual("
+        "$number$, this->$name$(i), output));\n"
+    "}\n");
 }
 
 void RepeatedMessageFieldGenerator::
 GenerateByteSize(io::Printer* printer) const {
   printer->Print(variables_,
-    "total_size += $tag_size$ * $name$_size();\n"
-    "for (int i = 0; i < $name$_size(); i++) {\n"
+    "total_size += $tag_size$ * this->$name$_size();\n"
+    "for (int i = 0; i < this->$name$_size(); i++) {\n"
     "  total_size +=\n"
     "    ::google::protobuf::internal::WireFormat::$declared_type$SizeNoVirtual(\n"
     "      this->$name$(i));\n"
diff --git a/src/google/protobuf/compiler/cpp/cpp_primitive_field.cc b/src/google/protobuf/compiler/cpp/cpp_primitive_field.cc
index ef4072f0..768d30cc 100644
--- a/src/google/protobuf/compiler/cpp/cpp_primitive_field.cc
+++ b/src/google/protobuf/compiler/cpp/cpp_primitive_field.cc
@@ -227,6 +227,11 @@ void RepeatedPrimitiveFieldGenerator::
 GeneratePrivateMembers(io::Printer* printer) const {
   printer->Print(variables_,
     "::google::protobuf::RepeatedField< $type$ > $name$_;\n");
+  if (descriptor_->options().packed() &&
+      descriptor_->file()->options().optimize_for() == FileOptions::SPEED) {
+    printer->Print(variables_,
+      "mutable int _$name$_cached_byte_size_;\n");
+  }
 }
 
 void RepeatedPrimitiveFieldGenerator::
@@ -283,33 +288,90 @@ GenerateInitializer(io::Printer* printer) const {
 
 void RepeatedPrimitiveFieldGenerator::
 GenerateMergeFromCodedStream(io::Printer* printer) const {
-  printer->Print(variables_,
-    "$type$ value;\n"
-    "DO_(::google::protobuf::internal::WireFormat::Read$declared_type$(input, &value));\n"
-    "add_$name$(value);\n");
+  if (descriptor_->options().packed()) {
+    printer->Print("{\n");
+    printer->Indent();
+    printer->Print(variables_,
+      "::google::protobuf::uint32 length;\n"
+      "DO_(input->ReadVarint32(&length));\n"
+      "::google::protobuf::io::CodedInputStream::Limit limit = "
+          "input->PushLimit(length);\n"
+      "while (input->BytesUntilLimit() > 0) {\n"
+      "  $type$ value;\n"
+      "  DO_(::google::protobuf::internal::WireFormat::Read$declared_type$("
+        "input, &value));\n"
+      "  add_$name$(value);\n"
+      "}\n"
+      "input->PopLimit(limit);\n");
+    printer->Outdent();
+    printer->Print("}\n");
+  } else {
+    printer->Print(variables_,
+      "$type$ value;\n"
+      "DO_(::google::protobuf::internal::WireFormat::Read$declared_type$("
+        "input, &value));\n"
+      "add_$name$(value);\n");
+  }
 }
 
 void RepeatedPrimitiveFieldGenerator::
 GenerateSerializeWithCachedSizes(io::Printer* printer) const {
+  if (descriptor_->options().packed()) {
+    // Write the tag and the size.
+    printer->Print(variables_,
+      "if (this->$name$_size() > 0) {\n"
+      "  DO_(::google::protobuf::internal::WireFormat::WriteTag("
+          "$number$, ::google::protobuf::internal::WireFormat::WIRETYPE_LENGTH_DELIMITED,"
+          "output));\n"
+      "  DO_(output->WriteVarint32(_$name$_cached_byte_size_));\n"
+      "}\n");
+  }
   printer->Print(variables_,
-    "DO_(::google::protobuf::internal::WireFormat::Write$declared_type$("
-      "$number$, this->$name$(i), output));\n");
+      "for (int i = 0; i < this->$name$_size(); i++) {\n");
+  if (descriptor_->options().packed()) {
+    printer->Print(variables_,
+      "  DO_(::google::protobuf::internal::WireFormat::Write$declared_type$NoTag("
+          "this->$name$(i), output));\n");
+  } else {
+    printer->Print(variables_,
+      "  DO_(::google::protobuf::internal::WireFormat::Write$declared_type$("
+          "$number$, this->$name$(i), output));\n");
+  }
+  printer->Print("}\n");
 }
 
 void RepeatedPrimitiveFieldGenerator::
 GenerateByteSize(io::Printer* printer) const {
+  printer->Print(variables_,
+    "{\n"
+    "  int data_size = 0;\n");
+  printer->Indent();
   int fixed_size = FixedSize(descriptor_->type());
   if (fixed_size == -1) {
     printer->Print(variables_,
-      "total_size += $tag_size$ * $name$_size();\n"
-      "for (int i = 0; i < $name$_size(); i++) {\n"
-      "  total_size += ::google::protobuf::internal::WireFormat::$declared_type$Size(\n"
+      "for (int i = 0; i < this->$name$_size(); i++) {\n"
+      "  data_size += ::google::protobuf::internal::WireFormat::$declared_type$Size(\n"
       "    this->$name$(i));\n"
       "}\n");
   } else {
     printer->Print(variables_,
-      "total_size += ($tag_size$ + $fixed_size$) * $name$_size();\n");
+      "data_size = $fixed_size$ * this->$name$_size();\n");
+  }
+
+  if (descriptor_->options().packed()) {
+    printer->Print(variables_,
+      "if (data_size > 0) {\n"
+      "  total_size += $tag_size$ + "
+        "::google::protobuf::internal::WireFormat::Int32Size(data_size);\n"
+      "}\n"
+      "_$name$_cached_byte_size_ = data_size;\n"
+      "total_size += data_size;\n");
+  } else {
+    printer->Print(variables_,
+      "total_size += $tag_size$ * this->$name$_size() + data_size;\n");
   }
+  printer->Outdent();
+  printer->Print("}\n");
 }
 
 }  // namespace cpp
diff --git a/src/google/protobuf/compiler/cpp/cpp_string_field.cc b/src/google/protobuf/compiler/cpp/cpp_string_field.cc
index 3e694ab7..200e3d68 100644
--- a/src/google/protobuf/compiler/cpp/cpp_string_field.cc
+++ b/src/google/protobuf/compiler/cpp/cpp_string_field.cc
@@ -374,15 +374,17 @@ GenerateMergeFromCodedStream(io::Printer* printer) const {
 void RepeatedStringFieldGenerator::
 GenerateSerializeWithCachedSizes(io::Printer* printer) const {
   printer->Print(variables_,
-    "DO_(::google::protobuf::internal::WireFormat::Write$declared_type$("
-      "$number$, this->$name$(i), output));\n");
+    "for (int i = 0; i < this->$name$_size(); i++) {\n"
+    "  DO_(::google::protobuf::internal::WireFormat::Write$declared_type$("
+        "$number$, this->$name$(i), output));\n"
+    "}\n");
 }
 
 void RepeatedStringFieldGenerator::
 GenerateByteSize(io::Printer* printer) const {
   printer->Print(variables_,
-    "total_size += $tag_size$ * $name$_size();\n"
-    "for (int i = 0; i < $name$_size(); i++) {\n"
+    "total_size += $tag_size$ * this->$name$_size();\n"
+    "for (int i = 0; i < this->$name$_size(); i++) {\n"
     "  total_size += ::google::protobuf::internal::WireFormat::$declared_type$Size(\n"
     "    this->$name$(i));\n"
     "}\n");
diff --git a/src/google/protobuf/compiler/cpp/cpp_test_bad_identifiers.proto b/src/google/protobuf/compiler/cpp/cpp_test_bad_identifiers.proto
index a4d96ac5..79971a95 100644
--- a/src/google/protobuf/compiler/cpp/cpp_test_bad_identifiers.proto
+++ b/src/google/protobuf/compiler/cpp/cpp_test_bad_identifiers.proto
@@ -54,6 +54,14 @@ message TestConflictingSymbolNames {
   optional int32 total_size = 6;
   optional int32 tag = 7;
 
+  enum TestEnum { FOO = 1; }
+  message Data1 { repeated int32 data = 1; }
+  message Data2 { repeated TestEnum data = 1; }
+  message Data3 { repeated string data = 1; }
+  message Data4 { repeated Data4 data = 1; }
+  message Data5 { repeated string data = 1 [ctype=STRING_PIECE]; }
+  message Data6 { repeated string data = 1 [ctype=CORD]; }
+
   optional int32 source = 8;
   optional int32 value = 9;
   optional int32 file = 10;
diff --git a/src/google/protobuf/compiler/cpp/cpp_unittest.cc b/src/google/protobuf/compiler/cpp/cpp_unittest.cc
index 393c923b..c7e4ee3d 100644
--- a/src/google/protobuf/compiler/cpp/cpp_unittest.cc
+++ b/src/google/protobuf/compiler/cpp/cpp_unittest.cc
@@ -225,7 +225,6 @@ TEST(GeneratedMessageTest, ClearOneField) {
 
 TEST(GeneratedMessageTest, CopyFrom) {
   unittest::TestAllTypes message1, message2;
-  string data;
 
   TestUtil::SetAllFields(&message1);
   message2.CopyFrom(message1);
@@ -413,6 +412,13 @@ TEST(GeneratedMessageTest, Serialization) {
   EXPECT_TRUE(message2.ParseFromString(data));
   TestUtil::ExpectAllFieldsSet(message2);
 
+
+  unittest::TestPackedTypes packed_message1, packed_message2;
+  string packed_data;
+  TestUtil::SetPackedFields(&packed_message1);
+  packed_message1.SerializeToString(&packed_data);
+  EXPECT_TRUE(packed_message2.ParseFromString(packed_data));
+  TestUtil::ExpectPackedFieldsSet(packed_message2);
 }