Push out changes from internal codebase.

All Languages
* Repeated fields of primitive types (types other that string, group, and
  nested messages) may now use the option [packed = true] to get a more
  efficient encoding.  In the new encoding, the entire list is written
  as a single byte blob using the "length-delimited" wire type.  Within
  this blob, the individual values are encoded the same way they would
  be normally except without a tag before each value (thus, they are
  tightly "packed").

C++
* UnknownFieldSet now supports STL-like iteration.
* Message interface has method ParseFromBoundedZeroCopyStream() which parses
  a limited number of bytes from an input stream rather than parsing until
  EOF.

Java
* Fixed bug where Message.mergeFrom(Message) failed to merge extensions.
* Message interface has new method toBuilder() which is equivalent to
  newBuilderForType().mergeFrom(this).
* All enums now implement the ProtocolMessageEnum interface.
* Setting a field to null now throws NullPointerException.
* Fixed tendency for TextFormat's parsing to overflow the stack when
  parsing large string values.  The underlying problem is with Java's
  regex implementation (which unfortunately uses recursive backtracking
  rather than building an NFA).  Worked around by making use of possesive
  quantifiers.

Python
* Updated RPC interfaces to allow for blocking operation.  A client may
  now pass None for a callback when making an RPC, in which case the
  call will block until the response is received, and the response
  object will be returned directly to the caller.  This interface change
  cannot be used in practice until RPC implementations are updated to
  implement it.

5 files changed, 245 insertions, 20 deletions

diff --git a/src/google/protobuf/compiler/java/java_enum.cc b/src/google/protobuf/compiler/java/java_enum.cc
index b1484763..4aac6493 100644
--- a/src/google/protobuf/compiler/java/java_enum.cc
+++ b/src/google/protobuf/compiler/java/java_enum.cc
@@ -71,7 +71,8 @@ void EnumGenerator::Generate(io::Printer* printer) {
     descriptor_->containing_type() == NULL &&
     descriptor_->file()->options().java_multiple_files();
   printer->Print(
-    "public $static$ enum $classname$ {\n",
+    "public $static$ enum $classname$\n"
+    "    implements com.google.protobuf.ProtocolMessageEnum {\n",
     "static", is_own_file ? "" : "static",
     "classname", descriptor_->name());
   printer->Indent();
diff --git a/src/google/protobuf/compiler/java/java_enum_field.cc b/src/google/protobuf/compiler/java/java_enum_field.cc
index e95fdab4..2153042d 100644
--- a/src/google/protobuf/compiler/java/java_enum_field.cc
+++ b/src/google/protobuf/compiler/java/java_enum_field.cc
@@ -39,6 +39,7 @@
 #include <google/protobuf/stubs/common.h>
 #include <google/protobuf/compiler/java/java_helpers.h>
 #include <google/protobuf/io/printer.h>
+#include <google/protobuf/wire_format_inl.h>
 #include <google/protobuf/stubs/strutil.h>
 
 namespace google {
@@ -64,6 +65,9 @@ void SetEnumVariables(const FieldDescriptor* descriptor,
   (*variables)["number"] = SimpleItoa(descriptor->number());
   (*variables)["type"] = type;
   (*variables)["default"] = type + "." + default_value->name();
+  (*variables)["tag"] = SimpleItoa(internal::WireFormat::MakeTag(descriptor));
+  (*variables)["tag_size"] = SimpleItoa(
+      internal::WireFormat::TagSize(descriptor->number(), descriptor->type()));
 }
 
 }  // namespace
@@ -97,6 +101,9 @@ GenerateBuilderMembers(io::Printer* printer) const {
     "  return result.get$capitalized_name$();\n"
     "}\n"
     "public Builder set$capitalized_name$($type$ value) {\n"
+    "  if (value == null) {\n"
+    "    throw new NullPointerException();\n"
+    "  }\n"
     "  result.has$capitalized_name$ = true;\n"
     "  result.$name$_ = value;\n"
     "  return this;\n"
@@ -176,6 +183,12 @@ GenerateMembers(io::Printer* printer) const {
     "public $type$ get$capitalized_name$(int index) {\n"
     "  return $name$_.get(index);\n"
     "}\n");
+
+  if (descriptor_->options().packed() &&
+      descriptor_->file()->options().optimize_for() == FileOptions::SPEED) {
+    printer->Print(variables_,
+      "private int $name$MemoizedSerializedSize;\n");
+  }
 }
 
 void RepeatedEnumFieldGenerator::
@@ -195,10 +208,16 @@ GenerateBuilderMembers(io::Printer* printer) const {
     "  return result.get$capitalized_name$(index);\n"
     "}\n"
     "public Builder set$capitalized_name$(int index, $type$ value) {\n"
+    "  if (value == null) {\n"
+    "    throw new NullPointerException();\n"
+    "  }\n"
     "  result.$name$_.set(index, value);\n"
     "  return this;\n"
     "}\n"
     "public Builder add$capitalized_name$($type$ value) {\n"
+    "  if (value == null) {\n"
+    "    throw new NullPointerException();\n"
+    "  }\n"
     "  if (result.$name$_.isEmpty()) {\n"
     "    result.$name$_ = new java.util.ArrayList<$type$>();\n"
     "  }\n"
@@ -241,6 +260,16 @@ GenerateBuildingCode(io::Printer* printer) const {
 
 void RepeatedEnumFieldGenerator::
 GenerateParsingCode(io::Printer* printer) const {
+  // If packed, set up the while loop
+  if (descriptor_->options().packed()) {
+    printer->Print(variables_,
+      "int length = input.readRawVarint32();\n"
+      "int oldLimit = input.pushLimit(length);\n"
+      "while(input.getBytesUntilLimit() > 0) {\n");
+    printer->Indent();
+  }
+
+  // Read and store the enum
   printer->Print(variables_,
     "int rawValue = input.readEnum();\n"
     "$type$ value = $type$.valueOf(rawValue);\n"
@@ -249,23 +278,68 @@ GenerateParsingCode(io::Printer* printer) const {
     "} else {\n"
     "  add$capitalized_name$(value);\n"
     "}\n");
+
+  if (descriptor_->options().packed()) {
+    printer->Outdent();
+    printer->Print(variables_,
+      "}\n"
+      "input.popLimit(oldLimit);\n");
+  }
 }
 
 void RepeatedEnumFieldGenerator::
 GenerateSerializationCode(io::Printer* printer) const {
-  printer->Print(variables_,
-    "for ($type$ element : get$capitalized_name$List()) {\n"
-    "  output.writeEnum($number$, element.getNumber());\n"
-    "}\n");
+  if (descriptor_->options().packed()) {
+    printer->Print(variables_,
+      "if (get$capitalized_name$List().size() > 0) {\n"
+      "  output.writeRawVarint32($tag$);\n"
+      "  output.writeRawVarint32($name$MemoizedSerializedSize);\n"
+      "}\n"
+      "for ($type$ element : get$capitalized_name$List()) {\n"
+      "  output.writeEnumNoTag(element.getNumber());\n"
+      "}\n");
+  } else {
+    printer->Print(variables_,
+      "for ($type$ element : get$capitalized_name$List()) {\n"
+      "  output.writeEnum($number$, element.getNumber());\n"
+      "}\n");
+  }
 }
 
 void RepeatedEnumFieldGenerator::
 GenerateSerializedSizeCode(io::Printer* printer) const {
   printer->Print(variables_,
+    "{\n"
+    "  int dataSize = 0;\n");
+  printer->Indent();
+
+  printer->Print(variables_,
     "for ($type$ element : get$capitalized_name$List()) {\n"
-    "  size += com.google.protobuf.CodedOutputStream\n"
-    "    .computeEnumSize($number$, element.getNumber());\n"
+    "  dataSize += com.google.protobuf.CodedOutputStream\n"
+    "    .computeEnumSizeNoTag(element.getNumber());\n"
     "}\n");
+  printer->Print(
+    "size += dataSize;\n");
+  if (descriptor_->options().packed()) {
+    printer->Print(variables_,
+      "if (!get$capitalized_name$List().isEmpty()) {"
+      "  size += $tag_size$;\n"
+      "  size += com.google.protobuf.CodedOutputStream\n"
+      "    .computeRawVarint32Size(dataSize);\n"
+      "}");
+  } else {
+    printer->Print(variables_,
+      "size += $tag_size$ * get$capitalized_name$List().size();\n");
+  }
+
+  // cache the data size for packed fields.
+  if (descriptor_->options().packed()) {
+    printer->Print(variables_,
+      "$name$MemoizedSerializedSize = dataSize;\n");
+  }
+
+  printer->Outdent();
+  printer->Print("}\n");
 }
 
 string RepeatedEnumFieldGenerator::GetBoxedType() const {
diff --git a/src/google/protobuf/compiler/java/java_message.cc b/src/google/protobuf/compiler/java/java_message.cc
index 3fdd1d57..9a4b2f79 100644
--- a/src/google/protobuf/compiler/java/java_message.cc
+++ b/src/google/protobuf/compiler/java/java_message.cc
@@ -41,7 +41,7 @@
 #include <google/protobuf/stubs/strutil.h>
 #include <google/protobuf/io/printer.h>
 #include <google/protobuf/io/coded_stream.h>
-#include <google/protobuf/wire_format.h>
+#include <google/protobuf/wire_format_inl.h>
 #include <google/protobuf/descriptor.pb.h>
 
 namespace google {
@@ -500,6 +500,7 @@ void MessageGenerator::GenerateBuilder(io::Printer* printer) {
     "public static Builder newBuilder($classname$ prototype) {\n"
     "  return new Builder().mergeFrom(prototype);\n"
     "}\n"
+    "public Builder toBuilder() { return newBuilder(this); }\n"
     "\n",
     "classname", ClassName(descriptor_));
 
@@ -634,6 +635,13 @@ void MessageGenerator::GenerateCommonBuilderMethods(io::Printer* printer) {
     }
 
     printer->Outdent();
+
+    // if message type has extensions
+    if (descriptor_->extension_range_count() > 0) {
+      printer->Print(
+        "  this.mergeExtensionFields(other);\n");
+    }
+
     printer->Print(
       "  this.mergeUnknownFields(other.getUnknownFields());\n"
       "  return this;\n"
@@ -692,7 +700,7 @@ void MessageGenerator::GenerateBuilderParsingMethods(io::Printer* printer) {
   for (int i = 0; i < descriptor_->field_count(); i++) {
     const FieldDescriptor* field = sorted_fields[i];
     uint32 tag = WireFormat::MakeTag(field->number(),
-      WireFormat::WireTypeForFieldType(field->type()));
+      WireFormat::WireTypeForField(field));
 
     printer->Print(
       "case $tag$: {\n",
diff --git a/src/google/protobuf/compiler/java/java_message_field.cc b/src/google/protobuf/compiler/java/java_message_field.cc
index c85a1598..bbddddde 100644
--- a/src/google/protobuf/compiler/java/java_message_field.cc
+++ b/src/google/protobuf/compiler/java/java_message_field.cc
@@ -94,6 +94,9 @@ GenerateBuilderMembers(io::Printer* printer) const {
     "  return result.get$capitalized_name$();\n"
     "}\n"
     "public Builder set$capitalized_name$($type$ value) {\n"
+    "  if (value == null) {\n"
+    "    throw new NullPointerException();\n"
+    "  }\n"
     "  result.has$capitalized_name$ = true;\n"
     "  result.$name$_ = value;\n"
     "  return this;\n"
@@ -216,6 +219,9 @@ GenerateBuilderMembers(io::Printer* printer) const {
     "  return result.get$capitalized_name$(index);\n"
     "}\n"
     "public Builder set$capitalized_name$(int index, $type$ value) {\n"
+    "  if (value == null) {\n"
+    "    throw new NullPointerException();\n"
+    "  }\n"
     "  result.$name$_.set(index, value);\n"
     "  return this;\n"
     "}\n"
@@ -225,6 +231,9 @@ GenerateBuilderMembers(io::Printer* printer) const {
     "  return this;\n"
     "}\n"
     "public Builder add$capitalized_name$($type$ value) {\n"
+    "  if (value == null) {\n"
+    "    throw new NullPointerException();\n"
+    "  }\n"
     "  if (result.$name$_.isEmpty()) {\n"
     "    result.$name$_ = new java.util.ArrayList<$type$>();\n"
     "  }\n"
diff --git a/src/google/protobuf/compiler/java/java_primitive_field.cc b/src/google/protobuf/compiler/java/java_primitive_field.cc
index fb4e650f..798e8608 100644
--- a/src/google/protobuf/compiler/java/java_primitive_field.cc
+++ b/src/google/protobuf/compiler/java/java_primitive_field.cc
@@ -39,6 +39,7 @@
 #include <google/protobuf/stubs/common.h>
 #include <google/protobuf/compiler/java/java_helpers.h>
 #include <google/protobuf/io/printer.h>
+#include <google/protobuf/wire_format_inl.h>
 #include <google/protobuf/stubs/strutil.h>
 #include <google/protobuf/stubs/substitute.h>
 
@@ -47,6 +48,8 @@ namespace protobuf {
 namespace compiler {
 namespace java {
 
+using internal::WireFormat;
+
 namespace {
 
 const char* PrimitiveTypeName(JavaType type) {
@@ -69,6 +72,26 @@ const char* PrimitiveTypeName(JavaType type) {
   return NULL;
 }
 
+bool IsReferenceType(JavaType type) {
+  switch (type) {
+    case JAVATYPE_INT    : return false;
+    case JAVATYPE_LONG   : return false;
+    case JAVATYPE_FLOAT  : return false;
+    case JAVATYPE_DOUBLE : return false;
+    case JAVATYPE_BOOLEAN: return false;
+    case JAVATYPE_STRING : return true;
+    case JAVATYPE_BYTES  : return true;
+    case JAVATYPE_ENUM   : return true;
+    case JAVATYPE_MESSAGE: return true;
+
+    // No default because we want the compiler to complain if any new
+    // JavaTypes are added.
+  }
+
+  GOOGLE_LOG(FATAL) << "Can't get here.";
+  return false;
+}
+
 const char* GetCapitalizedType(const FieldDescriptor* field) {
   switch (field->type()) {
     case FieldDescriptor::TYPE_INT32   : return "Int32"   ;
@@ -108,6 +131,38 @@ bool AllPrintableAscii(const string& text) {
   return true;
 }
 
+// For encodings with fixed sizes, returns that size in bytes.  Otherwise
+// returns -1.
+int FixedSize(FieldDescriptor::Type type) {
+  switch (type) {
+    case FieldDescriptor::TYPE_INT32   : return -1;
+    case FieldDescriptor::TYPE_INT64   : return -1;
+    case FieldDescriptor::TYPE_UINT32  : return -1;
+    case FieldDescriptor::TYPE_UINT64  : return -1;
+    case FieldDescriptor::TYPE_SINT32  : return -1;
+    case FieldDescriptor::TYPE_SINT64  : return -1;
+    case FieldDescriptor::TYPE_FIXED32 : return WireFormat::kFixed32Size;
+    case FieldDescriptor::TYPE_FIXED64 : return WireFormat::kFixed64Size;
+    case FieldDescriptor::TYPE_SFIXED32: return WireFormat::kSFixed32Size;
+    case FieldDescriptor::TYPE_SFIXED64: return WireFormat::kSFixed64Size;
+    case FieldDescriptor::TYPE_FLOAT   : return WireFormat::kFloatSize;
+    case FieldDescriptor::TYPE_DOUBLE  : return WireFormat::kDoubleSize;
+
+    case FieldDescriptor::TYPE_BOOL    : return WireFormat::kBoolSize;
+    case FieldDescriptor::TYPE_ENUM    : return -1;
+
+    case FieldDescriptor::TYPE_STRING  : return -1;
+    case FieldDescriptor::TYPE_BYTES   : return -1;
+    case FieldDescriptor::TYPE_GROUP   : return -1;
+    case FieldDescriptor::TYPE_MESSAGE : return -1;
+
+    // No default because we want the compiler to complain if any new
+    // types are added.
+  }
+  GOOGLE_LOG(FATAL) << "Can't get here.";
+  return -1;
+}
+
 string DefaultValue(const FieldDescriptor* field) {
   // Switch on cpp_type since we need to know which default_value_* method
   // of FieldDescriptor to call.
@@ -177,8 +232,22 @@ void SetPrimitiveVariables(const FieldDescriptor* descriptor,
   (*variables)["boxed_type"] = BoxedPrimitiveTypeName(GetJavaType(descriptor));
   (*variables)["default"] = DefaultValue(descriptor);
   (*variables)["capitalized_type"] = GetCapitalizedType(descriptor);
+  (*variables)["tag"] = SimpleItoa(WireFormat::MakeTag(descriptor));
+  (*variables)["tag_size"] = SimpleItoa(
+      WireFormat::TagSize(descriptor->number(), descriptor->type()));
+  if (IsReferenceType(GetJavaType(descriptor))) {
+    (*variables)["null_check"] =
+        "  if (value == null) {\n"
+        "    throw new NullPointerException();\n"
+        "  }\n";
+  } else {
+    (*variables)["null_check"] = "";
+  }
+  int fixed_size = FixedSize(descriptor->type());
+  if (fixed_size != -1) {
+    (*variables)["fixed_size"] = SimpleItoa(fixed_size);
+  }
 }
-
 }  // namespace
 
 // ===================================================================
@@ -210,6 +279,7 @@ GenerateBuilderMembers(io::Printer* printer) const {
     "  return result.get$capitalized_name$();\n"
     "}\n"
     "public Builder set$capitalized_name$($type$ value) {\n"
+    "$null_check$"
     "  result.has$capitalized_name$ = true;\n"
     "  result.$name$_ = value;\n"
     "  return this;\n"
@@ -283,6 +353,12 @@ GenerateMembers(io::Printer* printer) const {
     "public $type$ get$capitalized_name$(int index) {\n"
     "  return $name$_.get(index);\n"
     "}\n");
+
+  if (descriptor_->options().packed() &&
+      descriptor_->file()->options().optimize_for() == FileOptions::SPEED) {
+    printer->Print(variables_,
+      "private int $name$MemoizedSerializedSize;\n");
+  }
 }
 
 void RepeatedPrimitiveFieldGenerator::
@@ -302,10 +378,12 @@ GenerateBuilderMembers(io::Printer* printer) const {
     "  return result.get$capitalized_name$(index);\n"
     "}\n"
     "public Builder set$capitalized_name$(int index, $type$ value) {\n"
+    "$null_check$"
     "  result.$name$_.set(index, value);\n"
     "  return this;\n"
     "}\n"
     "public Builder add$capitalized_name$($type$ value) {\n"
+    "$null_check$"
     "  if (result.$name$_.isEmpty()) {\n"
     "    result.$name$_ = new java.util.ArrayList<$boxed_type$>();\n"
     "  }\n"
@@ -348,25 +426,80 @@ GenerateBuildingCode(io::Printer* printer) const {
 
 void RepeatedPrimitiveFieldGenerator::
 GenerateParsingCode(io::Printer* printer) const {
-  printer->Print(variables_,
-    "add$capitalized_name$(input.read$capitalized_type$());\n");
+  if (descriptor_->options().packed()) {
+    printer->Print(variables_,
+      "int length = input.readRawVarint32();\n"
+      "int limit = input.pushLimit(length);\n"
+      "while (input.getBytesUntilLimit() > 0) {\n"
+      "  add$capitalized_name$(input.read$capitalized_type$());\n"
+      "}\n"
+      "input.popLimit(limit);\n");
+  } else {
+    printer->Print(variables_,
+      "add$capitalized_name$(input.read$capitalized_type$());\n");
+  }
 }
 
 void RepeatedPrimitiveFieldGenerator::
 GenerateSerializationCode(io::Printer* printer) const {
-  printer->Print(variables_,
-    "for ($type$ element : get$capitalized_name$List()) {\n"
-    "  output.write$capitalized_type$($number$, element);\n"
-    "}\n");
+  if (descriptor_->options().packed()) {
+    printer->Print(variables_,
+      "if (get$capitalized_name$List().size() > 0) {\n"
+      "  output.writeRawVarint32($tag$);\n"
+      "  output.writeRawVarint32($name$MemoizedSerializedSize);\n"
+      "}\n"
+      "for ($type$ element : get$capitalized_name$List()) {\n"
+      "  output.write$capitalized_type$NoTag(element);\n"
+      "}\n");
+  } else {
+    printer->Print(variables_,
+      "for ($type$ element : get$capitalized_name$List()) {\n"
+      "  output.write$capitalized_type$($number$, element);\n"
+      "}\n");
+  }
 }
 
 void RepeatedPrimitiveFieldGenerator::
 GenerateSerializedSizeCode(io::Printer* printer) const {
   printer->Print(variables_,
-    "for ($type$ element : get$capitalized_name$List()) {\n"
-    "  size += com.google.protobuf.CodedOutputStream\n"
-    "    .compute$capitalized_type$Size($number$, element);\n"
-    "}\n");
+    "{\n"
+    "  int dataSize = 0;\n");
+  printer->Indent();
+
+  if (FixedSize(descriptor_->type()) == -1) {
+    printer->Print(variables_,
+      "for ($type$ element : get$capitalized_name$List()) {\n"
+      "  dataSize += com.google.protobuf.CodedOutputStream\n"
+      "    .compute$capitalized_type$SizeNoTag(element);\n"
+      "}\n");
+  } else {
+    printer->Print(variables_,
+      "dataSize = $fixed_size$ * get$capitalized_name$List().size();\n");
+  }
+
+  printer->Print(
+      "size += dataSize;\n");
+
+  if (descriptor_->options().packed()) {
+    printer->Print(variables_,
+      "if (!get$capitalized_name$List().isEmpty()) {\n"
+      "  size += $tag_size$;\n"
+      "  size += com.google.protobuf.CodedOutputStream\n"
+      "      .computeInt32SizeNoTag(dataSize);\n"
+      "}\n");
+  } else {
+    printer->Print(variables_,
+      "size += $tag_size$ * get$capitalized_name$List().size();\n");
+  }
+
+  // cache the data size for packed fields.
+  if (descriptor_->options().packed()) {
+    printer->Print(variables_,
+      "$name$MemoizedSerializedSize = dataSize;\n");
+  }
+
+  printer->Outdent();
+  printer->Print("}\n");
 }
 
 string RepeatedPrimitiveFieldGenerator::GetBoxedType() const {