Push out changes from internal codebase.

All Languages
* Repeated fields of primitive types (types other that string, group, and
  nested messages) may now use the option [packed = true] to get a more
  efficient encoding.  In the new encoding, the entire list is written
  as a single byte blob using the "length-delimited" wire type.  Within
  this blob, the individual values are encoded the same way they would
  be normally except without a tag before each value (thus, they are
  tightly "packed").

C++
* UnknownFieldSet now supports STL-like iteration.
* Message interface has method ParseFromBoundedZeroCopyStream() which parses
  a limited number of bytes from an input stream rather than parsing until
  EOF.

Java
* Fixed bug where Message.mergeFrom(Message) failed to merge extensions.
* Message interface has new method toBuilder() which is equivalent to
  newBuilderForType().mergeFrom(this).
* All enums now implement the ProtocolMessageEnum interface.
* Setting a field to null now throws NullPointerException.
* Fixed tendency for TextFormat's parsing to overflow the stack when
  parsing large string values.  The underlying problem is with Java's
  regex implementation (which unfortunately uses recursive backtracking
  rather than building an NFA).  Worked around by making use of possesive
  quantifiers.

Python
* Updated RPC interfaces to allow for blocking operation.  A client may
  now pass None for a callback when making an RPC, in which case the
  call will block until the response is received, and the response
  object will be returned directly to the caller.  This interface change
  cannot be used in practice until RPC implementations are updated to
  implement it.

14 files changed, 432 insertions, 71 deletions

diff --git a/src/google/protobuf/compiler/command_line_interface.cc b/src/google/protobuf/compiler/command_line_interface.cc
index 089844f4..09473ec7 100644
--- a/src/google/protobuf/compiler/command_line_interface.cc
+++ b/src/google/protobuf/compiler/command_line_interface.cc
@@ -751,7 +751,8 @@ bool CommandLineInterface::GenerateOutput(
   if (!output_directive.generator->Generate(
       parsed_file, output_directive.parameter, &output_directory, &error)) {
     // Generator returned an error.
-    cerr << output_directive.name << ": " << error << endl;
+    cerr << parsed_file->name() << ": " << output_directive.name << ": "
+         << error << endl;
     return false;
   }
 
diff --git a/src/google/protobuf/compiler/cpp/cpp_enum_field.cc b/src/google/protobuf/compiler/cpp/cpp_enum_field.cc
index c998f20b..19779a8a 100644
--- a/src/google/protobuf/compiler/cpp/cpp_enum_field.cc
+++ b/src/google/protobuf/compiler/cpp/cpp_enum_field.cc
@@ -158,7 +158,13 @@ RepeatedEnumFieldGenerator::~RepeatedEnumFieldGenerator() {}
 
 void RepeatedEnumFieldGenerator::
 GeneratePrivateMembers(io::Printer* printer) const {
-  printer->Print(variables_, "::google::protobuf::RepeatedField<int> $name$_;\n");
+  printer->Print(variables_,
+    "::google::protobuf::RepeatedField<int> $name$_;\n");
+  if (descriptor_->options().packed() &&
+      descriptor_->file()->options().optimize_for() == FileOptions::SPEED) {
+    printer->Print(variables_,
+      "mutable int _$name$_cached_byte_size_;\n");
+  }
 }
 
 void RepeatedEnumFieldGenerator::
@@ -217,31 +223,84 @@ GenerateInitializer(io::Printer* printer) const {
 
 void RepeatedEnumFieldGenerator::
 GenerateMergeFromCodedStream(io::Printer* printer) const {
-  printer->Print(variables_,
-    "int value;\n"
-    "DO_(::google::protobuf::internal::WireFormat::ReadEnum(input, &value));\n"
-    "if ($type$_IsValid(value)) {\n"
-    "  add_$name$(static_cast< $type$ >(value));\n"
-    "} else {\n"
-    "  mutable_unknown_fields()->AddField($number$)->add_varint(value);\n"
-    "}\n");
+  if (descriptor_->options().packed()) {
+    printer->Print(variables_,
+      "::google::protobuf::uint32 length;\n"
+      "DO_(input->ReadVarint32(&length));\n"
+      "::google::protobuf::io::CodedInputStream::Limit limit = "
+          "input->PushLimit(length);\n"
+      "while (input->BytesUntilLimit() > 0) {\n"
+      "  int value;\n"
+      "  DO_(::google::protobuf::internal::WireFormat::ReadEnum(input, &value));\n"
+      "  if ($type$_IsValid(value)) {\n"
+      "    add_$name$(static_cast< $type$ >(value));\n"
+      "  }\n"
+      "}\n"
+      "input->PopLimit(limit);\n");
+  } else {
+    printer->Print(variables_,
+      "int value;\n"
+      "DO_(::google::protobuf::internal::WireFormat::ReadEnum(input, &value));\n"
+      "if ($type$_IsValid(value)) {\n"
+      "  add_$name$(static_cast< $type$ >(value));\n"
+      "} else {\n"
+      "  mutable_unknown_fields()->AddField($number$)->add_varint(value);\n"
+      "}\n");
+  }
 }
 
 void RepeatedEnumFieldGenerator::
 GenerateSerializeWithCachedSizes(io::Printer* printer) const {
+  if (descriptor_->options().packed()) {
+    // Write the tag and the size.
+    printer->Print(variables_,
+      "if (this->$name$_size() > 0) {\n"
+      "  DO_(::google::protobuf::internal::WireFormat::WriteTag("
+          "$number$, ::google::protobuf::internal::WireFormat::WIRETYPE_LENGTH_DELIMITED,"
+          "output));\n"
+      "  DO_(output->WriteVarint32(_$name$_cached_byte_size_));\n"
+      "}\n");
+  }
   printer->Print(variables_,
-    "DO_(::google::protobuf::internal::WireFormat::WriteEnum("
-      "$number$, this->$name$(i), output));\n");
+      "for (int i = 0; i < this->$name$_size(); i++) {\n");
+  if (descriptor_->options().packed()) {
+    printer->Print(variables_,
+      "  DO_(::google::protobuf::internal::WireFormat::WriteEnumNoTag("
+          "this->$name$(i), output));\n");
+  } else {
+    printer->Print(variables_,
+      "  DO_(::google::protobuf::internal::WireFormat::WriteEnum("
+          "$number$, this->$name$(i), output));\n");
+  }
+  printer->Print("}\n");
 }
 
 void RepeatedEnumFieldGenerator::
 GenerateByteSize(io::Printer* printer) const {
   printer->Print(variables_,
-    "total_size += $tag_size$ * $name$_size();\n"
-    "for (int i = 0; i < $name$_size(); i++) {\n"
-    "  total_size += ::google::protobuf::internal::WireFormat::EnumSize(\n"
-    "    this->$name$(i));\n"
-    "}\n");
+    "{\n"
+    "  int data_size = 0;\n");
+  printer->Indent();
+  printer->Print(variables_,
+      "for (int i = 0; i < this->$name$_size(); i++) {\n"
+      "  data_size += ::google::protobuf::internal::WireFormat::EnumSize(\n"
+      "    this->$name$(i));\n"
+      "}\n");
+
+  if (descriptor_->options().packed()) {
+    printer->Print(variables_,
+      "if (data_size > 0) {\n"
+      "  total_size += $tag_size$ + "
+        "::google::protobuf::internal::WireFormat::Int32Size(data_size);\n"
+      "}\n"
+      "_$name$_cached_byte_size_ = data_size;\n"
+      "total_size += data_size;\n");
+  } else {
+    printer->Print(variables_,
+      "total_size += $tag_size$ * this->$name$_size() + data_size;\n");
+  }
+  printer->Outdent();
+  printer->Print("}\n");
 }
 
 }  // namespace cpp
diff --git a/src/google/protobuf/compiler/cpp/cpp_message.cc b/src/google/protobuf/compiler/cpp/cpp_message.cc
index eacceeaf..c6843e93 100644
--- a/src/google/protobuf/compiler/cpp/cpp_message.cc
+++ b/src/google/protobuf/compiler/cpp/cpp_message.cc
@@ -41,7 +41,7 @@
 #include <google/protobuf/stubs/strutil.h>
 #include <google/protobuf/io/printer.h>
 #include <google/protobuf/io/coded_stream.h>
-#include <google/protobuf/wire_format.h>
+#include <google/protobuf/wire_format_inl.h>
 #include <google/protobuf/descriptor.pb.h>
 
 namespace google {
@@ -1169,10 +1169,9 @@ GenerateMergeFromCodedStream(io::Printer* printer) {
         "    goto handle_uninterpreted;\n"
         "  }\n",
         "number", SimpleItoa(field->number()),
-        "wiretype", kWireTypeNames[
-          WireFormat::WireTypeForFieldType(field->type())]);
+        "wiretype", kWireTypeNames[WireFormat::WireTypeForField(field)]);
 
-      if (i > 0 || field->is_repeated()) {
+      if (i > 0 || (field->is_repeated() && !field->options().packed())) {
         printer->Print(
           " parse_$name$:\n",
           "name", field->name());
@@ -1184,7 +1183,7 @@ GenerateMergeFromCodedStream(io::Printer* printer) {
 
       // switch() is slow since it can't be predicted well.  Insert some if()s
       // here that attempt to predict the next tag.
-      if (field->is_repeated()) {
+      if (field->is_repeated() && !field->options().packed()) {
         // Expect repeats of this field.
         printer->Print(
           "if (input->ExpectTag($tag$)) goto parse_$name$;\n",
@@ -1283,22 +1282,20 @@ void MessageGenerator::GenerateSerializeOneField(
     io::Printer* printer, const FieldDescriptor* field) {
   PrintFieldComment(printer, field);
 
-  if (field->is_repeated()) {
-    printer->Print(
-      "for (int i = 0; i < $name$_.size(); i++) {\n",
-      "name", FieldName(field));
-  } else {
+  if (!field->is_repeated()) {
     printer->Print(
       "if (_has_bit($index$)) {\n",
       "index", SimpleItoa(field->index()));
+    printer->Indent();
   }
 
-  printer->Indent();
-
   field_generators_.get(field).GenerateSerializeWithCachedSizes(printer);
 
-  printer->Outdent();
-  printer->Print("}\n\n");
+  if (!field->is_repeated()) {
+    printer->Outdent();
+    printer->Print("}\n");
+  }
+  printer->Print("\n");
 }
 
 void MessageGenerator::GenerateSerializeOneExtensionRange(
diff --git a/src/google/protobuf/compiler/cpp/cpp_message_field.cc b/src/google/protobuf/compiler/cpp/cpp_message_field.cc
index d1c31067..7d57a6df 100644
--- a/src/google/protobuf/compiler/cpp/cpp_message_field.cc
+++ b/src/google/protobuf/compiler/cpp/cpp_message_field.cc
@@ -232,15 +232,17 @@ GenerateMergeFromCodedStream(io::Printer* printer) const {
 void RepeatedMessageFieldGenerator::
 GenerateSerializeWithCachedSizes(io::Printer* printer) const {
   printer->Print(variables_,
-    "DO_(::google::protobuf::internal::WireFormat::Write$declared_type$NoVirtual("
-      "$number$, this->$name$(i), output));\n");
+    "for (int i = 0; i < this->$name$_size(); i++) {\n"
+    "  DO_(::google::protobuf::internal::WireFormat::Write$declared_type$NoVirtual("
+        "$number$, this->$name$(i), output));\n"
+    "}\n");
 }
 
 void RepeatedMessageFieldGenerator::
 GenerateByteSize(io::Printer* printer) const {
   printer->Print(variables_,
-    "total_size += $tag_size$ * $name$_size();\n"
-    "for (int i = 0; i < $name$_size(); i++) {\n"
+    "total_size += $tag_size$ * this->$name$_size();\n"
+    "for (int i = 0; i < this->$name$_size(); i++) {\n"
     "  total_size +=\n"
     "    ::google::protobuf::internal::WireFormat::$declared_type$SizeNoVirtual(\n"
     "      this->$name$(i));\n"
diff --git a/src/google/protobuf/compiler/cpp/cpp_primitive_field.cc b/src/google/protobuf/compiler/cpp/cpp_primitive_field.cc
index ef4072f0..768d30cc 100644
--- a/src/google/protobuf/compiler/cpp/cpp_primitive_field.cc
+++ b/src/google/protobuf/compiler/cpp/cpp_primitive_field.cc
@@ -227,6 +227,11 @@ void RepeatedPrimitiveFieldGenerator::
 GeneratePrivateMembers(io::Printer* printer) const {
   printer->Print(variables_,
     "::google::protobuf::RepeatedField< $type$ > $name$_;\n");
+  if (descriptor_->options().packed() &&
+      descriptor_->file()->options().optimize_for() == FileOptions::SPEED) {
+    printer->Print(variables_,
+      "mutable int _$name$_cached_byte_size_;\n");
+  }
 }
 
 void RepeatedPrimitiveFieldGenerator::
@@ -283,33 +288,90 @@ GenerateInitializer(io::Printer* printer) const {
 
 void RepeatedPrimitiveFieldGenerator::
 GenerateMergeFromCodedStream(io::Printer* printer) const {
-  printer->Print(variables_,
-    "$type$ value;\n"
-    "DO_(::google::protobuf::internal::WireFormat::Read$declared_type$(input, &value));\n"
-    "add_$name$(value);\n");
+  if (descriptor_->options().packed()) {
+    printer->Print("{\n");
+    printer->Indent();
+    printer->Print(variables_,
+      "::google::protobuf::uint32 length;\n"
+      "DO_(input->ReadVarint32(&length));\n"
+      "::google::protobuf::io::CodedInputStream::Limit limit = "
+          "input->PushLimit(length);\n"
+      "while (input->BytesUntilLimit() > 0) {\n"
+      "  $type$ value;\n"
+      "  DO_(::google::protobuf::internal::WireFormat::Read$declared_type$("
+        "input, &value));\n"
+      "  add_$name$(value);\n"
+      "}\n"
+      "input->PopLimit(limit);\n");
+    printer->Outdent();
+    printer->Print("}\n");
+  } else {
+    printer->Print(variables_,
+      "$type$ value;\n"
+      "DO_(::google::protobuf::internal::WireFormat::Read$declared_type$("
+        "input, &value));\n"
+      "add_$name$(value);\n");
+  }
 }
 
 void RepeatedPrimitiveFieldGenerator::
 GenerateSerializeWithCachedSizes(io::Printer* printer) const {
+  if (descriptor_->options().packed()) {
+    // Write the tag and the size.
+    printer->Print(variables_,
+      "if (this->$name$_size() > 0) {\n"
+      "  DO_(::google::protobuf::internal::WireFormat::WriteTag("
+          "$number$, ::google::protobuf::internal::WireFormat::WIRETYPE_LENGTH_DELIMITED,"
+          "output));\n"
+      "  DO_(output->WriteVarint32(_$name$_cached_byte_size_));\n"
+      "}\n");
+  }
   printer->Print(variables_,
-    "DO_(::google::protobuf::internal::WireFormat::Write$declared_type$("
-      "$number$, this->$name$(i), output));\n");
+      "for (int i = 0; i < this->$name$_size(); i++) {\n");
+  if (descriptor_->options().packed()) {
+    printer->Print(variables_,
+      "  DO_(::google::protobuf::internal::WireFormat::Write$declared_type$NoTag("
+          "this->$name$(i), output));\n");
+  } else {
+    printer->Print(variables_,
+      "  DO_(::google::protobuf::internal::WireFormat::Write$declared_type$("
+          "$number$, this->$name$(i), output));\n");
+  }
+  printer->Print("}\n");
 }
 
 void RepeatedPrimitiveFieldGenerator::
 GenerateByteSize(io::Printer* printer) const {
+  printer->Print(variables_,
+    "{\n"
+    "  int data_size = 0;\n");
+  printer->Indent();
   int fixed_size = FixedSize(descriptor_->type());
   if (fixed_size == -1) {
     printer->Print(variables_,
-      "total_size += $tag_size$ * $name$_size();\n"
-      "for (int i = 0; i < $name$_size(); i++) {\n"
-      "  total_size += ::google::protobuf::internal::WireFormat::$declared_type$Size(\n"
+      "for (int i = 0; i < this->$name$_size(); i++) {\n"
+      "  data_size += ::google::protobuf::internal::WireFormat::$declared_type$Size(\n"
       "    this->$name$(i));\n"
       "}\n");
   } else {
     printer->Print(variables_,
-      "total_size += ($tag_size$ + $fixed_size$) * $name$_size();\n");
+      "data_size = $fixed_size$ * this->$name$_size();\n");
+  }
+
+  if (descriptor_->options().packed()) {
+    printer->Print(variables_,
+      "if (data_size > 0) {\n"
+      "  total_size += $tag_size$ + "
+        "::google::protobuf::internal::WireFormat::Int32Size(data_size);\n"
+      "}\n"
+      "_$name$_cached_byte_size_ = data_size;\n"
+      "total_size += data_size;\n");
+  } else {
+    printer->Print(variables_,
+      "total_size += $tag_size$ * this->$name$_size() + data_size;\n");
   }
+  printer->Outdent();
+  printer->Print("}\n");
 }
 
 }  // namespace cpp
diff --git a/src/google/protobuf/compiler/cpp/cpp_string_field.cc b/src/google/protobuf/compiler/cpp/cpp_string_field.cc
index 3e694ab7..200e3d68 100644
--- a/src/google/protobuf/compiler/cpp/cpp_string_field.cc
+++ b/src/google/protobuf/compiler/cpp/cpp_string_field.cc
@@ -374,15 +374,17 @@ GenerateMergeFromCodedStream(io::Printer* printer) const {
 void RepeatedStringFieldGenerator::
 GenerateSerializeWithCachedSizes(io::Printer* printer) const {
   printer->Print(variables_,
-    "DO_(::google::protobuf::internal::WireFormat::Write$declared_type$("
-      "$number$, this->$name$(i), output));\n");
+    "for (int i = 0; i < this->$name$_size(); i++) {\n"
+    "  DO_(::google::protobuf::internal::WireFormat::Write$declared_type$("
+        "$number$, this->$name$(i), output));\n"
+    "}\n");
 }
 
 void RepeatedStringFieldGenerator::
 GenerateByteSize(io::Printer* printer) const {
   printer->Print(variables_,
-    "total_size += $tag_size$ * $name$_size();\n"
-    "for (int i = 0; i < $name$_size(); i++) {\n"
+    "total_size += $tag_size$ * this->$name$_size();\n"
+    "for (int i = 0; i < this->$name$_size(); i++) {\n"
     "  total_size += ::google::protobuf::internal::WireFormat::$declared_type$Size(\n"
     "    this->$name$(i));\n"
     "}\n");
diff --git a/src/google/protobuf/compiler/cpp/cpp_test_bad_identifiers.proto b/src/google/protobuf/compiler/cpp/cpp_test_bad_identifiers.proto
index a4d96ac5..79971a95 100644
--- a/src/google/protobuf/compiler/cpp/cpp_test_bad_identifiers.proto
+++ b/src/google/protobuf/compiler/cpp/cpp_test_bad_identifiers.proto
@@ -54,6 +54,14 @@ message TestConflictingSymbolNames {
   optional int32 total_size = 6;
   optional int32 tag = 7;
 
+  enum TestEnum { FOO = 1; }
+  message Data1 { repeated int32 data = 1; }
+  message Data2 { repeated TestEnum data = 1; }
+  message Data3 { repeated string data = 1; }
+  message Data4 { repeated Data4 data = 1; }
+  message Data5 { repeated string data = 1 [ctype=STRING_PIECE]; }
+  message Data6 { repeated string data = 1 [ctype=CORD]; }
+
   optional int32 source = 8;
   optional int32 value = 9;
   optional int32 file = 10;
diff --git a/src/google/protobuf/compiler/cpp/cpp_unittest.cc b/src/google/protobuf/compiler/cpp/cpp_unittest.cc
index 393c923b..c7e4ee3d 100644
--- a/src/google/protobuf/compiler/cpp/cpp_unittest.cc
+++ b/src/google/protobuf/compiler/cpp/cpp_unittest.cc
@@ -225,7 +225,6 @@ TEST(GeneratedMessageTest, ClearOneField) {
 
 TEST(GeneratedMessageTest, CopyFrom) {
   unittest::TestAllTypes message1, message2;
-  string data;
 
   TestUtil::SetAllFields(&message1);
   message2.CopyFrom(message1);
@@ -413,6 +412,13 @@ TEST(GeneratedMessageTest, Serialization) {
   EXPECT_TRUE(message2.ParseFromString(data));
   TestUtil::ExpectAllFieldsSet(message2);
 
+
+  unittest::TestPackedTypes packed_message1, packed_message2;
+  string packed_data;
+  TestUtil::SetPackedFields(&packed_message1);
+  packed_message1.SerializeToString(&packed_data);
+  EXPECT_TRUE(packed_message2.ParseFromString(packed_data));
+  TestUtil::ExpectPackedFieldsSet(packed_message2);
 }
 
 
diff --git a/src/google/protobuf/compiler/java/java_enum.cc b/src/google/protobuf/compiler/java/java_enum.cc
index b1484763..4aac6493 100644
--- a/src/google/protobuf/compiler/java/java_enum.cc
+++ b/src/google/protobuf/compiler/java/java_enum.cc
@@ -71,7 +71,8 @@ void EnumGenerator::Generate(io::Printer* printer) {
     descriptor_->containing_type() == NULL &&
     descriptor_->file()->options().java_multiple_files();
   printer->Print(
-    "public $static$ enum $classname$ {\n",
+    "public $static$ enum $classname$\n"
+    "    implements com.google.protobuf.ProtocolMessageEnum {\n",
     "static", is_own_file ? "" : "static",
     "classname", descriptor_->name());
   printer->Indent();
diff --git a/src/google/protobuf/compiler/java/java_enum_field.cc b/src/google/protobuf/compiler/java/java_enum_field.cc
index e95fdab4..2153042d 100644
--- a/src/google/protobuf/compiler/java/java_enum_field.cc
+++ b/src/google/protobuf/compiler/java/java_enum_field.cc
@@ -39,6 +39,7 @@
 #include <google/protobuf/stubs/common.h>
 #include <google/protobuf/compiler/java/java_helpers.h>
 #include <google/protobuf/io/printer.h>
+#include <google/protobuf/wire_format_inl.h>
 #include <google/protobuf/stubs/strutil.h>
 
 namespace google {
@@ -64,6 +65,9 @@ void SetEnumVariables(const FieldDescriptor* descriptor,
   (*variables)["number"] = SimpleItoa(descriptor->number());
   (*variables)["type"] = type;
   (*variables)["default"] = type + "." + default_value->name();
+  (*variables)["tag"] = SimpleItoa(internal::WireFormat::MakeTag(descriptor));
+  (*variables)["tag_size"] = SimpleItoa(
+      internal::WireFormat::TagSize(descriptor->number(), descriptor->type()));
 }
 
 }  // namespace
@@ -97,6 +101,9 @@ GenerateBuilderMembers(io::Printer* printer) const {
     "  return result.get$capitalized_name$();\n"
     "}\n"
     "public Builder set$capitalized_name$($type$ value) {\n"
+    "  if (value == null) {\n"
+    "    throw new NullPointerException();\n"
+    "  }\n"
     "  result.has$capitalized_name$ = true;\n"
     "  result.$name$_ = value;\n"
     "  return this;\n"
@@ -176,6 +183,12 @@ GenerateMembers(io::Printer* printer) const {
     "public $type$ get$capitalized_name$(int index) {\n"
     "  return $name$_.get(index);\n"
     "}\n");
+
+  if (descriptor_->options().packed() &&
+      descriptor_->file()->options().optimize_for() == FileOptions::SPEED) {
+    printer->Print(variables_,
+      "private int $name$MemoizedSerializedSize;\n");
+  }
 }
 
 void RepeatedEnumFieldGenerator::
@@ -195,10 +208,16 @@ GenerateBuilderMembers(io::Printer* printer) const {
     "  return result.get$capitalized_name$(index);\n"
     "}\n"
     "public Builder set$capitalized_name$(int index, $type$ value) {\n"
+    "  if (value == null) {\n"
+    "    throw new NullPointerException();\n"
+    "  }\n"
     "  result.$name$_.set(index, value);\n"
     "  return this;\n"
     "}\n"
     "public Builder add$capitalized_name$($type$ value) {\n"
+    "  if (value == null) {\n"
+    "    throw new NullPointerException();\n"
+    "  }\n"
     "  if (result.$name$_.isEmpty()) {\n"
     "    result.$name$_ = new java.util.ArrayList<$type$>();\n"
     "  }\n"
@@ -241,6 +260,16 @@ GenerateBuildingCode(io::Printer* printer) const {
 
 void RepeatedEnumFieldGenerator::
 GenerateParsingCode(io::Printer* printer) const {
+  // If packed, set up the while loop
+  if (descriptor_->options().packed()) {
+    printer->Print(variables_,
+      "int length = input.readRawVarint32();\n"
+      "int oldLimit = input.pushLimit(length);\n"
+      "while(input.getBytesUntilLimit() > 0) {\n");
+    printer->Indent();
+  }
+
+  // Read and store the enum
   printer->Print(variables_,
     "int rawValue = input.readEnum();\n"
     "$type$ value = $type$.valueOf(rawValue);\n"
@@ -249,23 +278,68 @@ GenerateParsingCode(io::Printer* printer) const {
     "} else {\n"
     "  add$capitalized_name$(value);\n"
     "}\n");
+
+  if (descriptor_->options().packed()) {
+    printer->Outdent();
+    printer->Print(variables_,
+      "}\n"
+      "input.popLimit(oldLimit);\n");
+  }
 }
 
 void RepeatedEnumFieldGenerator::
 GenerateSerializationCode(io::Printer* printer) const {
-  printer->Print(variables_,
-    "for ($type$ element : get$capitalized_name$List()) {\n"
-    "  output.writeEnum($number$, element.getNumber());\n"
-    "}\n");
+  if (descriptor_->options().packed()) {
+    printer->Print(variables_,
+      "if (get$capitalized_name$List().size() > 0) {\n"
+      "  output.writeRawVarint32($tag$);\n"
+      "  output.writeRawVarint32($name$MemoizedSerializedSize);\n"
+      "}\n"
+      "for ($type$ element : get$capitalized_name$List()) {\n"
+      "  output.writeEnumNoTag(element.getNumber());\n"
+      "}\n");
+  } else {
+    printer->Print(variables_,
+      "for ($type$ element : get$capitalized_name$List()) {\n"
+      "  output.writeEnum($number$, element.getNumber());\n"
+      "}\n");
+  }
 }
 
 void RepeatedEnumFieldGenerator::
 GenerateSerializedSizeCode(io::Printer* printer) const {
   printer->Print(variables_,
+    "{\n"
+    "  int dataSize = 0;\n");
+  printer->Indent();
+
+  printer->Print(variables_,
     "for ($type$ element : get$capitalized_name$List()) {\n"
-    "  size += com.google.protobuf.CodedOutputStream\n"
-    "    .computeEnumSize($number$, element.getNumber());\n"
+    "  dataSize += com.google.protobuf.CodedOutputStream\n"
+    "    .computeEnumSizeNoTag(element.getNumber());\n"
     "}\n");
+  printer->Print(
+    "size += dataSize;\n");
+  if (descriptor_->options().packed()) {
+    printer->Print(variables_,
+      "if (!get$capitalized_name$List().isEmpty()) {"
+      "  size += $tag_size$;\n"
+      "  size += com.google.protobuf.CodedOutputStream\n"
+      "    .computeRawVarint32Size(dataSize);\n"
+      "}");
+  } else {
+    printer->Print(variables_,
+      "size += $tag_size$ * get$capitalized_name$List().size();\n");
+  }
+
+  // cache the data size for packed fields.
+  if (descriptor_->options().packed()) {
+    printer->Print(variables_,
+      "$name$MemoizedSerializedSize = dataSize;\n");
+  }
+
+  printer->Outdent();
+  printer->Print("}\n");
 }
 
 string RepeatedEnumFieldGenerator::GetBoxedType() const {
diff --git a/src/google/protobuf/compiler/java/java_message.cc b/src/google/protobuf/compiler/java/java_message.cc
index 3fdd1d57..9a4b2f79 100644
--- a/src/google/protobuf/compiler/java/java_message.cc
+++ b/src/google/protobuf/compiler/java/java_message.cc
@@ -41,7 +41,7 @@
 #include <google/protobuf/stubs/strutil.h>
 #include <google/protobuf/io/printer.h>
 #include <google/protobuf/io/coded_stream.h>
-#include <google/protobuf/wire_format.h>
+#include <google/protobuf/wire_format_inl.h>
 #include <google/protobuf/descriptor.pb.h>
 
 namespace google {
@@ -500,6 +500,7 @@ void MessageGenerator::GenerateBuilder(io::Printer* printer) {
     "public static Builder newBuilder($classname$ prototype) {\n"
     "  return new Builder().mergeFrom(prototype);\n"
     "}\n"
+    "public Builder toBuilder() { return newBuilder(this); }\n"
     "\n",
     "classname", ClassName(descriptor_));
 
@@ -634,6 +635,13 @@ void MessageGenerator::GenerateCommonBuilderMethods(io::Printer* printer) {
     }
 
     printer->Outdent();
+
+    // if message type has extensions
+    if (descriptor_->extension_range_count() > 0) {
+      printer->Print(
+        "  this.mergeExtensionFields(other);\n");
+    }
+
     printer->Print(
       "  this.mergeUnknownFields(other.getUnknownFields());\n"
       "  return this;\n"
@@ -692,7 +700,7 @@ void MessageGenerator::GenerateBuilderParsingMethods(io::Printer* printer) {
   for (int i = 0; i < descriptor_->field_count(); i++) {
     const FieldDescriptor* field = sorted_fields[i];
     uint32 tag = WireFormat::MakeTag(field->number(),
-      WireFormat::WireTypeForFieldType(field->type()));
+      WireFormat::WireTypeForField(field));
 
     printer->Print(
       "case $tag$: {\n",
diff --git a/src/google/protobuf/compiler/java/java_message_field.cc b/src/google/protobuf/compiler/java/java_message_field.cc
index c85a1598..bbddddde 100644
--- a/src/google/protobuf/compiler/java/java_message_field.cc
+++ b/src/google/protobuf/compiler/java/java_message_field.cc
@@ -94,6 +94,9 @@ GenerateBuilderMembers(io::Printer* printer) const {
     "  return result.get$capitalized_name$();\n"
     "}\n"
     "public Builder set$capitalized_name$($type$ value) {\n"
+    "  if (value == null) {\n"
+    "    throw new NullPointerException();\n"
+    "  }\n"
     "  result.has$capitalized_name$ = true;\n"
     "  result.$name$_ = value;\n"
     "  return this;\n"
@@ -216,6 +219,9 @@ GenerateBuilderMembers(io::Printer* printer) const {
     "  return result.get$capitalized_name$(index);\n"
     "}\n"
     "public Builder set$capitalized_name$(int index, $type$ value) {\n"
+    "  if (value == null) {\n"
+    "    throw new NullPointerException();\n"
+    "  }\n"
     "  result.$name$_.set(index, value);\n"
     "  return this;\n"
     "}\n"
@@ -225,6 +231,9 @@ GenerateBuilderMembers(io::Printer* printer) const {
     "  return this;\n"
     "}\n"
     "public Builder add$capitalized_name$($type$ value) {\n"
+    "  if (value == null) {\n"
+    "    throw new NullPointerException();\n"
+    "  }\n"
     "  if (result.$name$_.isEmpty()) {\n"
     "    result.$name$_ = new java.util.ArrayList<$type$>();\n"
     "  }\n"
diff --git a/src/google/protobuf/compiler/java/java_primitive_field.cc b/src/google/protobuf/compiler/java/java_primitive_field.cc
index fb4e650f..798e8608 100644
--- a/src/google/protobuf/compiler/java/java_primitive_field.cc
+++ b/src/google/protobuf/compiler/java/java_primitive_field.cc
@@ -39,6 +39,7 @@
 #include <google/protobuf/stubs/common.h>
 #include <google/protobuf/compiler/java/java_helpers.h>
 #include <google/protobuf/io/printer.h>
+#include <google/protobuf/wire_format_inl.h>
 #include <google/protobuf/stubs/strutil.h>
 #include <google/protobuf/stubs/substitute.h>
 
@@ -47,6 +48,8 @@ namespace protobuf {
 namespace compiler {
 namespace java {
 
+using internal::WireFormat;
+
 namespace {
 
 const char* PrimitiveTypeName(JavaType type) {
@@ -69,6 +72,26 @@ const char* PrimitiveTypeName(JavaType type) {
   return NULL;
 }
 
+bool IsReferenceType(JavaType type) {
+  switch (type) {
+    case JAVATYPE_INT    : return false;
+    case JAVATYPE_LONG   : return false;
+    case JAVATYPE_FLOAT  : return false;
+    case JAVATYPE_DOUBLE : return false;
+    case JAVATYPE_BOOLEAN: return false;
+    case JAVATYPE_STRING : return true;
+    case JAVATYPE_BYTES  : return true;
+    case JAVATYPE_ENUM   : return true;
+    case JAVATYPE_MESSAGE: return true;
+
+    // No default because we want the compiler to complain if any new
+    // JavaTypes are added.
+  }
+
+  GOOGLE_LOG(FATAL) << "Can't get here.";
+  return false;
+}
+
 const char* GetCapitalizedType(const FieldDescriptor* field) {
   switch (field->type()) {
     case FieldDescriptor::TYPE_INT32   : return "Int32"   ;
@@ -108,6 +131,38 @@ bool AllPrintableAscii(const string& text) {
   return true;
 }
 
+// For encodings with fixed sizes, returns that size in bytes.  Otherwise
+// returns -1.
+int FixedSize(FieldDescriptor::Type type) {
+  switch (type) {
+    case FieldDescriptor::TYPE_INT32   : return -1;
+    case FieldDescriptor::TYPE_INT64   : return -1;
+    case FieldDescriptor::TYPE_UINT32  : return -1;
+    case FieldDescriptor::TYPE_UINT64  : return -1;
+    case FieldDescriptor::TYPE_SINT32  : return -1;
+    case FieldDescriptor::TYPE_SINT64  : return -1;
+    case FieldDescriptor::TYPE_FIXED32 : return WireFormat::kFixed32Size;
+    case FieldDescriptor::TYPE_FIXED64 : return WireFormat::kFixed64Size;
+    case FieldDescriptor::TYPE_SFIXED32: return WireFormat::kSFixed32Size;
+    case FieldDescriptor::TYPE_SFIXED64: return WireFormat::kSFixed64Size;
+    case FieldDescriptor::TYPE_FLOAT   : return WireFormat::kFloatSize;
+    case FieldDescriptor::TYPE_DOUBLE  : return WireFormat::kDoubleSize;
+
+    case FieldDescriptor::TYPE_BOOL    : return WireFormat::kBoolSize;
+    case FieldDescriptor::TYPE_ENUM    : return -1;
+
+    case FieldDescriptor::TYPE_STRING  : return -1;
+    case FieldDescriptor::TYPE_BYTES   : return -1;
+    case FieldDescriptor::TYPE_GROUP   : return -1;
+    case FieldDescriptor::TYPE_MESSAGE : return -1;
+
+    // No default because we want the compiler to complain if any new
+    // types are added.
+  }
+  GOOGLE_LOG(FATAL) << "Can't get here.";
+  return -1;
+}
+
 string DefaultValue(const FieldDescriptor* field) {
   // Switch on cpp_type since we need to know which default_value_* method
   // of FieldDescriptor to call.
@@ -177,8 +232,22 @@ void SetPrimitiveVariables(const FieldDescriptor* descriptor,
   (*variables)["boxed_type"] = BoxedPrimitiveTypeName(GetJavaType(descriptor));
   (*variables)["default"] = DefaultValue(descriptor);
   (*variables)["capitalized_type"] = GetCapitalizedType(descriptor);
+  (*variables)["tag"] = SimpleItoa(WireFormat::MakeTag(descriptor));
+  (*variables)["tag_size"] = SimpleItoa(
+      WireFormat::TagSize(descriptor->number(), descriptor->type()));
+  if (IsReferenceType(GetJavaType(descriptor))) {
+    (*variables)["null_check"] =
+        "  if (value == null) {\n"
+        "    throw new NullPointerException();\n"
+        "  }\n";
+  } else {
+    (*variables)["null_check"] = "";
+  }
+  int fixed_size = FixedSize(descriptor->type());
+  if (fixed_size != -1) {
+    (*variables)["fixed_size"] = SimpleItoa(fixed_size);
+  }
 }
-
 }  // namespace
 
 // ===================================================================
@@ -210,6 +279,7 @@ GenerateBuilderMembers(io::Printer* printer) const {
     "  return result.get$capitalized_name$();\n"
     "}\n"
     "public Builder set$capitalized_name$($type$ value) {\n"
+    "$null_check$"
     "  result.has$capitalized_name$ = true;\n"
     "  result.$name$_ = value;\n"
     "  return this;\n"
@@ -283,6 +353,12 @@ GenerateMembers(io::Printer* printer) const {
     "public $type$ get$capitalized_name$(int index) {\n"
     "  return $name$_.get(index);\n"
     "}\n");
+
+  if (descriptor_->options().packed() &&
+      descriptor_->file()->options().optimize_for() == FileOptions::SPEED) {
+    printer->Print(variables_,
+      "private int $name$MemoizedSerializedSize;\n");
+  }
 }
 
 void RepeatedPrimitiveFieldGenerator::
@@ -302,10 +378,12 @@ GenerateBuilderMembers(io::Printer* printer) const {
     "  return result.get$capitalized_name$(index);\n"
     "}\n"
     "public Builder set$capitalized_name$(int index, $type$ value) {\n"
+    "$null_check$"
     "  result.$name$_.set(index, value);\n"
     "  return this;\n"
     "}\n"
     "public Builder add$capitalized_name$($type$ value) {\n"
+    "$null_check$"
     "  if (result.$name$_.isEmpty()) {\n"
     "    result.$name$_ = new java.util.ArrayList<$boxed_type$>();\n"
     "  }\n"
@@ -348,25 +426,80 @@ GenerateBuildingCode(io::Printer* printer) const {
 
 void RepeatedPrimitiveFieldGenerator::
 GenerateParsingCode(io::Printer* printer) const {
-  printer->Print(variables_,
-    "add$capitalized_name$(input.read$capitalized_type$());\n");
+  if (descriptor_->options().packed()) {
+    printer->Print(variables_,
+      "int length = input.readRawVarint32();\n"
+      "int limit = input.pushLimit(length);\n"
+      "while (input.getBytesUntilLimit() > 0) {\n"
+      "  add$capitalized_name$(input.read$capitalized_type$());\n"
+      "}\n"
+      "input.popLimit(limit);\n");
+  } else {
+    printer->Print(variables_,
+      "add$capitalized_name$(input.read$capitalized_type$());\n");
+  }
 }
 
 void RepeatedPrimitiveFieldGenerator::
 GenerateSerializationCode(io::Printer* printer) const {
-  printer->Print(variables_,
-    "for ($type$ element : get$capitalized_name$List()) {\n"
-    "  output.write$capitalized_type$($number$, element);\n"
-    "}\n");
+  if (descriptor_->options().packed()) {
+    printer->Print(variables_,
+      "if (get$capitalized_name$List().size() > 0) {\n"
+      "  output.writeRawVarint32($tag$);\n"
+      "  output.writeRawVarint32($name$MemoizedSerializedSize);\n"
+      "}\n"
+      "for ($type$ element : get$capitalized_name$List()) {\n"
+      "  output.write$capitalized_type$NoTag(element);\n"
+      "}\n");
+  } else {
+    printer->Print(variables_,
+      "for ($type$ element : get$capitalized_name$List()) {\n"
+      "  output.write$capitalized_type$($number$, element);\n"
+      "}\n");
+  }
 }
 
 void RepeatedPrimitiveFieldGenerator::
 GenerateSerializedSizeCode(io::Printer* printer) const {
   printer->Print(variables_,
-    "for ($type$ element : get$capitalized_name$List()) {\n"
-    "  size += com.google.protobuf.CodedOutputStream\n"
-    "    .compute$capitalized_type$Size($number$, element);\n"
-    "}\n");
+    "{\n"
+    "  int dataSize = 0;\n");
+  printer->Indent();
+
+  if (FixedSize(descriptor_->type()) == -1) {
+    printer->Print(variables_,
+      "for ($type$ element : get$capitalized_name$List()) {\n"
+      "  dataSize += com.google.protobuf.CodedOutputStream\n"
+      "    .compute$capitalized_type$SizeNoTag(element);\n"
+      "}\n");
+  } else {
+    printer->Print(variables_,
+      "dataSize = $fixed_size$ * get$capitalized_name$List().size();\n");
+  }
+
+  printer->Print(
+      "size += dataSize;\n");
+
+  if (descriptor_->options().packed()) {
+    printer->Print(variables_,
+      "if (!get$capitalized_name$List().isEmpty()) {\n"
+      "  size += $tag_size$;\n"
+      "  size += com.google.protobuf.CodedOutputStream\n"
+      "      .computeInt32SizeNoTag(dataSize);\n"
+      "}\n");
+  } else {
+    printer->Print(variables_,
+      "size += $tag_size$ * get$capitalized_name$List().size();\n");
+  }
+
+  // cache the data size for packed fields.
+  if (descriptor_->options().packed()) {
+    printer->Print(variables_,
+      "$name$MemoizedSerializedSize = dataSize;\n");
+  }
+
+  printer->Outdent();
+  printer->Print("}\n");
 }
 
 string RepeatedPrimitiveFieldGenerator::GetBoxedType() const {
diff --git a/src/google/protobuf/compiler/python/python_generator.cc b/src/google/protobuf/compiler/python/python_generator.cc
index 754dcbda..ca69fd4c 100644
--- a/src/google/protobuf/compiler/python/python_generator.cc
+++ b/src/google/protobuf/compiler/python/python_generator.cc
@@ -111,7 +111,6 @@ void PrintTopBoilerplate(
     io::Printer* printer, const FileDescriptor* file, bool descriptor_proto) {
   // TODO(robinson): Allow parameterization of Python version?
   printer->Print(
-      "#!/usr/bin/python2.4\n"
       "# Generated by the protocol buffer compiler.  DO NOT EDIT!\n"
       "\n"
       "from google.protobuf import descriptor\n"