Submit recent changes from internal branch, including "lite mode" for

C++ and Java.  See CHANGES.txt for more details.

11 files changed, 649 insertions, 27 deletions

diff --git a/python/google/protobuf/internal/containers.py b/python/google/protobuf/internal/containers.py
index fa1e3402..d8a825df 100755
--- a/python/google/protobuf/internal/containers.py
+++ b/python/google/protobuf/internal/containers.py
@@ -112,9 +112,11 @@ class RepeatedScalarFieldContainer(BaseContainer):
       return
 
     orig_empty = len(self._values) == 0
+    new_values = []
     for elem in elem_seq:
       self._type_checker.CheckValue(elem)
-    self._values.extend(elem_seq)
+      new_values.append(elem)
+    self._values.extend(new_values)
     self._message_listener.ByteSizeDirty()
     if orig_empty:
       self._message_listener.TransitionToNonempty()
@@ -139,9 +141,11 @@ class RepeatedScalarFieldContainer(BaseContainer):
 
   def __setslice__(self, start, stop, values):
     """Sets the subset of items from between the specified indices."""
+    new_values = []
     for value in values:
       self._type_checker.CheckValue(value)
-    self._values[start:stop] = list(values)
+      new_values.append(value)
+    self._values[start:stop] = new_values
     self._message_listener.ByteSizeDirty()
 
   def __delitem__(self, key):
diff --git a/python/google/protobuf/internal/decoder.py b/python/google/protobuf/internal/decoder.py
index 0bee6101..83d6fe0c 100755
--- a/python/google/protobuf/internal/decoder.py
+++ b/python/google/protobuf/internal/decoder.py
@@ -135,12 +135,12 @@ class Decoder(object):
   def ReadFloat(self):
     """Reads and returns a 4-byte floating-point number."""
     serialized = self._stream.ReadBytes(4)
-    return struct.unpack('f', serialized)[0]
+    return struct.unpack(wire_format.FORMAT_FLOAT_LITTLE_ENDIAN, serialized)[0]
 
   def ReadDouble(self):
     """Reads and returns an 8-byte floating-point number."""
     serialized = self._stream.ReadBytes(8)
-    return struct.unpack('d', serialized)[0]
+    return struct.unpack(wire_format.FORMAT_DOUBLE_LITTLE_ENDIAN, serialized)[0]
 
   def ReadBool(self):
     """Reads and returns a bool."""
diff --git a/python/google/protobuf/internal/decoder_test.py b/python/google/protobuf/internal/decoder_test.py
index 9bae888c..98e46472 100755
--- a/python/google/protobuf/internal/decoder_test.py
+++ b/python/google/protobuf/internal/decoder_test.py
@@ -36,12 +36,12 @@ __author__ = 'robinson@google.com (Will Robinson)'
 
 import struct
 import unittest
-from google.protobuf.internal import wire_format
-from google.protobuf.internal import encoder
 from google.protobuf.internal import decoder
-import logging
+from google.protobuf.internal import encoder
 from google.protobuf.internal import input_stream
+from google.protobuf.internal import wire_format
 from google.protobuf import message
+import logging
 import mox
 
 
@@ -110,6 +110,10 @@ class DecoderTest(unittest.TestCase):
     self.mox.VerifyAll()
     self.mox.ResetAll()
 
+  VAL = 1.125  # Perfectly representable as a float (no rounding error).
+  LITTLE_FLOAT_VAL = '\x00\x00\x90?'
+  LITTLE_DOUBLE_VAL = '\x00\x00\x00\x00\x00\x00\xf2?'
+
   def testReadScalars(self):
     test_string = 'I can feel myself getting sutpider.'
     scalar_tests = [
@@ -125,10 +129,10 @@ class DecoderTest(unittest.TestCase):
          'ReadLittleEndian32', long(0xffffffff)],
         ['sfixed64', decoder.Decoder.ReadSFixed64, long(-1),
          'ReadLittleEndian64', 0xffffffffffffffff],
-        ['float', decoder.Decoder.ReadFloat, 0.0,
-         'ReadBytes', struct.pack('f', 0.0), 4],
-        ['double', decoder.Decoder.ReadDouble, 0.0,
-         'ReadBytes', struct.pack('d', 0.0), 8],
+        ['float', decoder.Decoder.ReadFloat, self.VAL,
+         'ReadBytes', self.LITTLE_FLOAT_VAL, 4],
+        ['double', decoder.Decoder.ReadDouble, self.VAL,
+         'ReadBytes', self.LITTLE_DOUBLE_VAL, 8],
         ['bool', decoder.Decoder.ReadBool, True, 'ReadVarUInt32', 1],
         ['enum', decoder.Decoder.ReadEnum, 23, 'ReadVarUInt32', 23],
         ['string', decoder.Decoder.ReadString,
diff --git a/python/google/protobuf/internal/encoder.py b/python/google/protobuf/internal/encoder.py
index eed8c8bd..3ec3b2b1 100755
--- a/python/google/protobuf/internal/encoder.py
+++ b/python/google/protobuf/internal/encoder.py
@@ -123,11 +123,13 @@ class Encoder(object):
 
   def AppendFloatNoTag(self, value):
     """Appends a floating-point number to our buffer."""
-    self._stream.AppendRawBytes(struct.pack('f', value))
+    self._stream.AppendRawBytes(
+        struct.pack(wire_format.FORMAT_FLOAT_LITTLE_ENDIAN, value))
 
   def AppendDoubleNoTag(self, value):
     """Appends a double-precision floating-point number to our buffer."""
-    self._stream.AppendRawBytes(struct.pack('d', value))
+    self._stream.AppendRawBytes(
+        struct.pack(wire_format.FORMAT_DOUBLE_LITTLE_ENDIAN, value))
 
   def AppendBoolNoTag(self, value):
     """Appends a boolean to our buffer."""
diff --git a/python/google/protobuf/internal/encoder_test.py b/python/google/protobuf/internal/encoder_test.py
index 83a21c39..bf75ea80 100755
--- a/python/google/protobuf/internal/encoder_test.py
+++ b/python/google/protobuf/internal/encoder_test.py
@@ -123,6 +123,10 @@ class EncoderTest(unittest.TestCase):
     self.mox.VerifyAll()
     self.mox.ResetAll()
 
+  VAL = 1.125  # Perfectly representable as a float (no rounding error).
+  LITTLE_FLOAT_VAL = '\x00\x00\x90?'
+  LITTLE_DOUBLE_VAL = '\x00\x00\x00\x00\x00\x00\xf2?'
+
   def testAppendScalars(self):
     utf8_bytes = '\xd0\xa2\xd0\xb5\xd1\x81\xd1\x82'
     utf8_string = unicode(utf8_bytes, 'utf-8')
@@ -144,9 +148,9 @@ class EncoderTest(unittest.TestCase):
         ['sfixed64', self.encoder.AppendSFixed64, 'AppendLittleEndian64',
          wire_format.WIRETYPE_FIXED64, -1, 0xffffffffffffffff],
         ['float', self.encoder.AppendFloat, 'AppendRawBytes',
-         wire_format.WIRETYPE_FIXED32, 0.0, struct.pack('f', 0.0)],
+         wire_format.WIRETYPE_FIXED32, self.VAL, self.LITTLE_FLOAT_VAL],
         ['double', self.encoder.AppendDouble, 'AppendRawBytes',
-         wire_format.WIRETYPE_FIXED64, 0.0, struct.pack('d', 0.0)],
+         wire_format.WIRETYPE_FIXED64, self.VAL, self.LITTLE_DOUBLE_VAL],
         ['bool', self.encoder.AppendBool, 'AppendVarint32',
          wire_format.WIRETYPE_VARINT, False],
         ['enum', self.encoder.AppendEnum, 'AppendVarint32',
@@ -185,9 +189,9 @@ class EncoderTest(unittest.TestCase):
         ['sfixed64', self.encoder.AppendSFixed64NoTag,
          'AppendLittleEndian64', None, 0],
         ['float', self.encoder.AppendFloatNoTag,
-         'AppendRawBytes', None, 0.0, struct.pack('f', 0.0)],
+         'AppendRawBytes', None, self.VAL, self.LITTLE_FLOAT_VAL],
         ['double', self.encoder.AppendDoubleNoTag,
-         'AppendRawBytes', None, 0.0, struct.pack('d', 0.0)],
+         'AppendRawBytes', None, self.VAL, self.LITTLE_DOUBLE_VAL],
         ['bool', self.encoder.AppendBoolNoTag, 'AppendVarint32', None, 0],
         ['enum', self.encoder.AppendEnumNoTag, 'AppendVarint32', None, 0],
         ['sint32', self.encoder.AppendSInt32NoTag,
diff --git a/python/google/protobuf/internal/message_test.py b/python/google/protobuf/internal/message_test.py
new file mode 100755
index 00000000..df344cf0
--- /dev/null
+++ b/python/google/protobuf/internal/message_test.py
@@ -0,0 +1,53 @@
+#! /usr/bin/python
+#
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# http://code.google.com/p/protobuf/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Tests python protocol buffers against the golden message."""
+
+__author__ = 'gps@google.com (Gregory P. Smith)'
+
+import unittest
+from google.protobuf import unittest_import_pb2
+from google.protobuf import unittest_pb2
+from google.protobuf.internal import test_util
+
+
+class MessageTest(test_util.GoldenMessageTestCase):
+
+  def testGoldenMessage(self):
+    golden_data = test_util.GoldenFile('golden_message').read()
+    golden_message = unittest_pb2.TestAllTypes()
+    golden_message.ParseFromString(golden_data)
+    self.ExpectAllFieldsSet(golden_message)
+
+
+if __name__ == '__main__':
+  unittest.main()
diff --git a/python/google/protobuf/internal/reflection_test.py b/python/google/protobuf/internal/reflection_test.py
index e2da769a..86101774 100755
--- a/python/google/protobuf/internal/reflection_test.py
+++ b/python/google/protobuf/internal/reflection_test.py
@@ -232,13 +232,14 @@ class ReflectionTest(unittest.TestCase):
     proto.repeated_string.extend(['foo', 'bar'])
     proto.repeated_string.extend([])
     proto.repeated_string.append('baz')
+    proto.repeated_string.extend(str(x) for x in xrange(2))
     proto.optional_int32 = 21
     self.assertEqual(
       [ (proto.DESCRIPTOR.fields_by_name['optional_int32'  ], 21),
         (proto.DESCRIPTOR.fields_by_name['repeated_int32'  ], [5, 11]),
         (proto.DESCRIPTOR.fields_by_name['repeated_fixed32'], [1]),
         (proto.DESCRIPTOR.fields_by_name['repeated_string' ],
-          ['foo', 'bar', 'baz']) ],
+          ['foo', 'bar', 'baz', '0', '1']) ],
       proto.ListFields())
 
   def testSingularListExtensions(self):
@@ -447,6 +448,10 @@ class ReflectionTest(unittest.TestCase):
     self.assertEqual([25, 20, 15], proto.repeated_int32[1:4])
     self.assertEqual([5, 25, 20, 15, 30], proto.repeated_int32[:])
 
+    # Test slice assignment with an iterator
+    proto.repeated_int32[1:4] = (i for i in xrange(3))
+    self.assertEqual([5, 0, 1, 2, 30], proto.repeated_int32)
+
     # Test slice assignment.
     proto.repeated_int32[1:4] = [35, 40, 45]
     self.assertEqual([5, 35, 40, 45, 30], proto.repeated_int32)
@@ -1739,13 +1744,14 @@ class SerializationTest(unittest.TestCase):
     self.assertEqual(2, proto2.b)
     self.assertEqual(3, proto2.c)
 
-  def testSerializedAllPackedFields(self):
+  def testSerializeAllPackedFields(self):
     first_proto = unittest_pb2.TestPackedTypes()
     second_proto = unittest_pb2.TestPackedTypes()
     test_util.SetAllPackedFields(first_proto)
     serialized = first_proto.SerializeToString()
     self.assertEqual(first_proto.ByteSize(), len(serialized))
-    second_proto.MergeFromString(serialized)
+    bytes_read = second_proto.MergeFromString(serialized)
+    self.assertEqual(second_proto.ByteSize(), bytes_read)
     self.assertEqual(first_proto, second_proto)
 
   def testSerializeAllPackedExtensions(self):
@@ -1753,7 +1759,8 @@ class SerializationTest(unittest.TestCase):
     second_proto = unittest_pb2.TestPackedExtensions()
     test_util.SetAllPackedExtensions(first_proto)
     serialized = first_proto.SerializeToString()
-    second_proto.MergeFromString(serialized)
+    bytes_read = second_proto.MergeFromString(serialized)
+    self.assertEqual(second_proto.ByteSize(), bytes_read)
     self.assertEqual(first_proto, second_proto)
 
   def testMergePackedFromStringWhenSomeFieldsAlreadySet(self):
@@ -1838,6 +1845,79 @@ class SerializationTest(unittest.TestCase):
     self.assertEqual(unittest_pb2.REPEATED_NESTED_ENUM_EXTENSION_FIELD_NUMBER,
       51)
 
+  def testInitKwargs(self):
+    proto = unittest_pb2.TestAllTypes(
+        optional_int32=1,
+        optional_string='foo',
+        optional_bool=True,
+        optional_bytes='bar',
+        optional_nested_message=unittest_pb2.TestAllTypes.NestedMessage(bb=1),
+        optional_foreign_message=unittest_pb2.ForeignMessage(c=1),
+        optional_nested_enum=unittest_pb2.TestAllTypes.FOO,
+        optional_foreign_enum=unittest_pb2.FOREIGN_FOO,
+        repeated_int32=[1, 2, 3])
+    self.assertTrue(proto.IsInitialized())
+    self.assertTrue(proto.HasField('optional_int32'))
+    self.assertTrue(proto.HasField('optional_string'))
+    self.assertTrue(proto.HasField('optional_bool'))
+    self.assertTrue(proto.HasField('optional_bytes'))
+    self.assertTrue(proto.HasField('optional_nested_message'))
+    self.assertTrue(proto.HasField('optional_foreign_message'))
+    self.assertTrue(proto.HasField('optional_nested_enum'))
+    self.assertTrue(proto.HasField('optional_foreign_enum'))
+    self.assertEqual(1, proto.optional_int32)
+    self.assertEqual('foo', proto.optional_string)
+    self.assertEqual(True, proto.optional_bool)
+    self.assertEqual('bar', proto.optional_bytes)
+    self.assertEqual(1, proto.optional_nested_message.bb)
+    self.assertEqual(1, proto.optional_foreign_message.c)
+    self.assertEqual(unittest_pb2.TestAllTypes.FOO,
+                     proto.optional_nested_enum)
+    self.assertEqual(unittest_pb2.FOREIGN_FOO, proto.optional_foreign_enum)
+    self.assertEqual([1, 2, 3], proto.repeated_int32)
+
+  def testInitArgsUnknownFieldName(self):
+    def InitalizeEmptyMessageWithExtraKeywordArg():
+      unused_proto = unittest_pb2.TestEmptyMessage(unknown='unknown')
+    self._CheckRaises(ValueError,
+                      InitalizeEmptyMessageWithExtraKeywordArg,
+                      'Protocol message has no "unknown" field.')
+
+  def testInitRequiredKwargs(self):
+    proto = unittest_pb2.TestRequired(a=1, b=1, c=1)
+    self.assertTrue(proto.IsInitialized())
+    self.assertTrue(proto.HasField('a'))
+    self.assertTrue(proto.HasField('b'))
+    self.assertTrue(proto.HasField('c'))
+    self.assertTrue(not proto.HasField('dummy2'))
+    self.assertEqual(1, proto.a)
+    self.assertEqual(1, proto.b)
+    self.assertEqual(1, proto.c)
+
+  def testInitRequiredForeignKwargs(self):
+    proto = unittest_pb2.TestRequiredForeign(
+        optional_message=unittest_pb2.TestRequired(a=1, b=1, c=1))
+    self.assertTrue(proto.IsInitialized())
+    self.assertTrue(proto.HasField('optional_message'))
+    self.assertTrue(proto.optional_message.IsInitialized())
+    self.assertTrue(proto.optional_message.HasField('a'))
+    self.assertTrue(proto.optional_message.HasField('b'))
+    self.assertTrue(proto.optional_message.HasField('c'))
+    self.assertTrue(not proto.optional_message.HasField('dummy2'))
+    self.assertEqual(unittest_pb2.TestRequired(a=1, b=1, c=1),
+                     proto.optional_message)
+    self.assertEqual(1, proto.optional_message.a)
+    self.assertEqual(1, proto.optional_message.b)
+    self.assertEqual(1, proto.optional_message.c)
+
+  def testInitRepeatedKwargs(self):
+    proto = unittest_pb2.TestAllTypes(repeated_int32=[1, 2, 3])
+    self.assertTrue(proto.IsInitialized())
+    self.assertEqual(1, proto.repeated_int32[0])
+    self.assertEqual(2, proto.repeated_int32[1])
+    self.assertEqual(3, proto.repeated_int32[2])
+
+
 class OptionsTest(unittest.TestCase):
 
   def testMessageOptions(self):
diff --git a/python/google/protobuf/internal/test_util.py b/python/google/protobuf/internal/test_util.py
index 2d50bc4a..1a0da552 100755
--- a/python/google/protobuf/internal/test_util.py
+++ b/python/google/protobuf/internal/test_util.py
@@ -38,6 +38,7 @@ __author__ = 'robinson@google.com (Will Robinson)'
 
 import os.path
 
+import unittest
 from google.protobuf import unittest_import_pb2
 from google.protobuf import unittest_pb2
 
@@ -351,6 +352,200 @@ def ExpectAllFieldsAndExtensionsInOrder(serialized):
   if expected != serialized:
     raise ValueError('Expected %r, found %r' % (expected, serialized))
 
+
+class GoldenMessageTestCase(unittest.TestCase):
+  """This adds methods to TestCase useful for verifying our Golden Message."""
+
+  def ExpectAllFieldsSet(self, message):
+    """Check all fields for correct values have after Set*Fields() is called."""
+    self.assertTrue(message.HasField('optional_int32'))
+    self.assertTrue(message.HasField('optional_int64'))
+    self.assertTrue(message.HasField('optional_uint32'))
+    self.assertTrue(message.HasField('optional_uint64'))
+    self.assertTrue(message.HasField('optional_sint32'))
+    self.assertTrue(message.HasField('optional_sint64'))
+    self.assertTrue(message.HasField('optional_fixed32'))
+    self.assertTrue(message.HasField('optional_fixed64'))
+    self.assertTrue(message.HasField('optional_sfixed32'))
+    self.assertTrue(message.HasField('optional_sfixed64'))
+    self.assertTrue(message.HasField('optional_float'))
+    self.assertTrue(message.HasField('optional_double'))
+    self.assertTrue(message.HasField('optional_bool'))
+    self.assertTrue(message.HasField('optional_string'))
+    self.assertTrue(message.HasField('optional_bytes'))
+
+    self.assertTrue(message.HasField('optionalgroup'))
+    self.assertTrue(message.HasField('optional_nested_message'))
+    self.assertTrue(message.HasField('optional_foreign_message'))
+    self.assertTrue(message.HasField('optional_import_message'))
+
+    self.assertTrue(message.optionalgroup.HasField('a'))
+    self.assertTrue(message.optional_nested_message.HasField('bb'))
+    self.assertTrue(message.optional_foreign_message.HasField('c'))
+    self.assertTrue(message.optional_import_message.HasField('d'))
+
+    self.assertTrue(message.HasField('optional_nested_enum'))
+    self.assertTrue(message.HasField('optional_foreign_enum'))
+    self.assertTrue(message.HasField('optional_import_enum'))
+
+    self.assertTrue(message.HasField('optional_string_piece'))
+    self.assertTrue(message.HasField('optional_cord'))
+
+    self.assertEqual(101, message.optional_int32)
+    self.assertEqual(102, message.optional_int64)
+    self.assertEqual(103, message.optional_uint32)
+    self.assertEqual(104, message.optional_uint64)
+    self.assertEqual(105, message.optional_sint32)
+    self.assertEqual(106, message.optional_sint64)
+    self.assertEqual(107, message.optional_fixed32)
+    self.assertEqual(108, message.optional_fixed64)
+    self.assertEqual(109, message.optional_sfixed32)
+    self.assertEqual(110, message.optional_sfixed64)
+    self.assertEqual(111, message.optional_float)
+    self.assertEqual(112, message.optional_double)
+    self.assertEqual(True, message.optional_bool)
+    self.assertEqual('115', message.optional_string)
+    self.assertEqual('116', message.optional_bytes)
+
+    self.assertEqual(117, message.optionalgroup.a);
+    self.assertEqual(118, message.optional_nested_message.bb)
+    self.assertEqual(119, message.optional_foreign_message.c)
+    self.assertEqual(120, message.optional_import_message.d)
+
+    self.assertEqual(unittest_pb2.TestAllTypes.BAZ,
+                     message.optional_nested_enum)
+    self.assertEqual(unittest_pb2.FOREIGN_BAZ, message.optional_foreign_enum)
+    self.assertEqual(unittest_import_pb2.IMPORT_BAZ,
+                     message.optional_import_enum)
+
+    # -----------------------------------------------------------------
+
+    self.assertEqual(2, len(message.repeated_int32))
+    self.assertEqual(2, len(message.repeated_int64))
+    self.assertEqual(2, len(message.repeated_uint32))
+    self.assertEqual(2, len(message.repeated_uint64))
+    self.assertEqual(2, len(message.repeated_sint32))
+    self.assertEqual(2, len(message.repeated_sint64))
+    self.assertEqual(2, len(message.repeated_fixed32))
+    self.assertEqual(2, len(message.repeated_fixed64))
+    self.assertEqual(2, len(message.repeated_sfixed32))
+    self.assertEqual(2, len(message.repeated_sfixed64))
+    self.assertEqual(2, len(message.repeated_float))
+    self.assertEqual(2, len(message.repeated_double))
+    self.assertEqual(2, len(message.repeated_bool))
+    self.assertEqual(2, len(message.repeated_string))
+    self.assertEqual(2, len(message.repeated_bytes))
+
+    self.assertEqual(2, len(message.repeatedgroup))
+    self.assertEqual(2, len(message.repeated_nested_message))
+    self.assertEqual(2, len(message.repeated_foreign_message))
+    self.assertEqual(2, len(message.repeated_import_message))
+    self.assertEqual(2, len(message.repeated_nested_enum))
+    self.assertEqual(2, len(message.repeated_foreign_enum))
+    self.assertEqual(2, len(message.repeated_import_enum))
+
+    self.assertEqual(2, len(message.repeated_string_piece))
+    self.assertEqual(2, len(message.repeated_cord))
+
+    self.assertEqual(201, message.repeated_int32[0])
+    self.assertEqual(202, message.repeated_int64[0])
+    self.assertEqual(203, message.repeated_uint32[0])
+    self.assertEqual(204, message.repeated_uint64[0])
+    self.assertEqual(205, message.repeated_sint32[0])
+    self.assertEqual(206, message.repeated_sint64[0])
+    self.assertEqual(207, message.repeated_fixed32[0])
+    self.assertEqual(208, message.repeated_fixed64[0])
+    self.assertEqual(209, message.repeated_sfixed32[0])
+    self.assertEqual(210, message.repeated_sfixed64[0])
+    self.assertEqual(211, message.repeated_float[0])
+    self.assertEqual(212, message.repeated_double[0])
+    self.assertEqual(True, message.repeated_bool[0])
+    self.assertEqual('215', message.repeated_string[0])
+    self.assertEqual('216', message.repeated_bytes[0])
+
+    self.assertEqual(217, message.repeatedgroup[0].a)
+    self.assertEqual(218, message.repeated_nested_message[0].bb)
+    self.assertEqual(219, message.repeated_foreign_message[0].c)
+    self.assertEqual(220, message.repeated_import_message[0].d)
+
+    self.assertEqual(unittest_pb2.TestAllTypes.BAR,
+                     message.repeated_nested_enum[0])
+    self.assertEqual(unittest_pb2.FOREIGN_BAR,
+                     message.repeated_foreign_enum[0])
+    self.assertEqual(unittest_import_pb2.IMPORT_BAR,
+                     message.repeated_import_enum[0])
+
+    self.assertEqual(301, message.repeated_int32[1])
+    self.assertEqual(302, message.repeated_int64[1])
+    self.assertEqual(303, message.repeated_uint32[1])
+    self.assertEqual(304, message.repeated_uint64[1])
+    self.assertEqual(305, message.repeated_sint32[1])
+    self.assertEqual(306, message.repeated_sint64[1])
+    self.assertEqual(307, message.repeated_fixed32[1])
+    self.assertEqual(308, message.repeated_fixed64[1])
+    self.assertEqual(309, message.repeated_sfixed32[1])
+    self.assertEqual(310, message.repeated_sfixed64[1])
+    self.assertEqual(311, message.repeated_float[1])
+    self.assertEqual(312, message.repeated_double[1])
+    self.assertEqual(False, message.repeated_bool[1])
+    self.assertEqual('315', message.repeated_string[1])
+    self.assertEqual('316', message.repeated_bytes[1])
+
+    self.assertEqual(317, message.repeatedgroup[1].a)
+    self.assertEqual(318, message.repeated_nested_message[1].bb)
+    self.assertEqual(319, message.repeated_foreign_message[1].c)
+    self.assertEqual(320, message.repeated_import_message[1].d)
+
+    self.assertEqual(unittest_pb2.TestAllTypes.BAZ,
+                     message.repeated_nested_enum[1])
+    self.assertEqual(unittest_pb2.FOREIGN_BAZ,
+                     message.repeated_foreign_enum[1])
+    self.assertEqual(unittest_import_pb2.IMPORT_BAZ,
+                     message.repeated_import_enum[1])
+
+    # -----------------------------------------------------------------
+
+    self.assertTrue(message.HasField('default_int32'))
+    self.assertTrue(message.HasField('default_int64'))
+    self.assertTrue(message.HasField('default_uint32'))
+    self.assertTrue(message.HasField('default_uint64'))
+    self.assertTrue(message.HasField('default_sint32'))
+    self.assertTrue(message.HasField('default_sint64'))
+    self.assertTrue(message.HasField('default_fixed32'))
+    self.assertTrue(message.HasField('default_fixed64'))
+    self.assertTrue(message.HasField('default_sfixed32'))
+    self.assertTrue(message.HasField('default_sfixed64'))
+    self.assertTrue(message.HasField('default_float'))
+    self.assertTrue(message.HasField('default_double'))
+    self.assertTrue(message.HasField('default_bool'))
+    self.assertTrue(message.HasField('default_string'))
+    self.assertTrue(message.HasField('default_bytes'))
+
+    self.assertTrue(message.HasField('default_nested_enum'))
+    self.assertTrue(message.HasField('default_foreign_enum'))
+    self.assertTrue(message.HasField('default_import_enum'))
+
+    self.assertEqual(401, message.default_int32)
+    self.assertEqual(402, message.default_int64)
+    self.assertEqual(403, message.default_uint32)
+    self.assertEqual(404, message.default_uint64)
+    self.assertEqual(405, message.default_sint32)
+    self.assertEqual(406, message.default_sint64)
+    self.assertEqual(407, message.default_fixed32)
+    self.assertEqual(408, message.default_fixed64)
+    self.assertEqual(409, message.default_sfixed32)
+    self.assertEqual(410, message.default_sfixed64)
+    self.assertEqual(411, message.default_float)
+    self.assertEqual(412, message.default_double)
+    self.assertEqual(False, message.default_bool)
+    self.assertEqual('415', message.default_string)
+    self.assertEqual('416', message.default_bytes)
+
+    self.assertEqual(unittest_pb2.TestAllTypes.FOO, message.default_nested_enum)
+    self.assertEqual(unittest_pb2.FOREIGN_FOO, message.default_foreign_enum)
+    self.assertEqual(unittest_import_pb2.IMPORT_FOO,
+                     message.default_import_enum)
+
 def GoldenFile(filename):
   """Finds the given golden file and returns a file object representing it."""
 
@@ -359,7 +554,8 @@ def GoldenFile(filename):
   while os.path.exists(path):
     if os.path.exists(os.path.join(path, 'src/google/protobuf')):
       # Found it.  Load the golden file from the testdata directory.
-      return file(os.path.join(path, 'src/google/protobuf/testdata', filename))
+      full_path = os.path.join(path, 'src/google/protobuf/testdata', filename)
+      return open(full_path, 'rb')
     path = os.path.join(path, '..')
 
   raise RuntimeError(
diff --git a/python/google/protobuf/internal/text_format_test.py b/python/google/protobuf/internal/text_format_test.py
index 871590e7..0cf27186 100755
--- a/python/google/protobuf/internal/text_format_test.py
+++ b/python/google/protobuf/internal/text_format_test.py
@@ -42,11 +42,16 @@ from google.protobuf.internal import test_util
 from google.protobuf import unittest_pb2
 from google.protobuf import unittest_mset_pb2
 
-class TextFormatTest(unittest.TestCase):
-  def CompareToGoldenFile(self, text, golden_filename):
+
+class TextFormatTest(test_util.GoldenMessageTestCase):
+  def ReadGolden(self, golden_filename):
     f = test_util.GoldenFile(golden_filename)
     golden_lines = f.readlines()
     f.close()
+    return golden_lines
+
+  def CompareToGoldenFile(self, text, golden_filename):
+    golden_lines = self.ReadGolden(golden_filename)
     self.CompareToGoldenLines(text, golden_lines)
 
   def CompareToGoldenText(self, text, golden_text):
@@ -117,6 +122,276 @@ class TextFormatTest(unittest.TestCase):
     return text.replace('e+0','e+').replace('e+0','e+') \
                .replace('e-0','e-').replace('e-0','e-')
 
+  def testMergeGolden(self):
+    golden_text = '\n'.join(self.ReadGolden('text_format_unittest_data.txt'))
+    parsed_message = unittest_pb2.TestAllTypes()
+    text_format.Merge(golden_text, parsed_message)
+
+    message = unittest_pb2.TestAllTypes()
+    test_util.SetAllFields(message)
+    self.assertEquals(message, parsed_message)
+
+  def testMergeGoldenExtensions(self):
+    golden_text = '\n'.join(self.ReadGolden(
+        'text_format_unittest_extensions_data.txt'))
+    parsed_message = unittest_pb2.TestAllExtensions()
+    text_format.Merge(golden_text, parsed_message)
+
+    message = unittest_pb2.TestAllExtensions()
+    test_util.SetAllExtensions(message)
+    self.assertEquals(message, parsed_message)
+
+  def testMergeAllFields(self):
+    message = unittest_pb2.TestAllTypes()
+    test_util.SetAllFields(message)
+    ascii_text = text_format.MessageToString(message)
+
+    parsed_message = unittest_pb2.TestAllTypes()
+    text_format.Merge(ascii_text, parsed_message)
+    self.assertEqual(message, parsed_message)
+    self.ExpectAllFieldsSet(message)
+
+  def testMergeAllExtensions(self):
+    message = unittest_pb2.TestAllExtensions()
+    test_util.SetAllExtensions(message)
+    ascii_text = text_format.MessageToString(message)
+
+    parsed_message = unittest_pb2.TestAllExtensions()
+    text_format.Merge(ascii_text, parsed_message)
+    self.assertEqual(message, parsed_message)
+
+  def testMergeMessageSet(self):
+    message = unittest_pb2.TestAllTypes()
+    text = ('repeated_uint64: 1\n'
+            'repeated_uint64: 2\n')
+    text_format.Merge(text, message)
+    self.assertEqual(1, message.repeated_uint64[0])
+    self.assertEqual(2, message.repeated_uint64[1])
+
+    message = unittest_mset_pb2.TestMessageSetContainer()
+    text = ('message_set {\n'
+            '  [protobuf_unittest.TestMessageSetExtension1] {\n'
+            '    i: 23\n'
+            '  }\n'
+            '  [protobuf_unittest.TestMessageSetExtension2] {\n'
+            '    str: \"foo\"\n'
+            '  }\n'
+            '}\n')
+    text_format.Merge(text, message)
+    ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension
+    ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension
+    self.assertEquals(23, message.message_set.Extensions[ext1].i)
+    self.assertEquals('foo', message.message_set.Extensions[ext2].str)
+
+  def testMergeExotic(self):
+    message = unittest_pb2.TestAllTypes()
+    text = ('repeated_int64: -9223372036854775808\n'
+            'repeated_uint64: 18446744073709551615\n'
+            'repeated_double: 123.456\n'
+            'repeated_double: 1.23e+22\n'
+            'repeated_double: 1.23e-18\n'
+            'repeated_string: \n'
+            '\"\\000\\001\\007\\010\\014\\n\\r\\t\\013\\\\\\\'\\\"\"\n')
+    text_format.Merge(text, message)
+
+    self.assertEqual(-9223372036854775808, message.repeated_int64[0])
+    self.assertEqual(18446744073709551615, message.repeated_uint64[0])
+    self.assertEqual(123.456, message.repeated_double[0])
+    self.assertEqual(1.23e22, message.repeated_double[1])
+    self.assertEqual(1.23e-18, message.repeated_double[2])
+    self.assertEqual(
+        '\000\001\a\b\f\n\r\t\v\\\'\"', message.repeated_string[0])
+
+  def testMergeUnknownField(self):
+    message = unittest_pb2.TestAllTypes()
+    text = 'unknown_field: 8\n'
+    self.assertRaisesWithMessage(
+        text_format.ParseError,
+        ('1:1 : Message type "protobuf_unittest.TestAllTypes" has no field named '
+         '"unknown_field".'),
+        text_format.Merge, text, message)
+
+  def testMergeBadExtension(self):
+    message = unittest_pb2.TestAllTypes()
+    text = '[unknown_extension]: 8\n'
+    self.assertRaisesWithMessage(
+        text_format.ParseError,
+        '1:2 : Extension "unknown_extension" not registered.',
+        text_format.Merge, text, message)
+
+  def testMergeGroupNotClosed(self):
+    message = unittest_pb2.TestAllTypes()
+    text = 'RepeatedGroup: <'
+    self.assertRaisesWithMessage(
+        text_format.ParseError, '1:16 : Expected ">".',
+        text_format.Merge, text, message)
+
+    text = 'RepeatedGroup: {'
+    self.assertRaisesWithMessage(
+        text_format.ParseError, '1:16 : Expected "}".',
+        text_format.Merge, text, message)
+
+  def testMergeBadEnumValue(self):
+    message = unittest_pb2.TestAllTypes()
+    text = 'optional_nested_enum: BARR'
+    self.assertRaisesWithMessage(
+        text_format.ParseError,
+        ('1:23 : Enum type "protobuf_unittest.TestAllTypes.NestedEnum" '
+         'has no value named BARR.'),
+        text_format.Merge, text, message)
+
+    message = unittest_pb2.TestAllTypes()
+    text = 'optional_nested_enum: 100'
+    self.assertRaisesWithMessage(
+        text_format.ParseError,
+        ('1:23 : Enum type "protobuf_unittest.TestAllTypes.NestedEnum" '
+         'has no value with number 100.'),
+        text_format.Merge, text, message)
+
+  def assertRaisesWithMessage(self, e_class, e, func, *args, **kwargs):
+    """Same as assertRaises, but also compares the exception message."""
+    if hasattr(e_class, '__name__'):
+      exc_name = e_class.__name__
+    else:
+      exc_name = str(e_class)
+
+    try:
+      func(*args, **kwargs)
+    except e_class, expr:
+      if str(expr) != e:
+        msg = '%s raised, but with wrong message: "%s" instead of "%s"'
+        raise self.failureException(msg % (exc_name,
+                                           str(expr).encode('string_escape'),
+                                           e.encode('string_escape')))
+      return
+    else:
+      raise self.failureException('%s not raised' % exc_name)
+
+
+class TokenizerTest(unittest.TestCase):
+
+  def testSimpleTokenCases(self):
+    text = ('identifier1:"string1"\n     \n\n'
+            'identifier2 : \n \n123  \n  identifier3 :\'string\'\n'
+            'identifiER_4 : 1.1e+2 ID5:-0.23 ID6:\'aaaa\\\'bbbb\'\n'
+            'ID7 : "aa\\"bb"\n\n\n\n ID8: {A:inf B:-inf C:true D:false}\n'
+            'ID9: 22 ID10: -111111111111111111 ID11: -22\n'
+            'ID12: 2222222222222222222')
+    tokenizer = text_format._Tokenizer(text)
+    methods = [(tokenizer.ConsumeIdentifier, 'identifier1'),
+               ':',
+               (tokenizer.ConsumeString, 'string1'),
+               (tokenizer.ConsumeIdentifier, 'identifier2'),
+               ':',
+               (tokenizer.ConsumeInt32, 123),
+               (tokenizer.ConsumeIdentifier, 'identifier3'),
+               ':',
+               (tokenizer.ConsumeString, 'string'),
+               (tokenizer.ConsumeIdentifier, 'identifiER_4'),
+               ':',
+               (tokenizer.ConsumeFloat, 1.1e+2),
+               (tokenizer.ConsumeIdentifier, 'ID5'),
+               ':',
+               (tokenizer.ConsumeFloat, -0.23),
+               (tokenizer.ConsumeIdentifier, 'ID6'),
+               ':',
+               (tokenizer.ConsumeString, 'aaaa\'bbbb'),
+               (tokenizer.ConsumeIdentifier, 'ID7'),
+               ':',
+               (tokenizer.ConsumeString, 'aa\"bb'),
+               (tokenizer.ConsumeIdentifier, 'ID8'),
+               ':',
+               '{',
+               (tokenizer.ConsumeIdentifier, 'A'),
+               ':',
+               (tokenizer.ConsumeFloat, float('inf')),
+               (tokenizer.ConsumeIdentifier, 'B'),
+               ':',
+               (tokenizer.ConsumeFloat, float('-inf')),
+               (tokenizer.ConsumeIdentifier, 'C'),
+               ':',
+               (tokenizer.ConsumeBool, True),
+               (tokenizer.ConsumeIdentifier, 'D'),
+               ':',
+               (tokenizer.ConsumeBool, False),
+               '}',
+               (tokenizer.ConsumeIdentifier, 'ID9'),
+               ':',
+               (tokenizer.ConsumeUint32, 22),
+               (tokenizer.ConsumeIdentifier, 'ID10'),
+               ':',
+               (tokenizer.ConsumeInt64, -111111111111111111),
+               (tokenizer.ConsumeIdentifier, 'ID11'),
+               ':',
+               (tokenizer.ConsumeInt32, -22),
+               (tokenizer.ConsumeIdentifier, 'ID12'),
+               ':',
+               (tokenizer.ConsumeUint64, 2222222222222222222)]
+
+    i = 0
+    while not tokenizer.AtEnd():
+      m = methods[i]
+      if type(m) == str:
+        token = tokenizer.token
+        self.assertEqual(token, m)
+        tokenizer.NextToken()
+      else:
+        self.assertEqual(m[1], m[0]())
+      i += 1
+
+  def testConsumeIntegers(self):
+    # This test only tests the failures in the integer parsing methods as well
+    # as the '0' special cases.
+    int64_max = (1 << 63) - 1
+    uint32_max = (1 << 32) - 1
+    text = '-1 %d %d' % (uint32_max + 1, int64_max + 1)
+    tokenizer = text_format._Tokenizer(text)
+    self.assertRaises(text_format.ParseError, tokenizer.ConsumeUint32)
+    self.assertRaises(text_format.ParseError, tokenizer.ConsumeUint64)
+    self.assertEqual(-1, tokenizer.ConsumeInt32())
+
+    self.assertRaises(text_format.ParseError, tokenizer.ConsumeUint32)
+    self.assertRaises(text_format.ParseError, tokenizer.ConsumeInt32)
+    self.assertEqual(uint32_max + 1, tokenizer.ConsumeInt64())
+
+    self.assertRaises(text_format.ParseError, tokenizer.ConsumeInt64)
+    self.assertEqual(int64_max + 1, tokenizer.ConsumeUint64())
+    self.assertTrue(tokenizer.AtEnd())
+
+    text = '-0 -0 0 0'
+    tokenizer = text_format._Tokenizer(text)
+    self.assertEqual(0, tokenizer.ConsumeUint32())
+    self.assertEqual(0, tokenizer.ConsumeUint64())
+    self.assertEqual(0, tokenizer.ConsumeUint32())
+    self.assertEqual(0, tokenizer.ConsumeUint64())
+    self.assertTrue(tokenizer.AtEnd())
+
+  def testConsumeByteString(self):
+    text = '"string1\''
+    tokenizer = text_format._Tokenizer(text)
+    self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)
+
+    text = 'string1"'
+    tokenizer = text_format._Tokenizer(text)
+    self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)
+
+    text = '\n"\\xt"'
+    tokenizer = text_format._Tokenizer(text)
+    self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)
+
+    text = '\n"\\"'
+    tokenizer = text_format._Tokenizer(text)
+    self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)
+
+    text = '\n"\\x"'
+    tokenizer = text_format._Tokenizer(text)
+    self.assertRaises(text_format.ParseError, tokenizer.ConsumeByteString)
+
+  def testConsumeBool(self):
+    text = 'not-a-bool'
+    tokenizer = text_format._Tokenizer(text)
+    self.assertRaises(text_format.ParseError, tokenizer.ConsumeBool)
+
+
 if __name__ == '__main__':
   unittest.main()
-
diff --git a/python/google/protobuf/internal/type_checkers.py b/python/google/protobuf/internal/type_checkers.py
index c009627f..a3bc57ff 100755
--- a/python/google/protobuf/internal/type_checkers.py
+++ b/python/google/protobuf/internal/type_checkers.py
@@ -122,8 +122,10 @@ class UnicodeValueChecker(object):
       try:
         unicode(proposed_value, 'ascii')
       except UnicodeDecodeError:
-        raise ValueError('%.1024r isn\'t in 7-bit ASCII encoding.'
-                         % (proposed_value))
+        raise ValueError('%.1024r has type str, but isn\'t in 7-bit ASCII '
+                         'encoding. Non-ASCII strings must be converted to '
+                         'unicode objects before being added.' %
+                         (proposed_value))
 
 
 class Int32ValueChecker(IntValueChecker):
diff --git a/python/google/protobuf/internal/wire_format.py b/python/google/protobuf/internal/wire_format.py
index 950267f9..da6464de 100755
--- a/python/google/protobuf/internal/wire_format.py
+++ b/python/google/protobuf/internal/wire_format.py
@@ -64,6 +64,8 @@ UINT64_MAX = (1 << 64) - 1
 # "struct" format strings that will encode/decode the specified formats.
 FORMAT_UINT32_LITTLE_ENDIAN = '<I'
 FORMAT_UINT64_LITTLE_ENDIAN = '<Q'
+FORMAT_FLOAT_LITTLE_ENDIAN = '<f'
+FORMAT_DOUBLE_LITTLE_ENDIAN = '<d'
 
 
 # We'll have to provide alternate implementations of AppendLittleEndian*() on