Integrated internal changes from Google

1 files changed, 193 insertions, 15 deletions

diff --git a/python/google/protobuf/internal/message_test.py b/python/google/protobuf/internal/message_test.py
index dda72cdd..4622f10f 100755
--- a/python/google/protobuf/internal/message_test.py
+++ b/python/google/protobuf/internal/message_test.py
@@ -61,6 +61,7 @@ try:
 except NameError:
   cmp = lambda x, y: (x > y) - (x < y)  # Python 3
 
+from google.protobuf import map_proto2_unittest_pb2
 from google.protobuf import map_unittest_pb2
 from google.protobuf import unittest_pb2
 from google.protobuf import unittest_proto3_arena_pb2
@@ -69,6 +70,7 @@ from google.protobuf import descriptor_pool
 from google.protobuf import message_factory
 from google.protobuf import text_format
 from google.protobuf.internal import api_implementation
+from google.protobuf.internal import encoder
 from google.protobuf.internal import packed_field_test_pb2
 from google.protobuf.internal import test_util
 from google.protobuf.internal import testing_refleaks
@@ -140,6 +142,18 @@ class MessageTest(BaseTestCase):
     golden_copy = copy.deepcopy(golden_message)
     self.assertEqual(golden_data, golden_copy.SerializeToString())
 
+  def testParseErrors(self, message_module):
+    msg = message_module.TestAllTypes()
+    self.assertRaises(TypeError, msg.FromString, 0)
+    self.assertRaises(Exception, msg.FromString, '0')
+    # TODO(jieluo): Fix cpp extension to check unexpected end-group tag.
+    # b/27494216
+    if api_implementation.Type() == 'python':
+      end_tag = encoder.TagBytes(1, 4)
+      with self.assertRaises(message.DecodeError) as context:
+        msg.FromString(end_tag)
+      self.assertEqual('Unexpected end-group tag.', str(context.exception))
+
   def testDeterminismParameters(self, message_module):
     # This message is always deterministically serialized, even if determinism
     # is disabled, so we can use it to verify that all the determinism
@@ -609,6 +623,13 @@ class MessageTest(BaseTestCase):
     self.assertIsInstance(m.repeated_nested_message,
                           collections.MutableSequence)
 
+  def testRepeatedFieldsNotHashable(self, message_module):
+    m = message_module.TestAllTypes()
+    with self.assertRaises(TypeError):
+      hash(m.repeated_int32)
+    with self.assertRaises(TypeError):
+      hash(m.repeated_nested_message)
+
   def testRepeatedFieldInsideNestedMessage(self, message_module):
     m = message_module.NestedTestAllTypes()
     m.payload.repeated_int32.extend([])
@@ -626,6 +647,7 @@ class MessageTest(BaseTestCase):
   def testOneofGetCaseNonexistingField(self, message_module):
     m = message_module.TestAllTypes()
     self.assertRaises(ValueError, m.WhichOneof, 'no_such_oneof_field')
+    self.assertRaises(Exception, m.WhichOneof, 0)
 
   def testOneofDefaultValues(self, message_module):
     m = message_module.TestAllTypes()
@@ -1001,6 +1023,8 @@ class MessageTest(BaseTestCase):
     m = message_module.TestAllTypes()
     with self.assertRaises(IndexError) as _:
       m.repeated_nested_message.pop()
+    with self.assertRaises(TypeError) as _:
+      m.repeated_nested_message.pop('0')
     for i in range(5):
       n = m.repeated_nested_message.add()
       n.bb = i
@@ -1009,6 +1033,39 @@ class MessageTest(BaseTestCase):
     self.assertEqual(2, m.repeated_nested_message.pop(1).bb)
     self.assertEqual([1, 3], [n.bb for n in m.repeated_nested_message])
 
+  def testRepeatedCompareWithSelf(self, message_module):
+    m = message_module.TestAllTypes()
+    for i in range(5):
+      m.repeated_int32.insert(i, i)
+      n = m.repeated_nested_message.add()
+      n.bb = i
+    self.assertSequenceEqual(m.repeated_int32, m.repeated_int32)
+    self.assertEqual(m.repeated_nested_message, m.repeated_nested_message)
+
+  def testReleasedNestedMessages(self, message_module):
+    """A case that lead to a segfault when a message detached from its parent
+    container has itself a child container.
+    """
+    m = message_module.NestedTestAllTypes()
+    m = m.repeated_child.add()
+    m = m.child
+    m = m.repeated_child.add()
+    self.assertEqual(m.payload.optional_int32, 0)
+
+  def testSetRepeatedComposite(self, message_module):
+    m = message_module.TestAllTypes()
+    with self.assertRaises(AttributeError):
+      m.repeated_int32 = []
+    m.repeated_int32.append(1)
+    if api_implementation.Type() == 'cpp':
+      # For test coverage: cpp has a different path if composite
+      # field is in cache
+      with self.assertRaises(TypeError):
+        m.repeated_int32 = []
+    else:
+      with self.assertRaises(AttributeError):
+        m.repeated_int32 = []
+
 
 # Class to test proto2-only features (required, extensions, etc.)
 class Proto2Test(BaseTestCase):
@@ -1061,18 +1118,46 @@ class Proto2Test(BaseTestCase):
     self.assertEqual(False, message.optional_bool)
     self.assertEqual(0, message.optional_nested_message.bb)
 
-  # TODO(tibell): The C++ implementations actually allows assignment
-  # of unknown enum values to *scalar* fields (but not repeated
-  # fields). Once checked enum fields becomes the default in the
-  # Python implementation, the C++ implementation should follow suit.
   def testAssignInvalidEnum(self):
-    """It should not be possible to assign an invalid enum number to an
-    enum field."""
+    """Assigning an invalid enum number is not allowed in proto2."""
     m = unittest_pb2.TestAllTypes()
 
+    # Proto2 can not assign unknown enum.
     with self.assertRaises(ValueError) as _:
       m.optional_nested_enum = 1234567
     self.assertRaises(ValueError, m.repeated_nested_enum.append, 1234567)
+    # Assignment is a different code path than append for the C++ impl.
+    m.repeated_nested_enum.append(2)
+    m.repeated_nested_enum[0] = 2
+    with self.assertRaises(ValueError):
+      m.repeated_nested_enum[0] = 123456
+
+    # Unknown enum value can be parsed but is ignored.
+    m2 = unittest_proto3_arena_pb2.TestAllTypes()
+    m2.optional_nested_enum = 1234567
+    m2.repeated_nested_enum.append(7654321)
+    serialized = m2.SerializeToString()
+
+    m3 = unittest_pb2.TestAllTypes()
+    m3.ParseFromString(serialized)
+    self.assertFalse(m3.HasField('optional_nested_enum'))
+    # 1 is the default value for optional_nested_enum.
+    self.assertEqual(1, m3.optional_nested_enum)
+    self.assertEqual(0, len(m3.repeated_nested_enum))
+    m2.Clear()
+    m2.ParseFromString(m3.SerializeToString())
+    self.assertEqual(1234567, m2.optional_nested_enum)
+    self.assertEqual(7654321, m2.repeated_nested_enum[0])
+
+  def testUnknownEnumMap(self):
+    m = map_proto2_unittest_pb2.TestEnumMap()
+    m.known_map_field[123] = 0
+    with self.assertRaises(ValueError):
+      m.unknown_map_field[1] = 123
+
+  def testExtensionsErrors(self):
+    msg = unittest_pb2.TestAllTypes()
+    self.assertRaises(AttributeError, getattr, msg, 'Extensions')
 
   def testGoldenExtensions(self):
     golden_data = test_util.GoldenFileData('golden_message')
@@ -1297,6 +1382,7 @@ class Proto3Test(BaseTestCase):
     """Assigning an unknown enum value is allowed and preserves the value."""
     m = unittest_proto3_arena_pb2.TestAllTypes()
 
+    # Proto3 can assign unknown enums.
     m.optional_nested_enum = 1234567
     self.assertEqual(1234567, m.optional_nested_enum)
     m.repeated_nested_enum.append(22334455)
@@ -1311,18 +1397,10 @@ class Proto3Test(BaseTestCase):
     self.assertEqual(1234567, m2.optional_nested_enum)
     self.assertEqual(7654321, m2.repeated_nested_enum[0])
 
-    # ParseFromString in Proto2 should accept unknown enums too.
-    m3 = unittest_pb2.TestAllTypes()
-    m3.ParseFromString(serialized)
-    m2.Clear()
-    m2.ParseFromString(m3.SerializeToString())
-    self.assertEqual(1234567, m2.optional_nested_enum)
-    self.assertEqual(7654321, m2.repeated_nested_enum[0])
-
   # Map isn't really a proto3-only feature. But there is no proto2 equivalent
   # of google/protobuf/map_unittest.proto right now, so it's not easy to
   # test both with the same test like we do for the other proto2/proto3 tests.
-  # (google/protobuf/map_protobuf_unittest.proto is very different in the set
+  # (google/protobuf/map_proto2_unittest.proto is very different in the set
   # of messages and fields it contains).
   def testScalarMapDefaults(self):
     msg = map_unittest_pb2.TestMap()
@@ -1383,12 +1461,21 @@ class Proto3Test(BaseTestCase):
 
     msg.map_int32_int32[5] = 15
     self.assertEqual(15, msg.map_int32_int32.get(5))
+    self.assertEqual(15, msg.map_int32_int32.get(5))
+    with self.assertRaises(TypeError):
+      msg.map_int32_int32.get('')
 
     self.assertIsNone(msg.map_int32_foreign_message.get(5))
     self.assertEqual(10, msg.map_int32_foreign_message.get(5, 10))
 
     submsg = msg.map_int32_foreign_message[5]
     self.assertIs(submsg, msg.map_int32_foreign_message.get(5))
+    # TODO(jieluo): Fix python and cpp extension diff.
+    if api_implementation.Type() == 'cpp':
+      with self.assertRaises(TypeError):
+        msg.map_int32_foreign_message.get('')
+    else:
+      self.assertEqual(None, msg.map_int32_foreign_message.get(''))
 
   def testScalarMap(self):
     msg = map_unittest_pb2.TestMap()
@@ -1400,8 +1487,13 @@ class Proto3Test(BaseTestCase):
     msg.map_int64_int64[-2**33] = -2**34
     msg.map_uint32_uint32[123] = 456
     msg.map_uint64_uint64[2**33] = 2**34
+    msg.map_int32_float[2] = 1.2
+    msg.map_int32_double[1] = 3.3
     msg.map_string_string['abc'] = '123'
+    msg.map_bool_bool[True] = True
     msg.map_int32_enum[888] = 2
+    # Unknown numeric enum is supported in proto3.
+    msg.map_int32_enum[123] = 456
 
     self.assertEqual([], msg.FindInitializationErrors())
 
@@ -1435,8 +1527,24 @@ class Proto3Test(BaseTestCase):
     self.assertEqual(-2**34, msg2.map_int64_int64[-2**33])
     self.assertEqual(456, msg2.map_uint32_uint32[123])
     self.assertEqual(2**34, msg2.map_uint64_uint64[2**33])
+    self.assertAlmostEqual(1.2, msg.map_int32_float[2])
+    self.assertEqual(3.3, msg.map_int32_double[1])
     self.assertEqual('123', msg2.map_string_string['abc'])
+    self.assertEqual(True, msg2.map_bool_bool[True])
     self.assertEqual(2, msg2.map_int32_enum[888])
+    self.assertEqual(456, msg2.map_int32_enum[123])
+    # TODO(jieluo): Add cpp extension support.
+    if api_implementation.Type() == 'python':
+      self.assertEqual('{-123: -456}',
+                       str(msg2.map_int32_int32))
+
+  def testMapEntryAlwaysSerialized(self):
+    msg = map_unittest_pb2.TestMap()
+    msg.map_int32_int32[0] = 0
+    msg.map_string_string[''] = ''
+    self.assertEqual(msg.ByteSize(), 12)
+    self.assertEqual(b'\n\x04\x08\x00\x10\x00r\x04\n\x00\x12\x00',
+                     msg.SerializeToString())
 
   def testStringUnicodeConversionInMap(self):
     msg = map_unittest_pb2.TestMap()
@@ -1489,6 +1597,11 @@ class Proto3Test(BaseTestCase):
     self.assertIn(123, msg2.map_int32_foreign_message)
     self.assertIn(-456, msg2.map_int32_foreign_message)
     self.assertEqual(2, len(msg2.map_int32_foreign_message))
+    # TODO(jieluo): Fix text format for message map.
+    # TODO(jieluo): Add cpp extension support.
+    if api_implementation.Type() == 'python':
+      self.assertEqual(15,
+                       len(str(msg2.map_int32_foreign_message)))
 
   def testNestedMessageMapItemDelete(self):
     msg = map_unittest_pb2.TestMap()
@@ -1572,6 +1685,12 @@ class Proto3Test(BaseTestCase):
 
     del msg2.map_int32_foreign_message[222]
     self.assertFalse(222 in msg2.map_int32_foreign_message)
+    if api_implementation.Type() == 'cpp':
+      with self.assertRaises(TypeError):
+        del msg2.map_int32_foreign_message['']
+    else:
+      with self.assertRaises(KeyError):
+        del msg2.map_int32_foreign_message['']
 
   def testMergeFromBadType(self):
     msg = map_unittest_pb2.TestMap()
@@ -1706,6 +1825,54 @@ class Proto3Test(BaseTestCase):
     matching_dict = {2: 4, 3: 6, 4: 8}
     self.assertMapIterEquals(msg.map_int32_int32.items(), matching_dict)
 
+  def testPython2Map(self):
+    if sys.version_info < (3,):
+      msg = map_unittest_pb2.TestMap()
+      msg.map_int32_int32[2] = 4
+      msg.map_int32_int32[3] = 6
+      msg.map_int32_int32[4] = 8
+      msg.map_int32_int32[5] = 10
+      map_int32 = msg.map_int32_int32
+      self.assertEqual(4, len(map_int32))
+      msg2 = map_unittest_pb2.TestMap()
+      msg2.ParseFromString(msg.SerializeToString())
+
+      def CheckItems(seq, iterator):
+        self.assertEqual(next(iterator), seq[0])
+        self.assertEqual(list(iterator), seq[1:])
+
+      CheckItems(map_int32.items(), map_int32.iteritems())
+      CheckItems(map_int32.keys(), map_int32.iterkeys())
+      CheckItems(map_int32.values(), map_int32.itervalues())
+
+      self.assertEqual(6, map_int32.get(3))
+      self.assertEqual(None, map_int32.get(999))
+      self.assertEqual(6, map_int32.pop(3))
+      self.assertEqual(0, map_int32.pop(3))
+      self.assertEqual(3, len(map_int32))
+      key, value = map_int32.popitem()
+      self.assertEqual(2 * key, value)
+      self.assertEqual(2, len(map_int32))
+      map_int32.clear()
+      self.assertEqual(0, len(map_int32))
+
+      with self.assertRaises(KeyError):
+        map_int32.popitem()
+
+      self.assertEqual(0, map_int32.setdefault(2))
+      self.assertEqual(1, len(map_int32))
+
+      map_int32.update(msg2.map_int32_int32)
+      self.assertEqual(4, len(map_int32))
+
+      with self.assertRaises(TypeError):
+        map_int32.update(msg2.map_int32_int32,
+                         msg2.map_int32_int32)
+      with self.assertRaises(TypeError):
+        map_int32.update(0)
+      with self.assertRaises(TypeError):
+        map_int32.update(value=12)
+
   def testMapItems(self):
     # Map items used to have strange behaviors when use c extension. Because
     # [] may reorder the map and invalidate any exsting iterators.
@@ -1836,6 +2003,9 @@ class Proto3Test(BaseTestCase):
     del msg.map_int32_int32[4]
     self.assertEqual(0, len(msg.map_int32_int32))
 
+    with self.assertRaises(KeyError):
+      del msg.map_int32_all_types[32]
+
   def testMapsAreMapping(self):
     msg = map_unittest_pb2.TestMap()
     self.assertIsInstance(msg.map_int32_int32, collections.Mapping)
@@ -1844,6 +2014,14 @@ class Proto3Test(BaseTestCase):
     self.assertIsInstance(msg.map_int32_foreign_message,
                           collections.MutableMapping)
 
+  def testMapsCompare(self):
+    msg = map_unittest_pb2.TestMap()
+    msg.map_int32_int32[-123] = -456
+    self.assertEqual(msg.map_int32_int32, msg.map_int32_int32)
+    self.assertEqual(msg.map_int32_foreign_message,
+                     msg.map_int32_foreign_message)
+    self.assertNotEqual(msg.map_int32_int32, 0)
+
   def testMapFindInitializationErrorsSmokeTest(self):
     msg = map_unittest_pb2.TestMap()
     msg.map_string_string['abc'] = '123'