Merge remote-tracking branch 'upstream/master' into py2_py3_straddle

Conflicts:
	python/google/protobuf/descriptor_pool.py
	python/google/protobuf/internal/api_implementation_default_test.py
	python/google/protobuf/internal/cpp_message.py
	python/google/protobuf/internal/descriptor_database_test.py
	python/google/protobuf/internal/descriptor_pool_test.py
	python/google/protobuf/internal/descriptor_python_test.py
	python/google/protobuf/internal/descriptor_test.py
	python/google/protobuf/internal/generator_test.py
	python/google/protobuf/internal/message_factory_python_test.py
	python/google/protobuf/internal/message_factory_test.py
	python/google/protobuf/internal/message_test.py
	python/google/protobuf/internal/proto_builder_test.py
	python/google/protobuf/internal/python_message.py
	python/google/protobuf/internal/reflection_test.py
	python/google/protobuf/internal/service_reflection_test.py
	python/google/protobuf/internal/symbol_database_test.py
	python/google/protobuf/internal/text_encoding_test.py
	python/google/protobuf/internal/text_format_test.py
	python/google/protobuf/internal/unknown_fields_test.py
	python/google/protobuf/internal/wire_format_test.py
	python/google/protobuf/pyext/descriptor_cpp2_test.py
	python/google/protobuf/pyext/message_factory_cpp2_test.py
	python/google/protobuf/pyext/reflection_cpp2_generated_test.py
	python/setup.py
	ruby/lib/google/protobuf/message_exts.rb

1 files changed, 114 insertions, 42 deletions

diff --git a/python/google/protobuf/descriptor.py b/python/google/protobuf/descriptor.py
index e7acdacd..6840d1f4 100755
--- a/python/google/protobuf/descriptor.py
+++ b/python/google/protobuf/descriptor.py
@@ -39,15 +39,13 @@ __author__ = 'robinson@google.com (Will Robinson)'
 from google.protobuf.internal import api_implementation
 
 
+_USE_C_DESCRIPTORS = False
 if api_implementation.Type() == 'cpp':
   # Used by MakeDescriptor in cpp mode
   import os
   import uuid
-
-  if api_implementation.Version() == 2:
-    from google.protobuf.pyext import _message
-  else:
-    from google.protobuf.internal import cpp_message
+  from google.protobuf.pyext import _message
+  _USE_C_DESCRIPTORS = getattr(_message, '_USE_C_DESCRIPTORS', False)
 
 
 class Error(Exception):
@@ -58,12 +56,29 @@ class TypeTransformationError(Error):
   """Error transforming between python proto type and corresponding C++ type."""
 
 
+if _USE_C_DESCRIPTORS:
+  # This metaclass allows to override the behavior of code like
+  #     isinstance(my_descriptor, FieldDescriptor)
+  # and make it return True when the descriptor is an instance of the extension
+  # type written in C++.
+  class DescriptorMetaclass(type):
+    def __instancecheck__(cls, obj):
+      if super(DescriptorMetaclass, cls).__instancecheck__(obj):
+        return True
+      if isinstance(obj, cls._C_DESCRIPTOR_CLASS):
+        return True
+      return False
+else:
+  # The standard metaclass; nothing changes.
+  DescriptorMetaclass = type
+
+
 class DescriptorBase(object):
 
   """Descriptors base class.
 
   This class is the base of all descriptor classes. It provides common options
-  related functionaility.
+  related functionality.
 
   Attributes:
     has_options:  True if the descriptor has non-default options.  Usually it
@@ -73,6 +88,12 @@ class DescriptorBase(object):
         avoid some bootstrapping issues.
   """
 
+  __metaclass__ = DescriptorMetaclass
+  if _USE_C_DESCRIPTORS:
+    # The class, or tuple of classes, that are considered as "virtual
+    # subclasses" of this descriptor class.
+    _C_DESCRIPTOR_CLASS = ()
+
   def __init__(self, options, options_class_name):
     """Initialize the descriptor given its options message and the name of the
     class of the options message. The name of the class is required in case
@@ -222,9 +243,6 @@ class Descriptor(_NestedDescriptorBase):
 
     is_extendable:  Does this type define any extension ranges?
 
-    options: (descriptor_pb2.MessageOptions) Protocol message options or None
-      to use default message options.
-
     oneofs: (list of OneofDescriptor) The list of descriptors for oneof fields
       in this message.
     oneofs_by_name: (dict str -> OneofDescriptor) Same objects as in |oneofs|,
@@ -233,13 +251,25 @@ class Descriptor(_NestedDescriptorBase):
     file: (FileDescriptor) Reference to file descriptor.
   """
 
+  if _USE_C_DESCRIPTORS:
+    _C_DESCRIPTOR_CLASS = _message.Descriptor
+
+    def __new__(cls, name, full_name, filename, containing_type, fields,
+                nested_types, enum_types, extensions, options=None,
+                is_extendable=True, extension_ranges=None, oneofs=None,
+                file=None, serialized_start=None, serialized_end=None,
+                syntax=None):
+      _message.Message._CheckCalledFromGeneratedFile()
+      return _message.default_pool.FindMessageTypeByName(full_name)
+
   # NOTE(tmarek): The file argument redefining a builtin is nothing we can
   # fix right now since we don't know how many clients already rely on the
   # name of the argument.
   def __init__(self, name, full_name, filename, containing_type, fields,
                nested_types, enum_types, extensions, options=None,
                is_extendable=True, extension_ranges=None, oneofs=None,
-               file=None, serialized_start=None, serialized_end=None):  # pylint:disable=redefined-builtin
+               file=None, serialized_start=None, serialized_end=None,
+               syntax=None):  # pylint:disable=redefined-builtin
     """Arguments to __init__() are as described in the description
     of Descriptor fields above.
 
@@ -284,6 +314,7 @@ class Descriptor(_NestedDescriptorBase):
     self.oneofs_by_name = dict((o.name, o) for o in self.oneofs)
     for oneof in self.oneofs:
       oneof.containing_type = self
+    self.syntax = syntax or "proto2"
 
   def EnumValueName(self, enum, value):
     """Returns the string name of an enum value.
@@ -450,6 +481,19 @@ class FieldDescriptor(DescriptorBase):
   FIRST_RESERVED_FIELD_NUMBER = 19000
   LAST_RESERVED_FIELD_NUMBER = 19999
 
+  if _USE_C_DESCRIPTORS:
+    _C_DESCRIPTOR_CLASS = _message.FieldDescriptor
+
+    def __new__(cls, name, full_name, index, number, type, cpp_type, label,
+                default_value, message_type, enum_type, containing_type,
+                is_extension, extension_scope, options=None,
+                has_default_value=True, containing_oneof=None):
+      _message.Message._CheckCalledFromGeneratedFile()
+      if is_extension:
+        return _message.default_pool.FindExtensionByName(full_name)
+      else:
+        return _message.default_pool.FindFieldByName(full_name)
+
   def __init__(self, name, full_name, index, number, type, cpp_type, label,
                default_value, message_type, enum_type, containing_type,
                is_extension, extension_scope, options=None,
@@ -479,20 +523,9 @@ class FieldDescriptor(DescriptorBase):
     self.containing_oneof = containing_oneof
     if api_implementation.Type() == 'cpp':
       if is_extension:
-        if api_implementation.Version() == 2:
-          # pylint: disable=protected-access
-          self._cdescriptor = (
-              _message.Message._GetExtensionDescriptor(full_name))
-          # pylint: enable=protected-access
-        else:
-          self._cdescriptor = cpp_message.GetExtensionDescriptor(full_name)
+        self._cdescriptor = _message.default_pool.FindExtensionByName(full_name)
       else:
-        if api_implementation.Version() == 2:
-          # pylint: disable=protected-access
-          self._cdescriptor = _message.Message._GetFieldDescriptor(full_name)
-          # pylint: enable=protected-access
-        else:
-          self._cdescriptor = cpp_message.GetFieldDescriptor(full_name)
+        self._cdescriptor = _message.default_pool.FindFieldByName(full_name)
     else:
       self._cdescriptor = None
 
@@ -542,6 +575,15 @@ class EnumDescriptor(_NestedDescriptorBase):
       None to use default enum options.
   """
 
+  if _USE_C_DESCRIPTORS:
+    _C_DESCRIPTOR_CLASS = _message.EnumDescriptor
+
+    def __new__(cls, name, full_name, filename, values,
+                containing_type=None, options=None, file=None,
+                serialized_start=None, serialized_end=None):
+      _message.Message._CheckCalledFromGeneratedFile()
+      return _message.default_pool.FindEnumTypeByName(full_name)
+
   def __init__(self, name, full_name, filename, values,
                containing_type=None, options=None, file=None,
                serialized_start=None, serialized_end=None):
@@ -586,6 +628,17 @@ class EnumValueDescriptor(DescriptorBase):
       None to use default enum value options options.
   """
 
+  if _USE_C_DESCRIPTORS:
+    _C_DESCRIPTOR_CLASS = _message.EnumValueDescriptor
+
+    def __new__(cls, name, index, number, type=None, options=None):
+      _message.Message._CheckCalledFromGeneratedFile()
+      # There is no way we can build a complete EnumValueDescriptor with the
+      # given parameters (the name of the Enum is not known, for example).
+      # Fortunately generated files just pass it to the EnumDescriptor()
+      # constructor, which will ignore it, so returning None is good enough.
+      return None
+
   def __init__(self, name, index, number, type=None, options=None):
     """Arguments are as described in the attribute description above."""
     super(EnumValueDescriptor, self).__init__(options, 'EnumValueOptions')
@@ -609,6 +662,13 @@ class OneofDescriptor(object):
       oneof can contain.
   """
 
+  if _USE_C_DESCRIPTORS:
+    _C_DESCRIPTOR_CLASS = _message.OneofDescriptor
+
+    def __new__(cls, name, full_name, index, containing_type, fields):
+      _message.Message._CheckCalledFromGeneratedFile()
+      return _message.default_pool.FindOneofByName(full_name)
+
   def __init__(self, name, full_name, index, containing_type, fields):
     """Arguments are as described in the attribute description above."""
     self.name = name
@@ -702,6 +762,7 @@ class FileDescriptor(DescriptorBase):
 
   name: name of file, relative to root of source tree.
   package: name of the package
+  syntax: string indicating syntax of the file (can be "proto2" or "proto3")
   serialized_pb: (str) Byte string of serialized
     descriptor_pb2.FileDescriptorProto.
   dependencies: List of other FileDescriptors this FileDescriptor depends on.
@@ -710,14 +771,27 @@ class FileDescriptor(DescriptorBase):
   extensions_by_name: Dict of extension names and their descriptors.
   """
 
+  if _USE_C_DESCRIPTORS:
+    _C_DESCRIPTOR_CLASS = _message.FileDescriptor
+
+    def __new__(cls, name, package, options=None, serialized_pb=None,
+                dependencies=None, syntax=None):
+      # FileDescriptor() is called from various places, not only from generated
+      # files, to register dynamic proto files and messages.
+      if serialized_pb:
+        return _message.default_pool.AddSerializedFile(serialized_pb)
+      else:
+        return super(FileDescriptor, cls).__new__(cls)
+
   def __init__(self, name, package, options=None, serialized_pb=None,
-               dependencies=None):
+               dependencies=None, syntax=None):
     """Constructor."""
     super(FileDescriptor, self).__init__(options, 'FileOptions')
 
     self.message_types_by_name = {}
     self.name = name
     self.package = package
+    self.syntax = syntax or "proto2"
     self.serialized_pb = serialized_pb
 
     self.enum_types_by_name = {}
@@ -726,12 +800,7 @@ class FileDescriptor(DescriptorBase):
 
     if (api_implementation.Type() == 'cpp' and
         self.serialized_pb is not None):
-      if api_implementation.Version() == 2:
-        # pylint: disable=protected-access
-        _message.Message._BuildFile(self.serialized_pb)
-        # pylint: enable=protected-access
-      else:
-        cpp_message.BuildFile(self.serialized_pb)
+      _message.default_pool.AddSerializedFile(self.serialized_pb)
 
   def CopyToProto(self, proto):
     """Copies this to a descriptor_pb2.FileDescriptorProto.
@@ -752,7 +821,8 @@ def _ParseOptions(message, string):
   return message
 
 
-def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True):
+def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True,
+                   syntax=None):
   """Make a protobuf Descriptor given a DescriptorProto protobuf.
 
   Handles nested descriptors. Note that this is limited to the scope of defining
@@ -764,6 +834,8 @@ def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True):
     package: Optional package name for the new message Descriptor (string).
     build_file_if_cpp: Update the C++ descriptor pool if api matches.
                        Set to False on recursion, so no duplicates are created.
+    syntax: The syntax/semantics that should be used.  Set to "proto3" to get
+            proto3 field presence semantics.
   Returns:
     A Descriptor for protobuf messages.
   """
@@ -776,10 +848,10 @@ def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True):
     file_descriptor_proto = descriptor_pb2.FileDescriptorProto()
     file_descriptor_proto.message_type.add().MergeFrom(desc_proto)
 
-    # Generate a random name for this proto file to prevent conflicts with
-    # any imported ones. We need to specify a file name so BuildFile accepts
-    # our FileDescriptorProto, but it is not important what that file name
-    # is actually set to.
+    # Generate a random name for this proto file to prevent conflicts with any
+    # imported ones. We need to specify a file name so the descriptor pool
+    # accepts our FileDescriptorProto, but it is not important what that file
+    # name is actually set to.
     proto_name = str(uuid.uuid4())
 
     if package:
@@ -789,12 +861,11 @@ def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True):
     else:
       file_descriptor_proto.name = proto_name + '.proto'
 
-    if api_implementation.Version() == 2:
-      # pylint: disable=protected-access
-      _message.Message._BuildFile(file_descriptor_proto.SerializeToString())
-      # pylint: enable=protected-access
-    else:
-      cpp_message.BuildFile(file_descriptor_proto.SerializeToString())
+    _message.default_pool.Add(file_descriptor_proto)
+    result = _message.default_pool.FindFileByName(file_descriptor_proto.name)
+
+    if _USE_C_DESCRIPTORS:
+      return result.message_types_by_name[desc_proto.name]
 
   full_message_name = [desc_proto.name]
   if package: full_message_name.insert(0, package)
@@ -817,7 +888,8 @@ def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True):
     # used by fields in the message, so no loops are possible here.
     nested_desc = MakeDescriptor(nested_proto,
                                  package='.'.join(full_message_name),
-                                 build_file_if_cpp=False)
+                                 build_file_if_cpp=False,
+                                 syntax=syntax)
     nested_types[full_name] = nested_desc
 
   fields = []