diff options
Diffstat (limited to 'python')
53 files changed, 1586 insertions, 351 deletions
diff --git a/python/README.md b/python/README.md index 8f3db785..4c194297 100644 --- a/python/README.md +++ b/python/README.md @@ -32,77 +32,84 @@ Installation 1) Make sure you have Python 2.6 or newer. If in doubt, run: - $ python -V + $ python -V 2) If you do not have setuptools installed, note that it will be - downloaded and installed automatically as soon as you run setup.py. + downloaded and installed automatically as soon as you run `setup.py`. If you would rather install it manually, you may do so by following - the instructions on this page: + the instructions on [this page](https://packaging.python.org/en/latest/installing.html#setup-for-installing-packages). - https://packaging.python.org/en/latest/installing.html#setup-for-installing-packages - -3) Build the C++ code, or install a binary distribution of protoc. If +3) Build the C++ code, or install a binary distribution of `protoc`. If you install a binary distribution, make sure that it is the same version as this package. If in doubt, run: - $ protoc --version + $ protoc --version 4) Build and run the tests: - $ python setup.py build - $ python setup.py test + $ python setup.py build + $ python setup.py test + + To build, test, and use the C++ implementation, you must first compile + `libprotobuf.so`: + + $ (cd .. && make) + + On OS X: + + If you are running a Homebrew-provided Python, you must make sure another + version of protobuf is not already installed, as Homebrew's Python will + search `/usr/local/lib` for `libprotobuf.so` before it searches + `../src/.libs`. - To build, test, and use the C++ implementation, you must first compile - libprotobuf.so: + You can either unlink Homebrew's protobuf or install the `libprotobuf` you + built earlier: - $ (cd .. && make) + $ brew unlink protobuf - On OS X: + or - If you are running a homebrew-provided python, you must make sure another - version of protobuf is not already installed, as homebrew's python will - search /usr/local/lib for libprotobuf.so before it searches ../src/.libs - You can either unlink homebrew's protobuf or install the libprotobuf you - built earlier: + $ (cd .. && make install) - $ brew unlink protobuf - or - $ (cd .. && make install) + On other *nix: - On other *nix: + You must make `libprotobuf.so` dynamically available. You can either + install libprotobuf you built earlier, or set `LD_LIBRARY_PATH`: - You must make libprotobuf.so dynamically available. You can either - install libprotobuf you built earlier, or set LD_LIBRARY_PATH: + $ export LD_LIBRARY_PATH=../src/.libs - $ export LD_LIBRARY_PATH=../src/.libs - or - $ (cd .. && make install) + or - To build the C++ implementation run: - $ python setup.py build --cpp_implementation + $ (cd .. && make install) - Then run the tests like so: - $ python setup.py test --cpp_implementation + To build the C++ implementation run: + + $ python setup.py build --cpp_implementation + + Then run the tests like so: + + $ python setup.py test --cpp_implementation If some tests fail, this library may not work correctly on your system. Continue at your own risk. Please note that there is a known problem with some versions of Python on Cygwin which causes the tests to fail after printing the - error: "sem_init: Resource temporarily unavailable". This appears - to be a bug either in Cygwin or in Python: - http://www.cygwin.com/ml/cygwin/2005-07/msg01378.html + error: `sem_init: Resource temporarily unavailable`. This appears + to be a [bug either in Cygwin or in + Python](http://www.cygwin.com/ml/cygwin/2005-07/msg01378.html). + We do not know if or when it might be fixed. We also do not know how likely it is that this bug will affect users in practice. 5) Install: - $ python setup.py install + $ python setup.py install - or: + or: - $ (cd .. && make install) - $ python setup.py install --cpp_implementation + $ (cd .. && make install) + $ python setup.py install --cpp_implementation This step may require superuser privileges. NOTE: To use C++ implementation, you need to export an environment diff --git a/python/compatibility_tests/v2.5.0/setup.py b/python/compatibility_tests/v2.5.0/setup.py index d8e34bc0..b41d54d4 100755 --- a/python/compatibility_tests/v2.5.0/setup.py +++ b/python/compatibility_tests/v2.5.0/setup.py @@ -64,7 +64,7 @@ if __name__ == '__main__': url='https://developers.google.com/protocol-buffers/', maintainer='protobuf@googlegroups.com', maintainer_email='protobuf@googlegroups.com', - license='New BSD License', + license='3-Clause BSD License', classifiers=[ "Programming Language :: Python", "Programming Language :: Python :: 2", diff --git a/python/compatibility_tests/v2.5.0/test.sh b/python/compatibility_tests/v2.5.0/test.sh index c640da80..78c16ad1 100755 --- a/python/compatibility_tests/v2.5.0/test.sh +++ b/python/compatibility_tests/v2.5.0/test.sh @@ -92,16 +92,10 @@ python setup.py test # Test A.3: # proto set 1: use old version # proto set 2 which may import protos in set 1: use new version -# Compatiblility test fail if the old verison is less than 3.0.0-alpha-1. -# Because module name aliases was added in v3.0.0-alpha-1 instead of -# fully-qualified module names to refer to dependencies: dot was replaced -# with _dot_. -if [ "$(printf "$OLD_VERSION\n3.0.0" | sort -V | head -n 1 )" = "3.0.0" ]; then - cp old_protoc protoc_1 - cp ../../../src/protoc protoc_2 - python setup.py build - python setup.py test -fi +cp old_protoc protoc_1 +cp ../../../src/protoc protoc_2 +python setup.py build +python setup.py test rm google -r -f rm build -r -f diff --git a/python/compatibility_tests/v2.5.0/tests/google/protobuf/internal/message_test.py b/python/compatibility_tests/v2.5.0/tests/google/protobuf/internal/message_test.py index 53e9d507..e71b295b 100755 --- a/python/compatibility_tests/v2.5.0/tests/google/protobuf/internal/message_test.py +++ b/python/compatibility_tests/v2.5.0/tests/google/protobuf/internal/message_test.py @@ -55,6 +55,11 @@ from google.protobuf.internal import api_implementation from google.protobuf.internal import test_util from google.protobuf import message +try: + cmp # Python 2 +except NameError: + cmp = lambda x, y: (x > y) - (x < y) # Python 3 + # Python pre-2.6 does not have isinf() or isnan() functions, so we have # to provide our own. def isnan(val): diff --git a/python/compatibility_tests/v2.5.0/tests/google/protobuf/internal/text_format_test.py b/python/compatibility_tests/v2.5.0/tests/google/protobuf/internal/text_format_test.py index 6bf7befb..8267cd2c 100755 --- a/python/compatibility_tests/v2.5.0/tests/google/protobuf/internal/text_format_test.py +++ b/python/compatibility_tests/v2.5.0/tests/google/protobuf/internal/text_format_test.py @@ -370,9 +370,8 @@ class TextFormatTest(unittest.TestCase): def testMergeBadExtension(self): message = unittest_pb2.TestAllExtensions() text = '[unknown_extension]: 8\n' - self.assertRaisesWithMessage( + self.assertRaises( text_format.ParseError, - '1:2 : Extension "unknown_extension" not registered.', text_format.Merge, text, message) message = unittest_pb2.TestAllTypes() self.assertRaisesWithMessage( diff --git a/python/google/protobuf/__init__.py b/python/google/protobuf/__init__.py index 0e2a89ff..d26da0df 100755 --- a/python/google/protobuf/__init__.py +++ b/python/google/protobuf/__init__.py @@ -30,7 +30,7 @@ # Copyright 2007 Google Inc. All Rights Reserved. -__version__ = '3.1.0' +__version__ = '3.4.0' if __name__ != '__main__': try: diff --git a/python/google/protobuf/descriptor.py b/python/google/protobuf/descriptor.py index e1f2e3b7..b1f3ca38 100755 --- a/python/google/protobuf/descriptor.py +++ b/python/google/protobuf/descriptor.py @@ -406,6 +406,8 @@ class FieldDescriptor(DescriptorBase): containing_oneof: (OneofDescriptor) If the field is a member of a oneof union, contains its descriptor. Otherwise, None. + + file: (FileDescriptor) Reference to file descriptor. """ # Must be consistent with C++ FieldDescriptor::Type enum in @@ -490,7 +492,8 @@ class FieldDescriptor(DescriptorBase): def __new__(cls, name, full_name, index, number, type, cpp_type, label, default_value, message_type, enum_type, containing_type, is_extension, extension_scope, options=None, - has_default_value=True, containing_oneof=None, json_name=None): + has_default_value=True, containing_oneof=None, json_name=None, + file=None): _message.Message._CheckCalledFromGeneratedFile() if is_extension: return _message.default_pool.FindExtensionByName(full_name) @@ -500,7 +503,8 @@ class FieldDescriptor(DescriptorBase): def __init__(self, name, full_name, index, number, type, cpp_type, label, default_value, message_type, enum_type, containing_type, is_extension, extension_scope, options=None, - has_default_value=True, containing_oneof=None, json_name=None): + has_default_value=True, containing_oneof=None, json_name=None, + file=None): """The arguments are as described in the description of FieldDescriptor attributes above. @@ -511,6 +515,7 @@ class FieldDescriptor(DescriptorBase): super(FieldDescriptor, self).__init__(options, 'FieldOptions') self.name = name self.full_name = full_name + self.file = file self._camelcase_name = None if json_name is None: self.json_name = _ToJsonName(name) diff --git a/python/google/protobuf/descriptor_database.py b/python/google/protobuf/descriptor_database.py index 1333f996..eb45e127 100644 --- a/python/google/protobuf/descriptor_database.py +++ b/python/google/protobuf/descriptor_database.py @@ -54,9 +54,9 @@ class DescriptorDatabase(object): Args: file_desc_proto: The FileDescriptorProto to add. Raises: - DescriptorDatabaseException: if an attempt is made to add a proto - with the same name but different definition than an exisiting - proto in the database. + DescriptorDatabaseConflictingDefinitionError: if an attempt is made to + add a proto with the same name but different definition than an + exisiting proto in the database. """ proto_name = file_desc_proto.name if proto_name not in self._file_desc_protos_by_file: @@ -65,7 +65,7 @@ class DescriptorDatabase(object): raise DescriptorDatabaseConflictingDefinitionError( '%s already added, but with different descriptor.' % proto_name) - # Add the top-level Message, Enum and Extension descriptors to the index. + # Add all the top-level descriptors to the index. package = file_desc_proto.package for message in file_desc_proto.message_type: self._file_desc_protos_by_symbol.update( @@ -76,6 +76,9 @@ class DescriptorDatabase(object): for extension in file_desc_proto.extension: self._file_desc_protos_by_symbol[ '.'.join((package, extension.name))] = file_desc_proto + for service in file_desc_proto.service: + self._file_desc_protos_by_symbol[ + '.'.join((package, service.name))] = file_desc_proto def FindFileByName(self, name): """Finds the file descriptor proto by file name. @@ -131,8 +134,7 @@ def _ExtractSymbols(desc_proto, package): Yields: The fully qualified name found in the descriptor. """ - - message_name = '.'.join((package, desc_proto.name)) + message_name = package + '.' + desc_proto.name if package else desc_proto.name yield message_name for nested_type in desc_proto.nested_type: for symbol in _ExtractSymbols(nested_type, message_name): diff --git a/python/google/protobuf/descriptor_pool.py b/python/google/protobuf/descriptor_pool.py index fc3a7f44..3dbe0fd0 100644 --- a/python/google/protobuf/descriptor_pool.py +++ b/python/google/protobuf/descriptor_pool.py @@ -124,8 +124,12 @@ class DescriptorPool(object): self._descriptor_db = descriptor_db self._descriptors = {} self._enum_descriptors = {} + self._service_descriptors = {} self._file_descriptors = {} self._toplevel_extensions = {} + # TODO(jieluo): Remove _file_desc_by_toplevel_extension when + # FieldDescriptor.file is added in code gen. + self._file_desc_by_toplevel_extension = {} # We store extensions in two two-level mappings: The first key is the # descriptor of the message being extended, the second key is the extension # full name or its tag number. @@ -169,12 +173,12 @@ class DescriptorPool(object): raise TypeError('Expected instance of descriptor.Descriptor.') self._descriptors[desc.full_name] = desc - self.AddFileDescriptor(desc.file) + self._AddFileDescriptor(desc.file) def AddEnumDescriptor(self, enum_desc): """Adds an EnumDescriptor to the pool. - This method also registers the FileDescriptor associated with the message. + This method also registers the FileDescriptor associated with the enum. Args: enum_desc: An EnumDescriptor. @@ -184,7 +188,19 @@ class DescriptorPool(object): raise TypeError('Expected instance of descriptor.EnumDescriptor.') self._enum_descriptors[enum_desc.full_name] = enum_desc - self.AddFileDescriptor(enum_desc.file) + self._AddFileDescriptor(enum_desc.file) + + def AddServiceDescriptor(self, service_desc): + """Adds a ServiceDescriptor to the pool. + + Args: + service_desc: A ServiceDescriptor. + """ + + if not isinstance(service_desc, descriptor.ServiceDescriptor): + raise TypeError('Expected instance of descriptor.ServiceDescriptor.') + + self._service_descriptors[service_desc.full_name] = service_desc def AddExtensionDescriptor(self, extension): """Adds a FieldDescriptor describing an extension to the pool. @@ -238,6 +254,23 @@ class DescriptorPool(object): file_desc: A FileDescriptor. """ + self._AddFileDescriptor(file_desc) + # TODO(jieluo): This is a temporary solution for FieldDescriptor.file. + # Remove it when FieldDescriptor.file is added in code gen. + for extension in file_desc.extensions_by_name.values(): + self._file_desc_by_toplevel_extension[ + extension.full_name] = file_desc + + def _AddFileDescriptor(self, file_desc): + """Adds a FileDescriptor to the pool, non-recursively. + + If the FileDescriptor contains messages or enums, the caller must explicitly + register them. + + Args: + file_desc: A FileDescriptor. + """ + if not isinstance(file_desc, descriptor.FileDescriptor): raise TypeError('Expected instance of descriptor.FileDescriptor.') self._file_descriptors[file_desc.name] = file_desc @@ -252,7 +285,7 @@ class DescriptorPool(object): A FileDescriptor for the named file. Raises: - KeyError: if the file can not be found in the pool. + KeyError: if the file cannot be found in the pool. """ try: @@ -281,7 +314,7 @@ class DescriptorPool(object): A FileDescriptor that contains the specified symbol. Raises: - KeyError: if the file can not be found in the pool. + KeyError: if the file cannot be found in the pool. """ symbol = _NormalizeFullyQualifiedName(symbol) @@ -296,15 +329,28 @@ class DescriptorPool(object): pass try: - file_proto = self._internal_db.FindFileContainingSymbol(symbol) - except KeyError as error: - if self._descriptor_db: - file_proto = self._descriptor_db.FindFileContainingSymbol(symbol) - else: - raise error - if not file_proto: + return self._service_descriptors[symbol].file + except KeyError: + pass + + try: + return self._FindFileContainingSymbolInDb(symbol) + except KeyError: + pass + + try: + return self._file_desc_by_toplevel_extension[symbol] + except KeyError: + pass + + # Try nested extensions inside a message. + message_name, _, extension_name = symbol.rpartition('.') + try: + message = self.FindMessageTypeByName(message_name) + assert message.extensions_by_name[extension_name] + return message.file + except KeyError: raise KeyError('Cannot find a file containing %s' % symbol) - return self._ConvertFileProtoToFileDescriptor(file_proto) def FindMessageTypeByName(self, full_name): """Loads the named descriptor from the pool. @@ -314,11 +360,14 @@ class DescriptorPool(object): Returns: The descriptor for the named type. + + Raises: + KeyError: if the message cannot be found in the pool. """ full_name = _NormalizeFullyQualifiedName(full_name) if full_name not in self._descriptors: - self.FindFileContainingSymbol(full_name) + self._FindFileContainingSymbolInDb(full_name) return self._descriptors[full_name] def FindEnumTypeByName(self, full_name): @@ -329,11 +378,14 @@ class DescriptorPool(object): Returns: The enum descriptor for the named type. + + Raises: + KeyError: if the enum cannot be found in the pool. """ full_name = _NormalizeFullyQualifiedName(full_name) if full_name not in self._enum_descriptors: - self.FindFileContainingSymbol(full_name) + self._FindFileContainingSymbolInDb(full_name) return self._enum_descriptors[full_name] def FindFieldByName(self, full_name): @@ -344,6 +396,9 @@ class DescriptorPool(object): Returns: The field descriptor for the named field. + + Raises: + KeyError: if the field cannot be found in the pool. """ full_name = _NormalizeFullyQualifiedName(full_name) message_name, _, field_name = full_name.rpartition('.') @@ -358,6 +413,9 @@ class DescriptorPool(object): Returns: A FieldDescriptor, describing the named extension. + + Raises: + KeyError: if the extension cannot be found in the pool. """ full_name = _NormalizeFullyQualifiedName(full_name) try: @@ -374,7 +432,7 @@ class DescriptorPool(object): scope = self.FindMessageTypeByName(message_name) except KeyError: # Some extensions are defined at file scope. - scope = self.FindFileContainingSymbol(full_name) + scope = self._FindFileContainingSymbolInDb(full_name) return scope.extensions_by_name[extension_name] def FindExtensionByNumber(self, message_descriptor, number): @@ -390,7 +448,7 @@ class DescriptorPool(object): Returns: A FieldDescriptor describing the extension. - Raise: + Raises: KeyError: when no extension with the given number is known for the specified message. """ @@ -410,6 +468,46 @@ class DescriptorPool(object): """ return list(self._extensions_by_number[message_descriptor].values()) + def FindServiceByName(self, full_name): + """Loads the named service descriptor from the pool. + + Args: + full_name: The full name of the service descriptor to load. + + Returns: + The service descriptor for the named service. + + Raises: + KeyError: if the service cannot be found in the pool. + """ + full_name = _NormalizeFullyQualifiedName(full_name) + if full_name not in self._service_descriptors: + self._FindFileContainingSymbolInDb(full_name) + return self._service_descriptors[full_name] + + def _FindFileContainingSymbolInDb(self, symbol): + """Finds the file in descriptor DB containing the specified symbol. + + Args: + symbol: The name of the symbol to search for. + + Returns: + A FileDescriptor that contains the specified symbol. + + Raises: + KeyError: if the file cannot be found in the descriptor database. + """ + try: + file_proto = self._internal_db.FindFileContainingSymbol(symbol) + except KeyError as error: + if self._descriptor_db: + file_proto = self._descriptor_db.FindFileContainingSymbol(symbol) + else: + raise error + if not file_proto: + raise KeyError('Cannot find a file containing %s' % symbol) + return self._ConvertFileProtoToFileDescriptor(file_proto) + def _ConvertFileProtoToFileDescriptor(self, file_proto): """Creates a FileDescriptor from a proto or returns a cached copy. @@ -463,7 +561,8 @@ class DescriptorPool(object): for index, extension_proto in enumerate(file_proto.extension): extension_desc = self._MakeFieldDescriptor( - extension_proto, file_proto.package, index, is_extension=True) + extension_proto, file_proto.package, index, file_descriptor, + is_extension=True) extension_desc.containing_type = self._GetTypeFromScope( file_descriptor.package, extension_proto.extendee, scope) self._SetFieldType(extension_proto, extension_desc, @@ -529,10 +628,10 @@ class DescriptorPool(object): enums = [ self._ConvertEnumDescriptor(enum, desc_name, file_desc, None, scope) for enum in desc_proto.enum_type] - fields = [self._MakeFieldDescriptor(field, desc_name, index) + fields = [self._MakeFieldDescriptor(field, desc_name, index, file_desc) for index, field in enumerate(desc_proto.field)] extensions = [ - self._MakeFieldDescriptor(extension, desc_name, index, + self._MakeFieldDescriptor(extension, desc_name, index, file_desc, is_extension=True) for index, extension in enumerate(desc_proto.extension)] oneofs = [ @@ -614,7 +713,7 @@ class DescriptorPool(object): return desc def _MakeFieldDescriptor(self, field_proto, message_name, index, - is_extension=False): + file_desc, is_extension=False): """Creates a field descriptor from a FieldDescriptorProto. For message and enum type fields, this method will do a look up @@ -627,6 +726,7 @@ class DescriptorPool(object): field_proto: The proto describing the field. message_name: The name of the containing message. index: Index of the field + file_desc: The file containing the field descriptor. is_extension: Indication that this field is for an extension. Returns: @@ -653,7 +753,8 @@ class DescriptorPool(object): default_value=None, is_extension=is_extension, extension_scope=None, - options=_OptionsOrNone(field_proto)) + options=_OptionsOrNone(field_proto), + file=file_desc) def _SetAllFieldTypes(self, package, desc_proto, scope): """Sets all the descriptor's fields's types. @@ -804,6 +905,7 @@ class DescriptorPool(object): methods=methods, options=_OptionsOrNone(service_proto), file=file_desc) + self._service_descriptors[service_name] = desc return desc def _MakeMethodDescriptor(self, method_proto, service_name, package, scope, diff --git a/python/google/protobuf/internal/api_implementation.py b/python/google/protobuf/internal/api_implementation.py index 460a4a6c..422af590 100755 --- a/python/google/protobuf/internal/api_implementation.py +++ b/python/google/protobuf/internal/api_implementation.py @@ -100,6 +100,27 @@ if _implementation_version_str != '2': _implementation_version = int(_implementation_version_str) +# Detect if serialization should be deterministic by default +try: + # The presence of this module in a build allows the proto implementation to + # be upgraded merely via build deps. + # + # NOTE: Merely importing this automatically enables deterministic proto + # serialization for C++ code, but we still need to export it as a boolean so + # that we can do the same for `_implementation_type == 'python'`. + # + # NOTE2: It is possible for C++ code to enable deterministic serialization by + # default _without_ affecting Python code, if the C++ implementation is not in + # use by this module. That is intended behavior, so we don't actually expose + # this boolean outside of this module. + # + # pylint: disable=g-import-not-at-top,unused-import + from google.protobuf import enable_deterministic_proto_serialization + _python_deterministic_proto_serialization = True +except ImportError: + _python_deterministic_proto_serialization = False + + # Usage of this function is discouraged. Clients shouldn't care which # implementation of the API is in use. Note that there is no guarantee # that differences between APIs will be maintained. @@ -111,3 +132,8 @@ def Type(): # See comment on 'Type' above. def Version(): return _implementation_version + + +# For internal use only +def IsPythonDefaultSerializationDeterministic(): + return _python_deterministic_proto_serialization diff --git a/python/google/protobuf/internal/containers.py b/python/google/protobuf/internal/containers.py index de13018e..68be9e54 100755 --- a/python/google/protobuf/internal/containers.py +++ b/python/google/protobuf/internal/containers.py @@ -275,7 +275,7 @@ class RepeatedScalarFieldContainer(BaseContainer): new_values = [self._type_checker.CheckValue(elem) for elem in elem_seq_iter] if new_values: self._values.extend(new_values) - self._message_listener.Modified() + self._message_listener.Modified() def MergeFrom(self, other): """Appends the contents of another repeated field of the same type to this diff --git a/python/google/protobuf/internal/descriptor_pool_test.py b/python/google/protobuf/internal/descriptor_pool_test.py index 1e710dcf..6015e6f8 100644 --- a/python/google/protobuf/internal/descriptor_pool_test.py +++ b/python/google/protobuf/internal/descriptor_pool_test.py @@ -63,6 +63,9 @@ from google.protobuf import symbol_database class DescriptorPoolTest(unittest.TestCase): def setUp(self): + # TODO(jieluo): Should make the pool which is created by + # serialized_pb same with generated pool. + # TODO(jieluo): More test coverage for the generated pool. self.pool = descriptor_pool.DescriptorPool() self.factory_test1_fd = descriptor_pb2.FileDescriptorProto.FromString( factory_test1_pb2.DESCRIPTOR.serialized_pb) @@ -71,6 +74,13 @@ class DescriptorPoolTest(unittest.TestCase): self.pool.Add(self.factory_test1_fd) self.pool.Add(self.factory_test2_fd) + self.pool.Add(descriptor_pb2.FileDescriptorProto.FromString( + unittest_import_public_pb2.DESCRIPTOR.serialized_pb)) + self.pool.Add(descriptor_pb2.FileDescriptorProto.FromString( + unittest_import_pb2.DESCRIPTOR.serialized_pb)) + self.pool.Add(descriptor_pb2.FileDescriptorProto.FromString( + unittest_pb2.DESCRIPTOR.serialized_pb)) + def testFindFileByName(self): name1 = 'google/protobuf/internal/factory_test1.proto' file_desc1 = self.pool.FindFileByName(name1) @@ -107,6 +117,34 @@ class DescriptorPoolTest(unittest.TestCase): self.assertEqual('google.protobuf.python.internal', file_desc2.package) self.assertIn('Factory2Message', file_desc2.message_types_by_name) + # Tests top level extension. + file_desc3 = self.pool.FindFileContainingSymbol( + 'google.protobuf.python.internal.another_field') + self.assertIsInstance(file_desc3, descriptor.FileDescriptor) + self.assertEqual('google/protobuf/internal/factory_test2.proto', + file_desc3.name) + + # Tests nested extension inside a message. + file_desc4 = self.pool.FindFileContainingSymbol( + 'google.protobuf.python.internal.Factory2Message.one_more_field') + self.assertIsInstance(file_desc4, descriptor.FileDescriptor) + self.assertEqual('google/protobuf/internal/factory_test2.proto', + file_desc4.name) + + file_desc5 = self.pool.FindFileContainingSymbol( + 'protobuf_unittest.TestService') + self.assertIsInstance(file_desc5, descriptor.FileDescriptor) + self.assertEqual('google/protobuf/unittest.proto', + file_desc5.name) + + # Tests the generated pool. + assert descriptor_pool.Default().FindFileContainingSymbol( + 'google.protobuf.python.internal.Factory2Message.one_more_field') + assert descriptor_pool.Default().FindFileContainingSymbol( + 'google.protobuf.python.internal.another_field') + assert descriptor_pool.Default().FindFileContainingSymbol( + 'protobuf_unittest.TestService') + def testFindFileContainingSymbolFailure(self): with self.assertRaises(KeyError): self.pool.FindFileContainingSymbol('Does not exist') @@ -311,6 +349,10 @@ class DescriptorPoolTest(unittest.TestCase): self.pool.FindExtensionByName( 'google.protobuf.python.internal.Factory1Message.list_value') + def testFindService(self): + service = self.pool.FindServiceByName('protobuf_unittest.TestService') + self.assertEqual(service.full_name, 'protobuf_unittest.TestService') + def testUserDefinedDB(self): db = descriptor_database.DescriptorDatabase() self.pool = descriptor_pool.DescriptorPool(db) @@ -472,10 +514,10 @@ class MessageType(object): subtype.CheckType(test, desc, name, file_desc) for index, (name, field) in enumerate(self.field_list): - field.CheckField(test, desc, name, index) + field.CheckField(test, desc, name, index, file_desc) for index, (name, field) in enumerate(self.extensions): - field.CheckField(test, desc, name, index) + field.CheckField(test, desc, name, index, file_desc) class EnumField(object): @@ -485,7 +527,7 @@ class EnumField(object): self.type_name = type_name self.default_value = default_value - def CheckField(self, test, msg_desc, name, index): + def CheckField(self, test, msg_desc, name, index, file_desc): field_desc = msg_desc.fields_by_name[name] enum_desc = msg_desc.enum_types_by_name[self.type_name] test.assertEqual(name, field_desc.name) @@ -502,6 +544,7 @@ class EnumField(object): test.assertFalse(enum_desc.values_by_name[self.default_value].has_options) test.assertEqual(msg_desc, field_desc.containing_type) test.assertEqual(enum_desc, field_desc.enum_type) + test.assertEqual(file_desc, enum_desc.file) class MessageField(object): @@ -510,7 +553,7 @@ class MessageField(object): self.number = number self.type_name = type_name - def CheckField(self, test, msg_desc, name, index): + def CheckField(self, test, msg_desc, name, index, file_desc): field_desc = msg_desc.fields_by_name[name] field_type_desc = msg_desc.nested_types_by_name[self.type_name] test.assertEqual(name, field_desc.name) @@ -524,6 +567,7 @@ class MessageField(object): test.assertFalse(field_desc.has_default_value) test.assertEqual(msg_desc, field_desc.containing_type) test.assertEqual(field_type_desc, field_desc.message_type) + test.assertEqual(file_desc, field_desc.file) class StringField(object): @@ -532,7 +576,7 @@ class StringField(object): self.number = number self.default_value = default_value - def CheckField(self, test, msg_desc, name, index): + def CheckField(self, test, msg_desc, name, index, file_desc): field_desc = msg_desc.fields_by_name[name] test.assertEqual(name, field_desc.name) expected_field_full_name = '.'.join([msg_desc.full_name, name]) @@ -544,6 +588,7 @@ class StringField(object): field_desc.cpp_type) test.assertTrue(field_desc.has_default_value) test.assertEqual(self.default_value, field_desc.default_value) + test.assertEqual(file_desc, field_desc.file) class ExtensionField(object): @@ -552,7 +597,7 @@ class ExtensionField(object): self.number = number self.extended_type = extended_type - def CheckField(self, test, msg_desc, name, index): + def CheckField(self, test, msg_desc, name, index, file_desc): field_desc = msg_desc.extensions_by_name[name] test.assertEqual(name, field_desc.name) expected_field_full_name = '.'.join([msg_desc.full_name, name]) @@ -567,6 +612,7 @@ class ExtensionField(object): test.assertEqual(msg_desc, field_desc.extension_scope) test.assertEqual(msg_desc, field_desc.message_type) test.assertEqual(self.extended_type, field_desc.containing_type.name) + test.assertEqual(file_desc, field_desc.file) class AddDescriptorTest(unittest.TestCase): @@ -645,6 +691,17 @@ class AddDescriptorTest(unittest.TestCase): @unittest.skipIf(api_implementation.Type() == 'cpp', 'With the cpp implementation, Add() must be called first') + def testService(self): + pool = descriptor_pool.DescriptorPool() + with self.assertRaises(KeyError): + pool.FindServiceByName('protobuf_unittest.TestService') + pool.AddServiceDescriptor(unittest_pb2._TESTSERVICE) + self.assertEqual( + 'protobuf_unittest.TestService', + pool.FindServiceByName('protobuf_unittest.TestService').full_name) + + @unittest.skipIf(api_implementation.Type() == 'cpp', + 'With the cpp implementation, Add() must be called first') def testFile(self): pool = descriptor_pool.DescriptorPool() pool.AddFileDescriptor(unittest_pb2.DESCRIPTOR) @@ -701,15 +758,10 @@ class AddDescriptorTest(unittest.TestCase): self.assertIs(options, file_descriptor.GetOptions()) -@unittest.skipIf( - api_implementation.Type() != 'cpp', - 'default_pool is only supported by the C++ implementation') class DefaultPoolTest(unittest.TestCase): def testFindMethods(self): - # pylint: disable=g-import-not-at-top - from google.protobuf.pyext import _message - pool = _message.default_pool + pool = descriptor_pool.Default() self.assertIs( pool.FindFileByName('google/protobuf/unittest.proto'), unittest_pb2.DESCRIPTOR) @@ -720,19 +772,22 @@ class DefaultPoolTest(unittest.TestCase): pool.FindFieldByName('protobuf_unittest.TestAllTypes.optional_int32'), unittest_pb2.TestAllTypes.DESCRIPTOR.fields_by_name['optional_int32']) self.assertIs( - pool.FindExtensionByName('protobuf_unittest.optional_int32_extension'), - unittest_pb2.DESCRIPTOR.extensions_by_name['optional_int32_extension']) - self.assertIs( pool.FindEnumTypeByName('protobuf_unittest.ForeignEnum'), unittest_pb2.ForeignEnum.DESCRIPTOR) + if api_implementation.Type() != 'cpp': + self.skipTest('Only the C++ implementation correctly indexes all types') + self.assertIs( + pool.FindExtensionByName('protobuf_unittest.optional_int32_extension'), + unittest_pb2.DESCRIPTOR.extensions_by_name['optional_int32_extension']) self.assertIs( pool.FindOneofByName('protobuf_unittest.TestAllTypes.oneof_field'), unittest_pb2.TestAllTypes.DESCRIPTOR.oneofs_by_name['oneof_field']) + self.assertIs( + pool.FindServiceByName('protobuf_unittest.TestService'), + unittest_pb2.DESCRIPTOR.services_by_name['TestService']) def testAddFileDescriptor(self): - # pylint: disable=g-import-not-at-top - from google.protobuf.pyext import _message - pool = _message.default_pool + pool = descriptor_pool.Default() file_desc = descriptor_pb2.FileDescriptorProto(name='some/file.proto') pool.Add(file_desc) pool.AddSerializedFile(file_desc.SerializeToString()) diff --git a/python/google/protobuf/internal/descriptor_test.py b/python/google/protobuf/internal/descriptor_test.py index 1f148ab9..c0010081 100755 --- a/python/google/protobuf/internal/descriptor_test.py +++ b/python/google/protobuf/internal/descriptor_test.py @@ -521,6 +521,12 @@ class GeneratedDescriptorTest(unittest.TestCase): del enum self.assertEqual('FOO', next(values_iter).name) + def testServiceDescriptor(self): + service_descriptor = unittest_pb2.DESCRIPTOR.services_by_name['TestService'] + self.assertEqual(service_descriptor.name, 'TestService') + self.assertEqual(service_descriptor.methods[0].name, 'Foo') + self.assertIs(service_descriptor.file, unittest_pb2.DESCRIPTOR) + class DescriptorCopyToProtoTest(unittest.TestCase): """Tests for CopyTo functions of Descriptor.""" diff --git a/python/google/protobuf/internal/encoder.py b/python/google/protobuf/internal/encoder.py index 48ef2df3..ebec42e5 100755 --- a/python/google/protobuf/internal/encoder.py +++ b/python/google/protobuf/internal/encoder.py @@ -340,7 +340,7 @@ def MessageSetItemSizer(field_number): # Map is special: it needs custom logic to compute its size properly. -def MapSizer(field_descriptor): +def MapSizer(field_descriptor, is_message_map): """Returns a sizer for a map field.""" # Can't look at field_descriptor.message_type._concrete_class because it may @@ -355,9 +355,12 @@ def MapSizer(field_descriptor): # It's wasteful to create the messages and throw them away one second # later since we'll do the same for the actual encode. But there's not an # obvious way to avoid this within the current design without tons of code - # duplication. + # duplication. For message map, value.ByteSize() should be called to + # update the status. entry_msg = message_type._concrete_class(key=key, value=value) total += message_sizer(entry_msg) + if is_message_map: + value.ByteSize() return total return FieldSize @@ -369,7 +372,7 @@ def MapSizer(field_descriptor): def _VarintEncoder(): """Return an encoder for a basic varint value (does not include tag).""" - def EncodeVarint(write, value): + def EncodeVarint(write, value, unused_deterministic): bits = value & 0x7f value >>= 7 while value: @@ -385,7 +388,7 @@ def _SignedVarintEncoder(): """Return an encoder for a basic signed varint value (does not include tag).""" - def EncodeSignedVarint(write, value): + def EncodeSignedVarint(write, value, unused_deterministic): if value < 0: value += (1 << 64) bits = value & 0x7f @@ -408,7 +411,7 @@ def _VarintBytes(value): called at startup time so it doesn't need to be fast.""" pieces = [] - _EncodeVarint(pieces.append, value) + _EncodeVarint(pieces.append, value, True) return b"".join(pieces) @@ -437,27 +440,27 @@ def _SimpleEncoder(wire_type, encode_value, compute_value_size): if is_packed: tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) local_EncodeVarint = _EncodeVarint - def EncodePackedField(write, value): + def EncodePackedField(write, value, deterministic): write(tag_bytes) size = 0 for element in value: size += compute_value_size(element) - local_EncodeVarint(write, size) + local_EncodeVarint(write, size, deterministic) for element in value: - encode_value(write, element) + encode_value(write, element, deterministic) return EncodePackedField elif is_repeated: tag_bytes = TagBytes(field_number, wire_type) - def EncodeRepeatedField(write, value): + def EncodeRepeatedField(write, value, deterministic): for element in value: write(tag_bytes) - encode_value(write, element) + encode_value(write, element, deterministic) return EncodeRepeatedField else: tag_bytes = TagBytes(field_number, wire_type) - def EncodeField(write, value): + def EncodeField(write, value, deterministic): write(tag_bytes) - return encode_value(write, value) + return encode_value(write, value, deterministic) return EncodeField return SpecificEncoder @@ -471,27 +474,27 @@ def _ModifiedEncoder(wire_type, encode_value, compute_value_size, modify_value): if is_packed: tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) local_EncodeVarint = _EncodeVarint - def EncodePackedField(write, value): + def EncodePackedField(write, value, deterministic): write(tag_bytes) size = 0 for element in value: size += compute_value_size(modify_value(element)) - local_EncodeVarint(write, size) + local_EncodeVarint(write, size, deterministic) for element in value: - encode_value(write, modify_value(element)) + encode_value(write, modify_value(element), deterministic) return EncodePackedField elif is_repeated: tag_bytes = TagBytes(field_number, wire_type) - def EncodeRepeatedField(write, value): + def EncodeRepeatedField(write, value, deterministic): for element in value: write(tag_bytes) - encode_value(write, modify_value(element)) + encode_value(write, modify_value(element), deterministic) return EncodeRepeatedField else: tag_bytes = TagBytes(field_number, wire_type) - def EncodeField(write, value): + def EncodeField(write, value, deterministic): write(tag_bytes) - return encode_value(write, modify_value(value)) + return encode_value(write, modify_value(value), deterministic) return EncodeField return SpecificEncoder @@ -512,22 +515,22 @@ def _StructPackEncoder(wire_type, format): if is_packed: tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) local_EncodeVarint = _EncodeVarint - def EncodePackedField(write, value): + def EncodePackedField(write, value, deterministic): write(tag_bytes) - local_EncodeVarint(write, len(value) * value_size) + local_EncodeVarint(write, len(value) * value_size, deterministic) for element in value: write(local_struct_pack(format, element)) return EncodePackedField elif is_repeated: tag_bytes = TagBytes(field_number, wire_type) - def EncodeRepeatedField(write, value): + def EncodeRepeatedField(write, value, unused_deterministic): for element in value: write(tag_bytes) write(local_struct_pack(format, element)) return EncodeRepeatedField else: tag_bytes = TagBytes(field_number, wire_type) - def EncodeField(write, value): + def EncodeField(write, value, unused_deterministic): write(tag_bytes) return write(local_struct_pack(format, value)) return EncodeField @@ -578,9 +581,9 @@ def _FloatingPointEncoder(wire_type, format): if is_packed: tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) local_EncodeVarint = _EncodeVarint - def EncodePackedField(write, value): + def EncodePackedField(write, value, deterministic): write(tag_bytes) - local_EncodeVarint(write, len(value) * value_size) + local_EncodeVarint(write, len(value) * value_size, deterministic) for element in value: # This try/except block is going to be faster than any code that # we could write to check whether element is finite. @@ -591,7 +594,7 @@ def _FloatingPointEncoder(wire_type, format): return EncodePackedField elif is_repeated: tag_bytes = TagBytes(field_number, wire_type) - def EncodeRepeatedField(write, value): + def EncodeRepeatedField(write, value, unused_deterministic): for element in value: write(tag_bytes) try: @@ -601,7 +604,7 @@ def _FloatingPointEncoder(wire_type, format): return EncodeRepeatedField else: tag_bytes = TagBytes(field_number, wire_type) - def EncodeField(write, value): + def EncodeField(write, value, unused_deterministic): write(tag_bytes) try: write(local_struct_pack(format, value)) @@ -647,9 +650,9 @@ def BoolEncoder(field_number, is_repeated, is_packed): if is_packed: tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) local_EncodeVarint = _EncodeVarint - def EncodePackedField(write, value): + def EncodePackedField(write, value, deterministic): write(tag_bytes) - local_EncodeVarint(write, len(value)) + local_EncodeVarint(write, len(value), deterministic) for element in value: if element: write(true_byte) @@ -658,7 +661,7 @@ def BoolEncoder(field_number, is_repeated, is_packed): return EncodePackedField elif is_repeated: tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_VARINT) - def EncodeRepeatedField(write, value): + def EncodeRepeatedField(write, value, unused_deterministic): for element in value: write(tag_bytes) if element: @@ -668,7 +671,7 @@ def BoolEncoder(field_number, is_repeated, is_packed): return EncodeRepeatedField else: tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_VARINT) - def EncodeField(write, value): + def EncodeField(write, value, unused_deterministic): write(tag_bytes) if value: return write(true_byte) @@ -684,18 +687,18 @@ def StringEncoder(field_number, is_repeated, is_packed): local_len = len assert not is_packed if is_repeated: - def EncodeRepeatedField(write, value): + def EncodeRepeatedField(write, value, deterministic): for element in value: encoded = element.encode('utf-8') write(tag) - local_EncodeVarint(write, local_len(encoded)) + local_EncodeVarint(write, local_len(encoded), deterministic) write(encoded) return EncodeRepeatedField else: - def EncodeField(write, value): + def EncodeField(write, value, deterministic): encoded = value.encode('utf-8') write(tag) - local_EncodeVarint(write, local_len(encoded)) + local_EncodeVarint(write, local_len(encoded), deterministic) return write(encoded) return EncodeField @@ -708,16 +711,16 @@ def BytesEncoder(field_number, is_repeated, is_packed): local_len = len assert not is_packed if is_repeated: - def EncodeRepeatedField(write, value): + def EncodeRepeatedField(write, value, deterministic): for element in value: write(tag) - local_EncodeVarint(write, local_len(element)) + local_EncodeVarint(write, local_len(element), deterministic) write(element) return EncodeRepeatedField else: - def EncodeField(write, value): + def EncodeField(write, value, deterministic): write(tag) - local_EncodeVarint(write, local_len(value)) + local_EncodeVarint(write, local_len(value), deterministic) return write(value) return EncodeField @@ -729,16 +732,16 @@ def GroupEncoder(field_number, is_repeated, is_packed): end_tag = TagBytes(field_number, wire_format.WIRETYPE_END_GROUP) assert not is_packed if is_repeated: - def EncodeRepeatedField(write, value): + def EncodeRepeatedField(write, value, deterministic): for element in value: write(start_tag) - element._InternalSerialize(write) + element._InternalSerialize(write, deterministic) write(end_tag) return EncodeRepeatedField else: - def EncodeField(write, value): + def EncodeField(write, value, deterministic): write(start_tag) - value._InternalSerialize(write) + value._InternalSerialize(write, deterministic) return write(end_tag) return EncodeField @@ -750,17 +753,17 @@ def MessageEncoder(field_number, is_repeated, is_packed): local_EncodeVarint = _EncodeVarint assert not is_packed if is_repeated: - def EncodeRepeatedField(write, value): + def EncodeRepeatedField(write, value, deterministic): for element in value: write(tag) - local_EncodeVarint(write, element.ByteSize()) - element._InternalSerialize(write) + local_EncodeVarint(write, element.ByteSize(), deterministic) + element._InternalSerialize(write, deterministic) return EncodeRepeatedField else: - def EncodeField(write, value): + def EncodeField(write, value, deterministic): write(tag) - local_EncodeVarint(write, value.ByteSize()) - return value._InternalSerialize(write) + local_EncodeVarint(write, value.ByteSize(), deterministic) + return value._InternalSerialize(write, deterministic) return EncodeField @@ -787,10 +790,10 @@ def MessageSetItemEncoder(field_number): end_bytes = TagBytes(1, wire_format.WIRETYPE_END_GROUP) local_EncodeVarint = _EncodeVarint - def EncodeField(write, value): + def EncodeField(write, value, deterministic): write(start_bytes) - local_EncodeVarint(write, value.ByteSize()) - value._InternalSerialize(write) + local_EncodeVarint(write, value.ByteSize(), deterministic) + value._InternalSerialize(write, deterministic) return write(end_bytes) return EncodeField @@ -815,9 +818,10 @@ def MapEncoder(field_descriptor): message_type = field_descriptor.message_type encode_message = MessageEncoder(field_descriptor.number, False, False) - def EncodeField(write, value): - for key in value: + def EncodeField(write, value, deterministic): + value_keys = sorted(value.keys()) if deterministic else value.keys() + for key in value_keys: entry_msg = message_type._concrete_class(key=key, value=value[key]) - encode_message(write, entry_msg) + encode_message(write, entry_msg, deterministic) return EncodeField diff --git a/python/google/protobuf/internal/factory_test2.proto b/python/google/protobuf/internal/factory_test2.proto index bb1b54ad..5fcbc5ac 100644 --- a/python/google/protobuf/internal/factory_test2.proto +++ b/python/google/protobuf/internal/factory_test2.proto @@ -97,3 +97,8 @@ message MessageWithNestedEnumOnly { extend Factory1Message { optional string another_field = 1002; } + +message MessageWithOption { + option no_standard_descriptor_accessor = true; + optional int32 field1 = 1; +} diff --git a/python/google/protobuf/internal/json_format_test.py b/python/google/protobuf/internal/json_format_test.py index 5ed65622..077b64db 100644 --- a/python/google/protobuf/internal/json_format_test.py +++ b/python/google/protobuf/internal/json_format_test.py @@ -49,6 +49,7 @@ from google.protobuf import field_mask_pb2 from google.protobuf import struct_pb2 from google.protobuf import timestamp_pb2 from google.protobuf import wrappers_pb2 +from google.protobuf import unittest_mset_pb2 from google.protobuf.internal import well_known_types from google.protobuf import json_format from google.protobuf.util import json_format_proto3_pb2 @@ -158,6 +159,84 @@ class JsonFormatTest(JsonFormatBase): json_format.Parse(text, parsed_message) self.assertEqual(message, parsed_message) + def testExtensionToJsonAndBack(self): + message = unittest_mset_pb2.TestMessageSetContainer() + ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension + ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension + message.message_set.Extensions[ext1].i = 23 + message.message_set.Extensions[ext2].str = 'foo' + message_text = json_format.MessageToJson( + message + ) + parsed_message = unittest_mset_pb2.TestMessageSetContainer() + json_format.Parse(message_text, parsed_message) + self.assertEqual(message, parsed_message) + + def testExtensionToDictAndBack(self): + message = unittest_mset_pb2.TestMessageSetContainer() + ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension + ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension + message.message_set.Extensions[ext1].i = 23 + message.message_set.Extensions[ext2].str = 'foo' + message_dict = json_format.MessageToDict( + message + ) + parsed_message = unittest_mset_pb2.TestMessageSetContainer() + json_format.ParseDict(message_dict, parsed_message) + self.assertEqual(message, parsed_message) + + def testExtensionSerializationDictMatchesProto3Spec(self): + """See go/proto3-json-spec for spec. + """ + message = unittest_mset_pb2.TestMessageSetContainer() + ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension + ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension + message.message_set.Extensions[ext1].i = 23 + message.message_set.Extensions[ext2].str = 'foo' + message_dict = json_format.MessageToDict( + message + ) + golden_dict = { + 'messageSet': { + '[protobuf_unittest.' + 'TestMessageSetExtension1.messageSetExtension]': { + 'i': 23, + }, + '[protobuf_unittest.' + 'TestMessageSetExtension2.messageSetExtension]': { + 'str': u'foo', + }, + }, + } + self.assertEqual(golden_dict, message_dict) + + + def testExtensionSerializationJsonMatchesProto3Spec(self): + """See go/proto3-json-spec for spec. + """ + message = unittest_mset_pb2.TestMessageSetContainer() + ext1 = unittest_mset_pb2.TestMessageSetExtension1.message_set_extension + ext2 = unittest_mset_pb2.TestMessageSetExtension2.message_set_extension + message.message_set.Extensions[ext1].i = 23 + message.message_set.Extensions[ext2].str = 'foo' + message_text = json_format.MessageToJson( + message + ) + ext1_text = ('protobuf_unittest.TestMessageSetExtension1.' + 'messageSetExtension') + ext2_text = ('protobuf_unittest.TestMessageSetExtension2.' + 'messageSetExtension') + golden_text = ('{"messageSet": {' + ' "[%s]": {' + ' "i": 23' + ' },' + ' "[%s]": {' + ' "str": "foo"' + ' }' + '}}') % (ext1_text, ext2_text) + self.assertEqual(json.loads(golden_text), json.loads(message_text)) + + def testJsonEscapeString(self): message = json_format_proto3_pb2.TestMessage() if sys.version_info[0] < 3: @@ -768,7 +847,7 @@ class JsonFormatTest(JsonFormatBase): text = '{"value": "0000-01-01T00:00:00Z"}' self.assertRaisesRegexp( json_format.ParseError, - 'Failed to parse value field: year is out of range.', + 'Failed to parse value field: year (0 )?is out of range.', json_format.Parse, text, message) # Time bigger than maxinum time. message.value.seconds = 253402300800 @@ -840,6 +919,12 @@ class JsonFormatTest(JsonFormatBase): json_format.Parse('{"int32_value": 12345}', message) self.assertEqual(12345, message.int32_value) + def testIndent(self): + message = json_format_proto3_pb2.TestMessage() + message.int32_value = 12345 + self.assertEqual('{\n"int32Value": 12345\n}', + json_format.MessageToJson(message, indent=0)) + def testParseDict(self): expected = 12345 js_dict = {'int32Value': expected} @@ -862,6 +947,22 @@ class JsonFormatTest(JsonFormatBase): parsed_message = json_format_proto3_pb2.TestCustomJsonName() self.CheckParseBack(message, parsed_message) + def testSortKeys(self): + # Testing sort_keys is not perfectly working, as by random luck we could + # get the output sorted. We just use a selection of names. + message = json_format_proto3_pb2.TestMessage(bool_value=True, + int32_value=1, + int64_value=3, + uint32_value=4, + string_value='bla') + self.assertEqual( + json_format.MessageToJson(message, sort_keys=True), + # We use json.dumps() instead of a hardcoded string due to differences + # between Python 2 and Python 3. + json.dumps({'boolValue': True, 'int32Value': 1, 'int64Value': '3', + 'uint32Value': 4, 'stringValue': 'bla'}, + indent=2, sort_keys=True)) + if __name__ == '__main__': unittest.main() diff --git a/python/google/protobuf/internal/message_factory_test.py b/python/google/protobuf/internal/message_factory_test.py index 4caa2443..aa7af2d0 100644 --- a/python/google/protobuf/internal/message_factory_test.py +++ b/python/google/protobuf/internal/message_factory_test.py @@ -183,7 +183,14 @@ class MessageFactoryTest(unittest.TestCase): with self.assertRaises(Exception) as cm: factory.GetMessages([f.name]) - self.assertIsInstance(cm.exception, (AssertionError, ValueError)) + self.assertIn(str(cm.exception), + ['Extensions ' + '"google.protobuf.python.internal.Duplicate.extension_field" and' + ' "google.protobuf.python.internal.Extension.extension_field"' + ' both try to extend message type' + ' "google.protobuf.python.internal.Container"' + ' with field number 2.', + 'Double registration of Extensions']) if __name__ == '__main__': diff --git a/python/google/protobuf/internal/message_test.py b/python/google/protobuf/internal/message_test.py index 9986c0d9..dda72cdd 100755 --- a/python/google/protobuf/internal/message_test.py +++ b/python/google/protobuf/internal/message_test.py @@ -53,9 +53,13 @@ import six import sys try: - import unittest2 as unittest #PY26 + import unittest2 as unittest # PY26 except ImportError: import unittest +try: + cmp # Python 2 +except NameError: + cmp = lambda x, y: (x > y) - (x < y) # Python 3 from google.protobuf import map_unittest_pb2 from google.protobuf import unittest_pb2 @@ -136,6 +140,42 @@ class MessageTest(BaseTestCase): golden_copy = copy.deepcopy(golden_message) self.assertEqual(golden_data, golden_copy.SerializeToString()) + def testDeterminismParameters(self, message_module): + # This message is always deterministically serialized, even if determinism + # is disabled, so we can use it to verify that all the determinism + # parameters work correctly. + golden_data = (b'\xe2\x02\nOne string' + b'\xe2\x02\nTwo string' + b'\xe2\x02\nRed string' + b'\xe2\x02\x0bBlue string') + golden_message = message_module.TestAllTypes() + golden_message.repeated_string.extend([ + 'One string', + 'Two string', + 'Red string', + 'Blue string', + ]) + self.assertEqual(golden_data, + golden_message.SerializeToString(deterministic=None)) + self.assertEqual(golden_data, + golden_message.SerializeToString(deterministic=False)) + self.assertEqual(golden_data, + golden_message.SerializeToString(deterministic=True)) + + class BadArgError(Exception): + pass + + class BadArg(object): + + def __nonzero__(self): + raise BadArgError() + + def __bool__(self): + raise BadArgError() + + with self.assertRaises(BadArgError): + golden_message.SerializeToString(deterministic=BadArg()) + def testPickleSupport(self, message_module): golden_data = test_util.GoldenFileData('golden_message') golden_message = message_module.TestAllTypes() @@ -377,6 +417,7 @@ class MessageTest(BaseTestCase): self.assertEqual(message.repeated_int32[0], 1) self.assertEqual(message.repeated_int32[1], 2) self.assertEqual(message.repeated_int32[2], 3) + self.assertEqual(str(message.repeated_int32), str([1, 2, 3])) message.repeated_float.append(1.1) message.repeated_float.append(1.3) @@ -393,6 +434,7 @@ class MessageTest(BaseTestCase): self.assertEqual(message.repeated_string[0], 'a') self.assertEqual(message.repeated_string[1], 'b') self.assertEqual(message.repeated_string[2], 'c') + self.assertEqual(str(message.repeated_string), str([u'a', u'b', u'c'])) message.repeated_bytes.append(b'a') message.repeated_bytes.append(b'c') @@ -401,6 +443,7 @@ class MessageTest(BaseTestCase): self.assertEqual(message.repeated_bytes[0], b'a') self.assertEqual(message.repeated_bytes[1], b'b') self.assertEqual(message.repeated_bytes[2], b'c') + self.assertEqual(str(message.repeated_bytes), str([b'a', b'b', b'c'])) def testSortingRepeatedScalarFieldsCustomComparator(self, message_module): """Check some different types with custom comparator.""" @@ -439,6 +482,8 @@ class MessageTest(BaseTestCase): self.assertEqual(message.repeated_nested_message[3].bb, 4) self.assertEqual(message.repeated_nested_message[4].bb, 5) self.assertEqual(message.repeated_nested_message[5].bb, 6) + self.assertEqual(str(message.repeated_nested_message), + '[bb: 1\n, bb: 2\n, bb: 3\n, bb: 4\n, bb: 5\n, bb: 6\n]') def testSortingRepeatedCompositeFieldsStable(self, message_module): """Check passing a custom comparator to sort a repeated composite field.""" @@ -564,6 +609,11 @@ class MessageTest(BaseTestCase): self.assertIsInstance(m.repeated_nested_message, collections.MutableSequence) + def testRepeatedFieldInsideNestedMessage(self, message_module): + m = message_module.NestedTestAllTypes() + m.payload.repeated_int32.extend([]) + self.assertTrue(m.HasField('payload')) + def ensureNestedMessageExists(self, msg, attribute): """Make sure that a nested message object exists. @@ -1261,6 +1311,14 @@ class Proto3Test(BaseTestCase): self.assertEqual(1234567, m2.optional_nested_enum) self.assertEqual(7654321, m2.repeated_nested_enum[0]) + # ParseFromString in Proto2 should accept unknown enums too. + m3 = unittest_pb2.TestAllTypes() + m3.ParseFromString(serialized) + m2.Clear() + m2.ParseFromString(m3.SerializeToString()) + self.assertEqual(1234567, m2.optional_nested_enum) + self.assertEqual(7654321, m2.repeated_nested_enum[0]) + # Map isn't really a proto3-only feature. But there is no proto2 equivalent # of google/protobuf/map_unittest.proto right now, so it's not easy to # test both with the same test like we do for the other proto2/proto3 tests. @@ -1432,6 +1490,35 @@ class Proto3Test(BaseTestCase): self.assertIn(-456, msg2.map_int32_foreign_message) self.assertEqual(2, len(msg2.map_int32_foreign_message)) + def testNestedMessageMapItemDelete(self): + msg = map_unittest_pb2.TestMap() + msg.map_int32_all_types[1].optional_nested_message.bb = 1 + del msg.map_int32_all_types[1] + msg.map_int32_all_types[2].optional_nested_message.bb = 2 + self.assertEqual(1, len(msg.map_int32_all_types)) + msg.map_int32_all_types[1].optional_nested_message.bb = 1 + self.assertEqual(2, len(msg.map_int32_all_types)) + + serialized = msg.SerializeToString() + msg2 = map_unittest_pb2.TestMap() + msg2.ParseFromString(serialized) + keys = [1, 2] + # The loop triggers PyErr_Occurred() in c extension. + for key in keys: + del msg2.map_int32_all_types[key] + + def testMapByteSize(self): + msg = map_unittest_pb2.TestMap() + msg.map_int32_int32[1] = 1 + size = msg.ByteSize() + msg.map_int32_int32[1] = 128 + self.assertEqual(msg.ByteSize(), size + 1) + + msg.map_int32_foreign_message[19].c = 1 + size = msg.ByteSize() + msg.map_int32_foreign_message[19].c = 128 + self.assertEqual(msg.ByteSize(), size + 1) + def testMergeFrom(self): msg = map_unittest_pb2.TestMap() msg.map_int32_int32[12] = 34 @@ -1456,7 +1543,15 @@ class Proto3Test(BaseTestCase): self.assertEqual(5, msg2.map_int32_foreign_message[111].c) self.assertEqual(10, msg2.map_int32_foreign_message[222].c) self.assertFalse(msg2.map_int32_foreign_message[222].HasField('d')) - self.assertEqual(15, old_map_value.c) + if api_implementation.Type() != 'cpp': + # During the call to MergeFrom(), the C++ implementation will have + # deallocated the underlying message, but this is very difficult to detect + # properly. The line below is likely to cause a segmentation fault. + # With the Python implementation, old_map_value is just 'detached' from + # the main message. Using it will not crash of course, but since it still + # have a reference to the parent message I'm sure we can find interesting + # ways to cause inconsistencies. + self.assertEqual(15, old_map_value.c) # Verify that there is only one entry per key, even though the MergeFrom # may have internally created multiple entries for a single key in the @@ -1626,6 +1721,35 @@ class Proto3Test(BaseTestCase): items2 = msg.map_string_string.items() self.assertEqual(items1, items2) + def testMapDeterministicSerialization(self): + golden_data = (b'r\x0c\n\x07init_op\x12\x01d' + b'r\n\n\x05item1\x12\x01e' + b'r\n\n\x05item2\x12\x01f' + b'r\n\n\x05item3\x12\x01g' + b'r\x0b\n\x05item4\x12\x02QQ' + b'r\x12\n\rlocal_init_op\x12\x01a' + b'r\x0e\n\tsummaries\x12\x01e' + b'r\x18\n\x13trainable_variables\x12\x01b' + b'r\x0e\n\tvariables\x12\x01c') + msg = map_unittest_pb2.TestMap() + msg.map_string_string['local_init_op'] = 'a' + msg.map_string_string['trainable_variables'] = 'b' + msg.map_string_string['variables'] = 'c' + msg.map_string_string['init_op'] = 'd' + msg.map_string_string['summaries'] = 'e' + msg.map_string_string['item1'] = 'e' + msg.map_string_string['item2'] = 'f' + msg.map_string_string['item3'] = 'g' + msg.map_string_string['item4'] = 'QQ' + + # If deterministic serialization is not working correctly, this will be + # "flaky" depending on the exact python dict hash seed. + # + # Fortunately, there are enough items in this map that it is extremely + # unlikely to ever hit the "right" in-order combination, so the test + # itself should fail reliably. + self.assertEqual(golden_data, msg.SerializeToString(deterministic=True)) + def testMapIterationClearMessage(self): # Iterator needs to work even if message and map are deleted. msg = map_unittest_pb2.TestMap() @@ -1807,8 +1931,9 @@ class PackedFieldTest(BaseTestCase): self.assertEqual(golden_data, message.SerializeToString()) -@unittest.skipIf(api_implementation.Type() != 'cpp', - 'explicit tests of the C++ implementation') +@unittest.skipIf(api_implementation.Type() != 'cpp' or + sys.version_info < (2, 7), + 'explicit tests of the C++ implementation for PY27 and above') class OversizeProtosTest(BaseTestCase): @classmethod diff --git a/python/google/protobuf/internal/more_extensions_dynamic.proto b/python/google/protobuf/internal/more_extensions_dynamic.proto index 11f85ef6..98fcbcb6 100644 --- a/python/google/protobuf/internal/more_extensions_dynamic.proto +++ b/python/google/protobuf/internal/more_extensions_dynamic.proto @@ -47,4 +47,5 @@ message DynamicMessageType { extend ExtendedMessage { optional int32 dynamic_int32_extension = 100; optional DynamicMessageType dynamic_message_extension = 101; + repeated DynamicMessageType repeated_dynamic_message_extension = 102; } diff --git a/python/google/protobuf/internal/python_message.py b/python/google/protobuf/internal/python_message.py index c1bd1f9c..c363d843 100755 --- a/python/google/protobuf/internal/python_message.py +++ b/python/google/protobuf/internal/python_message.py @@ -56,19 +56,9 @@ import sys import weakref import six -try: - import six.moves.copyreg as copyreg -except ImportError: - # On some platforms, for example gMac, we run native Python because there is - # nothing like hermetic Python. This means lesser control on the system and - # the six.moves package may be missing (is missing on 20150321 on gMac). Be - # extra conservative and try to load the old replacement if it fails. - try: - import copy_reg as copyreg #PY26 - except ImportError: - import copyreg # We use "as" to avoid name collisions with variables. +from google.protobuf.internal import api_implementation from google.protobuf.internal import containers from google.protobuf.internal import decoder from google.protobuf.internal import encoder @@ -179,7 +169,6 @@ class GeneratedProtocolMessageType(type): _AddStaticMethods(cls) _AddMessageMethods(descriptor, cls) _AddPrivateHelperMethods(descriptor, cls) - copyreg.pickle(cls, lambda obj: (cls, (), obj.__getstate__())) superclass = super(GeneratedProtocolMessageType, cls) superclass.__init__(name, bases, dictionary) @@ -300,7 +289,8 @@ def _AttachFieldHelpers(cls, field_descriptor): if is_map_entry: field_encoder = encoder.MapEncoder(field_descriptor) - sizer = encoder.MapSizer(field_descriptor) + sizer = encoder.MapSizer(field_descriptor, + _IsMessageMapField(field_descriptor)) elif _IsMessageSetExtension(field_descriptor): field_encoder = encoder.MessageSetItemEncoder(field_descriptor.number) sizer = encoder.MessageSetItemSizer(field_descriptor.number) @@ -903,7 +893,7 @@ def _AddHasExtensionMethod(cls): def _InternalUnpackAny(msg): """Unpacks Any message and returns the unpacked message. - This internal method is differnt from public Any Unpack method which takes + This internal method is different from public Any Unpack method which takes the target message as argument. _InternalUnpackAny method does not have target message type and need to find the message type in descriptor pool. @@ -1037,29 +1027,34 @@ def _AddByteSizeMethod(message_descriptor, cls): def _AddSerializeToStringMethod(message_descriptor, cls): """Helper for _AddMessageMethods().""" - def SerializeToString(self): + def SerializeToString(self, **kwargs): # Check if the message has all of its required fields set. errors = [] if not self.IsInitialized(): raise message_mod.EncodeError( 'Message %s is missing required fields: %s' % ( self.DESCRIPTOR.full_name, ','.join(self.FindInitializationErrors()))) - return self.SerializePartialToString() + return self.SerializePartialToString(**kwargs) cls.SerializeToString = SerializeToString def _AddSerializePartialToStringMethod(message_descriptor, cls): """Helper for _AddMessageMethods().""" - def SerializePartialToString(self): + def SerializePartialToString(self, **kwargs): out = BytesIO() - self._InternalSerialize(out.write) + self._InternalSerialize(out.write, **kwargs) return out.getvalue() cls.SerializePartialToString = SerializePartialToString - def InternalSerialize(self, write_bytes): + def InternalSerialize(self, write_bytes, deterministic=None): + if deterministic is None: + deterministic = ( + api_implementation.IsPythonDefaultSerializationDeterministic()) + else: + deterministic = bool(deterministic) for field_descriptor, field_value in self.ListFields(): - field_descriptor._encoder(write_bytes, field_value) + field_descriptor._encoder(write_bytes, field_value, deterministic) for tag_bytes, value_bytes in self._unknown_fields: write_bytes(tag_bytes) write_bytes(value_bytes) @@ -1271,6 +1266,12 @@ def _AddWhichOneofMethod(message_descriptor, cls): cls.WhichOneof = WhichOneof +def _AddReduceMethod(cls): + def __reduce__(self): # pylint: disable=invalid-name + return (type(self), (), self.__getstate__()) + cls.__reduce__ = __reduce__ + + def _Clear(self): # Clear fields. self._fields = {} @@ -1316,6 +1317,7 @@ def _AddMessageMethods(message_descriptor, cls): _AddIsInitializedMethod(message_descriptor, cls) _AddMergeFromMethod(cls) _AddWhichOneofMethod(message_descriptor, cls) + _AddReduceMethod(cls) # Adds methods which do not depend on cls. cls.Clear = _Clear cls.DiscardUnknownFields = _DiscardUnknownFields diff --git a/python/google/protobuf/internal/python_protobuf.cc b/python/google/protobuf/internal/python_protobuf.cc new file mode 100644 index 00000000..f90cc438 --- /dev/null +++ b/python/google/protobuf/internal/python_protobuf.cc @@ -0,0 +1,63 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: qrczak@google.com (Marcin Kowalczyk) + +#include <google/protobuf/python/python_protobuf.h> + +namespace google { +namespace protobuf { +namespace python { + +static const Message* GetCProtoInsidePyProtoStub(PyObject* msg) { + return NULL; +} +static Message* MutableCProtoInsidePyProtoStub(PyObject* msg) { + return NULL; +} + +// This is initialized with a default, stub implementation. +// If python-google.protobuf.cc is loaded, the function pointer is overridden +// with a full implementation. +const Message* (*GetCProtoInsidePyProtoPtr)(PyObject* msg) = + GetCProtoInsidePyProtoStub; +Message* (*MutableCProtoInsidePyProtoPtr)(PyObject* msg) = + MutableCProtoInsidePyProtoStub; + +const Message* GetCProtoInsidePyProto(PyObject* msg) { + return GetCProtoInsidePyProtoPtr(msg); +} +Message* MutableCProtoInsidePyProto(PyObject* msg) { + return MutableCProtoInsidePyProtoPtr(msg); +} + +} // namespace python +} // namespace protobuf +} // namespace google diff --git a/python/google/protobuf/internal/reflection_test.py b/python/google/protobuf/internal/reflection_test.py index 0e881015..5ab5225e 100755 --- a/python/google/protobuf/internal/reflection_test.py +++ b/python/google/protobuf/internal/reflection_test.py @@ -40,6 +40,7 @@ import gc import operator import six import struct +import sys try: import unittest2 as unittest #PY26 @@ -686,8 +687,8 @@ class ReflectionTest(BaseTestCase): self.assertEqual(expected_min, getattr(pb, field_name)) setattr(pb, field_name, expected_max) self.assertEqual(expected_max, getattr(pb, field_name)) - self.assertRaises(ValueError, setattr, pb, field_name, expected_min - 1) - self.assertRaises(ValueError, setattr, pb, field_name, expected_max + 1) + self.assertRaises(Exception, setattr, pb, field_name, expected_min - 1) + self.assertRaises(Exception, setattr, pb, field_name, expected_max + 1) TestMinAndMaxIntegers('optional_int32', -(1 << 31), (1 << 31) - 1) TestMinAndMaxIntegers('optional_uint32', 0, 0xffffffff) @@ -696,7 +697,7 @@ class ReflectionTest(BaseTestCase): # A bit of white-box testing since -1 is an int and not a long in C++ and # so goes down a different path. pb = unittest_pb2.TestAllTypes() - with self.assertRaises(ValueError): + with self.assertRaises(Exception): pb.optional_uint64 = integer_fn(-(1 << 63)) pb = unittest_pb2.TestAllTypes() @@ -1551,7 +1552,14 @@ class ReflectionTest(BaseTestCase): container = copy.deepcopy(proto1.repeated_int32) self.assertEqual([2, 3], container) - # TODO(anuraag): Implement deepcopy for repeated composite / extension dict + message1 = proto1.repeated_nested_message.add() + message1.bb = 1 + messages = copy.deepcopy(proto1.repeated_nested_message) + self.assertEqual(proto1.repeated_nested_message, messages) + message1.bb = 2 + self.assertNotEqual(proto1.repeated_nested_message, messages) + + # TODO(anuraag): Implement deepcopy for extension dict def testClear(self): proto = unittest_pb2.TestAllTypes() diff --git a/python/google/protobuf/internal/symbol_database_test.py b/python/google/protobuf/internal/symbol_database_test.py index 4f5173b2..af42681a 100644 --- a/python/google/protobuf/internal/symbol_database_test.py +++ b/python/google/protobuf/internal/symbol_database_test.py @@ -60,6 +60,7 @@ class SymbolDatabaseTest(unittest.TestCase): db.RegisterMessage(unittest_pb2.TestAllTypes.RepeatedGroup) db.RegisterEnumDescriptor(unittest_pb2.ForeignEnum.DESCRIPTOR) db.RegisterEnumDescriptor(unittest_pb2.TestAllTypes.NestedEnum.DESCRIPTOR) + db.RegisterServiceDescriptor(unittest_pb2._TESTSERVICE) return db def testGetPrototype(self): @@ -109,7 +110,13 @@ class SymbolDatabaseTest(unittest.TestCase): self._Database().pool.FindMessageTypeByName( 'protobuf_unittest.TestAllTypes.NestedMessage').full_name) - def testFindFindContainingSymbol(self): + def testFindServiceByName(self): + self.assertEqual( + 'protobuf_unittest.TestService', + self._Database().pool.FindServiceByName( + 'protobuf_unittest.TestService').full_name) + + def testFindFileContainingSymbol(self): # Lookup based on either enum or message. self.assertEqual( 'google/protobuf/unittest.proto', diff --git a/python/google/protobuf/internal/test_util.py b/python/google/protobuf/internal/test_util.py index 269d0e2d..9434b7b1 100755 --- a/python/google/protobuf/internal/test_util.py +++ b/python/google/protobuf/internal/test_util.py @@ -39,11 +39,15 @@ __author__ = 'robinson@google.com (Will Robinson)' import numbers import operator import os.path -import sys from google.protobuf import unittest_import_pb2 from google.protobuf import unittest_pb2 -from google.protobuf import descriptor_pb2 + +try: + long # Python 2 +except NameError: + long = int # Python 3 + # Tests whether the given TestAllTypes message is proto2 or not. # This is used to gate several fields/features that only exist @@ -51,6 +55,7 @@ from google.protobuf import descriptor_pb2 def IsProto2(message): return message.DESCRIPTOR.syntax == "proto2" + def SetAllNonLazyFields(message): """Sets every non-lazy field in the message to a unique value. @@ -707,8 +712,8 @@ class NonStandardInteger(numbers.Integral): NonStandardInteger is the minimal legal specification for a custom Integral. As such, it does not support 0 < x < 5 and it is not hashable. - Note: This is added here instead of relying on numpy or a similar library with - custom integers to limit dependencies. + Note: This is added here instead of relying on numpy or a similar library + with custom integers to limit dependencies. """ def __init__(self, val, error_string_on_conversion=None): @@ -845,4 +850,3 @@ class NonStandardInteger(numbers.Integral): def __repr__(self): return 'NonStandardInteger(%s)' % self.val - diff --git a/python/google/protobuf/internal/text_format_test.py b/python/google/protobuf/internal/text_format_test.py index 176cbd15..424b29cc 100755 --- a/python/google/protobuf/internal/text_format_test.py +++ b/python/google/protobuf/internal/text_format_test.py @@ -452,16 +452,18 @@ class TextFormatTest(TextFormatBase): text_format.Parse(text_format.MessageToString(m), m2) self.assertEqual('oneof_uint32', m2.WhichOneof('oneof_field')) + def testMergeMultipleOneof(self, message_module): + m_string = '\n'.join(['oneof_uint32: 11', 'oneof_string: "foo"']) + m2 = message_module.TestAllTypes() + text_format.Merge(m_string, m2) + self.assertEqual('oneof_string', m2.WhichOneof('oneof_field')) + def testParseMultipleOneof(self, message_module): m_string = '\n'.join(['oneof_uint32: 11', 'oneof_string: "foo"']) m2 = message_module.TestAllTypes() - if message_module is unittest_pb2: - with self.assertRaisesRegexp(text_format.ParseError, - ' is specified along with field '): - text_format.Parse(m_string, m2) - else: + with self.assertRaisesRegexp(text_format.ParseError, + ' is specified along with field '): text_format.Parse(m_string, m2) - self.assertEqual('oneof_string', m2.WhichOneof('oneof_field')) # These are tests that aren't fundamentally specific to proto2, but are at @@ -1026,8 +1028,7 @@ class Proto3Tests(unittest.TestCase): packed_message.data = 'string1' message.repeated_any_value.add().Pack(packed_message) self.assertEqual( - text_format.MessageToString(message, - descriptor_pool=descriptor_pool.Default()), + text_format.MessageToString(message), 'repeated_any_value {\n' ' [type.googleapis.com/protobuf_unittest.OneString] {\n' ' data: "string0"\n' @@ -1039,18 +1040,6 @@ class Proto3Tests(unittest.TestCase): ' }\n' '}\n') - def testPrintMessageExpandAnyNoDescriptorPool(self): - packed_message = unittest_pb2.OneString() - packed_message.data = 'string' - message = any_test_pb2.TestAny() - message.any_value.Pack(packed_message) - self.assertEqual( - text_format.MessageToString(message, descriptor_pool=None), - 'any_value {\n' - ' type_url: "type.googleapis.com/protobuf_unittest.OneString"\n' - ' value: "\\n\\006string"\n' - '}\n') - def testPrintMessageExpandAnyDescriptorPoolMissingType(self): packed_message = unittest_pb2.OneString() packed_message.data = 'string' @@ -1071,8 +1060,7 @@ class Proto3Tests(unittest.TestCase): message.any_value.Pack(packed_message) self.assertEqual( text_format.MessageToString(message, - pointy_brackets=True, - descriptor_pool=descriptor_pool.Default()), + pointy_brackets=True), 'any_value <\n' ' [type.googleapis.com/protobuf_unittest.OneString] <\n' ' data: "string"\n' @@ -1086,8 +1074,7 @@ class Proto3Tests(unittest.TestCase): message.any_value.Pack(packed_message) self.assertEqual( text_format.MessageToString(message, - as_one_line=True, - descriptor_pool=descriptor_pool.Default()), + as_one_line=True), 'any_value {' ' [type.googleapis.com/protobuf_unittest.OneString]' ' { data: "string" } ' @@ -1115,7 +1102,12 @@ class Proto3Tests(unittest.TestCase): ' data: "string"\n' ' }\n' '}\n') - text_format.Merge(text, message, descriptor_pool=descriptor_pool.Default()) + text_format.Merge(text, message) + packed_message = unittest_pb2.OneString() + message.any_value.Unpack(packed_message) + self.assertEqual('string', packed_message.data) + message.Clear() + text_format.Parse(text, message) packed_message = unittest_pb2.OneString() message.any_value.Unpack(packed_message) self.assertEqual('string', packed_message.data) @@ -1132,7 +1124,7 @@ class Proto3Tests(unittest.TestCase): ' data: "string1"\n' ' }\n' '}\n') - text_format.Merge(text, message, descriptor_pool=descriptor_pool.Default()) + text_format.Merge(text, message) packed_message = unittest_pb2.OneString() message.repeated_any_value[0].Unpack(packed_message) self.assertEqual('string0', packed_message.data) @@ -1146,22 +1138,22 @@ class Proto3Tests(unittest.TestCase): ' data: "string"\n' ' >\n' '}\n') - text_format.Merge(text, message, descriptor_pool=descriptor_pool.Default()) + text_format.Merge(text, message) packed_message = unittest_pb2.OneString() message.any_value.Unpack(packed_message) self.assertEqual('string', packed_message.data) - def testMergeExpandedAnyNoDescriptorPool(self): + def testMergeAlternativeUrl(self): message = any_test_pb2.TestAny() text = ('any_value {\n' - ' [type.googleapis.com/protobuf_unittest.OneString] {\n' + ' [type.otherapi.com/protobuf_unittest.OneString] {\n' ' data: "string"\n' ' }\n' '}\n') - with self.assertRaises(text_format.ParseError) as e: - text_format.Merge(text, message, descriptor_pool=None) - self.assertEqual(str(e.exception), - 'Descriptor pool required to parse expanded Any field') + text_format.Merge(text, message) + packed_message = unittest_pb2.OneString() + self.assertEqual('type.otherapi.com/protobuf_unittest.OneString', + message.any_value.type_url) def testMergeExpandedAnyDescriptorPoolMissingType(self): message = any_test_pb2.TestAny() @@ -1373,6 +1365,148 @@ class TokenizerTest(unittest.TestCase): self.assertEqual('# some comment', tokenizer.ConsumeComment()) self.assertTrue(tokenizer.AtEnd()) + def testConsumeLineComment(self): + tokenizer = text_format.Tokenizer('# some comment'.splitlines(), + skip_comments=False) + self.assertFalse(tokenizer.AtEnd()) + self.assertEqual((False, '# some comment'), + tokenizer.ConsumeCommentOrTrailingComment()) + self.assertTrue(tokenizer.AtEnd()) + + def testConsumeTwoLineComments(self): + text = '# some comment\n# another comment' + tokenizer = text_format.Tokenizer(text.splitlines(), skip_comments=False) + self.assertEqual((False, '# some comment'), + tokenizer.ConsumeCommentOrTrailingComment()) + self.assertFalse(tokenizer.AtEnd()) + self.assertEqual((False, '# another comment'), + tokenizer.ConsumeCommentOrTrailingComment()) + self.assertTrue(tokenizer.AtEnd()) + + def testConsumeAndCheckTrailingComment(self): + text = 'some_number: 4 # some comment' # trailing comment on the same line + tokenizer = text_format.Tokenizer(text.splitlines(), skip_comments=False) + self.assertRaises(text_format.ParseError, + tokenizer.ConsumeCommentOrTrailingComment) + + self.assertEqual('some_number', tokenizer.ConsumeIdentifier()) + self.assertEqual(tokenizer.token, ':') + tokenizer.NextToken() + self.assertRaises(text_format.ParseError, + tokenizer.ConsumeCommentOrTrailingComment) + self.assertEqual(4, tokenizer.ConsumeInteger()) + self.assertFalse(tokenizer.AtEnd()) + + self.assertEqual((True, '# some comment'), + tokenizer.ConsumeCommentOrTrailingComment()) + self.assertTrue(tokenizer.AtEnd()) + + def testHashinComment(self): + text = 'some_number: 4 # some comment # not a new comment' + tokenizer = text_format.Tokenizer(text.splitlines(), skip_comments=False) + self.assertEqual('some_number', tokenizer.ConsumeIdentifier()) + self.assertEqual(tokenizer.token, ':') + tokenizer.NextToken() + self.assertEqual(4, tokenizer.ConsumeInteger()) + self.assertEqual((True, '# some comment # not a new comment'), + tokenizer.ConsumeCommentOrTrailingComment()) + self.assertTrue(tokenizer.AtEnd()) + + +# Tests for pretty printer functionality. +@_parameterized.Parameters((unittest_pb2), (unittest_proto3_arena_pb2)) +class PrettyPrinterTest(TextFormatBase): + + def testPrettyPrintNoMatch(self, message_module): + + def printer(message, indent, as_one_line): + del message, indent, as_one_line + return None + + message = message_module.TestAllTypes() + msg = message.repeated_nested_message.add() + msg.bb = 42 + self.CompareToGoldenText( + text_format.MessageToString( + message, as_one_line=True, message_formatter=printer), + 'repeated_nested_message { bb: 42 }') + + def testPrettyPrintOneLine(self, message_module): + + def printer(m, indent, as_one_line): + del indent, as_one_line + if m.DESCRIPTOR == message_module.TestAllTypes.NestedMessage.DESCRIPTOR: + return 'My lucky number is %s' % m.bb + + message = message_module.TestAllTypes() + msg = message.repeated_nested_message.add() + msg.bb = 42 + self.CompareToGoldenText( + text_format.MessageToString( + message, as_one_line=True, message_formatter=printer), + 'repeated_nested_message { My lucky number is 42 }') + + def testPrettyPrintMultiLine(self, message_module): + + def printer(m, indent, as_one_line): + if m.DESCRIPTOR == message_module.TestAllTypes.NestedMessage.DESCRIPTOR: + line_deliminator = (' ' if as_one_line else '\n') + ' ' * indent + return 'My lucky number is:%s%s' % (line_deliminator, m.bb) + return None + + message = message_module.TestAllTypes() + msg = message.repeated_nested_message.add() + msg.bb = 42 + self.CompareToGoldenText( + text_format.MessageToString( + message, as_one_line=True, message_formatter=printer), + 'repeated_nested_message { My lucky number is: 42 }') + self.CompareToGoldenText( + text_format.MessageToString( + message, as_one_line=False, message_formatter=printer), + 'repeated_nested_message {\n My lucky number is:\n 42\n}\n') + + def testPrettyPrintEntireMessage(self, message_module): + + def printer(m, indent, as_one_line): + del indent, as_one_line + if m.DESCRIPTOR == message_module.TestAllTypes.DESCRIPTOR: + return 'The is the message!' + return None + + message = message_module.TestAllTypes() + self.CompareToGoldenText( + text_format.MessageToString( + message, as_one_line=False, message_formatter=printer), + 'The is the message!\n') + self.CompareToGoldenText( + text_format.MessageToString( + message, as_one_line=True, message_formatter=printer), + 'The is the message!') + + def testPrettyPrintMultipleParts(self, message_module): + + def printer(m, indent, as_one_line): + del indent, as_one_line + if m.DESCRIPTOR == message_module.TestAllTypes.NestedMessage.DESCRIPTOR: + return 'My lucky number is %s' % m.bb + return None + + message = message_module.TestAllTypes() + message.optional_int32 = 61 + msg = message.repeated_nested_message.add() + msg.bb = 42 + msg = message.repeated_nested_message.add() + msg.bb = 99 + msg = message.optional_nested_message + msg.bb = 1 + self.CompareToGoldenText( + text_format.MessageToString( + message, as_one_line=True, message_formatter=printer), + ('optional_int32: 61 ' + 'optional_nested_message { My lucky number is 1 } ' + 'repeated_nested_message { My lucky number is 42 } ' + 'repeated_nested_message { My lucky number is 99 }')) if __name__ == '__main__': unittest.main() diff --git a/python/google/protobuf/internal/well_known_types.py b/python/google/protobuf/internal/well_known_types.py index d631abee..d0c7ffda 100644 --- a/python/google/protobuf/internal/well_known_types.py +++ b/python/google/protobuf/internal/well_known_types.py @@ -350,12 +350,12 @@ class Duration(object): self.nanos, _NANOS_PER_MICROSECOND)) def FromTimedelta(self, td): - """Convertd timedelta to Duration.""" + """Converts timedelta to Duration.""" self._NormalizeDuration(td.seconds + td.days * _SECONDS_PER_DAY, td.microseconds * _NANOS_PER_MICROSECOND) def _NormalizeDuration(self, seconds, nanos): - """Set Duration by seconds and nonas.""" + """Set Duration by seconds and nanos.""" # Force nanos to be negative if the duration is negative. if seconds < 0 and nanos > 0: seconds += 1 diff --git a/python/google/protobuf/internal/well_known_types_test.py b/python/google/protobuf/internal/well_known_types_test.py index 077f630f..123a537c 100644 --- a/python/google/protobuf/internal/well_known_types_test.py +++ b/python/google/protobuf/internal/well_known_types_test.py @@ -284,7 +284,7 @@ class TimeUtilTest(TimeUtilTestBase): '1972-01-01T01:00:00.01+08',) self.assertRaisesRegexp( ValueError, - 'year is out of range', + 'year (0 )?is out of range', message.FromJsonString, '0000-01-01T00:00:00Z') message.seconds = 253402300800 diff --git a/python/google/protobuf/json_format.py b/python/google/protobuf/json_format.py index d02cb091..801eed60 100644 --- a/python/google/protobuf/json_format.py +++ b/python/google/protobuf/json_format.py @@ -74,6 +74,9 @@ _UNPAIRED_SURROGATE_PATTERN = re.compile(six.u( r'[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]' )) +_VALID_EXTENSION_NAME = re.compile(r'\[[a-zA-Z0-9\._]*\]$') + + class Error(Exception): """Top-level module error for json_format.""" @@ -88,7 +91,9 @@ class ParseError(Error): def MessageToJson(message, including_default_value_fields=False, - preserving_proto_field_name=False): + preserving_proto_field_name=False, + indent=2, + sort_keys=False): """Converts protobuf message to JSON format. Args: @@ -100,19 +105,24 @@ def MessageToJson(message, preserving_proto_field_name: If True, use the original proto field names as defined in the .proto file. If False, convert the field names to lowerCamelCase. + indent: The JSON object will be pretty-printed with this indent level. + An indent level of 0 or negative will only insert newlines. + sort_keys: If True, then the output will be sorted by field names. Returns: A string containing the JSON formatted protocol buffer message. """ printer = _Printer(including_default_value_fields, preserving_proto_field_name) - return printer.ToJsonString(message) + return printer.ToJsonString(message, indent, sort_keys) def MessageToDict(message, including_default_value_fields=False, preserving_proto_field_name=False): - """Converts protobuf message to a JSON dictionary. + """Converts protobuf message to a dictionary. + + When the dictionary is encoded to JSON, it conforms to proto3 JSON spec. Args: message: The protocol buffers message instance to serialize. @@ -125,7 +135,7 @@ def MessageToDict(message, names to lowerCamelCase. Returns: - A dict representation of the JSON formatted protocol buffer message. + A dict representation of the protocol buffer message. """ printer = _Printer(including_default_value_fields, preserving_proto_field_name) @@ -148,9 +158,9 @@ class _Printer(object): self.including_default_value_fields = including_default_value_fields self.preserving_proto_field_name = preserving_proto_field_name - def ToJsonString(self, message): + def ToJsonString(self, message, indent, sort_keys): js = self._MessageToJsonObject(message) - return json.dumps(js, indent=2) + return json.dumps(js, indent=indent, sort_keys=sort_keys) def _MessageToJsonObject(self, message): """Converts message to an object according to Proto3 JSON Specification.""" @@ -192,6 +202,14 @@ class _Printer(object): # Convert a repeated field. js[name] = [self._FieldToJsonObject(field, k) for k in value] + elif field.is_extension: + f = field + if (f.containing_type.GetOptions().message_set_wire_format and + f.type == descriptor.FieldDescriptor.TYPE_MESSAGE and + f.label == descriptor.FieldDescriptor.LABEL_OPTIONAL): + f = f.message_type + name = '[%s.%s]' % (f.full_name, name) + js[name] = self._FieldToJsonObject(field, value) else: js[name] = self._FieldToJsonObject(field, value) @@ -433,12 +451,23 @@ class _Parser(object): field = fields_by_json_name.get(name, None) if not field: field = message_descriptor.fields_by_name.get(name, None) + if not field and _VALID_EXTENSION_NAME.match(name): + if not message_descriptor.is_extendable: + raise ParseError('Message type {0} does not have extensions'.format( + message_descriptor.full_name)) + identifier = name[1:-1] # strip [] brackets + identifier = '.'.join(identifier.split('.')[:-1]) + # pylint: disable=protected-access + field = message.Extensions._FindExtensionByName(identifier) + # pylint: enable=protected-access if not field: if self.ignore_unknown_fields: continue raise ParseError( - 'Message type "{0}" has no field named "{1}".'.format( - message_descriptor.full_name, name)) + ('Message type "{0}" has no field named "{1}".\n' + ' Available Fields(except extensions): {2}').format( + message_descriptor.full_name, name, + message_descriptor.fields)) if name in names: raise ParseError('Message type "{0}" should not have multiple ' '"{1}" fields.'.format( @@ -491,7 +520,10 @@ class _Parser(object): getattr(message, field.name).append( _ConvertScalarFieldValue(item, field)) elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: - sub_message = getattr(message, field.name) + if field.is_extension: + sub_message = message.Extensions[field] + else: + sub_message = getattr(message, field.name) sub_message.SetInParent() self.ConvertMessage(value, sub_message) else: @@ -532,8 +564,8 @@ class _Parser(object): def _ConvertGenericMessage(self, value, message): """Convert a JSON representation into message with FromJsonString.""" - # Durantion, Timestamp, FieldMask have FromJsonString method to do the - # convert. Users can also call the method directly. + # Duration, Timestamp, FieldMask have a FromJsonString method to do the + # conversion. Users can also call the method directly. message.FromJsonString(value) def _ConvertValueMessage(self, value, message): diff --git a/python/google/protobuf/message.py b/python/google/protobuf/message.py index aab250e4..eeb0d576 100755 --- a/python/google/protobuf/message.py +++ b/python/google/protobuf/message.py @@ -184,9 +184,15 @@ class Message(object): self.Clear() self.MergeFromString(serialized) - def SerializeToString(self): + def SerializeToString(self, **kwargs): """Serializes the protocol message to a binary string. + Arguments: + **kwargs: Keyword arguments to the serialize method, accepts + the following keyword args: + deterministic: If true, requests deterministic serialization of the + protobuf, with predictable ordering of map keys. + Returns: A binary string representation of the message if all of the required fields in the message are set (i.e. the message is initialized). @@ -196,12 +202,18 @@ class Message(object): """ raise NotImplementedError - def SerializePartialToString(self): + def SerializePartialToString(self, **kwargs): """Serializes the protocol message to a binary string. This method is similar to SerializeToString but doesn't check if the message is initialized. + Arguments: + **kwargs: Keyword arguments to the serialize method, accepts + the following keyword args: + deterministic: If true, requests deterministic serialization of the + protobuf, with predictable ordering of map keys. + Returns: A string representation of the partial message. """ diff --git a/python/google/protobuf/message_factory.py b/python/google/protobuf/message_factory.py index 8ab1c513..15740280 100644 --- a/python/google/protobuf/message_factory.py +++ b/python/google/protobuf/message_factory.py @@ -66,7 +66,7 @@ class MessageFactory(object): Returns: A class describing the passed in descriptor. """ - if descriptor.full_name not in self._classes: + if descriptor not in self._classes: descriptor_name = descriptor.name if str is bytes: # PY2 descriptor_name = descriptor.name.encode('ascii', 'ignore') @@ -75,16 +75,16 @@ class MessageFactory(object): (message.Message,), {'DESCRIPTOR': descriptor, '__module__': None}) # If module not set, it wrongly points to the reflection.py module. - self._classes[descriptor.full_name] = result_class + self._classes[descriptor] = result_class for field in descriptor.fields: if field.message_type: self.GetPrototype(field.message_type) for extension in result_class.DESCRIPTOR.extensions: - if extension.containing_type.full_name not in self._classes: + if extension.containing_type not in self._classes: self.GetPrototype(extension.containing_type) - extended_class = self._classes[extension.containing_type.full_name] + extended_class = self._classes[extension.containing_type] extended_class.RegisterExtension(extension) - return self._classes[descriptor.full_name] + return self._classes[descriptor] def GetMessages(self, files): """Gets all the messages from a specified file. @@ -116,9 +116,9 @@ class MessageFactory(object): # an error if they were different. for extension in file_desc.extensions_by_name.values(): - if extension.containing_type.full_name not in self._classes: + if extension.containing_type not in self._classes: self.GetPrototype(extension.containing_type) - extended_class = self._classes[extension.containing_type.full_name] + extended_class = self._classes[extension.containing_type] extended_class.RegisterExtension(extension) return result diff --git a/python/google/protobuf/pyext/descriptor.cc b/python/google/protobuf/pyext/descriptor.cc index 924ae0b9..9634ea05 100644 --- a/python/google/protobuf/pyext/descriptor.cc +++ b/python/google/protobuf/pyext/descriptor.cc @@ -32,6 +32,7 @@ #include <Python.h> #include <frameobject.h> +#include <google/protobuf/stubs/hash.h> #include <string> #include <google/protobuf/io/coded_stream.h> @@ -708,6 +709,10 @@ static PyObject* GetJsonName(PyBaseDescriptor* self, void *closure) { return PyString_FromCppString(_GetDescriptor(self)->json_name()); } +static PyObject* GetFile(PyBaseDescriptor *self, void *closure) { + return PyFileDescriptor_FromDescriptor(_GetDescriptor(self)->file()); +} + static PyObject* GetType(PyBaseDescriptor *self, void *closure) { return PyInt_FromLong(_GetDescriptor(self)->type()); } @@ -898,6 +903,7 @@ static PyGetSetDef Getters[] = { { "name", (getter)GetName, NULL, "Unqualified name"}, { "camelcase_name", (getter)GetCamelcaseName, NULL, "Camelcase name"}, { "json_name", (getter)GetJsonName, NULL, "Json name"}, + { "file", (getter)GetFile, NULL, "File Descriptor"}, { "type", (getter)GetType, NULL, "C++ Type"}, { "cpp_type", (getter)GetCppType, NULL, "C++ Type"}, { "label", (getter)GetLabel, NULL, "Label"}, @@ -1569,6 +1575,10 @@ static PyObject* GetFullName(PyBaseDescriptor* self, void *closure) { return PyString_FromCppString(_GetDescriptor(self)->full_name()); } +static PyObject* GetFile(PyBaseDescriptor *self, void *closure) { + return PyFileDescriptor_FromDescriptor(_GetDescriptor(self)->file()); +} + static PyObject* GetIndex(PyBaseDescriptor *self, void *closure) { return PyInt_FromLong(_GetDescriptor(self)->index()); } @@ -1610,6 +1620,7 @@ static PyObject* CopyToProto(PyBaseDescriptor *self, PyObject *target) { static PyGetSetDef Getters[] = { { "name", (getter)GetName, NULL, "Name", NULL}, { "full_name", (getter)GetFullName, NULL, "Full name", NULL}, + { "file", (getter)GetFile, NULL, "File descriptor"}, { "index", (getter)GetIndex, NULL, "Index", NULL}, { "methods", (getter)GetMethods, NULL, "Methods", NULL}, @@ -1666,6 +1677,15 @@ PyObject* PyServiceDescriptor_FromDescriptor( &PyServiceDescriptor_Type, service_descriptor, NULL); } +const ServiceDescriptor* PyServiceDescriptor_AsDescriptor(PyObject* obj) { + if (!PyObject_TypeCheck(obj, &PyServiceDescriptor_Type)) { + PyErr_SetString(PyExc_TypeError, "Not a ServiceDescriptor"); + return NULL; + } + return reinterpret_cast<const ServiceDescriptor*>( + reinterpret_cast<PyBaseDescriptor*>(obj)->descriptor); +} + namespace method_descriptor { // Unchecked accessor to the C++ pointer. @@ -1769,6 +1789,15 @@ PyObject* PyMethodDescriptor_FromDescriptor( &PyMethodDescriptor_Type, method_descriptor, NULL); } +const MethodDescriptor* PyMethodDescriptor_AsDescriptor(PyObject* obj) { + if (!PyObject_TypeCheck(obj, &PyMethodDescriptor_Type)) { + PyErr_SetString(PyExc_TypeError, "Not a MethodDescriptor"); + return NULL; + } + return reinterpret_cast<const MethodDescriptor*>( + reinterpret_cast<PyBaseDescriptor*>(obj)->descriptor); +} + // Add a enum values to a type dictionary. static bool AddEnumValues(PyTypeObject *type, const EnumDescriptor* enum_descriptor) { diff --git a/python/google/protobuf/pyext/descriptor.h b/python/google/protobuf/pyext/descriptor.h index 1ae0e672..f081df84 100644 --- a/python/google/protobuf/pyext/descriptor.h +++ b/python/google/protobuf/pyext/descriptor.h @@ -80,6 +80,8 @@ const Descriptor* PyMessageDescriptor_AsDescriptor(PyObject* obj); const FieldDescriptor* PyFieldDescriptor_AsDescriptor(PyObject* obj); const EnumDescriptor* PyEnumDescriptor_AsDescriptor(PyObject* obj); const FileDescriptor* PyFileDescriptor_AsDescriptor(PyObject* obj); +const ServiceDescriptor* PyServiceDescriptor_AsDescriptor(PyObject* obj); +const MethodDescriptor* PyMethodDescriptor_AsDescriptor(PyObject* obj); // Returns the raw C++ pointer. const void* PyDescriptor_AsVoidPtr(PyObject* obj); diff --git a/python/google/protobuf/pyext/descriptor_pool.cc b/python/google/protobuf/pyext/descriptor_pool.cc index fa66bf9a..16f4d49d 100644 --- a/python/google/protobuf/pyext/descriptor_pool.cc +++ b/python/google/protobuf/pyext/descriptor_pool.cc @@ -39,6 +39,7 @@ #include <google/protobuf/pyext/message.h> #include <google/protobuf/pyext/message_factory.h> #include <google/protobuf/pyext/scoped_pyobject_ptr.h> +#include <google/protobuf/stubs/hash.h> #if PY_MAJOR_VERSION >= 3 #define PyString_FromStringAndSize PyUnicode_FromStringAndSize @@ -437,8 +438,23 @@ PyObject* AddExtensionDescriptor(PyDescriptorPool* self, PyObject* descriptor) { Py_RETURN_NONE; } -// The code below loads new Descriptors from a serialized FileDescriptorProto. +PyObject* AddServiceDescriptor(PyDescriptorPool* self, PyObject* descriptor) { + const ServiceDescriptor* service_descriptor = + PyServiceDescriptor_AsDescriptor(descriptor); + if (!service_descriptor) { + return NULL; + } + if (service_descriptor != + self->pool->FindServiceByName(service_descriptor->full_name())) { + PyErr_Format(PyExc_ValueError, + "The service descriptor %s does not belong to this pool", + service_descriptor->full_name().c_str()); + return NULL; + } + Py_RETURN_NONE; +} +// The code below loads new Descriptors from a serialized FileDescriptorProto. // Collects errors that occur during proto file building to allow them to be // propagated in the python exception instead of only living in ERROR logs. @@ -538,6 +554,8 @@ static PyMethodDef Methods[] = { "No-op. Add() must have been called before." }, { "AddExtensionDescriptor", (PyCFunction)AddExtensionDescriptor, METH_O, "No-op. Add() must have been called before." }, + { "AddServiceDescriptor", (PyCFunction)AddServiceDescriptor, METH_O, + "No-op. Add() must have been called before." }, { "FindFileByName", (PyCFunction)FindFileByName, METH_O, "Searches for a file descriptor by its .proto name." }, diff --git a/python/google/protobuf/pyext/descriptor_pool.h b/python/google/protobuf/pyext/descriptor_pool.h index c4d7d403..53ee53dc 100644 --- a/python/google/protobuf/pyext/descriptor_pool.h +++ b/python/google/protobuf/pyext/descriptor_pool.h @@ -85,6 +85,7 @@ extern PyTypeObject PyDescriptorPool_Type; namespace cdescriptor_pool { + // Looks up a message by name. // Returns a message Descriptor, or NULL if not found. const Descriptor* FindMessageTypeByName(PyDescriptorPool* self, diff --git a/python/google/protobuf/pyext/extension_dict.cc b/python/google/protobuf/pyext/extension_dict.cc index 9423c1d8..43ee5d15 100644 --- a/python/google/protobuf/pyext/extension_dict.cc +++ b/python/google/protobuf/pyext/extension_dict.cc @@ -126,6 +126,8 @@ PyObject* subscript(ExtensionDict* self, PyObject* key) { CMessageClass* message_class = message_factory::GetOrCreateMessageClass( cmessage::GetFactoryForMessage(self->parent), descriptor->message_type()); + ScopedPyObjectPtr message_class_handler( + reinterpret_cast<PyObject*>(message_class)); if (message_class == NULL) { return NULL; } diff --git a/python/google/protobuf/pyext/map_container.cc b/python/google/protobuf/pyext/map_container.cc index 088ddf93..43be0701 100644 --- a/python/google/protobuf/pyext/map_container.cc +++ b/python/google/protobuf/pyext/map_container.cc @@ -712,8 +712,30 @@ int MapReflectionFriend::MessageMapSetItem(PyObject* _self, PyObject* key, } // Delete key from map. - if (reflection->DeleteMapValue(message, self->parent_field_descriptor, + if (reflection->ContainsMapKey(*message, self->parent_field_descriptor, map_key)) { + // Delete key from CMessage dict. + MapValueRef value; + reflection->InsertOrLookupMapValue(message, self->parent_field_descriptor, + map_key, &value); + ScopedPyObjectPtr key(PyLong_FromVoidPtr(value.MutableMessageValue())); + + // PyDict_DelItem will have key error if the key is not in the map. We do + // not want to call PyErr_Clear() which may clear other errors. Thus + // PyDict_Contains() check is called before delete. + int contains = PyDict_Contains(self->message_dict, key.get()); + if (contains < 0) { + return -1; + } + if (contains) { + if (PyDict_DelItem(self->message_dict, key.get()) < 0) { + return -1; + } + } + + // Delete key from map. + reflection->DeleteMapValue(message, self->parent_field_descriptor, + map_key); return 0; } else { PyErr_Format(PyExc_KeyError, "Key not present in map"); diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc index ecd3d287..43482c54 100644 --- a/python/google/protobuf/pyext/message.cc +++ b/python/google/protobuf/pyext/message.cc @@ -52,6 +52,7 @@ #include <google/protobuf/stubs/common.h> #include <google/protobuf/stubs/logging.h> #include <google/protobuf/io/coded_stream.h> +#include <google/protobuf/io/zero_copy_stream_impl_lite.h> #include <google/protobuf/util/message_differencer.h> #include <google/protobuf/descriptor.h> #include <google/protobuf/message.h> @@ -789,7 +790,7 @@ PyObject* CheckString(PyObject* arg, const FieldDescriptor* descriptor) { encoded_string = arg; // Already encoded. Py_INCREF(encoded_string); } else { - encoded_string = PyUnicode_AsEncodedObject(arg, "utf-8", NULL); + encoded_string = PyUnicode_AsEncodedString(arg, "utf-8", NULL); } } else { // In this case field type is "bytes". @@ -1065,13 +1066,15 @@ int InternalDeleteRepeatedField( if (PySlice_Check(slice)) { from = to = step = slice_length = 0; - PySlice_GetIndicesEx( #if PY_MAJOR_VERSION < 3 + PySlice_GetIndicesEx( reinterpret_cast<PySliceObject*>(slice), + length, &from, &to, &step, &slice_length); #else + PySlice_GetIndicesEx( slice, -#endif length, &from, &to, &step, &slice_length); +#endif if (from < to) { min = from; max = to - 1; @@ -1818,8 +1821,25 @@ static string GetMessageName(CMessage* self) { } } -static PyObject* SerializeToString(CMessage* self, PyObject* args) { - if (!self->message->IsInitialized()) { +static PyObject* InternalSerializeToString( + CMessage* self, PyObject* args, PyObject* kwargs, + bool require_initialized) { + // Parse the "deterministic" kwarg; defaults to False. + static char* kwlist[] = { "deterministic", 0 }; + PyObject* deterministic_obj = Py_None; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O", kwlist, + &deterministic_obj)) { + return NULL; + } + // Preemptively convert to a bool first, so we don't need to back out of + // allocating memory if this raises an exception. + // NOTE: This is unused later if deterministic == Py_None, but that's fine. + int deterministic = PyObject_IsTrue(deterministic_obj); + if (deterministic < 0) { + return NULL; + } + + if (require_initialized && !self->message->IsInitialized()) { ScopedPyObjectPtr errors(FindInitializationErrors(self)); if (errors == NULL) { return NULL; @@ -1857,24 +1877,36 @@ static PyObject* SerializeToString(CMessage* self, PyObject* args) { GetMessageName(self).c_str(), PyString_AsString(joined.get())); return NULL; } - int size = self->message->ByteSize(); - if (size <= 0) { + + // Ok, arguments parsed and errors checked, now encode to a string + const size_t size = self->message->ByteSizeLong(); + if (size == 0) { return PyBytes_FromString(""); } PyObject* result = PyBytes_FromStringAndSize(NULL, size); if (result == NULL) { return NULL; } - char* buffer = PyBytes_AS_STRING(result); - self->message->SerializeWithCachedSizesToArray( - reinterpret_cast<uint8*>(buffer)); + io::ArrayOutputStream out(PyBytes_AS_STRING(result), size); + io::CodedOutputStream coded_out(&out); + if (deterministic_obj != Py_None) { + coded_out.SetSerializationDeterministic(deterministic); + } + self->message->SerializeWithCachedSizes(&coded_out); + GOOGLE_CHECK(!coded_out.HadError()); return result; } -static PyObject* SerializePartialToString(CMessage* self) { - string contents; - self->message->SerializePartialToString(&contents); - return PyBytes_FromStringAndSize(contents.c_str(), contents.size()); +static PyObject* SerializeToString( + CMessage* self, PyObject* args, PyObject* kwargs) { + return InternalSerializeToString(self, args, kwargs, + /*require_initialized=*/true); +} + +static PyObject* SerializePartialToString( + CMessage* self, PyObject* args, PyObject* kwargs) { + return InternalSerializeToString(self, args, kwargs, + /*require_initialized=*/false); } // Formats proto fields for ascii dumps using python formatting functions where @@ -2352,8 +2384,10 @@ PyObject* InternalGetSubMessage( const Message& sub_message = reflection->GetMessage( *self->message, field_descriptor, factory->message_factory); - CMessageClass* message_class = message_factory::GetMessageClass( + CMessageClass* message_class = message_factory::GetOrCreateMessageClass( factory, field_descriptor->message_type()); + ScopedPyObjectPtr message_class_handler( + reinterpret_cast<PyObject*>(message_class)); if (message_class == NULL) { return NULL; } @@ -2543,7 +2577,10 @@ PyObject* Reduce(CMessage* self) { if (state == NULL) { return NULL; } - ScopedPyObjectPtr serialized(SerializePartialToString(self)); + string contents; + self->message->SerializePartialToString(&contents); + ScopedPyObjectPtr serialized( + PyBytes_FromStringAndSize(contents.c_str(), contents.size())); if (serialized == NULL) { return NULL; } @@ -2664,9 +2701,10 @@ static PyMethodDef Methods[] = { { "RegisterExtension", (PyCFunction)RegisterExtension, METH_O | METH_CLASS, "Registers an extension with the current message." }, { "SerializePartialToString", (PyCFunction)SerializePartialToString, - METH_NOARGS, + METH_VARARGS | METH_KEYWORDS, "Serializes the message to a string, even if it isn't initialized." }, - { "SerializeToString", (PyCFunction)SerializeToString, METH_NOARGS, + { "SerializeToString", (PyCFunction)SerializeToString, + METH_VARARGS | METH_KEYWORDS, "Serializes the message to a string, only for initialized messages." }, { "SetInParent", (PyCFunction)SetInParent, METH_NOARGS, "Sets the has bit of the given field in its parent message." }, diff --git a/python/google/protobuf/pyext/message_factory.cc b/python/google/protobuf/pyext/message_factory.cc index e0b45bf2..571bae2b 100644 --- a/python/google/protobuf/pyext/message_factory.cc +++ b/python/google/protobuf/pyext/message_factory.cc @@ -133,11 +133,7 @@ int RegisterMessageClass(PyMessageFactory* self, CMessageClass* GetOrCreateMessageClass(PyMessageFactory* self, const Descriptor* descriptor) { // This is the same implementation as MessageFactory.GetPrototype(). - ScopedPyObjectPtr py_descriptor( - PyMessageDescriptor_FromDescriptor(descriptor)); - if (py_descriptor == NULL) { - return NULL; - } + // Do not create a MessageClass that already exists. hash_map<const Descriptor*, CMessageClass*>::iterator it = self->classes_by_descriptor->find(descriptor); @@ -145,6 +141,11 @@ CMessageClass* GetOrCreateMessageClass(PyMessageFactory* self, Py_INCREF(it->second); return it->second; } + ScopedPyObjectPtr py_descriptor( + PyMessageDescriptor_FromDescriptor(descriptor)); + if (py_descriptor == NULL) { + return NULL; + } // Create a new message class. ScopedPyObjectPtr args(Py_BuildValue( "s(){sOsOsO}", descriptor->name().c_str(), diff --git a/python/google/protobuf/pyext/repeated_composite_container.cc b/python/google/protobuf/pyext/repeated_composite_container.cc index 43a2bc12..5ad71db5 100644 --- a/python/google/protobuf/pyext/repeated_composite_container.cc +++ b/python/google/protobuf/pyext/repeated_composite_container.cc @@ -46,7 +46,9 @@ #include <google/protobuf/pyext/descriptor.h> #include <google/protobuf/pyext/descriptor_pool.h> #include <google/protobuf/pyext/message.h> +#include <google/protobuf/pyext/message_factory.h> #include <google/protobuf/pyext/scoped_pyobject_ptr.h> +#include <google/protobuf/reflection.h> #if PY_MAJOR_VERSION >= 3 #define PyInt_Check PyLong_Check @@ -136,9 +138,12 @@ static PyObject* AddToAttached(RepeatedCompositeContainer* self, if (cmessage::AssureWritable(self->parent) == -1) return NULL; Message* message = self->message; + Message* sub_message = - message->GetReflection()->AddMessage(message, - self->parent_field_descriptor); + message->GetReflection()->AddMessage( + message, + self->parent_field_descriptor, + self->child_message_class->py_message_factory->message_factory); CMessage* cmsg = cmessage::NewEmptyMessage(self->child_message_class); if (cmsg == NULL) return NULL; @@ -265,10 +270,11 @@ int AssignSubscript(RepeatedCompositeContainer* self, if (PySlice_Check(slice)) { #if PY_MAJOR_VERSION >= 3 if (PySlice_GetIndicesEx(slice, + length, &from, &to, &step, &slicelength) == -1) { #else if (PySlice_GetIndicesEx(reinterpret_cast<PySliceObject*>(slice), -#endif length, &from, &to, &step, &slicelength) == -1) { +#endif return -1; } return PySequence_DelSlice(self->child_messages, from, to); @@ -334,6 +340,18 @@ static PyObject* RichCompare(RepeatedCompositeContainer* self, } } +static PyObject* ToStr(RepeatedCompositeContainer* self) { + ScopedPyObjectPtr full_slice(PySlice_New(NULL, NULL, NULL)); + if (full_slice == NULL) { + return NULL; + } + ScopedPyObjectPtr list(Subscript(self, full_slice.get())); + if (list == NULL) { + return NULL; + } + return PyObject_Repr(list.get()); +} + // --------------------------------------------------------------------- // sort() @@ -485,6 +503,32 @@ int Release(RepeatedCompositeContainer* self) { return 0; } +PyObject* DeepCopy(RepeatedCompositeContainer* self, PyObject* arg) { + ScopedPyObjectPtr cloneObj( + PyType_GenericAlloc(&RepeatedCompositeContainer_Type, 0)); + if (cloneObj == NULL) { + return NULL; + } + RepeatedCompositeContainer* clone = + reinterpret_cast<RepeatedCompositeContainer*>(cloneObj.get()); + + Message* new_message = self->message->New(); + clone->parent = NULL; + clone->parent_field_descriptor = self->parent_field_descriptor; + clone->message = new_message; + clone->owner.reset(new_message); + Py_INCREF(self->child_message_class); + clone->child_message_class = self->child_message_class; + clone->child_messages = PyList_New(0); + + new_message->GetReflection() + ->GetMutableRepeatedFieldRef<Message>(new_message, + self->parent_field_descriptor) + .MergeFrom(self->message->GetReflection()->GetRepeatedFieldRef<Message>( + *self->message, self->parent_field_descriptor)); + return cloneObj.release(); +} + int SetOwner(RepeatedCompositeContainer* self, const shared_ptr<Message>& new_owner) { GOOGLE_CHECK_ATTACHED(self); @@ -551,6 +595,8 @@ static PyMappingMethods MpMethods = { }; static PyMethodDef Methods[] = { + { "__deepcopy__", (PyCFunction)DeepCopy, METH_VARARGS, + "Makes a deep copy of the class." }, { "add", (PyCFunction) Add, METH_VARARGS | METH_KEYWORDS, "Adds an object to the repeated container." }, { "extend", (PyCFunction) Extend, METH_O, @@ -578,7 +624,7 @@ PyTypeObject RepeatedCompositeContainer_Type = { 0, // tp_getattr 0, // tp_setattr 0, // tp_compare - 0, // tp_repr + (reprfunc)repeated_composite_container::ToStr, // tp_repr 0, // tp_as_number &repeated_composite_container::SqMethods, // tp_as_sequence &repeated_composite_container::MpMethods, // tp_as_mapping diff --git a/python/google/protobuf/pyext/repeated_scalar_container.cc b/python/google/protobuf/pyext/repeated_scalar_container.cc index 95da85f8..54998800 100644 --- a/python/google/protobuf/pyext/repeated_scalar_container.cc +++ b/python/google/protobuf/pyext/repeated_scalar_container.cc @@ -305,10 +305,12 @@ static PyObject* Subscript(RepeatedScalarContainer* self, PyObject* slice) { length = Len(self); #if PY_MAJOR_VERSION >= 3 if (PySlice_GetIndicesEx(slice, + length, &from, &to, &step, &slicelength) == -1) { #else if (PySlice_GetIndicesEx(reinterpret_cast<PySliceObject*>(slice), -#endif length, &from, &to, &step, &slicelength) == -1) { + +#endif return NULL; } return_list = true; @@ -458,10 +460,11 @@ static int AssSubscript(RepeatedScalarContainer* self, length = reflection->FieldSize(*message, field_descriptor); #if PY_MAJOR_VERSION >= 3 if (PySlice_GetIndicesEx(slice, + length, &from, &to, &step, &slicelength) == -1) { #else if (PySlice_GetIndicesEx(reinterpret_cast<PySliceObject*>(slice), -#endif length, &from, &to, &step, &slicelength) == -1) { +#endif return -1; } create_list = true; @@ -656,6 +659,18 @@ static PyObject* Pop(RepeatedScalarContainer* self, return item; } +static PyObject* ToStr(RepeatedScalarContainer* self) { + ScopedPyObjectPtr full_slice(PySlice_New(NULL, NULL, NULL)); + if (full_slice == NULL) { + return NULL; + } + ScopedPyObjectPtr list(Subscript(self, full_slice.get())); + if (list == NULL) { + return NULL; + } + return PyObject_Repr(list.get()); +} + // The private constructor of RepeatedScalarContainer objects. PyObject *NewContainer( CMessage* parent, const FieldDescriptor* parent_field_descriptor) { @@ -778,7 +793,7 @@ PyTypeObject RepeatedScalarContainer_Type = { 0, // tp_getattr 0, // tp_setattr 0, // tp_compare - 0, // tp_repr + (reprfunc)repeated_scalar_container::ToStr, // tp_repr 0, // tp_as_number &repeated_scalar_container::SqMethods, // tp_as_sequence &repeated_scalar_container::MpMethods, // tp_as_mapping diff --git a/python/google/protobuf/pyext/scoped_pyobject_ptr.h b/python/google/protobuf/pyext/scoped_pyobject_ptr.h index a128cd4c..a2afa7f1 100644 --- a/python/google/protobuf/pyext/scoped_pyobject_ptr.h +++ b/python/google/protobuf/pyext/scoped_pyobject_ptr.h @@ -36,61 +36,70 @@ #include <google/protobuf/stubs/common.h> #include <Python.h> - namespace google { -class ScopedPyObjectPtr { +namespace protobuf { +namespace python { + +// Owns a python object and decrements the reference count on destruction. +// This class is not threadsafe. +template <typename PyObjectStruct> +class ScopedPythonPtr { public: - // Constructor. Defaults to initializing with NULL. - // There is no way to create an uninitialized ScopedPyObjectPtr. - explicit ScopedPyObjectPtr(PyObject* p = NULL) : ptr_(p) { } + // Takes the ownership of the specified object to ScopedPythonPtr. + // The reference count of the specified py_object is not incremented. + explicit ScopedPythonPtr(PyObjectStruct* py_object = NULL) + : ptr_(py_object) {} - // Destructor. If there is a PyObject object, delete it. - ~ScopedPyObjectPtr() { - Py_XDECREF(ptr_); - } + // If a PyObject is owned, decrement its reference count. + ~ScopedPythonPtr() { Py_XDECREF(ptr_); } - // Reset. Deletes the current owned object, if any. - // Then takes ownership of a new object, if given. + // Deletes the current owned object, if any. + // Then takes ownership of a new object without incrementing the reference + // count. // This function must be called with a reference that you own. // this->reset(this->get()) is wrong! // this->reset(this->release()) is OK. - PyObject* reset(PyObject* p = NULL) { + PyObjectStruct* reset(PyObjectStruct* p = NULL) { Py_XDECREF(ptr_); ptr_ = p; return ptr_; } - // Releases ownership of the object. + // Releases ownership of the object without decrementing the reference count. // The caller now owns the returned reference. - PyObject* release() { + PyObjectStruct* release() { PyObject* p = ptr_; ptr_ = NULL; return p; } - PyObject* operator->() const { + PyObjectStruct* operator->() const { assert(ptr_ != NULL); return ptr_; } - PyObject* get() const { return ptr_; } + PyObjectStruct* get() const { return ptr_; } - Py_ssize_t refcnt() const { return Py_REFCNT(ptr_); } + PyObject* as_pyobject() const { return reinterpret_cast<PyObject*>(ptr_); } + // Increments the reference count fo the current object. + // Should not be called when no object is held. void inc() const { Py_INCREF(ptr_); } - // Comparison operators. - // These return whether a ScopedPyObjectPtr and a raw pointer - // refer to the same object, not just to two different but equal - // objects. - bool operator==(const PyObject* p) const { return ptr_ == p; } - bool operator!=(const PyObject* p) const { return ptr_ != p; } + // True when a ScopedPyObjectPtr and a raw pointer refer to the same object. + // Comparison operators are non reflexive. + bool operator==(const PyObjectStruct* p) const { return ptr_ == p; } + bool operator!=(const PyObjectStruct* p) const { return ptr_ != p; } private: - PyObject* ptr_; + PyObjectStruct* ptr_; - GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ScopedPyObjectPtr); + GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ScopedPythonPtr); }; +typedef ScopedPythonPtr<PyObject> ScopedPyObjectPtr; + +} // namespace python +} // namespace protobuf } // namespace google #endif // GOOGLE_PROTOBUF_PYTHON_CPP_SCOPED_PYOBJECT_PTR_H__ diff --git a/python/google/protobuf/pyext/python_protobuf.h b/python/google/protobuf/python_protobuf.h index beb6e460..beb6e460 100644 --- a/python/google/protobuf/pyext/python_protobuf.h +++ b/python/google/protobuf/python_protobuf.h diff --git a/python/google/protobuf/reflection.py b/python/google/protobuf/reflection.py index 51c83321..f4ce8caf 100755 --- a/python/google/protobuf/reflection.py +++ b/python/google/protobuf/reflection.py @@ -61,6 +61,8 @@ else: # Part of the public interface, but normally only used by message factories. GeneratedProtocolMessageType = message_impl.GeneratedProtocolMessageType +MESSAGE_CLASS_CACHE = {} + def ParseMessage(descriptor, byte_str): """Generate a new Message instance from this Descriptor and a byte string. @@ -104,11 +106,16 @@ def MakeClass(descriptor): Returns: The Message class object described by the descriptor. """ + if descriptor in MESSAGE_CLASS_CACHE: + return MESSAGE_CLASS_CACHE[descriptor] + attributes = {} for name, nested_type in descriptor.nested_types_by_name.items(): attributes[name] = MakeClass(nested_type) attributes[GeneratedProtocolMessageType._DESCRIPTOR_KEY] = descriptor - return GeneratedProtocolMessageType(str(descriptor.name), (message.Message,), - attributes) + result = GeneratedProtocolMessageType( + str(descriptor.name), (message.Message,), attributes) + MESSAGE_CLASS_CACHE[descriptor] = result + return result diff --git a/python/google/protobuf/symbol_database.py b/python/google/protobuf/symbol_database.py index ecbef211..5ad869f4 100644 --- a/python/google/protobuf/symbol_database.py +++ b/python/google/protobuf/symbol_database.py @@ -78,10 +78,18 @@ class SymbolDatabase(message_factory.MessageFactory): """ desc = message.DESCRIPTOR - self._classes[desc.full_name] = message - self.pool.AddDescriptor(desc) + self._classes[desc] = message + self.RegisterMessageDescriptor(desc) return message + def RegisterMessageDescriptor(self, message_descriptor): + """Registers the given message descriptor in the local database. + + Args: + message_descriptor: a descriptor.MessageDescriptor. + """ + self.pool.AddDescriptor(message_descriptor) + def RegisterEnumDescriptor(self, enum_descriptor): """Registers the given enum descriptor in the local database. @@ -94,6 +102,17 @@ class SymbolDatabase(message_factory.MessageFactory): self.pool.AddEnumDescriptor(enum_descriptor) return enum_descriptor + def RegisterServiceDescriptor(self, service_descriptor): + """Registers the given service descriptor in the local database. + + Args: + service_descriptor: a descriptor.ServiceDescriptor. + + Returns: + The provided descriptor. + """ + self.pool.AddServiceDescriptor(service_descriptor) + def RegisterFileDescriptor(self, file_descriptor): """Registers the given file descriptor in the local database. @@ -121,7 +140,7 @@ class SymbolDatabase(message_factory.MessageFactory): KeyError: if the symbol could not be found. """ - return self._classes[symbol] + return self._classes[self.pool.FindMessageTypeByName(symbol)] def GetMessages(self, files): # TODO(amauryfa): Fix the differences with MessageFactory. @@ -142,20 +161,20 @@ class SymbolDatabase(message_factory.MessageFactory): KeyError: if a file could not be found. """ - def _GetAllMessageNames(desc): + def _GetAllMessages(desc): """Walk a message Descriptor and recursively yields all message names.""" - yield desc.full_name + yield desc for msg_desc in desc.nested_types: - for full_name in _GetAllMessageNames(msg_desc): - yield full_name + for nested_desc in _GetAllMessages(msg_desc): + yield nested_desc result = {} for file_name in files: file_desc = self.pool.FindFileByName(file_name) for msg_desc in file_desc.message_types_by_name.values(): - for full_name in _GetAllMessageNames(msg_desc): + for desc in _GetAllMessages(msg_desc): try: - result[full_name] = self._classes[full_name] + result[desc.full_name] = self._classes[desc] except KeyError: # This descriptor has no registered class, skip it. pass diff --git a/python/google/protobuf/text_format.py b/python/google/protobuf/text_format.py index 90f6ce42..aaca78ad 100755 --- a/python/google/protobuf/text_format.py +++ b/python/google/protobuf/text_format.py @@ -126,7 +126,8 @@ def MessageToString(message, float_format=None, use_field_number=False, descriptor_pool=None, - indent=0): + indent=0, + message_formatter=None): """Convert protobuf message to text format. Floating point values can be formatted compactly with 15 digits of @@ -148,6 +149,9 @@ def MessageToString(message, use_field_number: If True, print field numbers instead of names. descriptor_pool: A DescriptorPool used to resolve Any types. indent: The indent level, in terms of spaces, for pretty print. + message_formatter: A function(message, indent, as_one_line): unicode|None + to custom format selected sub-messages (usually based on message type). + Use to pretty print parts of the protobuf for easier diffing. Returns: A string of the text formatted protocol buffer message. @@ -155,7 +159,7 @@ def MessageToString(message, out = TextWriter(as_utf8) printer = _Printer(out, indent, as_utf8, as_one_line, pointy_brackets, use_index_order, float_format, use_field_number, - descriptor_pool) + descriptor_pool, message_formatter) printer.PrintMessage(message) result = out.getvalue() out.close() @@ -179,10 +183,11 @@ def PrintMessage(message, use_index_order=False, float_format=None, use_field_number=False, - descriptor_pool=None): + descriptor_pool=None, + message_formatter=None): printer = _Printer(out, indent, as_utf8, as_one_line, pointy_brackets, use_index_order, float_format, use_field_number, - descriptor_pool) + descriptor_pool, message_formatter) printer.PrintMessage(message) @@ -194,10 +199,11 @@ def PrintField(field, as_one_line=False, pointy_brackets=False, use_index_order=False, - float_format=None): + float_format=None, + message_formatter=None): """Print a single field name/value pair.""" printer = _Printer(out, indent, as_utf8, as_one_line, pointy_brackets, - use_index_order, float_format) + use_index_order, float_format, message_formatter) printer.PrintField(field, value) @@ -209,10 +215,11 @@ def PrintFieldValue(field, as_one_line=False, pointy_brackets=False, use_index_order=False, - float_format=None): + float_format=None, + message_formatter=None): """Print a single field value (not including name).""" printer = _Printer(out, indent, as_utf8, as_one_line, pointy_brackets, - use_index_order, float_format) + use_index_order, float_format, message_formatter) printer.PrintFieldValue(field, value) @@ -228,6 +235,9 @@ def _BuildMessageFromTypeName(type_name, descriptor_pool): wasn't found matching type_name. """ # pylint: disable=g-import-not-at-top + if descriptor_pool is None: + from google.protobuf import descriptor_pool as pool_mod + descriptor_pool = pool_mod.Default() from google.protobuf import symbol_database database = symbol_database.Default() try: @@ -250,7 +260,8 @@ class _Printer(object): use_index_order=False, float_format=None, use_field_number=False, - descriptor_pool=None): + descriptor_pool=None, + message_formatter=None): """Initialize the Printer. Floating point values can be formatted compactly with 15 digits of @@ -273,6 +284,9 @@ class _Printer(object): used. use_field_number: If True, print field numbers instead of names. descriptor_pool: A DescriptorPool used to resolve Any types. + message_formatter: A function(message, indent, as_one_line): unicode|None + to custom format selected sub-messages (usually based on message type). + Use to pretty print parts of the protobuf for easier diffing. """ self.out = out self.indent = indent @@ -283,6 +297,7 @@ class _Printer(object): self.float_format = float_format self.use_field_number = use_field_number self.descriptor_pool = descriptor_pool + self.message_formatter = message_formatter def _TryPrintAsAnyMessage(self, message): """Serializes if message is a google.protobuf.Any field.""" @@ -297,14 +312,27 @@ class _Printer(object): else: return False + def _TryCustomFormatMessage(self, message): + formatted = self.message_formatter(message, self.indent, self.as_one_line) + if formatted is None: + return False + + out = self.out + out.write(' ' * self.indent) + out.write(formatted) + out.write(' ' if self.as_one_line else '\n') + return True + def PrintMessage(self, message): """Convert protobuf message to text format. Args: message: The protocol buffers message. """ + if self.message_formatter and self._TryCustomFormatMessage(message): + return if (message.DESCRIPTOR.full_name == _ANY_FULL_TYPE_NAME and - self.descriptor_pool and self._TryPrintAsAnyMessage(message)): + self._TryPrintAsAnyMessage(message)): return fields = message.ListFields() if self.use_index_order: @@ -422,15 +450,33 @@ class _Printer(object): def Parse(text, message, allow_unknown_extension=False, - allow_field_number=False): + allow_field_number=False, + descriptor_pool=None): """Parses a text representation of a protocol message into a message. + NOTE: for historical reasons this function does not clear the input + message. This is different from what the binary msg.ParseFrom(...) does. + + Example + a = MyProto() + a.repeated_field.append('test') + b = MyProto() + + text_format.Parse(repr(a), b) + text_format.Parse(repr(a), b) # repeated_field contains ["test", "test"] + + # Binary version: + b.ParseFromString(a.SerializeToString()) # repeated_field is now "test" + + Caller is responsible for clearing the message as needed. + Args: text: Message text representation. message: A protocol buffer message to merge into. allow_unknown_extension: if True, skip over missing extensions and keep parsing allow_field_number: if True, both field number and field name are allowed. + descriptor_pool: A DescriptorPool used to resolve Any types. Returns: The same message passed as argument. @@ -440,8 +486,11 @@ def Parse(text, """ if not isinstance(text, str): text = text.decode('utf-8') - return ParseLines( - text.split('\n'), message, allow_unknown_extension, allow_field_number) + return ParseLines(text.split('\n'), + message, + allow_unknown_extension, + allow_field_number, + descriptor_pool=descriptor_pool) def Merge(text, @@ -479,7 +528,8 @@ def Merge(text, def ParseLines(lines, message, allow_unknown_extension=False, - allow_field_number=False): + allow_field_number=False, + descriptor_pool=None): """Parses a text representation of a protocol message into a message. Args: @@ -496,7 +546,9 @@ def ParseLines(lines, Raises: ParseError: On text parsing problems. """ - parser = _Parser(allow_unknown_extension, allow_field_number) + parser = _Parser(allow_unknown_extension, + allow_field_number, + descriptor_pool=descriptor_pool) return parser.ParseLines(lines, message) @@ -513,6 +565,7 @@ def MergeLines(lines, allow_unknown_extension: if True, skip over missing extensions and keep parsing allow_field_number: if True, both field number and field name are allowed. + descriptor_pool: A DescriptorPool used to resolve Any types. Returns: The same message passed as argument. @@ -584,11 +637,6 @@ class _Parser(object): ParseError: In case of text parsing problems. """ message_descriptor = message.DESCRIPTOR - if (hasattr(message_descriptor, 'syntax') and - message_descriptor.syntax == 'proto3'): - # Proto3 doesn't represent presence so we can't test if multiple - # scalars have occurred. We have to allow them. - self._allow_multiple_scalars = True if tokenizer.TryConsume('['): name = [tokenizer.ConsumeIdentifier()] while tokenizer.TryConsume('.'): @@ -607,7 +655,11 @@ class _Parser(object): field = None else: raise tokenizer.ParseErrorPreviousToken( - 'Extension "%s" not registered.' % name) + 'Extension "%s" not registered. ' + 'Did you import the _pb2 module which defines it? ' + 'If you are trying to place the extension in the MessageSet ' + 'field of another message that is in an Any or MessageSet field, ' + 'that message\'s _pb2 module must be imported as well' % name) elif message_descriptor != field.containing_type: raise tokenizer.ParseErrorPreviousToken( 'Extension "%s" does not extend message type "%s".' % @@ -686,17 +738,17 @@ class _Parser(object): def _ConsumeAnyTypeUrl(self, tokenizer): """Consumes a google.protobuf.Any type URL and returns the type name.""" # Consume "type.googleapis.com/". - tokenizer.ConsumeIdentifier() + prefix = [tokenizer.ConsumeIdentifier()] tokenizer.Consume('.') - tokenizer.ConsumeIdentifier() + prefix.append(tokenizer.ConsumeIdentifier()) tokenizer.Consume('.') - tokenizer.ConsumeIdentifier() + prefix.append(tokenizer.ConsumeIdentifier()) tokenizer.Consume('/') # Consume the fully-qualified type name. name = [tokenizer.ConsumeIdentifier()] while tokenizer.TryConsume('.'): name.append(tokenizer.ConsumeIdentifier()) - return '.'.join(name) + return '.'.join(prefix), '.'.join(name) def _MergeMessageField(self, tokenizer, message, field): """Merges a single scalar field into a message. @@ -719,7 +771,7 @@ class _Parser(object): if (field.message_type.full_name == _ANY_FULL_TYPE_NAME and tokenizer.TryConsume('[')): - packed_type_name = self._ConsumeAnyTypeUrl(tokenizer) + type_url_prefix, packed_type_name = self._ConsumeAnyTypeUrl(tokenizer) tokenizer.Consume(']') tokenizer.TryConsume(':') if tokenizer.TryConsume('<'): @@ -727,8 +779,6 @@ class _Parser(object): else: tokenizer.Consume('{') expanded_any_end_token = '}' - if not self.descriptor_pool: - raise ParseError('Descriptor pool required to parse expanded Any field') expanded_any_sub_message = _BuildMessageFromTypeName(packed_type_name, self.descriptor_pool) if not expanded_any_sub_message: @@ -743,7 +793,8 @@ class _Parser(object): any_message = getattr(message, field.name).add() else: any_message = getattr(message, field.name) - any_message.Pack(expanded_any_sub_message) + any_message.Pack(expanded_any_sub_message, + type_url_prefix=type_url_prefix) elif field.label == descriptor.FieldDescriptor.LABEL_REPEATED: if field.is_extension: sub_message = message.Extensions[field].add() @@ -771,6 +822,12 @@ class _Parser(object): else: getattr(message, field.name)[sub_message.key] = sub_message.value + @staticmethod + def _IsProto3Syntax(message): + message_descriptor = message.DESCRIPTOR + return (hasattr(message_descriptor, 'syntax') and + message_descriptor.syntax == 'proto3') + def _MergeScalarField(self, tokenizer, message, field): """Merges a single scalar field into a message. @@ -820,15 +877,20 @@ class _Parser(object): else: getattr(message, field.name).append(value) else: + # Proto3 doesn't represent presence so we can't test if multiple scalars + # have occurred. We have to allow them. + can_check_presence = not self._IsProto3Syntax(message) if field.is_extension: - if not self._allow_multiple_scalars and message.HasExtension(field): + if (not self._allow_multiple_scalars and can_check_presence and + message.HasExtension(field)): raise tokenizer.ParseErrorPreviousToken( 'Message type "%s" should not have multiple "%s" extensions.' % (message.DESCRIPTOR.full_name, field.full_name)) else: message.Extensions[field] = value else: - if not self._allow_multiple_scalars and message.HasField(field.name): + if (not self._allow_multiple_scalars and can_check_presence and + message.HasField(field.name)): raise tokenizer.ParseErrorPreviousToken( 'Message type "%s" should not have multiple "%s" fields.' % (message.DESCRIPTOR.full_name, field.name)) @@ -1023,6 +1085,22 @@ class Tokenizer(object): self.NextToken() return result + def ConsumeCommentOrTrailingComment(self): + """Consumes a comment, returns a 2-tuple (trailing bool, comment str).""" + + # Tokenizer initializes _previous_line and _previous_column to 0. As the + # tokenizer starts, it looks like there is a previous token on the line. + just_started = self._line == 0 and self._column == 0 + + before_parsing = self._previous_line + comment = self.ConsumeComment() + + # A trailing comment is a comment on the same line than the previous token. + trailing = (self._previous_line == before_parsing + and not just_started) + + return trailing, comment + def TryConsumeIdentifier(self): try: self.ConsumeIdentifier() @@ -1063,7 +1141,7 @@ class Tokenizer(object): """ result = self.token if not self._IDENTIFIER_OR_NUMBER.match(result): - raise self.ParseError('Expected identifier or number.') + raise self.ParseError('Expected identifier or number, got %s.' % result) self.NextToken() return result diff --git a/python/mox.py b/python/mox.py index 257468e5..43db0219 100755 --- a/python/mox.py +++ b/python/mox.py @@ -778,7 +778,7 @@ class Comparator: rhs: any python object """ - raise NotImplementedError, 'method must be implemented by a subclass.' + raise NotImplementedError('method must be implemented by a subclass.') def __eq__(self, rhs): return self.equals(rhs) diff --git a/python/release.sh b/python/release.sh new file mode 100755 index 00000000..b0bf3b3f --- /dev/null +++ b/python/release.sh @@ -0,0 +1,114 @@ +#!/bin/bash + +set -ex + +function get_source_version() { + grep "__version__ = '.*'" python/google/protobuf/__init__.py | sed -r "s/__version__ = '(.*)'/\1/" +} + +function run_install_test() { + local VERSION=$1 + local PYTHON=$2 + local PYPI=$3 + + virtualenv --no-site-packages -p `which $PYTHON` test-venv + + # Intentionally put a broken protoc in the path to make sure installation + # doesn't require protoc installed. + touch test-venv/bin/protoc + chmod +x test-venv/bin/protoc + + source test-venv/bin/activate + pip install -i ${PYPI} protobuf==${VERSION} + deactivate + rm -fr test-venv +} + + +[ $# -lt 1 ] && { + echo "Usage: $0 VERSION [" + echo "" + echo "Examples:" + echo " Test 3.3.0 release using version number 3.3.0.dev1:" + echo " $0 3.0.0 dev1" + echo " Actually release 3.3.0 to PyPI:" + echo " $0 3.3.0" + exit 1 +} +VERSION=$1 +DEV=$2 + +# Make sure we are in a protobuf source tree. +[ -f "python/google/protobuf/__init__.py" ] || { + echo "This script must be ran under root of protobuf source tree." + exit 1 +} + +# Make sure all files are world-readable. +find python -type d -exec chmod a+r,a+x {} + +find python -type f -exec chmod a+r {} + + +# Check that the supplied version number matches what's inside the source code. +SOURCE_VERSION=`get_source_version` + +[ "${VERSION}" == "${SOURCE_VERSION}" -o "${VERSION}.${DEV}" == "${SOURCE_VERSION}" ] || { + echo "Version number specified on the command line ${VERSION} doesn't match" + echo "the actual version number in the source code: ${SOURCE_VERSION}" + exit 1 +} + +TESTING_ONLY=1 +TESTING_VERSION=${VERSION}.${DEV} +if [ -z "${DEV}" ]; then + read -p "You are releasing ${VERSION} to PyPI. Are you sure? [y/n]" -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + exit 1 + fi + TESTING_ONLY=0 + TESTING_VERSION=${VERSION} +else + # Use dev version number for testing. + sed -i -r "s/__version__ = '.*'/__version__ = '${VERSION}.${DEV}'/" python/google/protobuf/__init__.py +fi + +cd python + +# Run tests locally. +python setup.py build +python setup.py test + +# Deploy source package to testing PyPI +python setup.py sdist upload -r https://test.pypi.org/legacy/ + +# Test locally with different python versions. +run_install_test ${TESTING_VERSION} python2.7 https://test.pypi.org/simple +run_install_test ${TESTING_VERSION} python3.4 https://test.pypi.org/simple + +# Deploy egg/wheel packages to testing PyPI and test again. +python setup.py bdist_egg bdist_wheel upload -r https://test.pypi.org/legacy/ +run_install_test ${TESTING_VERSION} python2.7 https://test.pypi.org/simple +run_install_test ${TESTING_VERSION} python3.4 https://test.pypi.org/simple + +echo "All install tests have passed using testing PyPI." + +if [ $TESTING_ONLY -eq 0 ]; then + read -p "Publish to PyPI? [y/n]" -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + exit 1 + fi + echo "Publishing to PyPI..." + # Be sure to run build before sdist, because otherwise sdist will not include + # well-known types. + python setup.py clean build sdist upload + # Be sure to run clean before bdist_xxx, because otherwise bdist_xxx will + # include files you may not want in the package. E.g., if you have built + # and tested with --cpp_implemenation, bdist_xxx will include the _message.so + # file even when you no longer pass the --cpp_implemenation flag. See: + # https://github.com/google/protobuf/issues/3042 + python setup.py clean build bdist_egg bdist_wheel upload +else + # Set the version number back (i.e., remove dev suffix). + sed -i -r "s/__version__ = '.*'/__version__ = '${VERSION}'/" google/protobuf/__init__.py +fi diff --git a/python/release/wheel/Dockerfile b/python/release/wheel/Dockerfile new file mode 100644 index 00000000..f38ec2f5 --- /dev/null +++ b/python/release/wheel/Dockerfile @@ -0,0 +1,6 @@ +FROM quay.io/pypa/manylinux1_x86_64 + +RUN yum install -y libtool +RUN /opt/python/cp27-cp27mu/bin/pip install twine + +COPY protobuf_optimized_pip.sh / diff --git a/python/release/wheel/README.md b/python/release/wheel/README.md new file mode 100644 index 00000000..edda2cd7 --- /dev/null +++ b/python/release/wheel/README.md @@ -0,0 +1,17 @@ +Description +------------------------------ +This directory is used to build released wheels according to PEP513 and upload +them to pypi. + +Usage +------------------------------ +For example, to release 3.3.0: + ./protobuf_optimized_pip.sh 3.3.0 PYPI_USERNAME PYPI_PASSWORD + +Structure +------------------------------ +| Source | Source | +|--------------------------------------|---------------------------------------------------| +| protobuf_optimized_pip.sh | Entry point. Calling Dockerfile and build_wheel_manylinux.sh | +| Dockerfile | Build docker image according to PEP513. | +| build_wheel_manylinux.sh | Build wheel packages in the docker container. | diff --git a/python/release/wheel/build_wheel_manylinux.sh b/python/release/wheel/build_wheel_manylinux.sh new file mode 100755 index 00000000..39fd8c12 --- /dev/null +++ b/python/release/wheel/build_wheel_manylinux.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Print usage and fail. +function usage() { + echo "Usage: protobuf_optimized_pip.sh PROTOBUF_VERSION PYPI_USERNAME PYPI_PASSWORD" >&2 + exit 1 # Causes caller to exit because we use -e. +} + +# Validate arguments. +if [ $0 != ./build_wheel_manylinux.sh ]; then + echo "Please run this script from the directory in which it is located." >&2 + exit 1 +fi + +if [ $# -lt 3 ]; then + usage + exit 1 +fi + +PROTOBUF_VERSION=$1 +PYPI_USERNAME=$2 +PYPI_PASSWORD=$3 + +docker rmi protobuf-python-wheel +docker build . -t protobuf-python-wheel +docker run --rm protobuf-python-wheel ./protobuf_optimized_pip.sh $PROTOBUF_VERSION $PYPI_USERNAME $PYPI_PASSWORD +docker rmi protobuf-python-wheel diff --git a/python/release/wheel/protobuf_optimized_pip.sh b/python/release/wheel/protobuf_optimized_pip.sh new file mode 100755 index 00000000..064d1d2a --- /dev/null +++ b/python/release/wheel/protobuf_optimized_pip.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +# DO NOT use this script manually! Called by docker. + +set -ex + +# Print usage and fail. +function usage() { + echo "Usage: protobuf_optimized_pip.sh PROTOBUF_VERSION PYPI_USERNAME PYPI_PASSWORD" >&2 + exit 1 # Causes caller to exit because we use -e. +} + +# Build wheel +function build_wheel() { + PYTHON_VERSION=$1 + PYTHON_BIN=/opt/python/${PYTHON_VERSION}/bin/python + + $PYTHON_BIN setup.py bdist_wheel --cpp_implementation --compile_static_extension + auditwheel repair dist/protobuf-${PROTOBUF_VERSION}-${PYTHON_VERSION}-linux_x86_64.whl +} + +# Validate arguments. +if [ $0 != ./protobuf_optimized_pip.sh ]; then + echo "Please run this script from the directory in which it is located." >&2 + exit 1 +fi + +if [ $# -lt 3 ]; then + usage + exit 1 +fi + +PROTOBUF_VERSION=$1 +PYPI_USERNAME=$2 +PYPI_PASSWORD=$3 + +DIR=${PWD}/'protobuf-python-build' +PYTHON_VERSIONS=('cp27-cp27mu' 'cp33-cp33m' 'cp34-cp34m' 'cp35-cp35m' 'cp36-cp36m') + +mkdir -p ${DIR} +cd ${DIR} +curl -SsL -O https://github.com/google/protobuf/archive/v${PROTOBUF_VERSION}.tar.gz +tar xzf v${PROTOBUF_VERSION}.tar.gz +cd $DIR/protobuf-${PROTOBUF_VERSION} + +# Autoconf on centos 5.11 cannot recognize AC_PROG_OBJC. +sed -i '/AC_PROG_OBJC/d' configure.ac +sed -i 's/conformance\/Makefile//g' configure.ac + +# Use the /usr/bin/autoconf and related tools to pick the correct aclocal macros +export PATH="/usr/bin:$PATH" + +# Build protoc +./autogen.sh +CXXFLAGS="-fPIC -g -O2" ./configure +make -j8 +export PROTOC=$DIR/src/protoc + +cd python + +for PYTHON_VERSION in "${PYTHON_VERSIONS[@]}" +do + build_wheel $PYTHON_VERSION +done + +/opt/python/cp27-cp27mu/bin/twine upload wheelhouse/* <<! +$PYPI_USERNAME +$PYPI_PASSWORD +! diff --git a/python/setup.py b/python/setup.py index ef1a31b4..70b7de5c 100755 --- a/python/setup.py +++ b/python/setup.py @@ -5,6 +5,7 @@ import glob import os import subprocess import sys +import platform # We must use setuptools, not distutils, because we need to use the # namespace_packages option for the "google" package. @@ -79,6 +80,7 @@ def GenerateUnittestProtos(): generate_proto("../src/google/protobuf/any_test.proto", False) generate_proto("../src/google/protobuf/map_unittest.proto", False) generate_proto("../src/google/protobuf/test_messages_proto3.proto", False) + generate_proto("../src/google/protobuf/test_messages_proto2.proto", False) generate_proto("../src/google/protobuf/unittest_arena.proto", False) generate_proto("../src/google/protobuf/unittest_no_arena.proto", False) generate_proto("../src/google/protobuf/unittest_no_arena_import.proto", False) @@ -189,6 +191,12 @@ if __name__ == '__main__': if "clang" in os.popen('$CC --version 2> /dev/null').read(): extra_compile_args.append('-Wno-shorten-64-to-32') + v, _, _ = platform.mac_ver() + if v: + v = float('.'.join(v.split('.')[:2])) + if v >= 10.12: + extra_compile_args.append('-std=c++11') + if warnings_as_errors in sys.argv: extra_compile_args.append('-Werror') sys.argv.remove(warnings_as_errors) @@ -227,7 +235,7 @@ if __name__ == '__main__': url='https://developers.google.com/protocol-buffers/', maintainer='protobuf@googlegroups.com', maintainer_email='protobuf@googlegroups.com', - license='New BSD License', + license='3-Clause BSD License', classifiers=[ "Programming Language :: Python", "Programming Language :: Python :: 2", diff --git a/python/tox.ini b/python/tox.ini index 1600db21..38a81b4f 100644 --- a/python/tox.ini +++ b/python/tox.ini @@ -1,6 +1,6 @@ [tox] envlist = - py{26,27,33,34}-{cpp,python} + py{27,33,34,35,36}-{cpp,python} [testenv] usedevelop=true @@ -9,6 +9,7 @@ setenv = cpp: LD_LIBRARY_PATH={toxinidir}/../src/.libs cpp: DYLD_LIBRARY_PATH={toxinidir}/../src/.libs cpp: PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp + python: PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python commands = python setup.py -q build_py python: python setup.py -q build |