diff options
38 files changed, 418 insertions, 153 deletions
@@ -51,6 +51,7 @@ cpp_test*.pb.* *.pyc *.egg-info *_pb2.py +python/*.egg python/.eggs/ python/build/ python/google/protobuf/compiler/ diff --git a/conformance/ConformanceJava.java b/conformance/ConformanceJava.java new file mode 100644 index 00000000..c1a53141 --- /dev/null +++ b/conformance/ConformanceJava.java @@ -0,0 +1,120 @@ + +import com.google.protobuf.conformance.Conformance; +import com.google.protobuf.InvalidProtocolBufferException; + +class ConformanceJava { + private int testCount = 0; + + private boolean readFromStdin(byte[] buf, int len) throws Exception { + int ofs = 0; + while (len > 0) { + int read = System.in.read(buf, ofs, len); + if (read == -1) { + return false; // EOF + } + ofs += read; + len -= read; + } + + return true; + } + + private void writeToStdout(byte[] buf) throws Exception { + System.out.write(buf); + } + + // Returns -1 on EOF (the actual values will always be positive). + private int readLittleEndianIntFromStdin() throws Exception { + byte[] buf = new byte[4]; + if (!readFromStdin(buf, 4)) { + return -1; + } + return buf[0] | (buf[1] << 1) | (buf[2] << 2) | (buf[3] << 3); + } + + private void writeLittleEndianIntToStdout(int val) throws Exception { + byte[] buf = new byte[4]; + buf[0] = (byte)val; + buf[1] = (byte)(val >> 8); + buf[2] = (byte)(val >> 16); + buf[3] = (byte)(val >> 24); + writeToStdout(buf); + } + + private Conformance.ConformanceResponse doTest(Conformance.ConformanceRequest request) { + Conformance.TestAllTypes testMessage; + + switch (request.getPayloadCase()) { + case PROTOBUF_PAYLOAD: { + try { + testMessage = Conformance.TestAllTypes.parseFrom(request.getProtobufPayload()); + } catch (InvalidProtocolBufferException e) { + return Conformance.ConformanceResponse.newBuilder().setParseError(e.getMessage()).build(); + } + break; + } + case JSON_PAYLOAD: { + return Conformance.ConformanceResponse.newBuilder().setRuntimeError("JSON not yet supported.").build(); + } + case PAYLOAD_NOT_SET: { + throw new RuntimeException("Request didn't have payload."); + } + + default: { + throw new RuntimeException("Unexpected payload case."); + } + } + + switch (request.getRequestedOutput()) { + case UNSPECIFIED: + throw new RuntimeException("Unspecified output format."); + + case PROTOBUF: + return Conformance.ConformanceResponse.newBuilder().setProtobufPayload(testMessage.toByteString()).build(); + + case JSON: + return Conformance.ConformanceResponse.newBuilder().setRuntimeError("JSON not yet supported.").build(); + + default: { + throw new RuntimeException("Unexpected request output."); + } + } + } + + private boolean doTestIo() throws Exception { + int bytes = readLittleEndianIntFromStdin(); + + if (bytes == -1) { + return false; // EOF + } + + byte[] serializedInput = new byte[bytes]; + + if (!readFromStdin(serializedInput, bytes)) { + throw new RuntimeException("Unexpected EOF from test program."); + } + + Conformance.ConformanceRequest request = + Conformance.ConformanceRequest.parseFrom(serializedInput); + Conformance.ConformanceResponse response = doTest(request); + byte[] serializedOutput = response.toByteArray(); + + writeLittleEndianIntToStdout(serializedOutput.length); + writeToStdout(serializedOutput); + + return true; + } + + public void run() throws Exception { + while (doTestIo()) { + // Empty. + } + + System.err.println("ConformanceJava: received EOF from test runner after " + + this.testCount + " tests"); + } + + public static void main(String[] args) throws Exception { + new ConformanceJava().run(); + } +} diff --git a/conformance/Makefile.am b/conformance/Makefile.am index 0c4eae75..59bb2576 100644 --- a/conformance/Makefile.am +++ b/conformance/Makefile.am @@ -21,30 +21,43 @@ conformance_cpp_CPPFLAGS = -I$(top_srcdir)/src if USE_EXTERNAL_PROTOC -unittest_proto_middleman: $(protoc_inputs) - $(PROTOC) -I$(srcdir) --cpp_out=. $^ - touch unittest_proto_middleman +protoc_middleman: $(protoc_inputs) + $(PROTOC) -I$(srcdir) --cpp_out=. --java_out=. $^ + touch protoc_middleman else # We have to cd to $(srcdir) before executing protoc because $(protoc_inputs) is # relative to srcdir, which may not be the same as the current directory when # building out-of-tree. -unittest_proto_middleman: $(top_srcdir)/src/protoc$(EXEEXT) $(protoc_inputs) - oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. --cpp_out=$$oldpwd $(protoc_inputs) ) - touch unittest_proto_middleman +protoc_middleman: $(top_srcdir)/src/protoc$(EXEEXT) $(protoc_inputs) + oldpwd=`pwd` && ( cd $(srcdir) && $$oldpwd/../src/protoc$(EXEEXT) -I. --cpp_out=$$oldpwd --java_out=$$oldpwd $(protoc_inputs) ) + touch protoc_middleman endif -$(protoc_outputs): unittest_proto_middleman +$(protoc_outputs): protoc_middleman BUILT_SOURCES = $(protoc_outputs) -CLEANFILES = $(protoc_outputs) unittest_proto_middleman +CLEANFILES = $(protoc_outputs) protoc_middleman javac_middleman conformance-java MAINTAINERCLEANFILES = \ Makefile.in +javac_middleman: ConformanceJava.java protoc_middleman + javac ConformanceJava.java com/google/protobuf/conformance/Conformance.java + @touch javac_middleman + +conformance-java: javac_middleman + @echo "Writing shortcut script conformance-java..." + @echo '#! /bin/sh' > conformance-java + @echo 'java -classpath .:$$CLASSPATH ConformanceJava "$$@"' >> conformance-java + @chmod +x conformance-java + # Targets for actually running tests. -test_cpp: unittest_proto_middleman conformance-test-runner conformance-cpp +test_cpp: protoc_middleman conformance-test-runner conformance-cpp ./conformance-test-runner ./conformance-cpp + +test_java: protoc_middleman conformance-test-runner conformance-java + ./conformance-test-runner ./conformance-java diff --git a/conformance/conformance.proto b/conformance/conformance.proto index 0fb66cf9..7b676898 100644 --- a/conformance/conformance.proto +++ b/conformance/conformance.proto @@ -30,6 +30,7 @@ syntax = "proto3"; package conformance; +option java_package = "com.google.protobuf.conformance"; // This defines the conformance testing protocol. This protocol exists between // the conformance test suite itself and the code being tested. For each test, diff --git a/conformance/conformance_test_runner.cc b/conformance/conformance_test_runner.cc index 21277700..ca5877bd 100644 --- a/conformance/conformance_test_runner.cc +++ b/conformance/conformance_test_runner.cc @@ -48,9 +48,9 @@ // Every test consists of a ConformanceRequest/ConformanceResponse // request/reply pair. The protocol on the pipe is simply: // -// 1. tester sends 4-byte length N +// 1. tester sends 4-byte length N (little endian) // 2. tester sends N bytes representing a ConformanceRequest proto -// 3. testee sends 4-byte length M +// 3. testee sends 4-byte length M (little endian) // 4. testee sends M bytes representing a ConformanceResponse proto #include <errno.h> diff --git a/examples/add_person.py b/examples/add_person.py index 78e56966..5a7b968e 100755 --- a/examples/add_person.py +++ b/examples/add_person.py @@ -1,4 +1,4 @@ -#! /usr/bin/python +#! /usr/bin/env python # See README.txt for information and build instructions. diff --git a/examples/list_people.py b/examples/list_people.py index f9f36b96..d470349a 100755 --- a/examples/list_people.py +++ b/examples/list_people.py @@ -1,4 +1,4 @@ -#! /usr/bin/python +#! /usr/bin/env python # See README.txt for information and build instructions. diff --git a/python/MANIFEST.in b/python/MANIFEST.in new file mode 100644 index 00000000..26088826 --- /dev/null +++ b/python/MANIFEST.in @@ -0,0 +1,14 @@ +prune google/protobuf/internal/import_test_package +exclude google/protobuf/internal/*_pb2.py +exclude google/protobuf/internal/*_test.py +exclude google/protobuf/internal/*.proto +exclude google/protobuf/internal/test_util.py + +recursive-exclude google *_test.py +recursive-exclude google *_test.proto +recursive-exclude google unittest*_pb2.py + +global-exclude *.dll +global-exclude *.pyc +global-exclude *.pyo +global-exclude *.so diff --git a/python/google/protobuf/internal/_parameterized.py b/python/google/protobuf/internal/_parameterized.py index c44e77e4..400b2216 100755 --- a/python/google/protobuf/internal/_parameterized.py +++ b/python/google/protobuf/internal/_parameterized.py @@ -1,4 +1,4 @@ -#! /usr/bin/python +#! /usr/bin/env python # # Protocol Buffers - Google's data interchange format # Copyright 2008 Google Inc. All rights reserved. diff --git a/python/google/protobuf/internal/descriptor_database_test.py b/python/google/protobuf/internal/descriptor_database_test.py index edf66a63..56fe14e9 100644 --- a/python/google/protobuf/internal/descriptor_database_test.py +++ b/python/google/protobuf/internal/descriptor_database_test.py @@ -1,4 +1,4 @@ -#! /usr/bin/python +#! /usr/bin/env python # # Protocol Buffers - Google's data interchange format # Copyright 2008 Google Inc. All rights reserved. diff --git a/python/google/protobuf/internal/descriptor_pool_test.py b/python/google/protobuf/internal/descriptor_pool_test.py index 6d04ebaa..7d145f42 100644 --- a/python/google/protobuf/internal/descriptor_pool_test.py +++ b/python/google/protobuf/internal/descriptor_pool_test.py @@ -1,4 +1,4 @@ -#! /usr/bin/python +#! /usr/bin/env python # # Protocol Buffers - Google's data interchange format # Copyright 2008 Google Inc. All rights reserved. diff --git a/python/google/protobuf/internal/descriptor_test.py b/python/google/protobuf/internal/descriptor_test.py index 549af088..335caee6 100755 --- a/python/google/protobuf/internal/descriptor_test.py +++ b/python/google/protobuf/internal/descriptor_test.py @@ -1,4 +1,4 @@ -#! /usr/bin/python +#! /usr/bin/env python # # Protocol Buffers - Google's data interchange format # Copyright 2008 Google Inc. All rights reserved. diff --git a/python/google/protobuf/internal/generator_test.py b/python/google/protobuf/internal/generator_test.py index 50fc1922..ccc5860b 100755 --- a/python/google/protobuf/internal/generator_test.py +++ b/python/google/protobuf/internal/generator_test.py @@ -1,4 +1,4 @@ -#! /usr/bin/python +#! /usr/bin/env python # # Protocol Buffers - Google's data interchange format # Copyright 2008 Google Inc. All rights reserved. diff --git a/python/google/protobuf/internal/message_factory_test.py b/python/google/protobuf/internal/message_factory_test.py index 45ddcd26..626c3fc9 100644 --- a/python/google/protobuf/internal/message_factory_test.py +++ b/python/google/protobuf/internal/message_factory_test.py @@ -1,4 +1,4 @@ -#! /usr/bin/python +#! /usr/bin/env python # # Protocol Buffers - Google's data interchange format # Copyright 2008 Google Inc. All rights reserved. diff --git a/python/google/protobuf/internal/message_test.py b/python/google/protobuf/internal/message_test.py index ce79b607..4ecaa1c7 100755 --- a/python/google/protobuf/internal/message_test.py +++ b/python/google/protobuf/internal/message_test.py @@ -1,4 +1,4 @@ -#! /usr/bin/python +#! /usr/bin/env python # # Protocol Buffers - Google's data interchange format # Copyright 2008 Google Inc. All rights reserved. diff --git a/python/google/protobuf/internal/proto_builder_test.py b/python/google/protobuf/internal/proto_builder_test.py index 9229205a..b1e57f35 100644 --- a/python/google/protobuf/internal/proto_builder_test.py +++ b/python/google/protobuf/internal/proto_builder_test.py @@ -1,4 +1,4 @@ -#! /usr/bin/python +#! /usr/bin/env python # # Protocol Buffers - Google's data interchange format # Copyright 2008 Google Inc. All rights reserved. diff --git a/python/google/protobuf/internal/reflection_test.py b/python/google/protobuf/internal/reflection_test.py index 8f28f4d9..ae79c78b 100755 --- a/python/google/protobuf/internal/reflection_test.py +++ b/python/google/protobuf/internal/reflection_test.py @@ -1,4 +1,4 @@ -#! /usr/bin/python +#! /usr/bin/env python # -*- coding: utf-8 -*- # # Protocol Buffers - Google's data interchange format diff --git a/python/google/protobuf/internal/service_reflection_test.py b/python/google/protobuf/internal/service_reflection_test.py index e3f71545..de462124 100755 --- a/python/google/protobuf/internal/service_reflection_test.py +++ b/python/google/protobuf/internal/service_reflection_test.py @@ -1,4 +1,4 @@ -#! /usr/bin/python +#! /usr/bin/env python # # Protocol Buffers - Google's data interchange format # Copyright 2008 Google Inc. All rights reserved. @@ -81,7 +81,7 @@ class FooUnitTest(unittest.TestCase): self.assertEqual('Method Bar not implemented.', rpc_controller.failure_message) self.assertEqual(None, self.callback_response) - + class MyServiceImpl(unittest_pb2.TestService): def Foo(self, rpc_controller, request, done): self.foo_called = True diff --git a/python/google/protobuf/internal/symbol_database_test.py b/python/google/protobuf/internal/symbol_database_test.py index a58cb1a4..c888aff7 100644 --- a/python/google/protobuf/internal/symbol_database_test.py +++ b/python/google/protobuf/internal/symbol_database_test.py @@ -1,4 +1,4 @@ -#! /usr/bin/python +#! /usr/bin/env python # # Protocol Buffers - Google's data interchange format # Copyright 2008 Google Inc. All rights reserved. diff --git a/python/google/protobuf/internal/text_encoding_test.py b/python/google/protobuf/internal/text_encoding_test.py index 48c30f01..27896b94 100755 --- a/python/google/protobuf/internal/text_encoding_test.py +++ b/python/google/protobuf/internal/text_encoding_test.py @@ -1,4 +1,4 @@ -#! /usr/bin/python +#! /usr/bin/env python # # Protocol Buffers - Google's data interchange format # Copyright 2008 Google Inc. All rights reserved. diff --git a/python/google/protobuf/internal/text_format_test.py b/python/google/protobuf/internal/text_format_test.py index 0135099d..bf7e06ee 100755 --- a/python/google/protobuf/internal/text_format_test.py +++ b/python/google/protobuf/internal/text_format_test.py @@ -1,4 +1,4 @@ -#! /usr/bin/python +#! /usr/bin/env python # # Protocol Buffers - Google's data interchange format # Copyright 2008 Google Inc. All rights reserved. diff --git a/python/google/protobuf/internal/unknown_fields_test.py b/python/google/protobuf/internal/unknown_fields_test.py index d116920e..9337ae8a 100755 --- a/python/google/protobuf/internal/unknown_fields_test.py +++ b/python/google/protobuf/internal/unknown_fields_test.py @@ -1,4 +1,4 @@ -#! /usr/bin/python +#! /usr/bin/env python # -*- coding: utf-8 -*- # # Protocol Buffers - Google's data interchange format diff --git a/python/google/protobuf/internal/wire_format_test.py b/python/google/protobuf/internal/wire_format_test.py index e40a40cc..5cd7fcb9 100755 --- a/python/google/protobuf/internal/wire_format_test.py +++ b/python/google/protobuf/internal/wire_format_test.py @@ -1,4 +1,4 @@ -#! /usr/bin/python +#! /usr/bin/env python # # Protocol Buffers - Google's data interchange format # Copyright 2008 Google Inc. All rights reserved. diff --git a/python/setup.py b/python/setup.py index fa98cd31..06aed94b 100755 --- a/python/setup.py +++ b/python/setup.py @@ -1,4 +1,4 @@ -#! /usr/bin/python +#! /usr/bin/env python # # See README for usage instructions. import sys @@ -8,28 +8,29 @@ import subprocess # We must use setuptools, not distutils, because we need to use the # namespace_packages option for the "google" package. try: - from setuptools import setup, Extension + from setuptools import setup, Extension, find_packages except ImportError: try: from ez_setup import use_setuptools use_setuptools() - from setuptools import setup, Extension + from setuptools import setup, Extension, find_packages except ImportError: sys.stderr.write( "Could not import setuptools; make sure you have setuptools or " - "ez_setup installed.\n") + "ez_setup installed.\n" + ) raise + from distutils.command.clean import clean as _clean -if sys.version_info[0] >= 3: - # Python 3 - from distutils.command.build_py import build_py_2to3 as _build_py + +if sys.version_info[0] == 3: + # Python 3 + from distutils.command.build_py import build_py_2to3 as _build_py else: - # Python 2 - from distutils.command.build_py import build_py as _build_py + # Python 2 + from distutils.command.build_py import build_py as _build_py from distutils.spawn import find_executable -maintainer_email = "protobuf@googlegroups.com" - # Find the Protocol Compiler. if 'PROTOC' in os.environ and os.path.exists(os.environ['PROTOC']): protoc = os.environ['PROTOC'] @@ -44,13 +45,13 @@ elif os.path.exists("../vsprojects/Release/protoc.exe"): else: protoc = find_executable("protoc") + def GetVersion(): """Gets the version from google/protobuf/__init__.py - Do not import google.protobuf.__init__ directly, because an installed protobuf - library may be loaded instead. + Do not import google.protobuf.__init__ directly, because an installed + protobuf library may be loaded instead.""" - """ with open(os.path.join('google', 'protobuf', '__init__.py')) as version_file: exec(version_file.read()) return __version__ @@ -66,22 +67,24 @@ def generate_proto(source): if (not os.path.exists(output) or (os.path.exists(source) and os.path.getmtime(source) > os.path.getmtime(output))): - print ("Generating %s..." % output) + print("Generating %s..." % output) if not os.path.exists(source): sys.stderr.write("Can't find required file: %s\n" % source) sys.exit(-1) - if protoc == None: + if protoc is None: sys.stderr.write( - "protoc is not installed nor found in ../src. Please compile it " - "or install the binary package.\n") + "protoc is not installed nor found in ../src. " + "Please compile it or install the binary package.\n" + ) sys.exit(-1) - protoc_command = [ protoc, "-I../src", "-I.", "--python_out=.", source ] + protoc_command = [protoc, "-I../src", "-I.", "--python_out=.", source] if subprocess.call(protoc_command) != 0: sys.exit(-1) + def GenerateUnittestProtos(): generate_proto("../src/google/protobuf/unittest.proto") generate_proto("../src/google/protobuf/unittest_custom_options.proto") @@ -92,17 +95,18 @@ def GenerateUnittestProtos(): generate_proto("../src/google/protobuf/unittest_proto3_arena.proto") generate_proto("google/protobuf/internal/descriptor_pool_test1.proto") generate_proto("google/protobuf/internal/descriptor_pool_test2.proto") - generate_proto("google/protobuf/internal/test_bad_identifiers.proto") - generate_proto("google/protobuf/internal/missing_enum_values.proto") - generate_proto("google/protobuf/internal/more_extensions.proto") - generate_proto("google/protobuf/internal/more_extensions_dynamic.proto") - generate_proto("google/protobuf/internal/more_messages.proto") generate_proto("google/protobuf/internal/factory_test1.proto") generate_proto("google/protobuf/internal/factory_test2.proto") generate_proto("google/protobuf/internal/import_test_package/inner.proto") generate_proto("google/protobuf/internal/import_test_package/outer.proto") + generate_proto("google/protobuf/internal/missing_enum_values.proto") + generate_proto("google/protobuf/internal/more_extensions.proto") + generate_proto("google/protobuf/internal/more_extensions_dynamic.proto") + generate_proto("google/protobuf/internal/more_messages.proto") + generate_proto("google/protobuf/internal/test_bad_identifiers.proto") generate_proto("google/protobuf/pyext/python.proto") + class clean(_clean): def run(self): # Delete generated files in the code tree. @@ -110,12 +114,13 @@ class clean(_clean): for filename in filenames: filepath = os.path.join(dirpath, filename) if filepath.endswith("_pb2.py") or filepath.endswith(".pyc") or \ - filepath.endswith(".so") or filepath.endswith(".o") or \ - filepath.endswith('google/protobuf/compiler/__init__.py'): + filepath.endswith(".so") or filepath.endswith(".o") or \ + filepath.endswith('google/protobuf/compiler/__init__.py'): os.remove(filepath) # _clean is an old-style class, so super() doesn't work. _clean.run(self) + class build_py(_build_py): def run(self): # Generate necessary .proto file if it doesn't exist. @@ -132,11 +137,11 @@ class build_py(_build_py): # _build_py is an old-style class, so super() doesn't work. _build_py.run(self) # TODO(mrovner): Subclass to run 2to3 on some files only. - # Tracing what https://wiki.python.org/moin/PortingPythonToPy3k's "Approach 2" - # section on how to get 2to3 to run on source files during install under - # Python 3. This class seems like a good place to put logic that calls - # python3's distutils.util.run_2to3 on the subset of the files we have in our - # release that are subject to conversion. + # Tracing what https://wiki.python.org/moin/PortingPythonToPy3k's + # "Approach 2" section on how to get 2to3 to run on source files during + # install under Python 3. This class seems like a good place to put logic + # that calls python3's distutils.util.run_2to3 on the subset of the files we + # have in our release that are subject to conversion. # See code reference in previous code review. if __name__ == '__main__': @@ -145,61 +150,49 @@ if __name__ == '__main__': if cpp_impl in sys.argv: sys.argv.remove(cpp_impl) # C++ implementation extension - ext_module_list.append(Extension( - "google.protobuf.pyext._message", - [ "google/protobuf/pyext/descriptor.cc", - "google/protobuf/pyext/descriptor_containers.cc", - "google/protobuf/pyext/descriptor_pool.cc", - "google/protobuf/pyext/message.cc", - "google/protobuf/pyext/extension_dict.cc", - "google/protobuf/pyext/repeated_scalar_container.cc", - "google/protobuf/pyext/repeated_composite_container.cc" ], - define_macros=[('GOOGLE_PROTOBUF_HAS_ONEOF', '1')], - include_dirs = [ ".", "../src" ], - libraries = [ "protobuf" ], - library_dirs = [ '../src/.libs' ], - )) + ext_module_list.append( + Extension( + "google.protobuf.pyext._message", + [ + "google/protobuf/pyext/descriptor.cc", + "google/protobuf/pyext/descriptor_containers.cc", + "google/protobuf/pyext/descriptor_pool.cc", + "google/protobuf/pyext/extension_dict.cc", + "google/protobuf/pyext/message.cc", + "google/protobuf/pyext/repeated_composite_container.cc", + "google/protobuf/pyext/repeated_scalar_container.cc", + ], + define_macros=[('GOOGLE_PROTOBUF_HAS_ONEOF', '1')], + include_dirs=[".", "../src"], + libraries=['protobuf'], + library_dirs=['../src/.libs'], + ) + ) os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp' - setup(name = 'protobuf', - version = GetVersion(), - packages = [ 'google' ], - namespace_packages = [ 'google' ], - test_suite = 'google.protobuf.internal', - # Must list modules explicitly so that we don't install tests. - py_modules = [ - 'google.protobuf.internal.api_implementation', - 'google.protobuf.internal.containers', - 'google.protobuf.internal.decoder', - 'google.protobuf.internal.encoder', - 'google.protobuf.internal.enum_type_wrapper', - 'google.protobuf.internal.message_listener', - 'google.protobuf.internal.python_message', - 'google.protobuf.internal.type_checkers', - 'google.protobuf.internal.wire_format', - 'google.protobuf.descriptor', - 'google.protobuf.descriptor_pb2', - 'google.protobuf.compiler.plugin_pb2', - 'google.protobuf.message', - 'google.protobuf.descriptor_database', - 'google.protobuf.descriptor_pool', - 'google.protobuf.message_factory', - 'google.protobuf.proto_builder', - 'google.protobuf.pyext.cpp_message', - 'google.protobuf.reflection', - 'google.protobuf.service', - 'google.protobuf.service_reflection', - 'google.protobuf.symbol_database', - 'google.protobuf.text_encoding', - 'google.protobuf.text_format'], - cmdclass = { 'clean': clean, 'build_py': build_py }, - install_requires = ['setuptools'], - ext_modules = ext_module_list, - url = 'https://developers.google.com/protocol-buffers/', - maintainer = maintainer_email, - maintainer_email = 'protobuf@googlegroups.com', - license = 'New BSD License', - description = 'Protocol Buffers', - long_description = - "Protocol Buffers are Google's data interchange format.", - ) + setup( + name='protobuf', + version=GetVersion(), + description='Protocol Buffers', + long_description="Protocol Buffers are Google's data interchange format", + url='https://developers.google.com/protocol-buffers/', + maintainer='protobuf@googlegroups.com', + maintainer_email='protobuf@googlegroups.com', + license='New BSD License', + classifiers=[ + 'Programming Language :: Python :: 2.7', + ], + namespace_packages=['google'], + packages=find_packages( + exclude=[ + 'import_test_package', + ], + ), + test_suite='google.protobuf.internal', + cmdclass={ + 'clean': clean, + 'build_py': build_py, + }, + install_requires=['setuptools'], + ext_modules=ext_module_list, + ) diff --git a/ruby/.gitignore b/ruby/.gitignore index 80c978f2..bd8745dd 100644 --- a/ruby/.gitignore +++ b/ruby/.gitignore @@ -4,3 +4,5 @@ tags lib/google/protobuf_java.jar protobuf-jruby.iml target/ +pkg/ +tmp/ diff --git a/ruby/Gemfile.lock b/ruby/Gemfile.lock index 89deb47d..6f349276 100644 --- a/ruby/Gemfile.lock +++ b/ruby/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - google-protobuf (3.0.0.alpha.2) + google-protobuf (3.0.0.alpha.3.1.pre) GEM remote: https://rubygems.org/ diff --git a/ruby/README.md b/ruby/README.md index d2fa76ab..16474322 100644 --- a/ruby/README.md +++ b/ruby/README.md @@ -63,7 +63,7 @@ To build this Ruby extension, you will need: To Build the JRuby extension, you will need: * Maven -* The latest version of the protobuf java library +* The latest version of the protobuf java library (see ../java/README.md) * Install JRuby via rbenv or RVM First switch to the desired platform with rbenv or RVM. @@ -75,8 +75,9 @@ Then install the required Ruby gems: Then build the Gem: - $ rake gem - $ gem install pkg/protobuf-$VERSION.gem + $ rake + $ rake clobber_package gem + $ gem install `ls pkg/google-protobuf-*.gem` To run the specs: diff --git a/ruby/Rakefile b/ruby/Rakefile index 7c1d8495..c25103d8 100644 --- a/ruby/Rakefile +++ b/ruby/Rakefile @@ -6,6 +6,9 @@ require "rake/testtask" spec = Gem::Specification.load("google-protobuf.gemspec") if RUBY_PLATFORM == "java" + if `which mvn` == '' + raise ArgumentError, "maven needs to be installed" + end task :clean do system("mvn clean") end diff --git a/ruby/ext/google/protobuf_c/repeated_field.c b/ruby/ext/google/protobuf_c/repeated_field.c index 8cf2e29b..5148ee87 100644 --- a/ruby/ext/google/protobuf_c/repeated_field.c +++ b/ruby/ext/google/protobuf_c/repeated_field.c @@ -47,6 +47,15 @@ RepeatedField* ruby_to_RepeatedField(VALUE _self) { return self; } +static int index_position(VALUE _index, RepeatedField* repeated_field) { + int index = NUM2INT(_index); + if (index < 0 && repeated_field->size > 0) { + index = repeated_field->size + index; + } + return index; +} + + /* * call-seq: * RepeatedField.each(&block) @@ -74,8 +83,7 @@ VALUE RepeatedField_each(VALUE _self) { * call-seq: * RepeatedField.[](index) => value * - * Accesses the element at the given index. Throws an exception on out-of-bounds - * errors. + * Accesses the element at the given index. Returns nil on out-of-bounds */ VALUE RepeatedField_index(VALUE _self, VALUE _index) { RepeatedField* self = ruby_to_RepeatedField(_self); @@ -83,9 +91,9 @@ VALUE RepeatedField_index(VALUE _self, VALUE _index) { upb_fieldtype_t field_type = self->field_type; VALUE field_type_class = self->field_type_class; - int index = NUM2INT(_index); + int index = index_position(_index, self); if (index < 0 || index >= self->size) { - rb_raise(rb_eRangeError, "Index out of range"); + return Qnil; } void* memory = (void *) (((uint8_t *)self->elements) + index * element_size); @@ -105,9 +113,9 @@ VALUE RepeatedField_index_set(VALUE _self, VALUE _index, VALUE val) { VALUE field_type_class = self->field_type_class; int element_size = native_slot_size(field_type); - int index = NUM2INT(_index); + int index = index_position(_index, self); if (index < 0 || index >= (INT_MAX - 1)) { - rb_raise(rb_eRangeError, "Index out of range"); + return Qnil; } if (index >= self->size) { RepeatedField_reserve(self, index + 1); diff --git a/ruby/ext/google/protobuf_c/storage.c b/ruby/ext/google/protobuf_c/storage.c index 5b1549d2..2ad8bd74 100644 --- a/ruby/ext/google/protobuf_c/storage.c +++ b/ruby/ext/google/protobuf_c/storage.c @@ -155,7 +155,9 @@ void native_slot_set_value_and_case(upb_fieldtype_t type, VALUE type_class, break; } case UPB_TYPE_MESSAGE: { - if (CLASS_OF(value) != type_class) { + if (CLASS_OF(value) == CLASS_OF(Qnil)) { + value = Qnil; + } else if (CLASS_OF(value) != type_class) { rb_raise(rb_eTypeError, "Invalid type %s to assign to submessage field.", rb_class2name(CLASS_OF(value))); diff --git a/ruby/google-protobuf.gemspec b/ruby/google-protobuf.gemspec index abbbde35..28cdebf5 100644 --- a/ruby/google-protobuf.gemspec +++ b/ruby/google-protobuf.gemspec @@ -4,10 +4,11 @@ Gem::Specification.new do |s| s.licenses = ["BSD"] s.summary = "Protocol Buffers" s.description = "Protocol Buffers are Google's data interchange format." + s.homepage = "https://developers.google.com/protocol-buffers" s.authors = ["Protobuf Authors"] s.email = "protobuf@googlegroups.com" s.require_paths = ["lib"] - s.files = ["lib/google/protobuf.rb"] + s.files = `git ls-files -z`.split("\x0").find_all{|f| f =~ /lib\/.+\.rb/} unless RUBY_PLATFORM == "java" s.files += `git ls-files "*.c" "*.h" extconf.rb Makefile`.split s.extensions= ["ext/google/protobuf_c/extconf.rb"] diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf.rb index 75869dd8..72797245 100644 --- a/ruby/lib/google/protobuf.rb +++ b/ruby/lib/google/protobuf.rb @@ -34,3 +34,5 @@ if RUBY_PLATFORM == "java" else require 'google/protobuf_c' end + +require 'google/protobuf/repeated_field' diff --git a/ruby/lib/google/protobuf/repeated_field.rb b/ruby/lib/google/protobuf/repeated_field.rb new file mode 100644 index 00000000..5b934e56 --- /dev/null +++ b/ruby/lib/google/protobuf/repeated_field.rb @@ -0,0 +1,40 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# add syntatic sugar on top of the core library +module Google + module Protobuf + class RepeatedField + + alias_method :size, :length + + end + end +end diff --git a/ruby/pom.xml b/ruby/pom.xml index 1630fe84..01f0e16b 100644 --- a/ruby/pom.xml +++ b/ruby/pom.xml @@ -78,7 +78,7 @@ <dependency> <groupId>com.google.protobuf</groupId> <artifactId>protobuf-java</artifactId> - <version>3.0.0-pre</version> + <version>3.0.0-alpha-3-pre</version> </dependency> </dependencies> </project> diff --git a/ruby/src/main/java/com/google/protobuf/jruby/RubyMessage.java b/ruby/src/main/java/com/google/protobuf/jruby/RubyMessage.java index 04bc0b76..c7fd7aa7 100644 --- a/ruby/src/main/java/com/google/protobuf/jruby/RubyMessage.java +++ b/ruby/src/main/java/com/google/protobuf/jruby/RubyMessage.java @@ -246,16 +246,15 @@ public class RubyMessage extends RubyObject { public IRubyObject dup(ThreadContext context) { RubyMessage dup = (RubyMessage) metaClass.newInstance(context, Block.NULL_BLOCK); IRubyObject value; - for (Descriptors.FieldDescriptor fieldDescriptor : builder.getAllFields().keySet()) { + for (Descriptors.FieldDescriptor fieldDescriptor : this.descriptor.getFields()) { if (fieldDescriptor.isRepeated()) { - dup.repeatedFields.put(fieldDescriptor, getRepeatedField(context, fieldDescriptor)); - } else if (builder.hasField(fieldDescriptor)) { - dup.fields.put(fieldDescriptor, wrapField(context, fieldDescriptor, builder.getField(fieldDescriptor))); + dup.addRepeatedField(fieldDescriptor, this.getRepeatedField(context, fieldDescriptor)); + } else if (fields.containsKey(fieldDescriptor)) { + dup.fields.put(fieldDescriptor, fields.get(fieldDescriptor)); + } else if (this.builder.hasField(fieldDescriptor)) { + dup.fields.put(fieldDescriptor, wrapField(context, fieldDescriptor, this.builder.getField(fieldDescriptor))); } } - for (Descriptors.FieldDescriptor fieldDescriptor : fields.keySet()) { - dup.fields.put(fieldDescriptor, fields.get(fieldDescriptor)); - } for (Descriptors.FieldDescriptor fieldDescriptor : maps.keySet()) { dup.maps.put(fieldDescriptor, maps.get(fieldDescriptor)); } @@ -411,6 +410,7 @@ public class RubyMessage extends RubyObject { for (int i = 0; i < count; i++) { ret.push(context, wrapField(context, fieldDescriptor, this.builder.getRepeatedField(fieldDescriptor, i))); } + addRepeatedField(fieldDescriptor, ret); return ret; } @@ -659,14 +659,14 @@ public class RubyMessage extends RubyObject { } else { Descriptors.FieldDescriptor.Type fieldType = fieldDescriptor.getType(); IRubyObject typeClass = context.runtime.getObject(); + boolean addValue = true; if (fieldType == Descriptors.FieldDescriptor.Type.MESSAGE) { typeClass = ((RubyDescriptor) getDescriptorForField(context, fieldDescriptor)).msgclass(context); + if (value.isNil()){ + addValue = false; + } } else if (fieldType == Descriptors.FieldDescriptor.Type.ENUM) { typeClass = ((RubyEnumDescriptor) getDescriptorForField(context, fieldDescriptor)).enummodule(context); - } - Utils.checkType(context, fieldType, value, (RubyModule) typeClass); - // Convert integer enum to symbol - if (fieldType == Descriptors.FieldDescriptor.Type.ENUM) { Descriptors.EnumDescriptor enumDescriptor = fieldDescriptor.getEnumType(); if (Utils.isRubyNum(value)) { Descriptors.EnumValueDescriptor val = @@ -674,7 +674,12 @@ public class RubyMessage extends RubyObject { if (val.getIndex() != -1) value = context.runtime.newSymbol(val.getName()); } } - this.fields.put(fieldDescriptor, value); + if (addValue) { + Utils.checkType(context, fieldType, value, (RubyModule) typeClass); + this.fields.put(fieldDescriptor, value); + } else { + this.fields.remove(fieldDescriptor); + } } } return context.runtime.getNil(); diff --git a/ruby/src/main/java/com/google/protobuf/jruby/RubyRepeatedField.java b/ruby/src/main/java/com/google/protobuf/jruby/RubyRepeatedField.java index 9788317a..84bf8956 100644 --- a/ruby/src/main/java/com/google/protobuf/jruby/RubyRepeatedField.java +++ b/ruby/src/main/java/com/google/protobuf/jruby/RubyRepeatedField.java @@ -108,8 +108,9 @@ public class RubyRepeatedField extends RubyObject { */ @JRubyMethod(name = "[]=") public IRubyObject indexSet(ThreadContext context, IRubyObject index, IRubyObject value) { + int arrIndex = normalizeArrayIndex(index); Utils.checkType(context, fieldType, value, (RubyModule) typeClass); - this.storage.set(RubyNumeric.num2int(index), value); + this.storage.set(arrIndex, value); return context.runtime.getNil(); } @@ -117,12 +118,15 @@ public class RubyRepeatedField extends RubyObject { * call-seq: * RepeatedField.[](index) => value * - * Accesses the element at the given index. Throws an exception on out-of-bounds - * errors. + * Accesses the element at the given index. Returns nil on out-of-bounds */ @JRubyMethod(name = "[]") public IRubyObject index(ThreadContext context, IRubyObject index) { - return this.storage.eltInternal(RubyNumeric.num2int(index)); + int arrIndex = normalizeArrayIndex(index); + if (arrIndex < 0 || arrIndex >= this.storage.size()) { + return context.runtime.getNil(); + } + return this.storage.eltInternal(arrIndex); } /* @@ -134,8 +138,7 @@ public class RubyRepeatedField extends RubyObject { @JRubyMethod(rest = true) public IRubyObject insert(ThreadContext context, IRubyObject[] args) { for (int i = 0; i < args.length; i++) { - Utils.checkType(context, fieldType, args[i], (RubyModule) typeClass); - this.storage.add(args[i]); + push(context, args[i]); } return context.runtime.getNil(); } @@ -385,6 +388,15 @@ public class RubyRepeatedField extends RubyObject { } } + private int normalizeArrayIndex(IRubyObject index) { + int arrIndex = RubyNumeric.num2int(index); + int arrSize = this.storage.size(); + if (arrIndex < 0 && arrSize > 0) { + arrIndex = arrSize + arrIndex; + } + return arrIndex; + } + private RubyArray storage; private Descriptors.FieldDescriptor.Type fieldType; private IRubyObject typeClass; diff --git a/ruby/tests/basic.rb b/ruby/tests/basic.rb index 1c3fb62c..1c2a03dc 100644 --- a/ruby/tests/basic.rb +++ b/ruby/tests/basic.rb @@ -154,6 +154,8 @@ module BasicTest assert m.optional_bytes == "world" m.optional_msg = TestMessage2.new(:foo => 42) assert m.optional_msg == TestMessage2.new(:foo => 42) + m.optional_msg = nil + assert m.optional_msg == nil end def test_ctor_args @@ -314,6 +316,17 @@ module BasicTest assert l4 == [0, 0, 0, 0, 0, 42, 100, 101, 102] end + def test_parent_rptfield + #make sure we set the RepeatedField and can add to it + m = TestMessage.new + assert m.repeated_string == [] + m.repeated_string << 'ok' + m.repeated_string.push('ok2') + assert m.repeated_string == ['ok', 'ok2'] + m.repeated_string += ['ok3'] + assert m.repeated_string == ['ok', 'ok2', 'ok3'] + end + def test_rptfield_msg l = Google::Protobuf::RepeatedField.new(:message, TestMessage) l.push TestMessage.new @@ -377,6 +390,39 @@ module BasicTest end end + def test_rptfield_array_ducktyping + l = Google::Protobuf::RepeatedField.new(:int32) + length_methods = %w(count length size) + length_methods.each do |lm| + assert l.send(lm) == 0 + end + # out of bounds returns a nil + assert l[0] == nil + assert l[1] == nil + assert l[-1] == nil + l.push 4 + length_methods.each do |lm| + assert l.send(lm) == 1 + end + assert l[0] == 4 + assert l[1] == nil + assert l[-1] == 4 + assert l[-2] == nil + + l.push 2 + length_methods.each do |lm| + assert l.send(lm) == 2 + end + assert l[0] == 4 + assert l[1] == 2 + assert l[2] == nil + assert l[-1] == 2 + assert l[-2] == 4 + assert l[-3] == nil + + #adding out of scope will backfill with empty objects + end + def test_map_basic # allowed key types: # :int32, :int64, :uint32, :uint64, :bool, :string, :bytes. @@ -712,9 +758,12 @@ module BasicTest m = TestMessage.new m.optional_string = "hello" m.optional_int32 = 42 - m.repeated_msg.push TestMessage2.new(:foo => 100) - m.repeated_msg.push TestMessage2.new(:foo => 200) - + tm1 = TestMessage2.new(:foo => 100) + tm2 = TestMessage2.new(:foo => 200) + m.repeated_msg.push tm1 + assert m.repeated_msg[-1] == tm1 + m.repeated_msg.push tm2 + assert m.repeated_msg[-1] == tm2 m2 = m.dup assert m == m2 m.optional_int32 += 1 @@ -827,7 +876,6 @@ module BasicTest assert m['a.b'] == 4 end - def test_int_ranges m = TestMessage.new @@ -933,7 +981,6 @@ module BasicTest assert_raise RangeError do m.optional_uint64 = 1.5 end - end def test_stress_test diff --git a/src/google/protobuf/arena_nc_test.py b/src/google/protobuf/arena_nc_test.py index fc510a48..f390df36 100644 --- a/src/google/protobuf/arena_nc_test.py +++ b/src/google/protobuf/arena_nc_test.py @@ -1,4 +1,4 @@ -#! /usr/bin/python +#! /usr/bin/env python # # Protocol Buffers - Google's data interchange format # Copyright 2008 Google Inc. All rights reserved. |