diff options
| -rw-r--r-- | PYTHON-MANIFEST.in | 1 | ||||
| -rw-r--r-- | setup.py | 42 | ||||
| -rw-r--r-- | src/python/grpcio/commands.py | 102 | ||||
| -rw-r--r-- | src/python/grpcio/precompiled.py | 102 | ||||
| -rwxr-xr-x | tools/run_tests/build_artifact_python.sh | 2 |
5 files changed, 128 insertions, 121 deletions
diff --git a/PYTHON-MANIFEST.in b/PYTHON-MANIFEST.in index 3c46d61170..072089ac51 100644 --- a/PYTHON-MANIFEST.in +++ b/PYTHON-MANIFEST.in @@ -7,6 +7,7 @@ graft third_party/zlib include src/python/grpcio/commands.py include src/python/grpcio/grpc_version.py include src/python/grpcio/grpc_core_dependencies.py +include src/python/grpcio/precompiled.py include src/python/grpcio/support.py include src/python/grpcio/README.rst include requirements.txt @@ -53,6 +53,7 @@ sys.path.insert(0, os.path.abspath(PYTHON_STEM)) # Break import-style to ensure we can actually find our in-repo dependencies. import commands +import precompiled import grpc_core_dependencies import grpc_version @@ -156,15 +157,14 @@ SETUP_REQUIRES = ( ) + INSTALL_REQUIRES COMMAND_CLASS = { - 'install': commands.Install, 'doc': commands.SphinxDocumentation, 'build_proto_modules': commands.BuildProtoModules, 'build_project_metadata': commands.BuildProjectMetadata, 'build_py': commands.BuildPy, 'build_ext': commands.BuildExt, + 'build_tagged_ext': precompiled.BuildTaggedExt, 'gather': commands.Gather, 'run_interop': commands.RunInterop, - 'bdist_wheel_grpc_custom': commands.BdistWheelCustomName, } # Ensure that package data is copied over before any commands have been run: @@ -205,9 +205,12 @@ PACKAGE_DATA = { 'grpc._adapter': [ 'credentials/roots.pem' ], + # Binaries that may or may not be present in the final installation, but are + # mentioned here for completeness. 'grpc._cython': [ '_windows/grpc_c.32.python', '_windows/grpc_c.64.python', + 'cygrpc.so', ], } if INSTALL_TESTS: @@ -217,19 +220,22 @@ else: PACKAGES = setuptools.find_packages( PYTHON_STEM, exclude=['tests', 'tests.*']) -setuptools.setup( - name='grpcio', - version=grpc_version.VERSION, - license=LICENSE, - ext_modules=CYTHON_EXTENSION_MODULES, - packages=list(PACKAGES), - package_dir=PACKAGE_DIRECTORIES, - package_data=PACKAGE_DATA, - install_requires=INSTALL_REQUIRES, - setup_requires=SETUP_REQUIRES, - cmdclass=COMMAND_CLASS, - tests_require=TESTS_REQUIRE, - test_suite=TEST_SUITE, - test_loader=TEST_LOADER, - test_runner=TEST_RUNNER, -) +setup_arguments = { + 'name': 'grpcio', + 'version': grpc_version.VERSION, + 'license': LICENSE, + 'ext_modules': CYTHON_EXTENSION_MODULES, + 'packages': list(PACKAGES), + 'package_dir': PACKAGE_DIRECTORIES, + 'package_data': PACKAGE_DATA, + 'install_requires': INSTALL_REQUIRES, + 'setup_requires': SETUP_REQUIRES, + 'cmdclass': COMMAND_CLASS, + 'tests_require': TESTS_REQUIRE, + 'test_loader': TEST_LOADER, + 'test_runner': TEST_RUNNER, +} + +precompiled.update_setup_arguments(setup_arguments) + +setuptools.setup(**setup_arguments) diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py index 58af6bebf5..aa29c728f2 100644 --- a/src/python/grpcio/commands.py +++ b/src/python/grpcio/commands.py @@ -46,21 +46,11 @@ from setuptools.command import build_py from setuptools.command import easy_install from setuptools.command import install from setuptools.command import test -from wheel import bdist_wheel import support PYTHON_STEM = os.path.dirname(os.path.abspath(__file__)) -BINARIES_REPOSITORY = os.environ.get( - 'GRPC_PYTHON_BINARIES_REPOSITORY', - 'https://storage.googleapis.com/grpc-precompiled-binaries/python') - -USE_GRPC_CUSTOM_BDIST = bool(int(os.environ.get( - 'GRPC_PYTHON_USE_CUSTOM_BDIST', '1'))) - -GRPC_CUSTOM_BDIST_EXT = '.whl' - CONF_PY_ADDENDUM = """ extensions.append('sphinx.ext.napoleon') napoleon_google_docstring = True @@ -111,98 +101,6 @@ def _get_grpc_custom_bdist(decorated_basename, target_bdist_basename): return bdist_path -class WheelNameMixin(object): - """Mixin for setuptools.Command classes to enable acquiring the bdist name.""" - - def wheel_custom_name(self): - base = self.wheel_name() - # Drop troublesome parts of the target tuple - base_split = base.split('-') - base = '-'.join(base_split[0:3] + base_split[4:]) - flavor = 'ucs2' if sys.maxunicode == 65535 else 'ucs4' - return '{base}-{flavor}'.format(base=base, flavor=flavor) - - def wheel_name(self): - wheel_command = self.get_finalized_command('bdist_wheel') - return wheel_command.get_archive_basename() - - -class Install(install.install, WheelNameMixin): - """Custom Install command for gRPC Python. - - This is for bdist shims and whatever else we might need a custom install - command for. - """ - - user_options = install.install.user_options + [ - # TODO(atash): remove this once PyPI has better Linux bdist support. See - # https://bitbucket.org/pypa/pypi/issues/120/binary-wheels-for-linux-are-not-supported - ('use-grpc-custom-bdist', None, - 'Whether to retrieve a binary from the gRPC binary repository instead ' - 'of building from source.'), - ] - - def initialize_options(self): - install.install.initialize_options(self) - self.use_grpc_custom_bdist = USE_GRPC_CUSTOM_BDIST - - def finalize_options(self): - install.install.finalize_options(self) - - def run(self): - if self.use_grpc_custom_bdist: - try: - try: - bdist_path = _get_grpc_custom_bdist(self.wheel_custom_name(), - self.wheel_name()) - except CommandError as error: - sys.stderr.write( - '\nWARNING: Failed to acquire grpcio prebuilt binary:\n' - '{}.\n\n'.format(error.message)) - raise - try: - self._run_bdist_retrieval_install(bdist_path) - except Exception as error: - # if anything else happens (and given how there's no way to really know - # what's happening in setuptools here, I mean *anything*), warn the user - # and fall back to building from source. - sys.stderr.write( - '{}\nWARNING: Failed to install grpcio prebuilt binary.\n\n' - .format(traceback.format_exc())) - raise - except Exception: - install.install.run(self) - else: - install.install.run(self) - - # TODO(atash): Remove this once PyPI has better Linux bdist support. See - # https://bitbucket.org/pypa/pypi/issues/120/binary-wheels-for-linux-are-not-supported - def _run_bdist_retrieval_install(self, bdist_path): - import pip - pip.main(['install', bdist_path]) - - -class BdistWheelCustomName(bdist_wheel.bdist_wheel, WheelNameMixin): - """Thin wrapper around the bdist command to build with our custom name.""" - - description = ("Create a gRPC custom-named wheel distribution. " - "Cannot be run with any other distribution-related command.") - - def run(self): - # TODO(atash): if the hack we use to support Linux binaries becomes - # 'supported' (i.e. - # https://bitbucket.org/pypa/pypi/issues/120/binary-wheels-for-linux-are-not-supported - # is not solved and we see users beginning to use this command, ill-advised - # as that may be) consider making the following capable of running with - # other distribution-related commands. Currently it depends on the (AFAIK - # undocumented, private) ordering of the distribution files. - bdist_wheel.bdist_wheel.run(self) - output = self.distribution.dist_files[-1][2] - target = os.path.join( - self.dist_dir, '{}.whl'.format(self.wheel_custom_name())) - shutil.move(output, target) - - class SphinxDocumentation(setuptools.Command): """Command to generate documentation via sphinx.""" diff --git a/src/python/grpcio/precompiled.py b/src/python/grpcio/precompiled.py new file mode 100644 index 0000000000..05c651b506 --- /dev/null +++ b/src/python/grpcio/precompiled.py @@ -0,0 +1,102 @@ +# Copyright 2015-2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import platform +import shutil +import sys + +import setuptools + +import commands +import grpc_version + +try: + from urllib2 import urlopen +except ImportError: + from urllib.request import urlopen + +PYTHON_STEM = os.path.dirname(os.path.abspath(__file__)) +BINARIES_REPOSITORY = os.environ.get( + 'GRPC_PYTHON_BINARIES_REPOSITORY', + 'https://storage.googleapis.com/grpc-precompiled-binaries/python') +USE_PRECOMPILED_BINARIES = bool(int(os.environ.get( + 'GRPC_PYTHON_USE_PRECOMPILED_BINARIES', '1'))) + +def _tagged_ext_name(base): + uname = platform.uname() + tags = '-'.join((grpc_version.VERSION, uname[0], uname[4])) + flavor = 'ucs2' if sys.maxunicode == 65535 else 'ucs4' + return '{base}-{tags}-{flavor}'.format(base=base, tags=tags, flavor=flavor) + + +class BuildTaggedExt(setuptools.Command): + + description = 'build the gRPC tagged extensions' + user_options = [] + + def initialize_options(self): + # distutils requires this override. + pass + + def finalize_options(self): + # distutils requires this override. + pass + + def run(self): + if 'linux' in sys.platform: + self.run_command('build_ext') + try: + os.makedirs('dist/') + except OSError: + pass + shutil.copyfile( + os.path.join(PYTHON_STEM, 'grpc/_cython/cygrpc.so'), + 'dist/{}.so'.format(_tagged_ext_name('cygrpc'))) + else: + sys.stderr.write('nothing to do for build_tagged_ext\n') + + +def update_setup_arguments(setup_arguments): + url = '{}/{}.so'.format(BINARIES_REPOSITORY, _tagged_ext_name('cygrpc')) + target_path = os.path.join(PYTHON_STEM, 'grpc/_cython/cygrpc.so') + try: + extension = urlopen(url).read() + except: + sys.stderr.write( + 'could not download precompiled extension: {}\n'.format(url)) + return + try: + with open(target_path, 'w') as target: + target.write(extension) + setup_arguments['ext_modules'] = [] + except: + sys.stderr.write( + 'could not write precompiled extension to directory: {} -> {}\n' + .format(url, target_path)) diff --git a/tools/run_tests/build_artifact_python.sh b/tools/run_tests/build_artifact_python.sh index 5768485ca2..6e7ab911d5 100755 --- a/tools/run_tests/build_artifact_python.sh +++ b/tools/run_tests/build_artifact_python.sh @@ -46,7 +46,7 @@ fi GRPC_PYTHON_USE_CUSTOM_BDIST=0 \ GRPC_PYTHON_BUILD_WITH_CYTHON=1 \ ${SETARCH_CMD} python setup.py \ - bdist_wheel_grpc_custom + build_tagged_ext GRPC_PYTHON_USE_CUSTOM_BDIST=0 \ GRPC_PYTHON_BUILD_WITH_CYTHON=1 \ |