From 3acddb20dbac343dcf12a4ba46bfdf7ebacf37f9 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Wed, 13 Jul 2016 18:38:33 -0700 Subject: Remove Python 'loader' hack --- setup.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'setup.py') diff --git a/setup.py b/setup.py index 700515b894..056e1fd033 100644 --- a/setup.py +++ b/setup.py @@ -90,10 +90,7 @@ CYTHON_EXTENSION_PACKAGE_NAMES = () CYTHON_EXTENSION_MODULE_NAMES = ('grpc._cython.cygrpc',) -CYTHON_HELPER_C_FILES = ( - os.path.join(PYTHON_STEM, 'grpc/_cython/loader.c'), - os.path.join(PYTHON_STEM, 'grpc/_cython/imports.generated.c'), -) +CYTHON_HELPER_C_FILES = () CORE_C_FILES = tuple(grpc_core_dependencies.CORE_SOURCE_FILES) -- cgit v1.2.3 From b236279832e0064ec36d80c71766681f3ea335a6 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Mon, 11 Jul 2016 15:10:02 -0700 Subject: Patch `spawn` for Python 'unix' compilers instead Before we patched the link command, now we just patch `spawn` as an updatable catch-all solution to ARG_MAX limitations on bash for MSYS and MinGW and friends. --- setup.py | 4 +- src/python/grpcio/_unixccompiler_patch.py | 98 ++++++++----------------------- 2 files changed, 27 insertions(+), 75 deletions(-) (limited to 'setup.py') diff --git a/setup.py b/setup.py index 056e1fd033..28aa5a5c8e 100644 --- a/setup.py +++ b/setup.py @@ -62,8 +62,8 @@ import commands import grpc_core_dependencies import grpc_version -# TODO(atash) make this conditional on being on a mingw32 build -_unixccompiler_patch.monkeypatch_unix_compiler() +if 'win32' in sys.platform: + _unixccompiler_patch.monkeypatch_unix_compiler() LICENSE = '3-clause BSD' diff --git a/src/python/grpcio/_unixccompiler_patch.py b/src/python/grpcio/_unixccompiler_patch.py index 9a697989b3..0ce5d63e98 100644 --- a/src/python/grpcio/_unixccompiler_patch.py +++ b/src/python/grpcio/_unixccompiler_patch.py @@ -38,84 +38,36 @@ import shutil import sys import tempfile +def _unix_commandfile_spawn(self, command): + """Wrapper around distutils.util.spawn that attempts to use command files. -def _unix_piecemeal_link( - self, target_desc, objects, output_filename, output_dir=None, - libraries=None, library_dirs=None, runtime_library_dirs=None, - export_symbols=None, debug=0, extra_preargs=None, extra_postargs=None, - build_temp=None, target_lang=None): - """`link` externalized method taken almost verbatim from UnixCCompiler. + Meant to replace the CCompiler method `spawn` on UnixCCompiler and its + derivatives (e.g. the MinGW32 compiler). - Modifies the link command for unix-like compilers by using a command file so - that long command line argument strings don't break the command shell's - ARG_MAX character limit. + Some commands like `gcc` (and friends like `clang`) support command files to + work around shell command length limits. """ - objects, output_dir = self._fix_object_args(objects, output_dir) - libraries, library_dirs, runtime_library_dirs = self._fix_lib_args( - libraries, library_dirs, runtime_library_dirs) - # filter out standard library paths, which are not explicitely needed - # for linking - library_dirs = [dir for dir in library_dirs - if not dir in ('/lib', '/lib64', '/usr/lib', '/usr/lib64')] - runtime_library_dirs = [dir for dir in runtime_library_dirs - if not dir in ('/lib', '/lib64', '/usr/lib', '/usr/lib64')] - lib_opts = ccompiler.gen_lib_options(self, library_dirs, runtime_library_dirs, - libraries) - if (not (isinstance(output_dir, str) or isinstance(output_dir, bytes)) - and output_dir is not None): - raise TypeError("'output_dir' must be a string or None") - if output_dir is not None: - output_filename = os.path.join(output_dir, output_filename) - - if self._need_link(objects, output_filename): - ld_args = (objects + self.objects + - lib_opts + ['-o', output_filename]) - if debug: - ld_args[:0] = ['-g'] - if extra_preargs: - ld_args[:0] = extra_preargs - if extra_postargs: - ld_args.extend(extra_postargs) - self.mkpath(os.path.dirname(output_filename)) - try: - if target_desc == ccompiler.CCompiler.EXECUTABLE: - linker = self.linker_exe[:] - else: - linker = self.linker_so[:] - if target_lang == "c++" and self.compiler_cxx: - # skip over environment variable settings if /usr/bin/env - # is used to set up the linker's environment. - # This is needed on OSX. Note: this assumes that the - # normal and C++ compiler have the same environment - # settings. - i = 0 - if os.path.basename(linker[0]) == "env": - i = 1 - while '=' in linker[i]: - i = i + 1 - - linker[i] = self.compiler_cxx[i] - - if sys.platform == 'darwin': - import _osx_support - linker = _osx_support.compiler_fixup(linker, ld_args) - - temporary_directory = tempfile.mkdtemp() - command_filename = os.path.abspath( - os.path.join(temporary_directory, 'command')) - with open(command_filename, 'w') as command_file: - escaped_ld_args = [arg.replace('\\', '\\\\') for arg in ld_args] - command_file.write(' '.join(escaped_ld_args)) - self.spawn(linker + ['@{}'.format(command_filename)]) - except errors.DistutilsExecError: - raise ccompiler.LinkError + command_base = os.path.basename(command[0].strip()) + if command_base == 'ccache': + command_base = command[:2] + command_args = command[2:] + elif command_base.startswith('ccache') or command_base in ['gcc', 'clang', 'clang++', 'g++']: + command_base = command[:1] + command_args = command[1:] else: - log.debug("skipping %s (up-to-date)", output_filename) + return ccompiler.CCompiler.spawn(self, command) + temporary_directory = tempfile.mkdtemp() + command_filename = os.path.abspath(os.path.join(temporary_directory, 'command')) + with open(command_filename, 'w') as command_file: + escaped_args = [arg.replace('\\', '\\\\') for arg in command_args] + command_file.write(' '.join(escaped_args)) + modified_command = command_base + ['@{}'.format(command_filename)] + result = ccompiler.CCompiler.spawn(self, modified_command) + shutil.rmtree(temporary_directory) + return result + -# TODO(atash) try replacing this monkeypatch of the compiler harness' link -# operation with a monkeypatch of the distutils `spawn` that applies -# command-argument-file hacks where it can. Might be cleaner. def monkeypatch_unix_compiler(): """Monkeypatching is dumb, but it's either that or we become maintainers of something much, much bigger.""" - unixccompiler.UnixCCompiler.link = _unix_piecemeal_link + unixccompiler.UnixCCompiler.spawn = _unix_commandfile_spawn -- cgit v1.2.3 From 58a0494cebb3d0d262557ec9ac25f60f2053fc86 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Sat, 16 Jul 2016 01:42:52 -0700 Subject: Use normalized path separators in `setup.py`s --- setup.py | 17 ++++++++++------- tools/distrib/python/grpcio_tools/setup.py | 27 ++++++++++++++++++--------- 2 files changed, 28 insertions(+), 16 deletions(-) (limited to 'setup.py') diff --git a/setup.py b/setup.py index 28aa5a5c8e..5d33756c19 100644 --- a/setup.py +++ b/setup.py @@ -47,10 +47,10 @@ from setuptools.command import egg_info egg_info.manifest_maker.template = 'PYTHON-MANIFEST.in' PY3 = sys.version_info.major == 3 -PYTHON_STEM = './src/python/grpcio' -CORE_INCLUDE = ('./include', '.',) -BORINGSSL_INCLUDE = ('./third_party/boringssl/include',) -ZLIB_INCLUDE = ('./third_party/zlib',) +PYTHON_STEM = os.path.join('src', 'python', 'grpcio') +CORE_INCLUDE = ('include', '.',) +BORINGSSL_INCLUDE = (os.path.join('third_party', 'boringssl', 'include'),) +ZLIB_INCLUDE = (os.path.join('third_party', 'zlib'),) # Ensure we're in the proper directory whether or not we're being used by pip. os.chdir(os.path.dirname(os.path.abspath(__file__))) @@ -105,7 +105,9 @@ if not "win32" in sys.platform: if "win32" in sys.platform: EXTENSION_LIBRARIES += ('ws2_32',) -DEFINE_MACROS = (('OPENSSL_NO_ASM', 1), ('_WIN32_WINNT', 0x600), ('GPR_BACKWARDS_COMPATIBILITY_MODE', 1),) +DEFINE_MACROS = ( + ('OPENSSL_NO_ASM', 1), ('_WIN32_WINNT', 0x600), + ('GPR_BACKWARDS_COMPATIBILITY_MODE', 1),) if "win32" in sys.platform: DEFINE_MACROS += (('OPENSSL_WINDOWS', 1), ('WIN32_LEAN_AND_MEAN', 1),) if '64bit' in platform.architecture()[0]: @@ -200,12 +202,13 @@ COMMAND_CLASS = { } # Ensure that package data is copied over before any commands have been run: -credentials_dir = os.path.join(PYTHON_STEM, 'grpc/_cython/_credentials') +credentials_dir = os.path.join(PYTHON_STEM, 'grpc', '_cython', '_credentials') try: os.mkdir(credentials_dir) except OSError: pass -shutil.copyfile('etc/roots.pem', os.path.join(credentials_dir, 'roots.pem')) +shutil.copyfile(os.path.join('etc', 'roots.pem'), + os.path.join(credentials_dir, 'roots.pem')) PACKAGE_DATA = { # Binaries that may or may not be present in the final installation, but are diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index d804f34fc6..a00ce01110 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -61,6 +61,13 @@ EXTRA_COMPILE_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_CFLAGS', EXTRA_LINK_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_LDFLAGS', '-lpthread')) +CC_FILES = [ + os.path.normpath(cc_file) for cc_file in protoc_lib_deps.CC_FILES] +PROTO_FILES = [ + os.path.normpath(proto_file) for proto_file in protoc_lib_deps.PROTO_FILES] +CC_INCLUDE = os.path.normpath(protoc_lib_deps.CC_INCLUDE) +PROTO_INCLUDE = os.path.normpath(protoc_lib_deps.PROTO_INCLUDE) + GRPC_PYTHON_TOOLS_PACKAGE = 'grpc.tools' GRPC_PYTHON_PROTO_RESOURCES_NAME = '_proto' @@ -82,8 +89,8 @@ def package_data(): proto_resources_path = os.path.join(tools_path, GRPC_PYTHON_PROTO_RESOURCES_NAME) proto_files = [] - for proto_file in protoc_lib_deps.PROTO_FILES: - source = os.path.join(protoc_lib_deps.PROTO_INCLUDE, proto_file) + for proto_file in PROTO_FILES: + source = os.path.join(PROTO_INCLUDE, proto_file) target = os.path.join(proto_resources_path, proto_file) relative_target = os.path.join(GRPC_PYTHON_PROTO_RESOURCES_NAME, proto_file) try: @@ -99,18 +106,20 @@ def package_data(): def protoc_ext_module(): plugin_sources = [ - 'grpc/tools/main.cc', - 'grpc_root/src/compiler/python_generator.cc'] + [ - os.path.join(protoc_lib_deps.CC_INCLUDE, cc_file) - for cc_file in protoc_lib_deps.CC_FILES] + os.path.join('grpc', 'tools', 'main.cc'), + os.path.join('grpc_root', 'src', 'compiler', 'python_generator.cc')] + [ + os.path.join(CC_INCLUDE, cc_file) + for cc_file in CC_FILES] plugin_ext = extension.Extension( name='grpc.tools._protoc_compiler', - sources=['grpc/tools/_protoc_compiler.pyx'] + plugin_sources, + sources=( + [os.path.join('grpc', 'tools', '_protoc_compiler.pyx')] + + plugin_sources), include_dirs=[ '.', 'grpc_root', - 'grpc_root/include', - protoc_lib_deps.CC_INCLUDE, + os.path.join('grpc_root', 'include'), + CC_INCLUDE, ], language='c++', define_macros=list(DEFINE_MACROS), -- cgit v1.2.3 From abdff3df896e4e0ed0bf76983831bff6484ee75f Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Mon, 18 Jul 2016 13:26:25 -0700 Subject: Clean imports in grpcio's `setup.py` --- setup.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'setup.py') diff --git a/setup.py b/setup.py index 5d33756c19..24414457cd 100644 --- a/setup.py +++ b/setup.py @@ -29,17 +29,16 @@ """A setup module for the GRPC Python package.""" +from distutils import extension as _extension import os import os.path +import pkg_resources import platform import shlex import shutil import sys import sysconfig -from distutils import core as _core -from distutils import extension as _extension -import pkg_resources import setuptools from setuptools.command import egg_info -- cgit v1.2.3 From 398b06e8432e5e9007349b7c89ae6b67e02c1035 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Fri, 15 Jul 2016 23:17:41 -0700 Subject: Force the Python host version on Mac OS X --- setup.py | 6 ++++++ tools/distrib/python/grpcio_tools/setup.py | 6 ++++++ 2 files changed, 12 insertions(+) (limited to 'setup.py') diff --git a/setup.py b/setup.py index 24414457cd..b43ec9ae3d 100644 --- a/setup.py +++ b/setup.py @@ -30,10 +30,12 @@ """A setup module for the GRPC Python package.""" from distutils import extension as _extension +from distutils import util import os import os.path import pkg_resources import platform +import re import shlex import shutil import sys @@ -133,6 +135,10 @@ if 'darwin' in sys.platform and PY3: if mac_target and (pkg_resources.parse_version(mac_target) < pkg_resources.parse_version('10.7.0')): os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.7' + os.environ['_PYTHON_HOST_PLATFORM'] = re.sub( + r'macosx-[0-9]+\.[0-9]+-(.+)', + r'macosx-10.7-\1', + util.get_platform()) def cython_extensions(module_names, extra_sources, include_dirs, diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index a00ce01110..8082c7a665 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -28,11 +28,13 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from distutils import extension +from distutils import util import errno import os import os.path import pkg_resources import platform +import re import shlex import shutil import sys @@ -83,6 +85,10 @@ if 'darwin' in sys.platform and PY3: if mac_target and (pkg_resources.parse_version(mac_target) < pkg_resources.parse_version('10.9.0')): os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.9' + os.environ['_PYTHON_HOST_PLATFORM'] = re.sub( + r'macosx-[0-9]+\.[0-9]+-(.+)', + r'macosx-10.9-\1', + util.get_platform()) def package_data(): tools_path = GRPC_PYTHON_TOOLS_PACKAGE.replace('.', os.path.sep) -- cgit v1.2.3