From 5998cd760c8ee73eba807614180e7f4c0ac3defe Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Wed, 10 Aug 2016 15:39:43 -0700 Subject: Add Python3.5 artifact targets --- PYTHON-MANIFEST.in | 2 +- setup.py | 62 ++++++++++++++++------- src/python/grpcio/_spawn_patch.py | 73 +++++++++++++++++++++++++++ src/python/grpcio/_unixccompiler_patch.py | 77 ----------------------------- tools/distrib/python/grpcio_tools/setup.py | 48 +++++++++++++++--- tools/distrib/python/make_grpcio_tools.py | 79 +++++------------------------- tools/run_tests/artifact_targets.py | 70 ++++++++++++++------------ tools/run_tests/build_artifact_python.bat | 51 ++++++++----------- tools/run_tests/build_artifact_python.sh | 36 ++++++++------ 9 files changed, 248 insertions(+), 250 deletions(-) create mode 100644 src/python/grpcio/_spawn_patch.py delete mode 100644 src/python/grpcio/_unixccompiler_patch.py diff --git a/PYTHON-MANIFEST.in b/PYTHON-MANIFEST.in index 96bcdb515c..adeb390649 100644 --- a/PYTHON-MANIFEST.in +++ b/PYTHON-MANIFEST.in @@ -7,7 +7,7 @@ graft include/grpc graft third_party/boringssl graft third_party/nanopb graft third_party/zlib -include src/python/grpcio/_unixccompiler_patch.py +include src/python/grpcio/_spawn_patch.py include src/python/grpcio/commands.py include src/python/grpcio/grpc_version.py include src/python/grpcio/grpc_core_dependencies.py diff --git a/setup.py b/setup.py index 1dddf55e5e..096e8bdd4b 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """A setup module for the GRPC Python package.""" - +from distutils import cygwinccompiler from distutils import extension as _extension from distutils import util import os @@ -58,14 +58,12 @@ os.chdir(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.abspath(PYTHON_STEM)) # Break import-style to ensure we can actually find our in-repo dependencies. -import _unixccompiler_patch +import _spawn_patch import commands import grpc_core_dependencies import grpc_version -if 'win32' in sys.platform: - _unixccompiler_patch.monkeypatch_unix_compiler() - +_spawn_patch.monkeypatch_spawn() LICENSE = '3-clause BSD' @@ -83,9 +81,38 @@ ENABLE_CYTHON_TRACING = os.environ.get( # entirely ignored/dropped/forgotten by distutils and its Cygwin/MinGW support. # We use these environment variables to thus get around that without locking # ourselves in w.r.t. the multitude of operating systems this ought to build on. -# By default we assume a GCC-like compiler. -EXTRA_COMPILE_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_CFLAGS', '')) -EXTRA_LINK_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_LDFLAGS', '')) +# We can also use these variables as a way to inject environment-specific +# compiler/linker flags. We assume GCC-like compilers and/or MinGW as a +# reasonable default. +EXTRA_ENV_COMPILE_ARGS = os.environ.get('GRPC_PYTHON_CFLAGS', None) +EXTRA_ENV_LINK_ARGS = os.environ.get('GRPC_PYTHON_LDFLAGS', None) +if EXTRA_ENV_COMPILE_ARGS is None: + EXTRA_ENV_COMPILE_ARGS = '' + if 'win32' in sys.platform and sys.version_info < (3, 5): + # We use define flags here and don't directly add to DEFINE_MACROS below to + # ensure that the expert user/builder has a way of turning it off (via the + # envvars) without adding yet more GRPC-specific envvars. + # See https://sourceforge.net/p/mingw-w64/bugs/363/ + if '32' in platform.architecture()[0]: + EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime32 -D_timeb=__timeb32 -D_ftime_s=_ftime32_s' + else: + EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime64 -D_timeb=__timeb64' + elif "linux" in sys.platform or "darwin" in sys.platform: + EXTRA_ENV_COMPILE_ARGS += ' -fno-wrapv' +if EXTRA_ENV_LINK_ARGS is None: + EXTRA_ENV_LINK_ARGS = '' + if "linux" in sys.platform or "darwin" in sys.platform: + EXTRA_ENV_LINK_ARGS += ' -lpthread' + elif "win32" in sys.platform and sys.version_info < (3, 5): + msvcr = cygwinccompiler.get_msvcr()[0] + # TODO(atash) sift through the GCC specs to see if libstdc++ can have any + # influence on the linkage outcome on MinGW for non-C++ programs. + EXTRA_ENV_LINK_ARGS += ( + ' -static-libgcc -static-libstdc++ -mcrtdll={msvcr} ' + '-static'.format(msvcr=msvcr)) + +EXTRA_COMPILE_ARGS = shlex.split(EXTRA_ENV_COMPILE_ARGS) +EXTRA_LINK_ARGS = shlex.split(EXTRA_ENV_LINK_ARGS) CYTHON_EXTENSION_PACKAGE_NAMES = () @@ -104,28 +131,29 @@ if "linux" in sys.platform: if not "win32" in sys.platform: EXTENSION_LIBRARIES += ('m',) if "win32" in sys.platform: - EXTENSION_LIBRARIES += ('ws2_32',) + EXTENSION_LIBRARIES += ('advapi32', 'ws2_32',) DEFINE_MACROS = ( ('OPENSSL_NO_ASM', 1), ('_WIN32_WINNT', 0x600), ('GPR_BACKWARDS_COMPATIBILITY_MODE', 1),) if "win32" in sys.platform: - DEFINE_MACROS += (('OPENSSL_WINDOWS', 1), ('WIN32_LEAN_AND_MEAN', 1),) + DEFINE_MACROS += (('WIN32_LEAN_AND_MEAN', 1),) if '64bit' in platform.architecture()[0]: DEFINE_MACROS += (('MS_WIN64', 1),) + elif sys.version_info >= (3, 5): + # For some reason, this is needed to get access to inet_pton/inet_ntop + # on msvc, but only for 32 bits + DEFINE_MACROS += (('NTDDI_VERSION', 0x06000000),) LDFLAGS = tuple(EXTRA_LINK_ARGS) CFLAGS = tuple(EXTRA_COMPILE_ARGS) if "linux" in sys.platform: LDFLAGS += ('-Wl,-wrap,memcpy',) if "linux" in sys.platform or "darwin" in sys.platform: - CFLAGS += ('-fvisibility=hidden',) - - pymodinit_type = 'PyObject*' if PY3 else 'void' - - pymodinit = '__attribute__((visibility ("default"))) {}'.format(pymodinit_type) - DEFINE_MACROS += (('PyMODINIT_FUNC', pymodinit),) - + CFLAGS += ('-fvisibility=hidden',) + pymodinit_type = 'PyObject*' if PY3 else 'void' + pymodinit = '__attribute__((visibility ("default"))) {}'.format(pymodinit_type) + DEFINE_MACROS += (('PyMODINIT_FUNC', pymodinit),) # By default, Python3 distutils enforces compatibility of # c plugins (.so files) with the OSX version Python3 was built with. diff --git a/src/python/grpcio/_spawn_patch.py b/src/python/grpcio/_spawn_patch.py new file mode 100644 index 0000000000..24306f0dd9 --- /dev/null +++ b/src/python/grpcio/_spawn_patch.py @@ -0,0 +1,73 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Patches the spawn() command for windows compilers. + +Windows has an 8191 character command line limit, but some compilers +support an @command_file directive where command_file is a file +containing the full command line. +""" + +from distutils import ccompiler +import os +import os.path +import shutil +import sys +import tempfile + +MAX_COMMAND_LENGTH = 8191 + +_classic_spawn = ccompiler.CCompiler.spawn + +def _commandfile_spawn(self, command): + command_length = sum([len(arg) for arg in command]) + if os.name == 'nt' and command_length > MAX_COMMAND_LENGTH: + # Even if this command doesn't support the @command_file, it will + # fail as is so we try blindly + print('Command line length exceeded, using command file') + print(' '.join(command)) + temporary_directory = tempfile.mkdtemp() + command_filename = os.path.abspath( + os.path.join(temporary_directory, 'command')) + with open(command_filename, 'w') as command_file: + escaped_args = ['"' + arg.replace('\\', '\\\\') + '"' for arg in command[1:]] + command_file.write(' '.join(escaped_args)) + modified_command = command[:1] + ['@{}'.format(command_filename)] + try: + _classic_spawn(self, modified_command) + finally: + shutil.rmtree(temporary_directory) + else: + _classic_spawn(self, command) + + +def monkeypatch_spawn(): + """Monkeypatching is dumb, but it's either that or we become maintainers of + something much, much bigger.""" + ccompiler.CCompiler.spawn = _commandfile_spawn diff --git a/src/python/grpcio/_unixccompiler_patch.py b/src/python/grpcio/_unixccompiler_patch.py deleted file mode 100644 index 894c3ef395..0000000000 --- a/src/python/grpcio/_unixccompiler_patch.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright 2016, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Covers inadequacies in distutils.""" - -from distutils import ccompiler -from distutils import errors -from distutils import unixccompiler -import os -import os.path -import shlex -import shutil -import sys -import tempfile - -def _unix_commandfile_spawn(self, command): - """Wrapper around distutils.util.spawn that attempts to use command files. - - Meant to replace the CCompiler method `spawn` on UnixCCompiler and its - derivatives (e.g. the MinGW32 compiler). - - Some commands like `gcc` (and friends like `clang`) support command files to - work around shell command length limits. - """ - # Sometimes distutils embeds the executables as full strings including some - # hard-coded flags rather than as lists. - command = list(shlex.split(command[0])) + list(command[1:]) - command_base = os.path.basename(command[0].strip()) - if command_base == 'ccache': - command_base = command[:2] - command_args = command[2:] - elif command_base.startswith('ccache') or command_base in ['gcc', 'clang', 'clang++', 'g++']: - command_base = command[:1] - command_args = command[1:] - else: - return ccompiler.CCompiler.spawn(self, command) - temporary_directory = tempfile.mkdtemp() - command_filename = os.path.abspath(os.path.join(temporary_directory, 'command')) - with open(command_filename, 'w') as command_file: - escaped_args = [arg.replace('\\', '\\\\') for arg in command_args] - command_file.write(' '.join(escaped_args)) - modified_command = command_base + ['@{}'.format(command_filename)] - result = ccompiler.CCompiler.spawn(self, modified_command) - shutil.rmtree(temporary_directory) - return result - - -def monkeypatch_unix_compiler(): - """Monkeypatching is dumb, but it's either that or we become maintainers of - something much, much bigger.""" - unixccompiler.UnixCCompiler.spawn = _unix_commandfile_spawn diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index 43d0f5518a..53d7eb0948 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -27,6 +27,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from distutils import cygwinccompiler from distutils import extension from distutils import util import errno @@ -57,11 +58,38 @@ PY3 = sys.version_info.major == 3 # entirely ignored/dropped/forgotten by distutils and its Cygwin/MinGW support. # We use these environment variables to thus get around that without locking # ourselves in w.r.t. the multitude of operating systems this ought to build on. -# By default we assume a GCC-like compiler. -EXTRA_COMPILE_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_CFLAGS', - '-fno-wrapv -frtti -std=c++11')) -EXTRA_LINK_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_LDFLAGS', - '-lpthread')) +# We can also use these variables as a way to inject environment-specific +# compiler/linker flags. We assume GCC-like compilers and/or MinGW as a +# reasonable default. +EXTRA_ENV_COMPILE_ARGS = os.environ.get('GRPC_PYTHON_CFLAGS', None) +EXTRA_ENV_LINK_ARGS = os.environ.get('GRPC_PYTHON_LDFLAGS', None) +if EXTRA_ENV_COMPILE_ARGS is None: + EXTRA_ENV_COMPILE_ARGS = '-std=c++11' + if 'win32' in sys.platform and sys.version_info < (3, 5): + # We use define flags here and don't directly add to DEFINE_MACROS below to + # ensure that the expert user/builder has a way of turning it off (via the + # envvars) without adding yet more GRPC-specific envvars. + # See https://sourceforge.net/p/mingw-w64/bugs/363/ + if '32' in platform.architecture()[0]: + EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime32 -D_timeb=__timeb32 -D_ftime_s=_ftime32_s' + else: + EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime64 -D_timeb=__timeb64' + elif "linux" in sys.platform or "darwin" in sys.platform: + EXTRA_ENV_COMPILE_ARGS += ' -fno-wrapv -frtti' +if EXTRA_ENV_LINK_ARGS is None: + EXTRA_ENV_LINK_ARGS = '' + if "linux" in sys.platform or "darwin" in sys.platform: + EXTRA_ENV_LINK_ARGS += ' -lpthread' + elif "win32" in sys.platform and sys.version_info < (3, 5): + msvcr = cygwinccompiler.get_msvcr()[0] + # TODO(atash) sift through the GCC specs to see if libstdc++ can have any + # influence on the linkage outcome on MinGW for non-C++ programs. + EXTRA_ENV_LINK_ARGS += ( + ' -static-libgcc -static-libstdc++ -mcrtdll={msvcr} ' + '-static'.format(msvcr=msvcr)) + +EXTRA_COMPILE_ARGS = shlex.split(EXTRA_ENV_COMPILE_ARGS) +EXTRA_LINK_ARGS = shlex.split(EXTRA_ENV_LINK_ARGS) CC_FILES = [ os.path.normpath(cc_file) for cc_file in protoc_lib_deps.CC_FILES] @@ -73,9 +101,13 @@ PROTO_INCLUDE = os.path.normpath(protoc_lib_deps.PROTO_INCLUDE) GRPC_PYTHON_TOOLS_PACKAGE = 'grpc.tools' GRPC_PYTHON_PROTO_RESOURCES_NAME = '_proto' -DEFINE_MACROS = (('HAVE_PTHREAD', 1),) -if "win32" in sys.platform and '64bit' in platform.architecture()[0]: - DEFINE_MACROS += (('MS_WIN64', 1),) +DEFINE_MACROS = () +if "win32" in sys.platform: + DEFINE_MACROS += (('WIN32_LEAN_AND_MEAN', 1),) + if '64bit' in platform.architecture()[0]: + DEFINE_MACROS += (('MS_WIN64', 1),) +elif "linux" in sys.platform or "darwin" in sys.platform: + DEFINE_MACROS += (('HAVE_PTHREAD', 1),) # By default, Python3 distutils enforces compatibility of # c plugins (.so files) with the OSX version Python3 was built with. diff --git a/tools/distrib/python/make_grpcio_tools.py b/tools/distrib/python/make_grpcio_tools.py index adf58445af..7413928eca 100755 --- a/tools/distrib/python/make_grpcio_tools.py +++ b/tools/distrib/python/make_grpcio_tools.py @@ -88,22 +88,23 @@ GRPC_ROOT = os.path.abspath( os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', '..')) -GRPC_PYTHON_ROOT = os.path.join(GRPC_ROOT, 'tools/distrib/python/grpcio_tools') +GRPC_PYTHON_ROOT = os.path.join(GRPC_ROOT, 'tools', 'distrib', + 'python', 'grpcio_tools') -GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT = 'third_party/protobuf/src' +GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT = os.path.join('third_party', 'protobuf', 'src') GRPC_PROTOBUF = os.path.join(GRPC_ROOT, GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT) -GRPC_PROTOC_PLUGINS = os.path.join(GRPC_ROOT, 'src/compiler') -GRPC_PYTHON_PROTOBUF = os.path.join(GRPC_PYTHON_ROOT, - 'third_party/protobuf/src') -GRPC_PYTHON_PROTOC_PLUGINS = os.path.join(GRPC_PYTHON_ROOT, - 'grpc_root/src/compiler') +GRPC_PROTOC_PLUGINS = os.path.join(GRPC_ROOT, 'src', 'compiler') +GRPC_PYTHON_PROTOBUF = os.path.join(GRPC_PYTHON_ROOT, 'third_party', 'protobuf', + 'src') +GRPC_PYTHON_PROTOC_PLUGINS = os.path.join(GRPC_PYTHON_ROOT, 'grpc_root', 'src', + 'compiler') GRPC_PYTHON_PROTOC_LIB_DEPS = os.path.join(GRPC_PYTHON_ROOT, 'protoc_lib_deps.py') GRPC_INCLUDE = os.path.join(GRPC_ROOT, 'include') -GRPC_PYTHON_INCLUDE = os.path.join(GRPC_PYTHON_ROOT, 'grpc_root/include') +GRPC_PYTHON_INCLUDE = os.path.join(GRPC_PYTHON_ROOT, 'grpc_root', 'include') -BAZEL_DEPS = os.path.join(GRPC_ROOT, 'tools/distrib/python/bazel_deps.sh') +BAZEL_DEPS = os.path.join(GRPC_ROOT, 'tools', 'distrib', 'python', 'bazel_deps.sh') BAZEL_DEPS_PROTOC_LIB_QUERY = '//:protoc_lib' BAZEL_DEPS_COMMON_PROTOS_QUERY = '//:well_known_protos' @@ -136,64 +137,6 @@ def long_path(path): else: return path -def atomic_file_copy(src, dst): - """Based on the lock-free-whack-a-mole algorithm, depending on filesystem - renaming being atomic. Described at http://stackoverflow.com/a/28090883. - """ - try: - if filecmp.cmp(src, dst): - return - except: - pass - dst_dir = os.path.abspath(os.path.dirname(dst)) - dst_base = os.path.basename(dst) - this_id = str(uuid.uuid4()).replace('.', '-') - temporary_file = os.path.join(dst_dir, '{}.{}.tmp'.format(dst_base, this_id)) - mole_file = os.path.join(dst_dir, '{}.{}.mole.tmp'.format(dst_base, this_id)) - mole_pattern = os.path.join(dst_dir, '{}.*.mole.tmp'.format(dst_base)) - src = long_path(src) - dst = long_path(dst) - temporary_file = long_path(temporary_file) - mole_file = long_path(mole_file) - mole_pattern = long_path(mole_pattern) - shutil.copy2(src, temporary_file) - try: - os.rename(temporary_file, mole_file) - except: - print('Error moving temporary file {} to {}'.format(temporary_file, mole_file), file=sys.stderr) - print('while trying to copy file {} to {}'.format(src, dst), file=sys.stderr) - raise - for other_file in glob.glob(mole_pattern): - other_id = other_file.split('.')[-3] - if this_id == other_id: - pass - elif this_id < other_id: - try: - os.remove(other_file) - except: - pass - else: - try: - os.remove(mole_file) - except: - pass - this_id = other_id - mole_file = other_file - try: - if filecmp.cmp(src, dst): - try: - os.remove(mole_file) - except: - pass - return - except: - pass - try: - os.rename(mole_file, dst) - except: - pass - - def main(): os.chdir(GRPC_ROOT) @@ -211,7 +154,7 @@ def main(): for relative_file in files: source_file = os.path.abspath(os.path.join(source_dir, relative_file)) target_file = os.path.abspath(os.path.join(target_dir, relative_file)) - atomic_file_copy(source_file, target_file) + shutil.copyfile(source_file, target_file) try: protoc_lib_deps_content = get_deps() diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifact_targets.py index 1c480f7a7f..0ba9f3b331 100644 --- a/tools/run_tests/artifact_targets.py +++ b/tools/run_tests/artifact_targets.py @@ -31,6 +31,8 @@ """Definition of targets to build artifacts.""" import os.path +import random +import string import sys import jobset @@ -79,27 +81,16 @@ _ARCH_FLAG_MAP = { 'x64': '-m64' } -python_windows_version_arch_map = { - ('x86', '2.7'): 'Python27_32bits', - ('x64', '2.7'): 'Python27', - ('x86', '3.4'): 'Python34_32bits', - ('x64', '3.4'): 'Python34', -} class PythonArtifact: """Builds Python artifacts.""" - def __init__(self, platform, arch, python_version, manylinux_build=None): - if manylinux_build: - self.name = 'python%s_%s_%s_%s' % (python_version, platform, arch, manylinux_build) - else: - self.name = 'python%s_%s_%s' % (python_version, platform, arch) + def __init__(self, platform, arch, py_version): + self.name = 'python_%s_%s_%s' % (platform, arch, py_version) self.platform = platform self.arch = arch - self.labels = ['artifact', 'python', python_version, platform, arch] - self.python_version = python_version - self.python_windows_prefix = python_windows_version_arch_map[arch, python_version] - self.manylinux_build = manylinux_build + self.labels = ['artifact', 'python', platform, arch, py_version] + self.py_version = py_version def pre_build_jobspecs(self): return [] @@ -111,8 +102,8 @@ class PythonArtifact: environ['SETARCH_CMD'] = 'linux32' # Inside the manylinux container, the python installations are located in # special places... - environ['PYTHON'] = '/opt/python/{}/bin/python'.format(self.manylinux_build) - environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.manylinux_build) + environ['PYTHON'] = '/opt/python/{}/bin/python'.format(self.py_version) + environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.py_version) # Platform autodetection for the manylinux1 image breaks so we set the # defines ourselves. # TODO(atash) get better platform-detection support in core so we don't @@ -126,14 +117,24 @@ class PythonArtifact: environ=environ, timeout_seconds=60*60) elif self.platform == 'windows': + if 'Python27' in self.py_version or 'Python34' in self.py_version: + environ['EXT_COMPILER'] = 'mingw32' + else: + environ['EXT_COMPILER'] = 'msvc' + # For some reason, the batch script %random% always runs with the same + # seed. We create a random temp-dir here + dir = ''.join(random.choice(string.ascii_uppercase) for _ in range(10)) return create_jobspec(self.name, ['tools\\run_tests\\build_artifact_python.bat', - self.python_windows_prefix, - '32' if self.arch == 'x86' else '64' + self.py_version, + '32' if self.arch == 'x86' else '64', + dir ], + environ=environ, shell=True) else: - environ['PYTHON'] = 'python{}'.format(self.python_version) + environ['PYTHON'] = self.py_version + environ['SKIP_PIP_INSTALL'] = 'TRUE' return create_jobspec(self.name, ['tools/run_tests/build_artifact_python.sh'], environ=environ) @@ -330,18 +331,23 @@ def targets(): for Cls in (CSharpExtArtifact, NodeExtArtifact, ProtocArtifact) for platform in ('linux', 'macos', 'windows') for arch in ('x86', 'x64')] + - [PythonArtifact('linux', 'x86', '2.7', 'cp27-cp27m'), - PythonArtifact('linux', 'x86', '2.7', 'cp27-cp27mu'), - PythonArtifact('linux', 'x64', '2.7', 'cp27-cp27m'), - PythonArtifact('linux', 'x64', '2.7', 'cp27-cp27mu'), - PythonArtifact('macos', 'x64', '2.7'), - PythonArtifact('windows', 'x86', '2.7'), - PythonArtifact('windows', 'x64', '2.7'), - PythonArtifact('linux', 'x86', '3.4', 'cp34-cp34m'), - PythonArtifact('linux', 'x64', '3.4', 'cp34-cp34m'), - PythonArtifact('macos', 'x64', '3.4'), - PythonArtifact('windows', 'x86', '3.4'), - PythonArtifact('windows', 'x64', '3.4'), + [PythonArtifact('linux', 'x86', 'cp27-cp27m'), + PythonArtifact('linux', 'x86', 'cp27-cp27mu'), + PythonArtifact('linux', 'x86', 'cp34-cp34m'), + PythonArtifact('linux', 'x86', 'cp35-cp35m'), + PythonArtifact('linux', 'x64', 'cp27-cp27m'), + PythonArtifact('linux', 'x64', 'cp27-cp27mu'), + PythonArtifact('linux', 'x64', 'cp34-cp34m'), + PythonArtifact('linux', 'x64', 'cp35-cp35m'), + PythonArtifact('macos', 'x64', 'python2.7'), + PythonArtifact('macos', 'x64', 'python3.4'), + PythonArtifact('macos', 'x64', 'python3.5'), + PythonArtifact('windows', 'x86', 'Python27_32bits'), + PythonArtifact('windows', 'x86', 'Python34_32bits'), + PythonArtifact('windows', 'x86', 'Python35_32bits'), + PythonArtifact('windows', 'x64', 'Python27'), + PythonArtifact('windows', 'x64', 'Python34'), + PythonArtifact('windows', 'x64', 'Python35'), RubyArtifact('linux', 'x86'), RubyArtifact('linux', 'x64'), RubyArtifact('macos', 'x64'), diff --git a/tools/run_tests/build_artifact_python.bat b/tools/run_tests/build_artifact_python.bat index d0b147b426..246713a6ce 100644 --- a/tools/run_tests/build_artifact_python.bat +++ b/tools/run_tests/build_artifact_python.bat @@ -34,56 +34,45 @@ pip install --upgrade six pip install --upgrade setuptools pip install -rrequirements.txt -@rem Because this is windows and *everything seems to hate Windows* we have to -@rem set all of these flags ourselves because Python won't help us (see the -@rem setup.py of the grpcio_tools project). -set GRPC_PYTHON_CFLAGS=-fno-wrapv -frtti -std=c++11 - -@rem See https://sourceforge.net/p/mingw-w64/bugs/363/ -if %2 == 32 ( - set GRPC_PYTHON_CFLAGS=%GRPC_PYTHON_CFLAGS% -D_ftime=_ftime32 -D_timeb=__timeb32 -D_ftime_s=_ftime32_s -) else ( - set GRPC_PYTHON_CFLAGS=%GRPC_PYTHON_CFLAGS% -D_ftime=_ftime64 -D_timeb=__timeb64 -) - -@rem Further confusing things, MSYS2's mingw64 tries to dynamically link -@rem libgcc, libstdc++, and winpthreads. We have to override this or our -@rem extensions end up linking to MSYS2 DLLs, which the normal Python on -@rem Windows user won't have... and ON TOP OF THIS, there's MinGW's GCC default -@rem behavior of linking msvcrt.dll as the C runtime library, which we need to -@rem override so that Python's distutils doesn't link us against multiple C -@rem runtimes. -python -c "from distutils.cygwinccompiler import get_msvcr; print(get_msvcr()[0])" > temp.txt -set /p PYTHON_MSVCR=