aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorGravatar Ken Payson <kpayson@google.com>2016-08-10 15:39:43 -0700
committerGravatar Ken Payson <kpayson@google.com>2016-08-15 10:57:47 -0700
commit5998cd760c8ee73eba807614180e7f4c0ac3defe (patch)
tree24848f04c3a146ebad7f1d97dfc55c9b3b3ccbaa
parent10776b99cfa17dceae79f2e96d6b694b3d636db2 (diff)
Add Python3.5 artifact targets
-rw-r--r--PYTHON-MANIFEST.in2
-rw-r--r--setup.py62
-rw-r--r--src/python/grpcio/_spawn_patch.py (renamed from src/python/grpcio/_unixccompiler_patch.py)62
-rw-r--r--tools/distrib/python/grpcio_tools/setup.py48
-rwxr-xr-xtools/distrib/python/make_grpcio_tools.py79
-rw-r--r--tools/run_tests/artifact_targets.py70
-rw-r--r--tools/run_tests/build_artifact_python.bat51
-rwxr-xr-xtools/run_tests/build_artifact_python.sh36
8 files changed, 204 insertions, 206 deletions
diff --git a/PYTHON-MANIFEST.in b/PYTHON-MANIFEST.in
index 96bcdb515c..adeb390649 100644
--- a/PYTHON-MANIFEST.in
+++ b/PYTHON-MANIFEST.in
@@ -7,7 +7,7 @@ graft include/grpc
graft third_party/boringssl
graft third_party/nanopb
graft third_party/zlib
-include src/python/grpcio/_unixccompiler_patch.py
+include src/python/grpcio/_spawn_patch.py
include src/python/grpcio/commands.py
include src/python/grpcio/grpc_version.py
include src/python/grpcio/grpc_core_dependencies.py
diff --git a/setup.py b/setup.py
index 1dddf55e5e..096e8bdd4b 100644
--- a/setup.py
+++ b/setup.py
@@ -28,7 +28,7 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""A setup module for the GRPC Python package."""
-
+from distutils import cygwinccompiler
from distutils import extension as _extension
from distutils import util
import os
@@ -58,14 +58,12 @@ os.chdir(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, os.path.abspath(PYTHON_STEM))
# Break import-style to ensure we can actually find our in-repo dependencies.
-import _unixccompiler_patch
+import _spawn_patch
import commands
import grpc_core_dependencies
import grpc_version
-if 'win32' in sys.platform:
- _unixccompiler_patch.monkeypatch_unix_compiler()
-
+_spawn_patch.monkeypatch_spawn()
LICENSE = '3-clause BSD'
@@ -83,9 +81,38 @@ ENABLE_CYTHON_TRACING = os.environ.get(
# entirely ignored/dropped/forgotten by distutils and its Cygwin/MinGW support.
# We use these environment variables to thus get around that without locking
# ourselves in w.r.t. the multitude of operating systems this ought to build on.
-# By default we assume a GCC-like compiler.
-EXTRA_COMPILE_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_CFLAGS', ''))
-EXTRA_LINK_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_LDFLAGS', ''))
+# We can also use these variables as a way to inject environment-specific
+# compiler/linker flags. We assume GCC-like compilers and/or MinGW as a
+# reasonable default.
+EXTRA_ENV_COMPILE_ARGS = os.environ.get('GRPC_PYTHON_CFLAGS', None)
+EXTRA_ENV_LINK_ARGS = os.environ.get('GRPC_PYTHON_LDFLAGS', None)
+if EXTRA_ENV_COMPILE_ARGS is None:
+ EXTRA_ENV_COMPILE_ARGS = ''
+ if 'win32' in sys.platform and sys.version_info < (3, 5):
+ # We use define flags here and don't directly add to DEFINE_MACROS below to
+ # ensure that the expert user/builder has a way of turning it off (via the
+ # envvars) without adding yet more GRPC-specific envvars.
+ # See https://sourceforge.net/p/mingw-w64/bugs/363/
+ if '32' in platform.architecture()[0]:
+ EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime32 -D_timeb=__timeb32 -D_ftime_s=_ftime32_s'
+ else:
+ EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime64 -D_timeb=__timeb64'
+ elif "linux" in sys.platform or "darwin" in sys.platform:
+ EXTRA_ENV_COMPILE_ARGS += ' -fno-wrapv'
+if EXTRA_ENV_LINK_ARGS is None:
+ EXTRA_ENV_LINK_ARGS = ''
+ if "linux" in sys.platform or "darwin" in sys.platform:
+ EXTRA_ENV_LINK_ARGS += ' -lpthread'
+ elif "win32" in sys.platform and sys.version_info < (3, 5):
+ msvcr = cygwinccompiler.get_msvcr()[0]
+ # TODO(atash) sift through the GCC specs to see if libstdc++ can have any
+ # influence on the linkage outcome on MinGW for non-C++ programs.
+ EXTRA_ENV_LINK_ARGS += (
+ ' -static-libgcc -static-libstdc++ -mcrtdll={msvcr} '
+ '-static'.format(msvcr=msvcr))
+
+EXTRA_COMPILE_ARGS = shlex.split(EXTRA_ENV_COMPILE_ARGS)
+EXTRA_LINK_ARGS = shlex.split(EXTRA_ENV_LINK_ARGS)
CYTHON_EXTENSION_PACKAGE_NAMES = ()
@@ -104,28 +131,29 @@ if "linux" in sys.platform:
if not "win32" in sys.platform:
EXTENSION_LIBRARIES += ('m',)
if "win32" in sys.platform:
- EXTENSION_LIBRARIES += ('ws2_32',)
+ EXTENSION_LIBRARIES += ('advapi32', 'ws2_32',)
DEFINE_MACROS = (
('OPENSSL_NO_ASM', 1), ('_WIN32_WINNT', 0x600),
('GPR_BACKWARDS_COMPATIBILITY_MODE', 1),)
if "win32" in sys.platform:
- DEFINE_MACROS += (('OPENSSL_WINDOWS', 1), ('WIN32_LEAN_AND_MEAN', 1),)
+ DEFINE_MACROS += (('WIN32_LEAN_AND_MEAN', 1),)
if '64bit' in platform.architecture()[0]:
DEFINE_MACROS += (('MS_WIN64', 1),)
+ elif sys.version_info >= (3, 5):
+ # For some reason, this is needed to get access to inet_pton/inet_ntop
+ # on msvc, but only for 32 bits
+ DEFINE_MACROS += (('NTDDI_VERSION', 0x06000000),)
LDFLAGS = tuple(EXTRA_LINK_ARGS)
CFLAGS = tuple(EXTRA_COMPILE_ARGS)
if "linux" in sys.platform:
LDFLAGS += ('-Wl,-wrap,memcpy',)
if "linux" in sys.platform or "darwin" in sys.platform:
- CFLAGS += ('-fvisibility=hidden',)
-
- pymodinit_type = 'PyObject*' if PY3 else 'void'
-
- pymodinit = '__attribute__((visibility ("default"))) {}'.format(pymodinit_type)
- DEFINE_MACROS += (('PyMODINIT_FUNC', pymodinit),)
-
+ CFLAGS += ('-fvisibility=hidden',)
+ pymodinit_type = 'PyObject*' if PY3 else 'void'
+ pymodinit = '__attribute__((visibility ("default"))) {}'.format(pymodinit_type)
+ DEFINE_MACROS += (('PyMODINIT_FUNC', pymodinit),)
# By default, Python3 distutils enforces compatibility of
# c plugins (.so files) with the OSX version Python3 was built with.
diff --git a/src/python/grpcio/_unixccompiler_patch.py b/src/python/grpcio/_spawn_patch.py
index 894c3ef395..24306f0dd9 100644
--- a/src/python/grpcio/_unixccompiler_patch.py
+++ b/src/python/grpcio/_spawn_patch.py
@@ -27,51 +27,47 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-"""Covers inadequacies in distutils."""
+"""Patches the spawn() command for windows compilers.
+
+Windows has an 8191 character command line limit, but some compilers
+support an @command_file directive where command_file is a file
+containing the full command line.
+"""
from distutils import ccompiler
-from distutils import errors
-from distutils import unixccompiler
import os
import os.path
-import shlex
import shutil
import sys
import tempfile
-def _unix_commandfile_spawn(self, command):
- """Wrapper around distutils.util.spawn that attempts to use command files.
+MAX_COMMAND_LENGTH = 8191
- Meant to replace the CCompiler method `spawn` on UnixCCompiler and its
- derivatives (e.g. the MinGW32 compiler).
+_classic_spawn = ccompiler.CCompiler.spawn
- Some commands like `gcc` (and friends like `clang`) support command files to
- work around shell command length limits.
- """
- # Sometimes distutils embeds the executables as full strings including some
- # hard-coded flags rather than as lists.
- command = list(shlex.split(command[0])) + list(command[1:])
- command_base = os.path.basename(command[0].strip())
- if command_base == 'ccache':
- command_base = command[:2]
- command_args = command[2:]
- elif command_base.startswith('ccache') or command_base in ['gcc', 'clang', 'clang++', 'g++']:
- command_base = command[:1]
- command_args = command[1:]
+def _commandfile_spawn(self, command):
+ command_length = sum([len(arg) for arg in command])
+ if os.name == 'nt' and command_length > MAX_COMMAND_LENGTH:
+ # Even if this command doesn't support the @command_file, it will
+ # fail as is so we try blindly
+ print('Command line length exceeded, using command file')
+ print(' '.join(command))
+ temporary_directory = tempfile.mkdtemp()
+ command_filename = os.path.abspath(
+ os.path.join(temporary_directory, 'command'))
+ with open(command_filename, 'w') as command_file:
+ escaped_args = ['"' + arg.replace('\\', '\\\\') + '"' for arg in command[1:]]
+ command_file.write(' '.join(escaped_args))
+ modified_command = command[:1] + ['@{}'.format(command_filename)]
+ try:
+ _classic_spawn(self, modified_command)
+ finally:
+ shutil.rmtree(temporary_directory)
else:
- return ccompiler.CCompiler.spawn(self, command)
- temporary_directory = tempfile.mkdtemp()
- command_filename = os.path.abspath(os.path.join(temporary_directory, 'command'))
- with open(command_filename, 'w') as command_file:
- escaped_args = [arg.replace('\\', '\\\\') for arg in command_args]
- command_file.write(' '.join(escaped_args))
- modified_command = command_base + ['@{}'.format(command_filename)]
- result = ccompiler.CCompiler.spawn(self, modified_command)
- shutil.rmtree(temporary_directory)
- return result
+ _classic_spawn(self, command)
-def monkeypatch_unix_compiler():
+def monkeypatch_spawn():
"""Monkeypatching is dumb, but it's either that or we become maintainers of
something much, much bigger."""
- unixccompiler.UnixCCompiler.spawn = _unix_commandfile_spawn
+ ccompiler.CCompiler.spawn = _commandfile_spawn
diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py
index 43d0f5518a..53d7eb0948 100644
--- a/tools/distrib/python/grpcio_tools/setup.py
+++ b/tools/distrib/python/grpcio_tools/setup.py
@@ -27,6 +27,7 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+from distutils import cygwinccompiler
from distutils import extension
from distutils import util
import errno
@@ -57,11 +58,38 @@ PY3 = sys.version_info.major == 3
# entirely ignored/dropped/forgotten by distutils and its Cygwin/MinGW support.
# We use these environment variables to thus get around that without locking
# ourselves in w.r.t. the multitude of operating systems this ought to build on.
-# By default we assume a GCC-like compiler.
-EXTRA_COMPILE_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_CFLAGS',
- '-fno-wrapv -frtti -std=c++11'))
-EXTRA_LINK_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_LDFLAGS',
- '-lpthread'))
+# We can also use these variables as a way to inject environment-specific
+# compiler/linker flags. We assume GCC-like compilers and/or MinGW as a
+# reasonable default.
+EXTRA_ENV_COMPILE_ARGS = os.environ.get('GRPC_PYTHON_CFLAGS', None)
+EXTRA_ENV_LINK_ARGS = os.environ.get('GRPC_PYTHON_LDFLAGS', None)
+if EXTRA_ENV_COMPILE_ARGS is None:
+ EXTRA_ENV_COMPILE_ARGS = '-std=c++11'
+ if 'win32' in sys.platform and sys.version_info < (3, 5):
+ # We use define flags here and don't directly add to DEFINE_MACROS below to
+ # ensure that the expert user/builder has a way of turning it off (via the
+ # envvars) without adding yet more GRPC-specific envvars.
+ # See https://sourceforge.net/p/mingw-w64/bugs/363/
+ if '32' in platform.architecture()[0]:
+ EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime32 -D_timeb=__timeb32 -D_ftime_s=_ftime32_s'
+ else:
+ EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime64 -D_timeb=__timeb64'
+ elif "linux" in sys.platform or "darwin" in sys.platform:
+ EXTRA_ENV_COMPILE_ARGS += ' -fno-wrapv -frtti'
+if EXTRA_ENV_LINK_ARGS is None:
+ EXTRA_ENV_LINK_ARGS = ''
+ if "linux" in sys.platform or "darwin" in sys.platform:
+ EXTRA_ENV_LINK_ARGS += ' -lpthread'
+ elif "win32" in sys.platform and sys.version_info < (3, 5):
+ msvcr = cygwinccompiler.get_msvcr()[0]
+ # TODO(atash) sift through the GCC specs to see if libstdc++ can have any
+ # influence on the linkage outcome on MinGW for non-C++ programs.
+ EXTRA_ENV_LINK_ARGS += (
+ ' -static-libgcc -static-libstdc++ -mcrtdll={msvcr} '
+ '-static'.format(msvcr=msvcr))
+
+EXTRA_COMPILE_ARGS = shlex.split(EXTRA_ENV_COMPILE_ARGS)
+EXTRA_LINK_ARGS = shlex.split(EXTRA_ENV_LINK_ARGS)
CC_FILES = [
os.path.normpath(cc_file) for cc_file in protoc_lib_deps.CC_FILES]
@@ -73,9 +101,13 @@ PROTO_INCLUDE = os.path.normpath(protoc_lib_deps.PROTO_INCLUDE)
GRPC_PYTHON_TOOLS_PACKAGE = 'grpc.tools'
GRPC_PYTHON_PROTO_RESOURCES_NAME = '_proto'
-DEFINE_MACROS = (('HAVE_PTHREAD', 1),)
-if "win32" in sys.platform and '64bit' in platform.architecture()[0]:
- DEFINE_MACROS += (('MS_WIN64', 1),)
+DEFINE_MACROS = ()
+if "win32" in sys.platform:
+ DEFINE_MACROS += (('WIN32_LEAN_AND_MEAN', 1),)
+ if '64bit' in platform.architecture()[0]:
+ DEFINE_MACROS += (('MS_WIN64', 1),)
+elif "linux" in sys.platform or "darwin" in sys.platform:
+ DEFINE_MACROS += (('HAVE_PTHREAD', 1),)
# By default, Python3 distutils enforces compatibility of
# c plugins (.so files) with the OSX version Python3 was built with.
diff --git a/tools/distrib/python/make_grpcio_tools.py b/tools/distrib/python/make_grpcio_tools.py
index adf58445af..7413928eca 100755
--- a/tools/distrib/python/make_grpcio_tools.py
+++ b/tools/distrib/python/make_grpcio_tools.py
@@ -88,22 +88,23 @@ GRPC_ROOT = os.path.abspath(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
'..', '..', '..'))
-GRPC_PYTHON_ROOT = os.path.join(GRPC_ROOT, 'tools/distrib/python/grpcio_tools')
+GRPC_PYTHON_ROOT = os.path.join(GRPC_ROOT, 'tools', 'distrib',
+ 'python', 'grpcio_tools')
-GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT = 'third_party/protobuf/src'
+GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT = os.path.join('third_party', 'protobuf', 'src')
GRPC_PROTOBUF = os.path.join(GRPC_ROOT, GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT)
-GRPC_PROTOC_PLUGINS = os.path.join(GRPC_ROOT, 'src/compiler')
-GRPC_PYTHON_PROTOBUF = os.path.join(GRPC_PYTHON_ROOT,
- 'third_party/protobuf/src')
-GRPC_PYTHON_PROTOC_PLUGINS = os.path.join(GRPC_PYTHON_ROOT,
- 'grpc_root/src/compiler')
+GRPC_PROTOC_PLUGINS = os.path.join(GRPC_ROOT, 'src', 'compiler')
+GRPC_PYTHON_PROTOBUF = os.path.join(GRPC_PYTHON_ROOT, 'third_party', 'protobuf',
+ 'src')
+GRPC_PYTHON_PROTOC_PLUGINS = os.path.join(GRPC_PYTHON_ROOT, 'grpc_root', 'src',
+ 'compiler')
GRPC_PYTHON_PROTOC_LIB_DEPS = os.path.join(GRPC_PYTHON_ROOT,
'protoc_lib_deps.py')
GRPC_INCLUDE = os.path.join(GRPC_ROOT, 'include')
-GRPC_PYTHON_INCLUDE = os.path.join(GRPC_PYTHON_ROOT, 'grpc_root/include')
+GRPC_PYTHON_INCLUDE = os.path.join(GRPC_PYTHON_ROOT, 'grpc_root', 'include')
-BAZEL_DEPS = os.path.join(GRPC_ROOT, 'tools/distrib/python/bazel_deps.sh')
+BAZEL_DEPS = os.path.join(GRPC_ROOT, 'tools', 'distrib', 'python', 'bazel_deps.sh')
BAZEL_DEPS_PROTOC_LIB_QUERY = '//:protoc_lib'
BAZEL_DEPS_COMMON_PROTOS_QUERY = '//:well_known_protos'
@@ -136,64 +137,6 @@ def long_path(path):
else:
return path
-def atomic_file_copy(src, dst):
- """Based on the lock-free-whack-a-mole algorithm, depending on filesystem
- renaming being atomic. Described at http://stackoverflow.com/a/28090883.
- """
- try:
- if filecmp.cmp(src, dst):
- return
- except:
- pass
- dst_dir = os.path.abspath(os.path.dirname(dst))
- dst_base = os.path.basename(dst)
- this_id = str(uuid.uuid4()).replace('.', '-')
- temporary_file = os.path.join(dst_dir, '{}.{}.tmp'.format(dst_base, this_id))
- mole_file = os.path.join(dst_dir, '{}.{}.mole.tmp'.format(dst_base, this_id))
- mole_pattern = os.path.join(dst_dir, '{}.*.mole.tmp'.format(dst_base))
- src = long_path(src)
- dst = long_path(dst)
- temporary_file = long_path(temporary_file)
- mole_file = long_path(mole_file)
- mole_pattern = long_path(mole_pattern)
- shutil.copy2(src, temporary_file)
- try:
- os.rename(temporary_file, mole_file)
- except:
- print('Error moving temporary file {} to {}'.format(temporary_file, mole_file), file=sys.stderr)
- print('while trying to copy file {} to {}'.format(src, dst), file=sys.stderr)
- raise
- for other_file in glob.glob(mole_pattern):
- other_id = other_file.split('.')[-3]
- if this_id == other_id:
- pass
- elif this_id < other_id:
- try:
- os.remove(other_file)
- except:
- pass
- else:
- try:
- os.remove(mole_file)
- except:
- pass
- this_id = other_id
- mole_file = other_file
- try:
- if filecmp.cmp(src, dst):
- try:
- os.remove(mole_file)
- except:
- pass
- return
- except:
- pass
- try:
- os.rename(mole_file, dst)
- except:
- pass
-
-
def main():
os.chdir(GRPC_ROOT)
@@ -211,7 +154,7 @@ def main():
for relative_file in files:
source_file = os.path.abspath(os.path.join(source_dir, relative_file))
target_file = os.path.abspath(os.path.join(target_dir, relative_file))
- atomic_file_copy(source_file, target_file)
+ shutil.copyfile(source_file, target_file)
try:
protoc_lib_deps_content = get_deps()
diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifact_targets.py
index 1c480f7a7f..0ba9f3b331 100644
--- a/tools/run_tests/artifact_targets.py
+++ b/tools/run_tests/artifact_targets.py
@@ -31,6 +31,8 @@
"""Definition of targets to build artifacts."""
import os.path
+import random
+import string
import sys
import jobset
@@ -79,27 +81,16 @@ _ARCH_FLAG_MAP = {
'x64': '-m64'
}
-python_windows_version_arch_map = {
- ('x86', '2.7'): 'Python27_32bits',
- ('x64', '2.7'): 'Python27',
- ('x86', '3.4'): 'Python34_32bits',
- ('x64', '3.4'): 'Python34',
-}
class PythonArtifact:
"""Builds Python artifacts."""
- def __init__(self, platform, arch, python_version, manylinux_build=None):
- if manylinux_build:
- self.name = 'python%s_%s_%s_%s' % (python_version, platform, arch, manylinux_build)
- else:
- self.name = 'python%s_%s_%s' % (python_version, platform, arch)
+ def __init__(self, platform, arch, py_version):
+ self.name = 'python_%s_%s_%s' % (platform, arch, py_version)
self.platform = platform
self.arch = arch
- self.labels = ['artifact', 'python', python_version, platform, arch]
- self.python_version = python_version
- self.python_windows_prefix = python_windows_version_arch_map[arch, python_version]
- self.manylinux_build = manylinux_build
+ self.labels = ['artifact', 'python', platform, arch, py_version]
+ self.py_version = py_version
def pre_build_jobspecs(self):
return []
@@ -111,8 +102,8 @@ class PythonArtifact:
environ['SETARCH_CMD'] = 'linux32'
# Inside the manylinux container, the python installations are located in
# special places...
- environ['PYTHON'] = '/opt/python/{}/bin/python'.format(self.manylinux_build)
- environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.manylinux_build)
+ environ['PYTHON'] = '/opt/python/{}/bin/python'.format(self.py_version)
+ environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.py_version)
# Platform autodetection for the manylinux1 image breaks so we set the
# defines ourselves.
# TODO(atash) get better platform-detection support in core so we don't
@@ -126,14 +117,24 @@ class PythonArtifact:
environ=environ,
timeout_seconds=60*60)
elif self.platform == 'windows':
+ if 'Python27' in self.py_version or 'Python34' in self.py_version:
+ environ['EXT_COMPILER'] = 'mingw32'
+ else:
+ environ['EXT_COMPILER'] = 'msvc'
+ # For some reason, the batch script %random% always runs with the same
+ # seed. We create a random temp-dir here
+ dir = ''.join(random.choice(string.ascii_uppercase) for _ in range(10))
return create_jobspec(self.name,
['tools\\run_tests\\build_artifact_python.bat',
- self.python_windows_prefix,
- '32' if self.arch == 'x86' else '64'
+ self.py_version,
+ '32' if self.arch == 'x86' else '64',
+ dir
],
+ environ=environ,
shell=True)
else:
- environ['PYTHON'] = 'python{}'.format(self.python_version)
+ environ['PYTHON'] = self.py_version
+ environ['SKIP_PIP_INSTALL'] = 'TRUE'
return create_jobspec(self.name,
['tools/run_tests/build_artifact_python.sh'],
environ=environ)
@@ -330,18 +331,23 @@ def targets():
for Cls in (CSharpExtArtifact, NodeExtArtifact, ProtocArtifact)
for platform in ('linux', 'macos', 'windows')
for arch in ('x86', 'x64')] +
- [PythonArtifact('linux', 'x86', '2.7', 'cp27-cp27m'),
- PythonArtifact('linux', 'x86', '2.7', 'cp27-cp27mu'),
- PythonArtifact('linux', 'x64', '2.7', 'cp27-cp27m'),
- PythonArtifact('linux', 'x64', '2.7', 'cp27-cp27mu'),
- PythonArtifact('macos', 'x64', '2.7'),
- PythonArtifact('windows', 'x86', '2.7'),
- PythonArtifact('windows', 'x64', '2.7'),
- PythonArtifact('linux', 'x86', '3.4', 'cp34-cp34m'),
- PythonArtifact('linux', 'x64', '3.4', 'cp34-cp34m'),
- PythonArtifact('macos', 'x64', '3.4'),
- PythonArtifact('windows', 'x86', '3.4'),
- PythonArtifact('windows', 'x64', '3.4'),
+ [PythonArtifact('linux', 'x86', 'cp27-cp27m'),
+ PythonArtifact('linux', 'x86', 'cp27-cp27mu'),
+ PythonArtifact('linux', 'x86', 'cp34-cp34m'),
+ PythonArtifact('linux', 'x86', 'cp35-cp35m'),
+ PythonArtifact('linux', 'x64', 'cp27-cp27m'),
+ PythonArtifact('linux', 'x64', 'cp27-cp27mu'),
+ PythonArtifact('linux', 'x64', 'cp34-cp34m'),
+ PythonArtifact('linux', 'x64', 'cp35-cp35m'),
+ PythonArtifact('macos', 'x64', 'python2.7'),
+ PythonArtifact('macos', 'x64', 'python3.4'),
+ PythonArtifact('macos', 'x64', 'python3.5'),
+ PythonArtifact('windows', 'x86', 'Python27_32bits'),
+ PythonArtifact('windows', 'x86', 'Python34_32bits'),
+ PythonArtifact('windows', 'x86', 'Python35_32bits'),
+ PythonArtifact('windows', 'x64', 'Python27'),
+ PythonArtifact('windows', 'x64', 'Python34'),
+ PythonArtifact('windows', 'x64', 'Python35'),
RubyArtifact('linux', 'x86'),
RubyArtifact('linux', 'x64'),
RubyArtifact('macos', 'x64'),
diff --git a/tools/run_tests/build_artifact_python.bat b/tools/run_tests/build_artifact_python.bat
index d0b147b426..246713a6ce 100644
--- a/tools/run_tests/build_artifact_python.bat
+++ b/tools/run_tests/build_artifact_python.bat
@@ -34,56 +34,45 @@ pip install --upgrade six
pip install --upgrade setuptools
pip install -rrequirements.txt
-@rem Because this is windows and *everything seems to hate Windows* we have to
-@rem set all of these flags ourselves because Python won't help us (see the
-@rem setup.py of the grpcio_tools project).
-set GRPC_PYTHON_CFLAGS=-fno-wrapv -frtti -std=c++11
-
-@rem See https://sourceforge.net/p/mingw-w64/bugs/363/
-if %2 == 32 (
- set GRPC_PYTHON_CFLAGS=%GRPC_PYTHON_CFLAGS% -D_ftime=_ftime32 -D_timeb=__timeb32 -D_ftime_s=_ftime32_s
-) else (
- set GRPC_PYTHON_CFLAGS=%GRPC_PYTHON_CFLAGS% -D_ftime=_ftime64 -D_timeb=__timeb64
-)
-
-@rem Further confusing things, MSYS2's mingw64 tries to dynamically link
-@rem libgcc, libstdc++, and winpthreads. We have to override this or our
-@rem extensions end up linking to MSYS2 DLLs, which the normal Python on
-@rem Windows user won't have... and ON TOP OF THIS, there's MinGW's GCC default
-@rem behavior of linking msvcrt.dll as the C runtime library, which we need to
-@rem override so that Python's distutils doesn't link us against multiple C
-@rem runtimes.
-python -c "from distutils.cygwinccompiler import get_msvcr; print(get_msvcr()[0])" > temp.txt
-set /p PYTHON_MSVCR=<temp.txt
-set GRPC_PYTHON_LDFLAGS=-static-libgcc -static-libstdc++ -mcrtdll=%PYTHON_MSVCR% -static -lpthread
-
set GRPC_PYTHON_BUILD_WITH_CYTHON=1
+@rem Multiple builds are running simultaneously, so to avoid distutils
+@rem file collisions, we build everything in a tmp directory
+if not exist "artifacts" mkdir "artifacts"
+set ARTIFACT_DIR=%cd%\artifacts
+set BUILD_DIR=C:\Windows\Temp\pygrpc-%3\
+mkdir %BUILD_DIR%
+xcopy /s/e/q %cd%\* %BUILD_DIR%
+pushd %BUILD_DIR%
+
@rem Set up gRPC Python tools
python tools\distrib\python\make_grpcio_tools.py
@rem Build gRPC Python extensions
-python setup.py build_ext -c mingw32
+python setup.py build_ext -c %EXT_COMPILER% || goto :error
pushd tools\distrib\python\grpcio_tools
-python setup.py build_ext -c mingw32
+python setup.py build_ext -c %EXT_COMPILER% || goto :error
popd
-
@rem Build gRPC Python distributions
-python setup.py bdist_wheel
+python setup.py bdist_wheel || goto :error
pushd tools\distrib\python\grpcio_tools
-python setup.py bdist_wheel
+python setup.py bdist_wheel || goto :error
popd
-mkdir artifacts
-xcopy /Y /I /S dist\* artifacts\ || goto :error
-xcopy /Y /I /S tools\distrib\python\grpcio_tools\dist\* artifacts\ || goto :error
+xcopy /Y /I /S dist\* %ARTIFACT_DIR% || goto :error
+xcopy /Y /I /S tools\distrib\python\grpcio_tools\dist\* %ARTIFACT_DIR% || goto :error
+
+popd
+rmdir /s /q %BUILD_DIR%
goto :EOF
:error
+popd
+rmdir /s /q %BUILD_DIR%
exit /b 1
diff --git a/tools/run_tests/build_artifact_python.sh b/tools/run_tests/build_artifact_python.sh
index 8f8330ef24..9fed7c5028 100755
--- a/tools/run_tests/build_artifact_python.sh
+++ b/tools/run_tests/build_artifact_python.sh
@@ -38,38 +38,42 @@ export PYTHON=${PYTHON:-python}
export PIP=${PIP:-pip}
export AUDITWHEEL=${AUDITWHEEL:-auditwheel}
+# Because multiple builds run in parallel, some distutils file
+# operations may collide. To avoid this, each build is run in
+# a temp directory
+mkdir -p artifacts
+ARTIFACT_DIR="$PWD/artifacts"
+BUILD_DIR=`mktemp -d "${TMPDIR:-/tmp}/pygrpc.XXXXXX"`
+trap "rm -rf $BUILD_DIR" EXIT
+cp -r * "$BUILD_DIR"
+cd "$BUILD_DIR"
# Build the source distribution first because MANIFEST.in cannot override
# exclusion of built shared objects among package resources (for some
# inexplicable reason).
-${SETARCH_CMD} ${PYTHON} setup.py \
- sdist
+${SETARCH_CMD} ${PYTHON} setup.py sdist
# Wheel has a bug where directories don't get excluded.
# https://bitbucket.org/pypa/wheel/issues/99/cannot-exclude-directory
-${SETARCH_CMD} ${PYTHON} setup.py \
- bdist_wheel
+${SETARCH_CMD} ${PYTHON} setup.py bdist_wheel
# Build gRPC tools package distribution
${PYTHON} tools/distrib/python/make_grpcio_tools.py
# Build gRPC tools package source distribution
-${SETARCH_CMD} ${PYTHON} tools/distrib/python/grpcio_tools/setup.py \
- sdist
+${SETARCH_CMD} ${PYTHON} tools/distrib/python/grpcio_tools/setup.py sdist
# Build gRPC tools package binary distribution
-CFLAGS="$CFLAGS -fno-wrapv" ${SETARCH_CMD} \
- ${PYTHON} tools/distrib/python/grpcio_tools/setup.py bdist_wheel
+${SETARCH_CMD} ${PYTHON} tools/distrib/python/grpcio_tools/setup.py bdist_wheel
-mkdir -p artifacts
if [ "$BUILD_MANYLINUX_WHEEL" != "" ]
then
for wheel in dist/*.whl; do
- ${AUDITWHEEL} repair $wheel -w artifacts/
+ ${AUDITWHEEL} repair $wheel -w "$ARTIFACT_DIR"
rm $wheel
done
for wheel in tools/distrib/python/grpcio_tools/dist/*.whl; do
- ${AUDITWHEEL} repair $wheel -w artifacts/
+ ${AUDITWHEEL} repair $wheel -w "$ARTIFACT_DIR"
rm $wheel
done
fi
@@ -81,14 +85,14 @@ fi
if [ "$BUILD_HEALTH_CHECKING" != "" ]
then
${PIP} install -rrequirements.txt
- ${PIP} install grpcio --no-index --find-links "file://${PWD}/artifacts/"
- ${PIP} install grpcio-tools --no-index --find-links "file://${PWD}/artifacts/"
+ ${PIP} install grpcio --no-index --find-links "file://$ARTIFACT_DIR/"
+ ${PIP} install grpcio-tools --no-index --find-links "file://$ARTIFACT_DIR/"
# Build gRPC health check source distribution
${SETARCH_CMD} ${PYTHON} src/python/grpcio_health_checking/setup.py \
preprocess build_package_protos sdist
- cp -r src/python/grpcio_health_checking/dist/* artifacts
+ cp -r src/python/grpcio_health_checking/dist/* "$ARTIFACT_DIR"
fi
-cp -r dist/* artifacts
-cp -r tools/distrib/python/grpcio_tools/dist/* artifacts
+cp -r dist/* "$ARTIFACT_DIR"
+cp -r tools/distrib/python/grpcio_tools/dist/* "$ARTIFACT_DIR"