aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
-rw-r--r--setup.py21
-rw-r--r--src/python/grpcio/_unixccompiler_patch.py102
-rw-r--r--tools/distrib/python/grpcio_tools/setup.py27
-rwxr-xr-xtools/distrib/python/make_grpcio_tools.py94
-rw-r--r--tools/run_tests/artifact_targets.py58
-rw-r--r--tools/run_tests/build_artifact_python.bat12
6 files changed, 188 insertions, 126 deletions
diff --git a/setup.py b/setup.py
index 056e1fd033..5d33756c19 100644
--- a/setup.py
+++ b/setup.py
@@ -47,10 +47,10 @@ from setuptools.command import egg_info
egg_info.manifest_maker.template = 'PYTHON-MANIFEST.in'
PY3 = sys.version_info.major == 3
-PYTHON_STEM = './src/python/grpcio'
-CORE_INCLUDE = ('./include', '.',)
-BORINGSSL_INCLUDE = ('./third_party/boringssl/include',)
-ZLIB_INCLUDE = ('./third_party/zlib',)
+PYTHON_STEM = os.path.join('src', 'python', 'grpcio')
+CORE_INCLUDE = ('include', '.',)
+BORINGSSL_INCLUDE = (os.path.join('third_party', 'boringssl', 'include'),)
+ZLIB_INCLUDE = (os.path.join('third_party', 'zlib'),)
# Ensure we're in the proper directory whether or not we're being used by pip.
os.chdir(os.path.dirname(os.path.abspath(__file__)))
@@ -62,8 +62,8 @@ import commands
import grpc_core_dependencies
import grpc_version
-# TODO(atash) make this conditional on being on a mingw32 build
-_unixccompiler_patch.monkeypatch_unix_compiler()
+if 'win32' in sys.platform:
+ _unixccompiler_patch.monkeypatch_unix_compiler()
LICENSE = '3-clause BSD'
@@ -105,7 +105,9 @@ if not "win32" in sys.platform:
if "win32" in sys.platform:
EXTENSION_LIBRARIES += ('ws2_32',)
-DEFINE_MACROS = (('OPENSSL_NO_ASM', 1), ('_WIN32_WINNT', 0x600), ('GPR_BACKWARDS_COMPATIBILITY_MODE', 1),)
+DEFINE_MACROS = (
+ ('OPENSSL_NO_ASM', 1), ('_WIN32_WINNT', 0x600),
+ ('GPR_BACKWARDS_COMPATIBILITY_MODE', 1),)
if "win32" in sys.platform:
DEFINE_MACROS += (('OPENSSL_WINDOWS', 1), ('WIN32_LEAN_AND_MEAN', 1),)
if '64bit' in platform.architecture()[0]:
@@ -200,12 +202,13 @@ COMMAND_CLASS = {
}
# Ensure that package data is copied over before any commands have been run:
-credentials_dir = os.path.join(PYTHON_STEM, 'grpc/_cython/_credentials')
+credentials_dir = os.path.join(PYTHON_STEM, 'grpc', '_cython', '_credentials')
try:
os.mkdir(credentials_dir)
except OSError:
pass
-shutil.copyfile('etc/roots.pem', os.path.join(credentials_dir, 'roots.pem'))
+shutil.copyfile(os.path.join('etc', 'roots.pem'),
+ os.path.join(credentials_dir, 'roots.pem'))
PACKAGE_DATA = {
# Binaries that may or may not be present in the final installation, but are
diff --git a/src/python/grpcio/_unixccompiler_patch.py b/src/python/grpcio/_unixccompiler_patch.py
index 9a697989b3..894c3ef395 100644
--- a/src/python/grpcio/_unixccompiler_patch.py
+++ b/src/python/grpcio/_unixccompiler_patch.py
@@ -34,88 +34,44 @@ from distutils import errors
from distutils import unixccompiler
import os
import os.path
+import shlex
import shutil
import sys
import tempfile
+def _unix_commandfile_spawn(self, command):
+ """Wrapper around distutils.util.spawn that attempts to use command files.
-def _unix_piecemeal_link(
- self, target_desc, objects, output_filename, output_dir=None,
- libraries=None, library_dirs=None, runtime_library_dirs=None,
- export_symbols=None, debug=0, extra_preargs=None, extra_postargs=None,
- build_temp=None, target_lang=None):
- """`link` externalized method taken almost verbatim from UnixCCompiler.
+ Meant to replace the CCompiler method `spawn` on UnixCCompiler and its
+ derivatives (e.g. the MinGW32 compiler).
- Modifies the link command for unix-like compilers by using a command file so
- that long command line argument strings don't break the command shell's
- ARG_MAX character limit.
+ Some commands like `gcc` (and friends like `clang`) support command files to
+ work around shell command length limits.
"""
- objects, output_dir = self._fix_object_args(objects, output_dir)
- libraries, library_dirs, runtime_library_dirs = self._fix_lib_args(
- libraries, library_dirs, runtime_library_dirs)
- # filter out standard library paths, which are not explicitely needed
- # for linking
- library_dirs = [dir for dir in library_dirs
- if not dir in ('/lib', '/lib64', '/usr/lib', '/usr/lib64')]
- runtime_library_dirs = [dir for dir in runtime_library_dirs
- if not dir in ('/lib', '/lib64', '/usr/lib', '/usr/lib64')]
- lib_opts = ccompiler.gen_lib_options(self, library_dirs, runtime_library_dirs,
- libraries)
- if (not (isinstance(output_dir, str) or isinstance(output_dir, bytes))
- and output_dir is not None):
- raise TypeError("'output_dir' must be a string or None")
- if output_dir is not None:
- output_filename = os.path.join(output_dir, output_filename)
-
- if self._need_link(objects, output_filename):
- ld_args = (objects + self.objects +
- lib_opts + ['-o', output_filename])
- if debug:
- ld_args[:0] = ['-g']
- if extra_preargs:
- ld_args[:0] = extra_preargs
- if extra_postargs:
- ld_args.extend(extra_postargs)
- self.mkpath(os.path.dirname(output_filename))
- try:
- if target_desc == ccompiler.CCompiler.EXECUTABLE:
- linker = self.linker_exe[:]
- else:
- linker = self.linker_so[:]
- if target_lang == "c++" and self.compiler_cxx:
- # skip over environment variable settings if /usr/bin/env
- # is used to set up the linker's environment.
- # This is needed on OSX. Note: this assumes that the
- # normal and C++ compiler have the same environment
- # settings.
- i = 0
- if os.path.basename(linker[0]) == "env":
- i = 1
- while '=' in linker[i]:
- i = i + 1
-
- linker[i] = self.compiler_cxx[i]
-
- if sys.platform == 'darwin':
- import _osx_support
- linker = _osx_support.compiler_fixup(linker, ld_args)
-
- temporary_directory = tempfile.mkdtemp()
- command_filename = os.path.abspath(
- os.path.join(temporary_directory, 'command'))
- with open(command_filename, 'w') as command_file:
- escaped_ld_args = [arg.replace('\\', '\\\\') for arg in ld_args]
- command_file.write(' '.join(escaped_ld_args))
- self.spawn(linker + ['@{}'.format(command_filename)])
- except errors.DistutilsExecError:
- raise ccompiler.LinkError
+ # Sometimes distutils embeds the executables as full strings including some
+ # hard-coded flags rather than as lists.
+ command = list(shlex.split(command[0])) + list(command[1:])
+ command_base = os.path.basename(command[0].strip())
+ if command_base == 'ccache':
+ command_base = command[:2]
+ command_args = command[2:]
+ elif command_base.startswith('ccache') or command_base in ['gcc', 'clang', 'clang++', 'g++']:
+ command_base = command[:1]
+ command_args = command[1:]
else:
- log.debug("skipping %s (up-to-date)", output_filename)
+ return ccompiler.CCompiler.spawn(self, command)
+ temporary_directory = tempfile.mkdtemp()
+ command_filename = os.path.abspath(os.path.join(temporary_directory, 'command'))
+ with open(command_filename, 'w') as command_file:
+ escaped_args = [arg.replace('\\', '\\\\') for arg in command_args]
+ command_file.write(' '.join(escaped_args))
+ modified_command = command_base + ['@{}'.format(command_filename)]
+ result = ccompiler.CCompiler.spawn(self, modified_command)
+ shutil.rmtree(temporary_directory)
+ return result
+
-# TODO(atash) try replacing this monkeypatch of the compiler harness' link
-# operation with a monkeypatch of the distutils `spawn` that applies
-# command-argument-file hacks where it can. Might be cleaner.
def monkeypatch_unix_compiler():
"""Monkeypatching is dumb, but it's either that or we become maintainers of
something much, much bigger."""
- unixccompiler.UnixCCompiler.link = _unix_piecemeal_link
+ unixccompiler.UnixCCompiler.spawn = _unix_commandfile_spawn
diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py
index d804f34fc6..a00ce01110 100644
--- a/tools/distrib/python/grpcio_tools/setup.py
+++ b/tools/distrib/python/grpcio_tools/setup.py
@@ -61,6 +61,13 @@ EXTRA_COMPILE_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_CFLAGS',
EXTRA_LINK_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_LDFLAGS',
'-lpthread'))
+CC_FILES = [
+ os.path.normpath(cc_file) for cc_file in protoc_lib_deps.CC_FILES]
+PROTO_FILES = [
+ os.path.normpath(proto_file) for proto_file in protoc_lib_deps.PROTO_FILES]
+CC_INCLUDE = os.path.normpath(protoc_lib_deps.CC_INCLUDE)
+PROTO_INCLUDE = os.path.normpath(protoc_lib_deps.PROTO_INCLUDE)
+
GRPC_PYTHON_TOOLS_PACKAGE = 'grpc.tools'
GRPC_PYTHON_PROTO_RESOURCES_NAME = '_proto'
@@ -82,8 +89,8 @@ def package_data():
proto_resources_path = os.path.join(tools_path,
GRPC_PYTHON_PROTO_RESOURCES_NAME)
proto_files = []
- for proto_file in protoc_lib_deps.PROTO_FILES:
- source = os.path.join(protoc_lib_deps.PROTO_INCLUDE, proto_file)
+ for proto_file in PROTO_FILES:
+ source = os.path.join(PROTO_INCLUDE, proto_file)
target = os.path.join(proto_resources_path, proto_file)
relative_target = os.path.join(GRPC_PYTHON_PROTO_RESOURCES_NAME, proto_file)
try:
@@ -99,18 +106,20 @@ def package_data():
def protoc_ext_module():
plugin_sources = [
- 'grpc/tools/main.cc',
- 'grpc_root/src/compiler/python_generator.cc'] + [
- os.path.join(protoc_lib_deps.CC_INCLUDE, cc_file)
- for cc_file in protoc_lib_deps.CC_FILES]
+ os.path.join('grpc', 'tools', 'main.cc'),
+ os.path.join('grpc_root', 'src', 'compiler', 'python_generator.cc')] + [
+ os.path.join(CC_INCLUDE, cc_file)
+ for cc_file in CC_FILES]
plugin_ext = extension.Extension(
name='grpc.tools._protoc_compiler',
- sources=['grpc/tools/_protoc_compiler.pyx'] + plugin_sources,
+ sources=(
+ [os.path.join('grpc', 'tools', '_protoc_compiler.pyx')] +
+ plugin_sources),
include_dirs=[
'.',
'grpc_root',
- 'grpc_root/include',
- protoc_lib_deps.CC_INCLUDE,
+ os.path.join('grpc_root', 'include'),
+ CC_INCLUDE,
],
language='c++',
define_macros=list(DEFINE_MACROS),
diff --git a/tools/distrib/python/make_grpcio_tools.py b/tools/distrib/python/make_grpcio_tools.py
index fd9b38b084..adf58445af 100755
--- a/tools/distrib/python/make_grpcio_tools.py
+++ b/tools/distrib/python/make_grpcio_tools.py
@@ -29,12 +29,18 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+from __future__ import print_function
+
+import errno
+import filecmp
+import glob
import os
import os.path
import shutil
import subprocess
import sys
import traceback
+import uuid
DEPS_FILE_CONTENT="""
# Copyright 2016, Google Inc.
@@ -124,20 +130,88 @@ def get_deps():
proto_include=repr(GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT))
return deps_file_content
+def long_path(path):
+ if os.name == 'nt':
+ return '\\\\?\\' + path
+ else:
+ return path
+
+def atomic_file_copy(src, dst):
+ """Based on the lock-free-whack-a-mole algorithm, depending on filesystem
+ renaming being atomic. Described at http://stackoverflow.com/a/28090883.
+ """
+ try:
+ if filecmp.cmp(src, dst):
+ return
+ except:
+ pass
+ dst_dir = os.path.abspath(os.path.dirname(dst))
+ dst_base = os.path.basename(dst)
+ this_id = str(uuid.uuid4()).replace('.', '-')
+ temporary_file = os.path.join(dst_dir, '{}.{}.tmp'.format(dst_base, this_id))
+ mole_file = os.path.join(dst_dir, '{}.{}.mole.tmp'.format(dst_base, this_id))
+ mole_pattern = os.path.join(dst_dir, '{}.*.mole.tmp'.format(dst_base))
+ src = long_path(src)
+ dst = long_path(dst)
+ temporary_file = long_path(temporary_file)
+ mole_file = long_path(mole_file)
+ mole_pattern = long_path(mole_pattern)
+ shutil.copy2(src, temporary_file)
+ try:
+ os.rename(temporary_file, mole_file)
+ except:
+ print('Error moving temporary file {} to {}'.format(temporary_file, mole_file), file=sys.stderr)
+ print('while trying to copy file {} to {}'.format(src, dst), file=sys.stderr)
+ raise
+ for other_file in glob.glob(mole_pattern):
+ other_id = other_file.split('.')[-3]
+ if this_id == other_id:
+ pass
+ elif this_id < other_id:
+ try:
+ os.remove(other_file)
+ except:
+ pass
+ else:
+ try:
+ os.remove(mole_file)
+ except:
+ pass
+ this_id = other_id
+ mole_file = other_file
+ try:
+ if filecmp.cmp(src, dst):
+ try:
+ os.remove(mole_file)
+ except:
+ pass
+ return
+ except:
+ pass
+ try:
+ os.rename(mole_file, dst)
+ except:
+ pass
+
def main():
os.chdir(GRPC_ROOT)
- for tree in [GRPC_PYTHON_PROTOBUF,
- GRPC_PYTHON_PROTOC_PLUGINS,
- GRPC_PYTHON_INCLUDE]:
- try:
- shutil.rmtree(tree)
- except Exception as _:
- pass
- shutil.copytree(GRPC_PROTOBUF, GRPC_PYTHON_PROTOBUF)
- shutil.copytree(GRPC_PROTOC_PLUGINS, GRPC_PYTHON_PROTOC_PLUGINS)
- shutil.copytree(GRPC_INCLUDE, GRPC_PYTHON_INCLUDE)
+ for source, target in [
+ (GRPC_PROTOBUF, GRPC_PYTHON_PROTOBUF),
+ (GRPC_PROTOC_PLUGINS, GRPC_PYTHON_PROTOC_PLUGINS),
+ (GRPC_INCLUDE, GRPC_PYTHON_INCLUDE)]:
+ for source_dir, _, files in os.walk(source):
+ target_dir = os.path.abspath(os.path.join(target, os.path.relpath(source_dir, source)))
+ try:
+ os.makedirs(target_dir)
+ except OSError as error:
+ if error.errno != errno.EEXIST:
+ raise
+ for relative_file in files:
+ source_file = os.path.abspath(os.path.join(source_dir, relative_file))
+ target_file = os.path.abspath(os.path.join(target_dir, relative_file))
+ atomic_file_copy(source_file, target_file)
try:
protoc_lib_deps_content = get_deps()
diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifact_targets.py
index bd1269ceb7..b145e0392c 100644
--- a/tools/run_tests/artifact_targets.py
+++ b/tools/run_tests/artifact_targets.py
@@ -30,11 +30,14 @@
"""Definition of targets to build artifacts."""
+import os.path
+import sys
+
import jobset
def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={},
- flake_retries=0, timeout_retries=0):
+ flake_retries=0, timeout_retries=0, timeout_seconds=30*60):
"""Creates jobspec for a task running under docker."""
environ = environ.copy()
environ['RUN_COMMAND'] = shell_command
@@ -49,20 +52,20 @@ def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={},
cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] + docker_args,
environ=docker_env,
shortname='build_artifact.%s' % (name),
- timeout_seconds=30*60,
+ timeout_seconds=timeout_seconds,
flake_retries=flake_retries,
timeout_retries=timeout_retries)
return jobspec
def create_jobspec(name, cmdline, environ=None, shell=False,
- flake_retries=0, timeout_retries=0):
+ flake_retries=0, timeout_retries=0, timeout_seconds=30*60):
"""Creates jobspec."""
jobspec = jobset.JobSpec(
cmdline=cmdline,
environ=environ,
shortname='build_artifact.%s' % (name),
- timeout_seconds=30*60,
+ timeout_seconds=timeout_seconds,
flake_retries=flake_retries,
timeout_retries=timeout_retries,
shell=shell)
@@ -76,27 +79,30 @@ _ARCH_FLAG_MAP = {
'x64': '-m64'
}
-python_version_arch_map = {
- 'x86': 'Python27_32bits',
- 'x64': 'Python27'
+python_windows_version_arch_map = {
+ ('x86', '2.7'): 'Python27_32bits',
+ ('x64', '2.7'): 'Python27',
+ ('x86', '3.4'): 'Python34_32bits',
+ ('x64', '3.4'): 'Python34',
}
class PythonArtifact:
"""Builds Python artifacts."""
- def __init__(self, platform, arch, manylinux_build=None):
+ def __init__(self, platform, arch, python_version, manylinux_build=None):
if manylinux_build:
- self.name = 'python_%s_%s_%s' % (platform, arch, manylinux_build)
+ self.name = 'python%s_%s_%s_%s' % (python_version, platform, arch, manylinux_build)
else:
- self.name = 'python_%s_%s' % (platform, arch)
+ self.name = 'python%s_%s_%s' % (python_version, platform, arch)
self.platform = platform
self.arch = arch
- self.labels = ['artifact', 'python', platform, arch]
- self.python_version = python_version_arch_map[arch]
+ self.labels = ['artifact', 'python', python_version, platform, arch]
+ self.python_version = python_version
+ self.python_windows_prefix = python_windows_version_arch_map[arch, python_version]
self.manylinux_build = manylinux_build
def pre_build_jobspecs(self):
- return []
+ return []
def build_jobspec(self):
environ = {}
@@ -107,7 +113,6 @@ class PythonArtifact:
# special places...
environ['PYTHON'] = '/opt/python/{}/bin/python'.format(self.manylinux_build)
environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.manylinux_build)
- # Our docker image has all the prerequisites pip-installed already.
environ['SKIP_PIP_INSTALL'] = '1'
# Platform autodetection for the manylinux1 image breaks so we set the
# defines ourselves.
@@ -117,16 +122,18 @@ class PythonArtifact:
return create_docker_jobspec(self.name,
'tools/dockerfile/grpc_artifact_python_manylinux_%s' % self.arch,
'tools/run_tests/build_artifact_python.sh',
- environ=environ)
+ environ=environ,
+ timeout_seconds=60*60)
elif self.platform == 'windows':
return create_jobspec(self.name,
['tools\\run_tests\\build_artifact_python.bat',
- self.python_version,
+ self.python_windows_prefix,
'32' if self.arch == 'x86' else '64'
],
shell=True)
else:
environ['SKIP_PIP_INSTALL'] = 'TRUE'
+ environ['PYTHON'] = 'python{}'.format(self.python_version)
return create_jobspec(self.name,
['tools/run_tests/build_artifact_python.sh'],
environ=environ)
@@ -323,13 +330,18 @@ def targets():
for Cls in (CSharpExtArtifact, NodeExtArtifact, ProtocArtifact)
for platform in ('linux', 'macos', 'windows')
for arch in ('x86', 'x64')] +
- [PythonArtifact('linux', 'x86', 'cp27-cp27m'),
- PythonArtifact('linux', 'x86', 'cp27-cp27mu'),
- PythonArtifact('linux', 'x64', 'cp27-cp27m'),
- PythonArtifact('linux', 'x64', 'cp27-cp27mu'),
- PythonArtifact('macos', 'x64'),
- PythonArtifact('windows', 'x86'),
- PythonArtifact('windows', 'x64'),
+ [PythonArtifact('linux', 'x86', '2.7', 'cp27-cp27m'),
+ PythonArtifact('linux', 'x86', '2.7', 'cp27-cp27mu'),
+ PythonArtifact('linux', 'x64', '2.7', 'cp27-cp27m'),
+ PythonArtifact('linux', 'x64', '2.7', 'cp27-cp27mu'),
+ PythonArtifact('macos', 'x64', '2.7'),
+ PythonArtifact('windows', 'x86', '2.7'),
+ PythonArtifact('windows', 'x64', '2.7'),
+ PythonArtifact('linux', 'x86', '3.4', 'cp34-cp34m'),
+ PythonArtifact('linux', 'x64', '3.4', 'cp34-cp34m'),
+ PythonArtifact('macos', 'x64', '3.4'),
+ PythonArtifact('windows', 'x86', '3.4'),
+ PythonArtifact('windows', 'x64', '3.4'),
RubyArtifact('linux', 'x86'),
RubyArtifact('linux', 'x64'),
RubyArtifact('macos', 'x64'),
diff --git a/tools/run_tests/build_artifact_python.bat b/tools/run_tests/build_artifact_python.bat
index 7c8c2aa12d..d0b147b426 100644
--- a/tools/run_tests/build_artifact_python.bat
+++ b/tools/run_tests/build_artifact_python.bat
@@ -65,11 +65,19 @@ python tools\distrib\python\make_grpcio_tools.py
@rem Build gRPC Python extensions
python setup.py build_ext -c mingw32
-python tools\distrib\python\grpcio_tools\setup.py build_ext -c mingw32
+
+pushd tools\distrib\python\grpcio_tools
+python setup.py build_ext -c mingw32
+popd
+
@rem Build gRPC Python distributions
python setup.py bdist_wheel
-python tools\distrib\python\grpcio_tools\setup.py bdist_wheel
+
+pushd tools\distrib\python\grpcio_tools
+python setup.py bdist_wheel
+popd
+
mkdir artifacts
xcopy /Y /I /S dist\* artifacts\ || goto :error