diff options
| -rw-r--r-- | setup.py | 21 | ||||
| -rw-r--r-- | src/python/grpcio/_unixccompiler_patch.py | 102 | ||||
| -rw-r--r-- | tools/distrib/python/grpcio_tools/setup.py | 27 | ||||
| -rwxr-xr-x | tools/distrib/python/make_grpcio_tools.py | 94 | ||||
| -rw-r--r-- | tools/run_tests/artifact_targets.py | 58 | ||||
| -rw-r--r-- | tools/run_tests/build_artifact_python.bat | 12 |
6 files changed, 188 insertions, 126 deletions
@@ -47,10 +47,10 @@ from setuptools.command import egg_info egg_info.manifest_maker.template = 'PYTHON-MANIFEST.in' PY3 = sys.version_info.major == 3 -PYTHON_STEM = './src/python/grpcio' -CORE_INCLUDE = ('./include', '.',) -BORINGSSL_INCLUDE = ('./third_party/boringssl/include',) -ZLIB_INCLUDE = ('./third_party/zlib',) +PYTHON_STEM = os.path.join('src', 'python', 'grpcio') +CORE_INCLUDE = ('include', '.',) +BORINGSSL_INCLUDE = (os.path.join('third_party', 'boringssl', 'include'),) +ZLIB_INCLUDE = (os.path.join('third_party', 'zlib'),) # Ensure we're in the proper directory whether or not we're being used by pip. os.chdir(os.path.dirname(os.path.abspath(__file__))) @@ -62,8 +62,8 @@ import commands import grpc_core_dependencies import grpc_version -# TODO(atash) make this conditional on being on a mingw32 build -_unixccompiler_patch.monkeypatch_unix_compiler() +if 'win32' in sys.platform: + _unixccompiler_patch.monkeypatch_unix_compiler() LICENSE = '3-clause BSD' @@ -105,7 +105,9 @@ if not "win32" in sys.platform: if "win32" in sys.platform: EXTENSION_LIBRARIES += ('ws2_32',) -DEFINE_MACROS = (('OPENSSL_NO_ASM', 1), ('_WIN32_WINNT', 0x600), ('GPR_BACKWARDS_COMPATIBILITY_MODE', 1),) +DEFINE_MACROS = ( + ('OPENSSL_NO_ASM', 1), ('_WIN32_WINNT', 0x600), + ('GPR_BACKWARDS_COMPATIBILITY_MODE', 1),) if "win32" in sys.platform: DEFINE_MACROS += (('OPENSSL_WINDOWS', 1), ('WIN32_LEAN_AND_MEAN', 1),) if '64bit' in platform.architecture()[0]: @@ -200,12 +202,13 @@ COMMAND_CLASS = { } # Ensure that package data is copied over before any commands have been run: -credentials_dir = os.path.join(PYTHON_STEM, 'grpc/_cython/_credentials') +credentials_dir = os.path.join(PYTHON_STEM, 'grpc', '_cython', '_credentials') try: os.mkdir(credentials_dir) except OSError: pass -shutil.copyfile('etc/roots.pem', os.path.join(credentials_dir, 'roots.pem')) +shutil.copyfile(os.path.join('etc', 'roots.pem'), + os.path.join(credentials_dir, 'roots.pem')) PACKAGE_DATA = { # Binaries that may or may not be present in the final installation, but are diff --git a/src/python/grpcio/_unixccompiler_patch.py b/src/python/grpcio/_unixccompiler_patch.py index 9a697989b3..894c3ef395 100644 --- a/src/python/grpcio/_unixccompiler_patch.py +++ b/src/python/grpcio/_unixccompiler_patch.py @@ -34,88 +34,44 @@ from distutils import errors from distutils import unixccompiler import os import os.path +import shlex import shutil import sys import tempfile +def _unix_commandfile_spawn(self, command): + """Wrapper around distutils.util.spawn that attempts to use command files. -def _unix_piecemeal_link( - self, target_desc, objects, output_filename, output_dir=None, - libraries=None, library_dirs=None, runtime_library_dirs=None, - export_symbols=None, debug=0, extra_preargs=None, extra_postargs=None, - build_temp=None, target_lang=None): - """`link` externalized method taken almost verbatim from UnixCCompiler. + Meant to replace the CCompiler method `spawn` on UnixCCompiler and its + derivatives (e.g. the MinGW32 compiler). - Modifies the link command for unix-like compilers by using a command file so - that long command line argument strings don't break the command shell's - ARG_MAX character limit. + Some commands like `gcc` (and friends like `clang`) support command files to + work around shell command length limits. """ - objects, output_dir = self._fix_object_args(objects, output_dir) - libraries, library_dirs, runtime_library_dirs = self._fix_lib_args( - libraries, library_dirs, runtime_library_dirs) - # filter out standard library paths, which are not explicitely needed - # for linking - library_dirs = [dir for dir in library_dirs - if not dir in ('/lib', '/lib64', '/usr/lib', '/usr/lib64')] - runtime_library_dirs = [dir for dir in runtime_library_dirs - if not dir in ('/lib', '/lib64', '/usr/lib', '/usr/lib64')] - lib_opts = ccompiler.gen_lib_options(self, library_dirs, runtime_library_dirs, - libraries) - if (not (isinstance(output_dir, str) or isinstance(output_dir, bytes)) - and output_dir is not None): - raise TypeError("'output_dir' must be a string or None") - if output_dir is not None: - output_filename = os.path.join(output_dir, output_filename) - - if self._need_link(objects, output_filename): - ld_args = (objects + self.objects + - lib_opts + ['-o', output_filename]) - if debug: - ld_args[:0] = ['-g'] - if extra_preargs: - ld_args[:0] = extra_preargs - if extra_postargs: - ld_args.extend(extra_postargs) - self.mkpath(os.path.dirname(output_filename)) - try: - if target_desc == ccompiler.CCompiler.EXECUTABLE: - linker = self.linker_exe[:] - else: - linker = self.linker_so[:] - if target_lang == "c++" and self.compiler_cxx: - # skip over environment variable settings if /usr/bin/env - # is used to set up the linker's environment. - # This is needed on OSX. Note: this assumes that the - # normal and C++ compiler have the same environment - # settings. - i = 0 - if os.path.basename(linker[0]) == "env": - i = 1 - while '=' in linker[i]: - i = i + 1 - - linker[i] = self.compiler_cxx[i] - - if sys.platform == 'darwin': - import _osx_support - linker = _osx_support.compiler_fixup(linker, ld_args) - - temporary_directory = tempfile.mkdtemp() - command_filename = os.path.abspath( - os.path.join(temporary_directory, 'command')) - with open(command_filename, 'w') as command_file: - escaped_ld_args = [arg.replace('\\', '\\\\') for arg in ld_args] - command_file.write(' '.join(escaped_ld_args)) - self.spawn(linker + ['@{}'.format(command_filename)]) - except errors.DistutilsExecError: - raise ccompiler.LinkError + # Sometimes distutils embeds the executables as full strings including some + # hard-coded flags rather than as lists. + command = list(shlex.split(command[0])) + list(command[1:]) + command_base = os.path.basename(command[0].strip()) + if command_base == 'ccache': + command_base = command[:2] + command_args = command[2:] + elif command_base.startswith('ccache') or command_base in ['gcc', 'clang', 'clang++', 'g++']: + command_base = command[:1] + command_args = command[1:] else: - log.debug("skipping %s (up-to-date)", output_filename) + return ccompiler.CCompiler.spawn(self, command) + temporary_directory = tempfile.mkdtemp() + command_filename = os.path.abspath(os.path.join(temporary_directory, 'command')) + with open(command_filename, 'w') as command_file: + escaped_args = [arg.replace('\\', '\\\\') for arg in command_args] + command_file.write(' '.join(escaped_args)) + modified_command = command_base + ['@{}'.format(command_filename)] + result = ccompiler.CCompiler.spawn(self, modified_command) + shutil.rmtree(temporary_directory) + return result + -# TODO(atash) try replacing this monkeypatch of the compiler harness' link -# operation with a monkeypatch of the distutils `spawn` that applies -# command-argument-file hacks where it can. Might be cleaner. def monkeypatch_unix_compiler(): """Monkeypatching is dumb, but it's either that or we become maintainers of something much, much bigger.""" - unixccompiler.UnixCCompiler.link = _unix_piecemeal_link + unixccompiler.UnixCCompiler.spawn = _unix_commandfile_spawn diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index d804f34fc6..a00ce01110 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -61,6 +61,13 @@ EXTRA_COMPILE_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_CFLAGS', EXTRA_LINK_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_LDFLAGS', '-lpthread')) +CC_FILES = [ + os.path.normpath(cc_file) for cc_file in protoc_lib_deps.CC_FILES] +PROTO_FILES = [ + os.path.normpath(proto_file) for proto_file in protoc_lib_deps.PROTO_FILES] +CC_INCLUDE = os.path.normpath(protoc_lib_deps.CC_INCLUDE) +PROTO_INCLUDE = os.path.normpath(protoc_lib_deps.PROTO_INCLUDE) + GRPC_PYTHON_TOOLS_PACKAGE = 'grpc.tools' GRPC_PYTHON_PROTO_RESOURCES_NAME = '_proto' @@ -82,8 +89,8 @@ def package_data(): proto_resources_path = os.path.join(tools_path, GRPC_PYTHON_PROTO_RESOURCES_NAME) proto_files = [] - for proto_file in protoc_lib_deps.PROTO_FILES: - source = os.path.join(protoc_lib_deps.PROTO_INCLUDE, proto_file) + for proto_file in PROTO_FILES: + source = os.path.join(PROTO_INCLUDE, proto_file) target = os.path.join(proto_resources_path, proto_file) relative_target = os.path.join(GRPC_PYTHON_PROTO_RESOURCES_NAME, proto_file) try: @@ -99,18 +106,20 @@ def package_data(): def protoc_ext_module(): plugin_sources = [ - 'grpc/tools/main.cc', - 'grpc_root/src/compiler/python_generator.cc'] + [ - os.path.join(protoc_lib_deps.CC_INCLUDE, cc_file) - for cc_file in protoc_lib_deps.CC_FILES] + os.path.join('grpc', 'tools', 'main.cc'), + os.path.join('grpc_root', 'src', 'compiler', 'python_generator.cc')] + [ + os.path.join(CC_INCLUDE, cc_file) + for cc_file in CC_FILES] plugin_ext = extension.Extension( name='grpc.tools._protoc_compiler', - sources=['grpc/tools/_protoc_compiler.pyx'] + plugin_sources, + sources=( + [os.path.join('grpc', 'tools', '_protoc_compiler.pyx')] + + plugin_sources), include_dirs=[ '.', 'grpc_root', - 'grpc_root/include', - protoc_lib_deps.CC_INCLUDE, + os.path.join('grpc_root', 'include'), + CC_INCLUDE, ], language='c++', define_macros=list(DEFINE_MACROS), diff --git a/tools/distrib/python/make_grpcio_tools.py b/tools/distrib/python/make_grpcio_tools.py index fd9b38b084..adf58445af 100755 --- a/tools/distrib/python/make_grpcio_tools.py +++ b/tools/distrib/python/make_grpcio_tools.py @@ -29,12 +29,18 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from __future__ import print_function + +import errno +import filecmp +import glob import os import os.path import shutil import subprocess import sys import traceback +import uuid DEPS_FILE_CONTENT=""" # Copyright 2016, Google Inc. @@ -124,20 +130,88 @@ def get_deps(): proto_include=repr(GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT)) return deps_file_content +def long_path(path): + if os.name == 'nt': + return '\\\\?\\' + path + else: + return path + +def atomic_file_copy(src, dst): + """Based on the lock-free-whack-a-mole algorithm, depending on filesystem + renaming being atomic. Described at http://stackoverflow.com/a/28090883. + """ + try: + if filecmp.cmp(src, dst): + return + except: + pass + dst_dir = os.path.abspath(os.path.dirname(dst)) + dst_base = os.path.basename(dst) + this_id = str(uuid.uuid4()).replace('.', '-') + temporary_file = os.path.join(dst_dir, '{}.{}.tmp'.format(dst_base, this_id)) + mole_file = os.path.join(dst_dir, '{}.{}.mole.tmp'.format(dst_base, this_id)) + mole_pattern = os.path.join(dst_dir, '{}.*.mole.tmp'.format(dst_base)) + src = long_path(src) + dst = long_path(dst) + temporary_file = long_path(temporary_file) + mole_file = long_path(mole_file) + mole_pattern = long_path(mole_pattern) + shutil.copy2(src, temporary_file) + try: + os.rename(temporary_file, mole_file) + except: + print('Error moving temporary file {} to {}'.format(temporary_file, mole_file), file=sys.stderr) + print('while trying to copy file {} to {}'.format(src, dst), file=sys.stderr) + raise + for other_file in glob.glob(mole_pattern): + other_id = other_file.split('.')[-3] + if this_id == other_id: + pass + elif this_id < other_id: + try: + os.remove(other_file) + except: + pass + else: + try: + os.remove(mole_file) + except: + pass + this_id = other_id + mole_file = other_file + try: + if filecmp.cmp(src, dst): + try: + os.remove(mole_file) + except: + pass + return + except: + pass + try: + os.rename(mole_file, dst) + except: + pass + def main(): os.chdir(GRPC_ROOT) - for tree in [GRPC_PYTHON_PROTOBUF, - GRPC_PYTHON_PROTOC_PLUGINS, - GRPC_PYTHON_INCLUDE]: - try: - shutil.rmtree(tree) - except Exception as _: - pass - shutil.copytree(GRPC_PROTOBUF, GRPC_PYTHON_PROTOBUF) - shutil.copytree(GRPC_PROTOC_PLUGINS, GRPC_PYTHON_PROTOC_PLUGINS) - shutil.copytree(GRPC_INCLUDE, GRPC_PYTHON_INCLUDE) + for source, target in [ + (GRPC_PROTOBUF, GRPC_PYTHON_PROTOBUF), + (GRPC_PROTOC_PLUGINS, GRPC_PYTHON_PROTOC_PLUGINS), + (GRPC_INCLUDE, GRPC_PYTHON_INCLUDE)]: + for source_dir, _, files in os.walk(source): + target_dir = os.path.abspath(os.path.join(target, os.path.relpath(source_dir, source))) + try: + os.makedirs(target_dir) + except OSError as error: + if error.errno != errno.EEXIST: + raise + for relative_file in files: + source_file = os.path.abspath(os.path.join(source_dir, relative_file)) + target_file = os.path.abspath(os.path.join(target_dir, relative_file)) + atomic_file_copy(source_file, target_file) try: protoc_lib_deps_content = get_deps() diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifact_targets.py index bd1269ceb7..b145e0392c 100644 --- a/tools/run_tests/artifact_targets.py +++ b/tools/run_tests/artifact_targets.py @@ -30,11 +30,14 @@ """Definition of targets to build artifacts.""" +import os.path +import sys + import jobset def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, - flake_retries=0, timeout_retries=0): + flake_retries=0, timeout_retries=0, timeout_seconds=30*60): """Creates jobspec for a task running under docker.""" environ = environ.copy() environ['RUN_COMMAND'] = shell_command @@ -49,20 +52,20 @@ def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] + docker_args, environ=docker_env, shortname='build_artifact.%s' % (name), - timeout_seconds=30*60, + timeout_seconds=timeout_seconds, flake_retries=flake_retries, timeout_retries=timeout_retries) return jobspec def create_jobspec(name, cmdline, environ=None, shell=False, - flake_retries=0, timeout_retries=0): + flake_retries=0, timeout_retries=0, timeout_seconds=30*60): """Creates jobspec.""" jobspec = jobset.JobSpec( cmdline=cmdline, environ=environ, shortname='build_artifact.%s' % (name), - timeout_seconds=30*60, + timeout_seconds=timeout_seconds, flake_retries=flake_retries, timeout_retries=timeout_retries, shell=shell) @@ -76,27 +79,30 @@ _ARCH_FLAG_MAP = { 'x64': '-m64' } -python_version_arch_map = { - 'x86': 'Python27_32bits', - 'x64': 'Python27' +python_windows_version_arch_map = { + ('x86', '2.7'): 'Python27_32bits', + ('x64', '2.7'): 'Python27', + ('x86', '3.4'): 'Python34_32bits', + ('x64', '3.4'): 'Python34', } class PythonArtifact: """Builds Python artifacts.""" - def __init__(self, platform, arch, manylinux_build=None): + def __init__(self, platform, arch, python_version, manylinux_build=None): if manylinux_build: - self.name = 'python_%s_%s_%s' % (platform, arch, manylinux_build) + self.name = 'python%s_%s_%s_%s' % (python_version, platform, arch, manylinux_build) else: - self.name = 'python_%s_%s' % (platform, arch) + self.name = 'python%s_%s_%s' % (python_version, platform, arch) self.platform = platform self.arch = arch - self.labels = ['artifact', 'python', platform, arch] - self.python_version = python_version_arch_map[arch] + self.labels = ['artifact', 'python', python_version, platform, arch] + self.python_version = python_version + self.python_windows_prefix = python_windows_version_arch_map[arch, python_version] self.manylinux_build = manylinux_build def pre_build_jobspecs(self): - return [] + return [] def build_jobspec(self): environ = {} @@ -107,7 +113,6 @@ class PythonArtifact: # special places... environ['PYTHON'] = '/opt/python/{}/bin/python'.format(self.manylinux_build) environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.manylinux_build) - # Our docker image has all the prerequisites pip-installed already. environ['SKIP_PIP_INSTALL'] = '1' # Platform autodetection for the manylinux1 image breaks so we set the # defines ourselves. @@ -117,16 +122,18 @@ class PythonArtifact: return create_docker_jobspec(self.name, 'tools/dockerfile/grpc_artifact_python_manylinux_%s' % self.arch, 'tools/run_tests/build_artifact_python.sh', - environ=environ) + environ=environ, + timeout_seconds=60*60) elif self.platform == 'windows': return create_jobspec(self.name, ['tools\\run_tests\\build_artifact_python.bat', - self.python_version, + self.python_windows_prefix, '32' if self.arch == 'x86' else '64' ], shell=True) else: environ['SKIP_PIP_INSTALL'] = 'TRUE' + environ['PYTHON'] = 'python{}'.format(self.python_version) return create_jobspec(self.name, ['tools/run_tests/build_artifact_python.sh'], environ=environ) @@ -323,13 +330,18 @@ def targets(): for Cls in (CSharpExtArtifact, NodeExtArtifact, ProtocArtifact) for platform in ('linux', 'macos', 'windows') for arch in ('x86', 'x64')] + - [PythonArtifact('linux', 'x86', 'cp27-cp27m'), - PythonArtifact('linux', 'x86', 'cp27-cp27mu'), - PythonArtifact('linux', 'x64', 'cp27-cp27m'), - PythonArtifact('linux', 'x64', 'cp27-cp27mu'), - PythonArtifact('macos', 'x64'), - PythonArtifact('windows', 'x86'), - PythonArtifact('windows', 'x64'), + [PythonArtifact('linux', 'x86', '2.7', 'cp27-cp27m'), + PythonArtifact('linux', 'x86', '2.7', 'cp27-cp27mu'), + PythonArtifact('linux', 'x64', '2.7', 'cp27-cp27m'), + PythonArtifact('linux', 'x64', '2.7', 'cp27-cp27mu'), + PythonArtifact('macos', 'x64', '2.7'), + PythonArtifact('windows', 'x86', '2.7'), + PythonArtifact('windows', 'x64', '2.7'), + PythonArtifact('linux', 'x86', '3.4', 'cp34-cp34m'), + PythonArtifact('linux', 'x64', '3.4', 'cp34-cp34m'), + PythonArtifact('macos', 'x64', '3.4'), + PythonArtifact('windows', 'x86', '3.4'), + PythonArtifact('windows', 'x64', '3.4'), RubyArtifact('linux', 'x86'), RubyArtifact('linux', 'x64'), RubyArtifact('macos', 'x64'), diff --git a/tools/run_tests/build_artifact_python.bat b/tools/run_tests/build_artifact_python.bat index 7c8c2aa12d..d0b147b426 100644 --- a/tools/run_tests/build_artifact_python.bat +++ b/tools/run_tests/build_artifact_python.bat @@ -65,11 +65,19 @@ python tools\distrib\python\make_grpcio_tools.py @rem Build gRPC Python extensions python setup.py build_ext -c mingw32 -python tools\distrib\python\grpcio_tools\setup.py build_ext -c mingw32 + +pushd tools\distrib\python\grpcio_tools +python setup.py build_ext -c mingw32 +popd + @rem Build gRPC Python distributions python setup.py bdist_wheel -python tools\distrib\python\grpcio_tools\setup.py bdist_wheel + +pushd tools\distrib\python\grpcio_tools +python setup.py bdist_wheel +popd + mkdir artifacts xcopy /Y /I /S dist\* artifacts\ || goto :error |