diff options
Diffstat (limited to 'tools')
29 files changed, 307 insertions, 90 deletions
diff --git a/tools/distrib/python/docgen.py b/tools/distrib/python/docgen.py index f5e89f1da6..15bd8d855f 100755 --- a/tools/distrib/python/docgen.py +++ b/tools/distrib/python/docgen.py @@ -71,6 +71,8 @@ environment.update({ subprocess_arguments_list = [ {'args': ['virtualenv', VIRTUALENV_DIR], 'env': environment}, + {'args': [VIRTUALENV_PIP_PATH, 'install', '--upgrade', 'pip'], + 'env': environment}, {'args': [VIRTUALENV_PIP_PATH, 'install', '-r', REQUIREMENTS_PATH], 'env': environment}, {'args': [VIRTUALENV_PYTHON_PATH, SETUP_PATH, 'build'], 'env': environment}, diff --git a/tools/distrib/python/grpcio_tools/README.rst b/tools/distrib/python/grpcio_tools/README.rst index 8946a1d5b3..55521d17bb 100644 --- a/tools/distrib/python/grpcio_tools/README.rst +++ b/tools/distrib/python/grpcio_tools/README.rst @@ -122,6 +122,7 @@ Help, I ... third_party/protobuf/src/google/protobuf/stubs/mathlimits.h:173:31: note: in expansion of macro 'SIGNED_INT_MAX' static const Type kPosMax = SIGNED_INT_MAX(Type); \\ ^ + And your toolchain is GCC (at the time of this writing, up through at least GCC 6.0), this is probably a bug where GCC chokes on constant expressions when the :code:`-fwrapv` flag is specified. You should consider setting your @@ -136,3 +137,42 @@ Given protobuf include directories :code:`$INCLUDE`, an output directory :: $ python -m grpc.tools.protoc -I$INCLUDE --python_out=$OUTPUT --grpc_python_out=$OUTPUT $PROTO_FILES + +To use as a build step in distutils-based projects, you may use the provided +command class in your :code:`setup.py`: + +:: + + setuptools.setup( + # ... + cmdclass={ + 'build_proto_modules': grpc.tools.command.BuildPackageProtos, + } + # ... + ) + +Invocation of the command will walk the project tree and transpile every +:code:`.proto` file into a :code:`_pb2.py` file in the same directory. + +Note that this particular approach requires :code:`grpcio-tools` to be +installed on the machine before the setup script is invoked (i.e. no +combination of :code:`setup_requires` or :code:`install_requires` will provide +access to :code:`grpc.tools.command.BuildPackageProtos` if it isn't already +installed). One way to work around this can be found in our +:code:`grpcio-health-checking` +`package <https://pypi.python.org/pypi/grpcio-health-checking>`_: + +:: + + class BuildPackageProtos(setuptools.Command): + """Command to generate project *_pb2.py modules from proto files.""" + # ... + def run(self): + from grpc.tools import command + command.build_package_protos(self.distribution.package_dir['']) + +Now including :code:`grpcio-tools` in :code:`setup_requires` will provide the +command on-setup as desired. + +For more information on command classes, consult :code:`distutils` and +:code:`setuptools` documentation. diff --git a/tools/distrib/python/grpcio_tools/grpc/tools/command.py b/tools/distrib/python/grpcio_tools/grpc/tools/command.py index ccf38b7d56..2520099835 100644 --- a/tools/distrib/python/grpcio_tools/grpc/tools/command.py +++ b/tools/distrib/python/grpcio_tools/grpc/tools/command.py @@ -35,7 +35,26 @@ import setuptools from grpc.tools import protoc -class BuildProtoModules(setuptools.Command): +def build_package_protos(package_root): + proto_files = [] + inclusion_root = os.path.abspath(package_root) + for root, _, files in os.walk(inclusion_root): + for filename in files: + if filename.endswith('.proto'): + proto_files.append(os.path.abspath(os.path.join(root, filename))) + + for proto_file in proto_files: + command = [ + 'grpc.tools.protoc', + '--proto_path={}'.format(inclusion_root), + '--python_out={}'.format(inclusion_root), + '--grpc_python_out={}'.format(inclusion_root), + ] + [proto_file] + if protoc.main(command) != 0: + sys.stderr.write('warning: {} failed'.format(command)) + + +class BuildPackageProtos(setuptools.Command): """Command to generate project *_pb2.py modules from proto files.""" description = 'build grpc protobuf modules' @@ -52,19 +71,4 @@ class BuildProtoModules(setuptools.Command): # directory is provided as an 'include' directory. We assume it's the '' key # to `self.distribution.package_dir` (and get a key error if it's not # there). - proto_files = [] - inclusion_root = os.path.abspath(self.distribution.package_dir['']) - for root, _, files in os.walk(inclusion_root): - for filename in files: - if filename.endswith('.proto'): - proto_files.append(os.path.abspath(os.path.join(root, filename))) - - for proto_file in proto_files: - command = [ - 'grpc.tools.protoc', - '--proto_path={}'.format(inclusion_root), - '--python_out={}'.format(inclusion_root), - '--grpc_python_out={}'.format(inclusion_root), - ] + [proto_file] - if protoc.main(command) != 0: - sys.stderr.write('warning: {} failed'.format(command)) + build_package_protos(self.distribution.package_dir['']) diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index bb2c71d843..bb1f1cf085 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -28,11 +28,13 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from distutils import extension +from distutils import util import errno import os import os.path import pkg_resources import platform +import re import shlex import shutil import sys @@ -89,6 +91,13 @@ if EXTRA_ENV_LINK_ARGS is None: EXTRA_COMPILE_ARGS = shlex.split(EXTRA_ENV_COMPILE_ARGS) EXTRA_LINK_ARGS = shlex.split(EXTRA_ENV_LINK_ARGS) +CC_FILES = [ + os.path.normpath(cc_file) for cc_file in protoc_lib_deps.CC_FILES] +PROTO_FILES = [ + os.path.normpath(proto_file) for proto_file in protoc_lib_deps.PROTO_FILES] +CC_INCLUDE = os.path.normpath(protoc_lib_deps.CC_INCLUDE) +PROTO_INCLUDE = os.path.normpath(protoc_lib_deps.PROTO_INCLUDE) + GRPC_PYTHON_TOOLS_PACKAGE = 'grpc.tools' GRPC_PYTHON_PROTO_RESOURCES_NAME = '_proto' @@ -104,14 +113,18 @@ if 'darwin' in sys.platform and PY3: if mac_target and (pkg_resources.parse_version(mac_target) < pkg_resources.parse_version('10.9.0')): os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.9' + os.environ['_PYTHON_HOST_PLATFORM'] = re.sub( + r'macosx-[0-9]+\.[0-9]+-(.+)', + r'macosx-10.9-\1', + util.get_platform()) def package_data(): tools_path = GRPC_PYTHON_TOOLS_PACKAGE.replace('.', os.path.sep) proto_resources_path = os.path.join(tools_path, GRPC_PYTHON_PROTO_RESOURCES_NAME) proto_files = [] - for proto_file in protoc_lib_deps.PROTO_FILES: - source = os.path.join(protoc_lib_deps.PROTO_INCLUDE, proto_file) + for proto_file in PROTO_FILES: + source = os.path.join(PROTO_INCLUDE, proto_file) target = os.path.join(proto_resources_path, proto_file) relative_target = os.path.join(GRPC_PYTHON_PROTO_RESOURCES_NAME, proto_file) try: @@ -127,22 +140,22 @@ def package_data(): def extension_modules(): if BUILD_WITH_CYTHON: - plugin_sources = ['grpc/tools/_protoc_compiler.pyx'] + plugin_sources = [os.path.join('grpc', 'tools', '_protoc_compiler.pyx')] else: - plugin_sources = ['grpc/tools/_protoc_compiler.cpp'] + plugin_sources = [os.path.join('grpc', 'tools', '_protoc_compiler.cpp')] plugin_sources += [ - 'grpc/tools/main.cc', - 'grpc_root/src/compiler/python_generator.cc'] + [ - os.path.join(protoc_lib_deps.CC_INCLUDE, cc_file) - for cc_file in protoc_lib_deps.CC_FILES] + os.path.join('grpc', 'tools', 'main.cc'), + os.path.join('grpc_root', 'src', 'compiler', 'python_generator.cc')] + [ + os.path.join(CC_INCLUDE, cc_file) + for cc_file in CC_FILES] plugin_ext = extension.Extension( name='grpc.tools._protoc_compiler', sources=plugin_sources, include_dirs=[ '.', 'grpc_root', - 'grpc_root/include', - protoc_lib_deps.CC_INCLUDE, + os.path.join('grpc_root', 'include'), + CC_INCLUDE, ], language='c++', define_macros=list(DEFINE_MACROS), diff --git a/tools/distrib/python/make_grpcio_tools.py b/tools/distrib/python/make_grpcio_tools.py index fd9b38b084..adf58445af 100755 --- a/tools/distrib/python/make_grpcio_tools.py +++ b/tools/distrib/python/make_grpcio_tools.py @@ -29,12 +29,18 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from __future__ import print_function + +import errno +import filecmp +import glob import os import os.path import shutil import subprocess import sys import traceback +import uuid DEPS_FILE_CONTENT=""" # Copyright 2016, Google Inc. @@ -124,20 +130,88 @@ def get_deps(): proto_include=repr(GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT)) return deps_file_content +def long_path(path): + if os.name == 'nt': + return '\\\\?\\' + path + else: + return path + +def atomic_file_copy(src, dst): + """Based on the lock-free-whack-a-mole algorithm, depending on filesystem + renaming being atomic. Described at http://stackoverflow.com/a/28090883. + """ + try: + if filecmp.cmp(src, dst): + return + except: + pass + dst_dir = os.path.abspath(os.path.dirname(dst)) + dst_base = os.path.basename(dst) + this_id = str(uuid.uuid4()).replace('.', '-') + temporary_file = os.path.join(dst_dir, '{}.{}.tmp'.format(dst_base, this_id)) + mole_file = os.path.join(dst_dir, '{}.{}.mole.tmp'.format(dst_base, this_id)) + mole_pattern = os.path.join(dst_dir, '{}.*.mole.tmp'.format(dst_base)) + src = long_path(src) + dst = long_path(dst) + temporary_file = long_path(temporary_file) + mole_file = long_path(mole_file) + mole_pattern = long_path(mole_pattern) + shutil.copy2(src, temporary_file) + try: + os.rename(temporary_file, mole_file) + except: + print('Error moving temporary file {} to {}'.format(temporary_file, mole_file), file=sys.stderr) + print('while trying to copy file {} to {}'.format(src, dst), file=sys.stderr) + raise + for other_file in glob.glob(mole_pattern): + other_id = other_file.split('.')[-3] + if this_id == other_id: + pass + elif this_id < other_id: + try: + os.remove(other_file) + except: + pass + else: + try: + os.remove(mole_file) + except: + pass + this_id = other_id + mole_file = other_file + try: + if filecmp.cmp(src, dst): + try: + os.remove(mole_file) + except: + pass + return + except: + pass + try: + os.rename(mole_file, dst) + except: + pass + def main(): os.chdir(GRPC_ROOT) - for tree in [GRPC_PYTHON_PROTOBUF, - GRPC_PYTHON_PROTOC_PLUGINS, - GRPC_PYTHON_INCLUDE]: - try: - shutil.rmtree(tree) - except Exception as _: - pass - shutil.copytree(GRPC_PROTOBUF, GRPC_PYTHON_PROTOBUF) - shutil.copytree(GRPC_PROTOC_PLUGINS, GRPC_PYTHON_PROTOC_PLUGINS) - shutil.copytree(GRPC_INCLUDE, GRPC_PYTHON_INCLUDE) + for source, target in [ + (GRPC_PROTOBUF, GRPC_PYTHON_PROTOBUF), + (GRPC_PROTOC_PLUGINS, GRPC_PYTHON_PROTOC_PLUGINS), + (GRPC_INCLUDE, GRPC_PYTHON_INCLUDE)]: + for source_dir, _, files in os.walk(source): + target_dir = os.path.abspath(os.path.join(target, os.path.relpath(source_dir, source))) + try: + os.makedirs(target_dir) + except OSError as error: + if error.errno != errno.EEXIST: + raise + for relative_file in files: + source_file = os.path.abspath(os.path.join(source_dir, relative_file)) + target_file = os.path.abspath(os.path.join(target_dir, relative_file)) + atomic_file_copy(source_file, target_file) try: protoc_lib_deps_content = get_deps() diff --git a/tools/dockerfile/distribtest/python_arch_x64/Dockerfile b/tools/dockerfile/distribtest/python_arch_x64/Dockerfile index 2f79cc3017..dff72eee97 100644 --- a/tools/dockerfile/distribtest/python_arch_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_arch_x64/Dockerfile @@ -33,4 +33,4 @@ RUN pacman --noconfirm -Syy RUN pacman --noconfirm -S openssl RUN pacman --noconfirm -S python2 RUN pacman --noconfirm -S python2-pip - +RUN pip install virtualenv diff --git a/tools/dockerfile/distribtest/python_centos6_x64/Dockerfile b/tools/dockerfile/distribtest/python_centos6_x64/Dockerfile index d4f473792e..967450156c 100644 --- a/tools/dockerfile/distribtest/python_centos6_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_centos6_x64/Dockerfile @@ -44,3 +44,5 @@ RUN curl https://bootstrap.pypa.io/get-pip.py | python - # "which" command required by python's run_distrib_test.sh RUN yum install -y which + +RUN pip install virtualenv diff --git a/tools/dockerfile/distribtest/python_centos7_x64/Dockerfile b/tools/dockerfile/distribtest/python_centos7_x64/Dockerfile index ca64fa7bea..0127fe1e28 100644 --- a/tools/dockerfile/distribtest/python_centos7_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_centos7_x64/Dockerfile @@ -32,4 +32,4 @@ FROM centos:7 RUN yum install -y python RUN yum install -y epel-release RUN yum install -y python-pip - +RUN pip install virtualenv diff --git a/tools/dockerfile/distribtest/python_fedora20_x64/Dockerfile b/tools/dockerfile/distribtest/python_fedora20_x64/Dockerfile index 8b0f769c26..3d3636e43d 100644 --- a/tools/dockerfile/distribtest/python_fedora20_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_fedora20_x64/Dockerfile @@ -35,3 +35,5 @@ RUN yum clean all && yum update -y && yum install -y python python-pip # Trying twice makes it work fine. # https://github.com/docker/docker/issues/10180 RUN pip install --upgrade six || pip install --upgrade six + +RUN pip install virtualenv diff --git a/tools/dockerfile/distribtest/python_fedora21_x64/Dockerfile b/tools/dockerfile/distribtest/python_fedora21_x64/Dockerfile index fcbe053f1f..0b1b6aeb35 100644 --- a/tools/dockerfile/distribtest/python_fedora21_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_fedora21_x64/Dockerfile @@ -40,3 +40,5 @@ RUN yum clean all && yum update -y && yum install -y python python-pip # Trying twice makes it work fine. # https://github.com/docker/docker/issues/10180 RUN pip2 install --upgrade six || pip2 install --upgrade six + +RUN pip2 install virtualenv diff --git a/tools/dockerfile/distribtest/python_fedora22_x64/Dockerfile b/tools/dockerfile/distribtest/python_fedora22_x64/Dockerfile index ddcacb4257..4d75034c15 100644 --- a/tools/dockerfile/distribtest/python_fedora22_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_fedora22_x64/Dockerfile @@ -30,3 +30,4 @@ FROM fedora:22 RUN yum clean all && yum update -y && yum install -y python python-pip +RUN pip install virtualenv diff --git a/tools/dockerfile/distribtest/python_fedora23_x64/Dockerfile b/tools/dockerfile/distribtest/python_fedora23_x64/Dockerfile index d45195e509..a1bc9ba8d6 100644 --- a/tools/dockerfile/distribtest/python_fedora23_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_fedora23_x64/Dockerfile @@ -30,3 +30,4 @@ FROM fedora:23 RUN yum clean all && yum update -y && yum install -y python python-pip +RUN pip install virtualenv diff --git a/tools/dockerfile/distribtest/python_jessie_x64/Dockerfile b/tools/dockerfile/distribtest/python_jessie_x64/Dockerfile index 83df4ed4fa..7dc32a088e 100644 --- a/tools/dockerfile/distribtest/python_jessie_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_jessie_x64/Dockerfile @@ -30,3 +30,4 @@ FROM debian:jessie RUN apt-get update && apt-get install -y python python-pip +RUN pip install virtualenv diff --git a/tools/dockerfile/distribtest/python_jessie_x86/Dockerfile b/tools/dockerfile/distribtest/python_jessie_x86/Dockerfile index 19addb2912..04c1402e72 100644 --- a/tools/dockerfile/distribtest/python_jessie_x86/Dockerfile +++ b/tools/dockerfile/distribtest/python_jessie_x86/Dockerfile @@ -31,6 +31,8 @@ FROM 32bit/debian:jessie RUN apt-get update && apt-get install -y python python-pip +RUN pip install virtualenv + # docker is running on a 64-bit machine, so we need to # override "uname -m" to report i686 instead of x86_64, otherwise # python will choose a wrong binary package to install. diff --git a/tools/dockerfile/distribtest/python_opensuse_x64/Dockerfile b/tools/dockerfile/distribtest/python_opensuse_x64/Dockerfile index fe1406be98..27159c72e3 100644 --- a/tools/dockerfile/distribtest/python_opensuse_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_opensuse_x64/Dockerfile @@ -38,3 +38,5 @@ RUN zypper --non-interactive install which # Without this, pip won't be able to connect to # https://pypi.python.org/simple/ RUN zypper --non-interactive install ca-certificates-mozilla + +RUN pip install virtualenv diff --git a/tools/dockerfile/distribtest/python_ubuntu1204_x64/Dockerfile b/tools/dockerfile/distribtest/python_ubuntu1204_x64/Dockerfile index 4068fbe2ba..7a8c91b79b 100644 --- a/tools/dockerfile/distribtest/python_ubuntu1204_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_ubuntu1204_x64/Dockerfile @@ -30,3 +30,5 @@ FROM ubuntu:12.04 RUN apt-get update -y && apt-get install -y python python-pip + +RUN pip install virtualenv diff --git a/tools/dockerfile/distribtest/python_ubuntu1404_x64/Dockerfile b/tools/dockerfile/distribtest/python_ubuntu1404_x64/Dockerfile index 0858fb0c06..65189a44de 100644 --- a/tools/dockerfile/distribtest/python_ubuntu1404_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_ubuntu1404_x64/Dockerfile @@ -30,3 +30,5 @@ FROM ubuntu:14.04 RUN apt-get update -y && apt-get install -y python python-pip + +RUN pip install virtualenv diff --git a/tools/dockerfile/distribtest/python_ubuntu1504_x64/Dockerfile b/tools/dockerfile/distribtest/python_ubuntu1504_x64/Dockerfile index ed6ffddbec..abf36c4a24 100644 --- a/tools/dockerfile/distribtest/python_ubuntu1504_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_ubuntu1504_x64/Dockerfile @@ -30,3 +30,5 @@ FROM ubuntu:15.04 RUN apt-get update -y && apt-get install -y python python-pip + +RUN pip install virtualenv diff --git a/tools/dockerfile/distribtest/python_ubuntu1510_x64/Dockerfile b/tools/dockerfile/distribtest/python_ubuntu1510_x64/Dockerfile index 9e3e0c260f..6e862d203b 100644 --- a/tools/dockerfile/distribtest/python_ubuntu1510_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_ubuntu1510_x64/Dockerfile @@ -30,3 +30,5 @@ FROM ubuntu:15.10 RUN apt-get update -y && apt-get install -y python python-pip + +RUN pip install virtualenv diff --git a/tools/dockerfile/distribtest/python_ubuntu1604_x64/Dockerfile b/tools/dockerfile/distribtest/python_ubuntu1604_x64/Dockerfile index 5098da8a26..59f4feab55 100644 --- a/tools/dockerfile/distribtest/python_ubuntu1604_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_ubuntu1604_x64/Dockerfile @@ -30,3 +30,5 @@ FROM ubuntu:16.04 RUN apt-get update -y && apt-get install -y python python-pip + +RUN pip install virtualenv diff --git a/tools/dockerfile/distribtest/python_wheezy_x64/Dockerfile b/tools/dockerfile/distribtest/python_wheezy_x64/Dockerfile index 66165ee929..bc8816d305 100644 --- a/tools/dockerfile/distribtest/python_wheezy_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_wheezy_x64/Dockerfile @@ -30,3 +30,5 @@ FROM debian:wheezy RUN apt-get update -y && apt-get install -y python python-pip + +RUN pip install virtualenv diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifact_targets.py index e9267be58b..8550ee7b84 100644 --- a/tools/run_tests/artifact_targets.py +++ b/tools/run_tests/artifact_targets.py @@ -30,11 +30,14 @@ """Definition of targets to build artifacts.""" +import os.path +import sys + import jobset def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, - flake_retries=0, timeout_retries=0): + flake_retries=0, timeout_retries=0, timeout_seconds=30*60): """Creates jobspec for a task running under docker.""" environ = environ.copy() environ['RUN_COMMAND'] = shell_command @@ -49,20 +52,20 @@ def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] + docker_args, environ=docker_env, shortname='build_artifact.%s' % (name), - timeout_seconds=30*60, + timeout_seconds=timeout_seconds, flake_retries=flake_retries, timeout_retries=timeout_retries) return jobspec def create_jobspec(name, cmdline, environ=None, shell=False, - flake_retries=0, timeout_retries=0): + flake_retries=0, timeout_retries=0, timeout_seconds=30*60): """Creates jobspec.""" jobspec = jobset.JobSpec( cmdline=cmdline, environ=environ, shortname='build_artifact.%s' % (name), - timeout_seconds=30*60, + timeout_seconds=timeout_seconds, flake_retries=flake_retries, timeout_retries=timeout_retries, shell=shell) @@ -76,27 +79,30 @@ _ARCH_FLAG_MAP = { 'x64': '-m64' } -python_version_arch_map = { - 'x86': 'Python27_32bits', - 'x64': 'Python27' +python_windows_version_arch_map = { + ('x86', '2.7'): 'Python27_32bits', + ('x64', '2.7'): 'Python27', + ('x86', '3.4'): 'Python34_32bits', + ('x64', '3.4'): 'Python34', } class PythonArtifact: """Builds Python artifacts.""" - def __init__(self, platform, arch, manylinux_build=None): + def __init__(self, platform, arch, python_version, manylinux_build=None): if manylinux_build: - self.name = 'python_%s_%s_%s' % (platform, arch, manylinux_build) + self.name = 'python%s_%s_%s_%s' % (python_version, platform, arch, manylinux_build) else: - self.name = 'python_%s_%s' % (platform, arch) + self.name = 'python%s_%s_%s' % (python_version, platform, arch) self.platform = platform self.arch = arch - self.labels = ['artifact', 'python', platform, arch] - self.python_version = python_version_arch_map[arch] + self.labels = ['artifact', 'python', python_version, platform, arch] + self.python_version = python_version + self.python_windows_prefix = python_windows_version_arch_map[arch, python_version] self.manylinux_build = manylinux_build def pre_build_jobspecs(self): - return [] + return [] def build_jobspec(self): environ = {} @@ -107,26 +113,27 @@ class PythonArtifact: # special places... environ['PYTHON'] = '/opt/python/{}/bin/python'.format(self.manylinux_build) environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.manylinux_build) - # Our docker image has all the prerequisites pip-installed already. - environ['SKIP_PIP_INSTALL'] = '1' # Platform autodetection for the manylinux1 image breaks so we set the # defines ourselves. # TODO(atash) get better platform-detection support in core so we don't # need to do this manually... environ['CFLAGS'] = '-DGPR_MANYLINUX1=1' + environ['BUILD_HEALTH_CHECKING'] = 'TRUE' + environ['BUILD_MANYLINUX_WHEEL'] = 'TRUE' return create_docker_jobspec(self.name, 'tools/dockerfile/grpc_artifact_python_manylinux_%s' % self.arch, 'tools/run_tests/build_artifact_python.sh', - environ=environ) + environ=environ, + timeout_seconds=60*60) elif self.platform == 'windows': return create_jobspec(self.name, ['tools\\run_tests\\build_artifact_python.bat', - self.python_version, + self.python_windows_prefix, '32' if self.arch == 'x86' else '64' ], shell=True) else: - environ['SKIP_PIP_INSTALL'] = 'TRUE' + environ['PYTHON'] = 'python{}'.format(self.python_version) return create_jobspec(self.name, ['tools/run_tests/build_artifact_python.sh'], environ=environ) @@ -323,13 +330,18 @@ def targets(): for Cls in (CSharpExtArtifact, NodeExtArtifact, ProtocArtifact) for platform in ('linux', 'macos', 'windows') for arch in ('x86', 'x64')] + - [PythonArtifact('linux', 'x86', 'cp27-cp27m'), - PythonArtifact('linux', 'x86', 'cp27-cp27mu'), - PythonArtifact('linux', 'x64', 'cp27-cp27m'), - PythonArtifact('linux', 'x64', 'cp27-cp27mu'), - PythonArtifact('macos', 'x64'), - PythonArtifact('windows', 'x86'), - PythonArtifact('windows', 'x64'), + [PythonArtifact('linux', 'x86', '2.7', 'cp27-cp27m'), + PythonArtifact('linux', 'x86', '2.7', 'cp27-cp27mu'), + PythonArtifact('linux', 'x64', '2.7', 'cp27-cp27m'), + PythonArtifact('linux', 'x64', '2.7', 'cp27-cp27mu'), + PythonArtifact('macos', 'x64', '2.7'), + PythonArtifact('windows', 'x86', '2.7'), + PythonArtifact('windows', 'x64', '2.7'), + PythonArtifact('linux', 'x86', '3.4', 'cp34-cp34m'), + PythonArtifact('linux', 'x64', '3.4', 'cp34-cp34m'), + PythonArtifact('macos', 'x64', '3.4'), + PythonArtifact('windows', 'x86', '3.4'), + PythonArtifact('windows', 'x64', '3.4'), RubyArtifact('linux', 'x86'), RubyArtifact('linux', 'x64'), RubyArtifact('macos', 'x64'), diff --git a/tools/run_tests/build_artifact_python.bat b/tools/run_tests/build_artifact_python.bat index a7b1a58284..074a3c6781 100644 --- a/tools/run_tests/build_artifact_python.bat +++ b/tools/run_tests/build_artifact_python.bat @@ -42,11 +42,19 @@ python tools\distrib\python\make_grpcio_tools.py @rem Build gRPC Python extensions python setup.py build_ext -c mingw32 -python tools\distrib\python\grpcio_tools\setup.py build_ext -c mingw32 + +pushd tools\distrib\python\grpcio_tools +python setup.py build_ext -c mingw32 +popd + @rem Build gRPC Python distributions python setup.py bdist_wheel -python tools\distrib\python\grpcio_tools\setup.py bdist_wheel + +pushd tools\distrib\python\grpcio_tools +python setup.py bdist_wheel +popd + mkdir artifacts xcopy /Y /I /S dist\* artifacts\ || goto :error diff --git a/tools/run_tests/build_artifact_python.sh b/tools/run_tests/build_artifact_python.sh index 55f8eb634b..8f8330ef24 100755 --- a/tools/run_tests/build_artifact_python.sh +++ b/tools/run_tests/build_artifact_python.sh @@ -39,15 +39,6 @@ export PIP=${PIP:-pip} export AUDITWHEEL=${AUDITWHEEL:-auditwheel} -if [ "$SKIP_PIP_INSTALL" == "" ] -then - ${PIP} install --upgrade six - # There's a bug in newer versions of setuptools (see - # https://bitbucket.org/pypa/setuptools/issues/503/pkg_resources_vendorpackagingrequirementsi) - ${PIP} pip install --upgrade 'setuptools==18' - ${PIP} install -rrequirements.txt -fi - # Build the source distribution first because MANIFEST.in cannot override # exclusion of built shared objects among package resources (for some # inexplicable reason). @@ -71,15 +62,33 @@ CFLAGS="$CFLAGS -fno-wrapv" ${SETARCH_CMD} \ ${PYTHON} tools/distrib/python/grpcio_tools/setup.py bdist_wheel mkdir -p artifacts -if command -v ${AUDITWHEEL} +if [ "$BUILD_MANYLINUX_WHEEL" != "" ] then for wheel in dist/*.whl; do ${AUDITWHEEL} repair $wheel -w artifacts/ + rm $wheel done for wheel in tools/distrib/python/grpcio_tools/dist/*.whl; do ${AUDITWHEEL} repair $wheel -w artifacts/ + rm $wheel done fi +# We need to use the built grpcio-tools/grpcio to compile the health proto +# Wheels are not supported by setup_requires/dependency_links, so we +# manually install the dependency. Note we should only do this if we +# are in a docker image or in a virtualenv. +if [ "$BUILD_HEALTH_CHECKING" != "" ] +then + ${PIP} install -rrequirements.txt + ${PIP} install grpcio --no-index --find-links "file://${PWD}/artifacts/" + ${PIP} install grpcio-tools --no-index --find-links "file://${PWD}/artifacts/" + + # Build gRPC health check source distribution + ${SETARCH_CMD} ${PYTHON} src/python/grpcio_health_checking/setup.py \ + preprocess build_package_protos sdist + cp -r src/python/grpcio_health_checking/dist/* artifacts +fi + cp -r dist/* artifacts cp -r tools/distrib/python/grpcio_tools/dist/* artifacts diff --git a/tools/run_tests/build_package_node.sh b/tools/run_tests/build_package_node.sh index ff4cfdb8bf..ef4a10cca7 100755 --- a/tools/run_tests/build_package_node.sh +++ b/tools/run_tests/build_package_node.sh @@ -61,7 +61,7 @@ mkdir -p $output_dir well_known_protos=( any api compiler/plugin descriptor duration empty field_mask source_context struct timestamp type wrappers ) for arch in {x86,x64}; do - case arch in + case $arch in x86) node_arch=ia32 ;; @@ -70,7 +70,7 @@ for arch in {x86,x64}; do ;; esac for plat in {windows,linux,macos}; do - case plat in + case $plat in windows) node_plat=win32 ;; diff --git a/tools/run_tests/build_python.sh b/tools/run_tests/build_python.sh index 9cb3cb12a9..c4b5d98a53 100755 --- a/tools/run_tests/build_python.sh +++ b/tools/run_tests/build_python.sh @@ -164,7 +164,8 @@ pip_install_dir $ROOT/tools/distrib/python/grpcio_tools # etc... pip_install_dir $ROOT $VENV_PYTHON $ROOT/src/python/grpcio_health_checking/setup.py preprocess +$VENV_PYTHON $ROOT/src/python/grpcio_health_checking/setup.py build_package_protos pip_install_dir $ROOT/src/python/grpcio_health_checking $VENV_PYTHON $ROOT/src/python/grpcio_tests/setup.py preprocess -$VENV_PYTHON $ROOT/src/python/grpcio_tests/setup.py build_proto_modules +$VENV_PYTHON $ROOT/src/python/grpcio_tests/setup.py build_package_protos pip_install_dir $ROOT/src/python/grpcio_tests diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 57fff2ec9c..bb3d090ae1 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -63,7 +63,9 @@ _ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) os.chdir(_ROOT) -_FORCE_ENVIRON_FOR_WRAPPERS = {} +_FORCE_ENVIRON_FOR_WRAPPERS = { + 'GRPC_VERBOSITY': 'DEBUG', +} _POLLING_STRATEGIES = { @@ -164,7 +166,8 @@ class CLanguage(object): for polling_strategy in polling_strategies: env={'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH': _ROOT + '/src/core/lib/tsi/test_creds/ca.pem', - 'GRPC_POLL_STRATEGY': polling_strategy} + 'GRPC_POLL_STRATEGY': polling_strategy, + 'GRPC_VERBOSITY': 'DEBUG'} shortname_ext = '' if polling_strategy=='all' else ' GRPC_POLL_STRATEGY=%s' % polling_strategy if self.config.build_config in target['exclude_configs']: continue diff --git a/tools/run_tests/sanity/check_sources_and_headers.py b/tools/run_tests/sanity/check_sources_and_headers.py index c028499ca6..28c1dc46d7 100755 --- a/tools/run_tests/sanity/check_sources_and_headers.py +++ b/tools/run_tests/sanity/check_sources_and_headers.py @@ -55,7 +55,8 @@ def target_has_header(target, name): for dep in target['deps']: if target_has_header(get_target(dep), name): return True - if name == 'src/core/lib/profiling/stap_probes.h': + if name in ['src/core/lib/profiling/stap_probes.h', + 'src/proto/grpc/reflection/v1alpha/reflection.grpc.pb.h']: return True return False diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 4ab784feff..9dc1c66eee 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -4362,25 +4362,19 @@ { "deps": [ "grpc++", - "grpc++_codegen_proto" + "grpc++_reflection_proto" ], "headers": [ "include/grpc++/ext/proto_server_reflection_plugin.h", - "include/grpc++/ext/reflection.grpc.pb.h", - "include/grpc++/ext/reflection.pb.h", "src/cpp/ext/proto_server_reflection.h" ], "language": "c++", "name": "grpc++_reflection", "src": [ "include/grpc++/ext/proto_server_reflection_plugin.h", - "include/grpc++/ext/reflection.grpc.pb.h", - "include/grpc++/ext/reflection.pb.h", "src/cpp/ext/proto_server_reflection.cc", "src/cpp/ext/proto_server_reflection.h", - "src/cpp/ext/proto_server_reflection_plugin.cc", - "src/cpp/ext/reflection.grpc.pb.cc", - "src/cpp/ext/reflection.pb.cc" + "src/cpp/ext/proto_server_reflection_plugin.cc" ], "third_party": false, "type": "lib" @@ -4388,6 +4382,18 @@ { "deps": [], "headers": [ + "src/proto/grpc/reflection/v1alpha/reflection.grpc.pb.h", + "src/proto/grpc/reflection/v1alpha/reflection.pb.h" + ], + "language": "c++", + "name": "grpc++_reflection_codegen", + "src": [], + "third_party": false, + "type": "lib" + }, + { + "deps": [], + "headers": [ "test/cpp/util/test_config.h" ], "language": "c++", @@ -6773,5 +6779,24 @@ ], "third_party": false, "type": "filegroup" + }, + { + "deps": [ + "grpc++_codegen_proto" + ], + "headers": [ + "include/grpc++/ext/reflection.grpc.pb.h", + "include/grpc++/ext/reflection.pb.h" + ], + "language": "c++", + "name": "grpc++_reflection_proto", + "src": [ + "include/grpc++/ext/reflection.grpc.pb.h", + "include/grpc++/ext/reflection.pb.h", + "src/cpp/ext/reflection.grpc.pb.cc", + "src/cpp/ext/reflection.pb.cc" + ], + "third_party": false, + "type": "filegroup" } ] |