aboutsummaryrefslogtreecommitdiffhomepage
path: root/tools
diff options
context:
space:
mode:
authorGravatar Mark D. Roth <roth@google.com>2016-07-26 11:17:32 -0700
committerGravatar Mark D. Roth <roth@google.com>2016-07-26 11:17:32 -0700
commit1212d3f453eda0bf04178b9f9f78440ab5cff42d (patch)
tree457cecded1dfb964bbc8253f4743624928b1c300 /tools
parent889e02e29da6ec5e663cf1e8009a7dc4996df3fe (diff)
parentb8ec23586dca270754bc035300204e2a202a5ae0 (diff)
Merge remote-tracking branch 'upstream/master' into http_connect
Diffstat (limited to 'tools')
-rwxr-xr-xtools/distrib/python/docgen.py2
-rw-r--r--tools/distrib/python/grpcio_tools/README.rst40
-rw-r--r--tools/distrib/python/grpcio_tools/grpc/tools/command.py38
-rw-r--r--tools/distrib/python/grpcio_tools/setup.py33
-rwxr-xr-xtools/distrib/python/make_grpcio_tools.py94
-rw-r--r--tools/dockerfile/distribtest/python_arch_x64/Dockerfile2
-rw-r--r--tools/dockerfile/distribtest/python_centos6_x64/Dockerfile2
-rw-r--r--tools/dockerfile/distribtest/python_centos7_x64/Dockerfile2
-rw-r--r--tools/dockerfile/distribtest/python_fedora20_x64/Dockerfile2
-rw-r--r--tools/dockerfile/distribtest/python_fedora21_x64/Dockerfile2
-rw-r--r--tools/dockerfile/distribtest/python_fedora22_x64/Dockerfile1
-rw-r--r--tools/dockerfile/distribtest/python_fedora23_x64/Dockerfile1
-rw-r--r--tools/dockerfile/distribtest/python_jessie_x64/Dockerfile1
-rw-r--r--tools/dockerfile/distribtest/python_jessie_x86/Dockerfile2
-rw-r--r--tools/dockerfile/distribtest/python_opensuse_x64/Dockerfile2
-rw-r--r--tools/dockerfile/distribtest/python_ubuntu1204_x64/Dockerfile2
-rw-r--r--tools/dockerfile/distribtest/python_ubuntu1404_x64/Dockerfile2
-rw-r--r--tools/dockerfile/distribtest/python_ubuntu1504_x64/Dockerfile2
-rw-r--r--tools/dockerfile/distribtest/python_ubuntu1510_x64/Dockerfile2
-rw-r--r--tools/dockerfile/distribtest/python_ubuntu1604_x64/Dockerfile2
-rw-r--r--tools/dockerfile/distribtest/python_wheezy_x64/Dockerfile2
-rw-r--r--tools/dockerfile/interoptest/grpc_interop_php/Dockerfile29
-rw-r--r--tools/dockerfile/interoptest/grpc_interop_php7/Dockerfile125
-rwxr-xr-xtools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh52
-rw-r--r--tools/dockerfile/stress_test/grpc_interop_stress_php/Dockerfile11
-rw-r--r--tools/dockerfile/test/multilang_jessie_x64/Dockerfile6
-rw-r--r--tools/dockerfile/test/php7_jessie_x64/Dockerfile105
-rw-r--r--tools/dockerfile/test/php_jessie_x64/Dockerfile6
-rw-r--r--tools/run_tests/artifact_targets.py62
-rw-r--r--tools/run_tests/build_artifact_python.bat12
-rwxr-xr-xtools/run_tests/build_artifact_python.sh29
-rwxr-xr-xtools/run_tests/build_package_node.sh4
-rwxr-xr-xtools/run_tests/build_python.sh3
-rwxr-xr-xtools/run_tests/run_interop_tests.py28
-rwxr-xr-xtools/run_tests/run_tests.py44
-rwxr-xr-xtools/run_tests/sanity/check_sources_and_headers.py3
-rw-r--r--tools/run_tests/sources_and_headers.json41
37 files changed, 655 insertions, 141 deletions
diff --git a/tools/distrib/python/docgen.py b/tools/distrib/python/docgen.py
index f5e89f1da6..15bd8d855f 100755
--- a/tools/distrib/python/docgen.py
+++ b/tools/distrib/python/docgen.py
@@ -71,6 +71,8 @@ environment.update({
subprocess_arguments_list = [
{'args': ['virtualenv', VIRTUALENV_DIR], 'env': environment},
+ {'args': [VIRTUALENV_PIP_PATH, 'install', '--upgrade', 'pip'],
+ 'env': environment},
{'args': [VIRTUALENV_PIP_PATH, 'install', '-r', REQUIREMENTS_PATH],
'env': environment},
{'args': [VIRTUALENV_PYTHON_PATH, SETUP_PATH, 'build'], 'env': environment},
diff --git a/tools/distrib/python/grpcio_tools/README.rst b/tools/distrib/python/grpcio_tools/README.rst
index 8946a1d5b3..55521d17bb 100644
--- a/tools/distrib/python/grpcio_tools/README.rst
+++ b/tools/distrib/python/grpcio_tools/README.rst
@@ -122,6 +122,7 @@ Help, I ...
third_party/protobuf/src/google/protobuf/stubs/mathlimits.h:173:31: note: in expansion of macro 'SIGNED_INT_MAX'
static const Type kPosMax = SIGNED_INT_MAX(Type); \\
^
+
And your toolchain is GCC (at the time of this writing, up through at least
GCC 6.0), this is probably a bug where GCC chokes on constant expressions
when the :code:`-fwrapv` flag is specified. You should consider setting your
@@ -136,3 +137,42 @@ Given protobuf include directories :code:`$INCLUDE`, an output directory
::
$ python -m grpc.tools.protoc -I$INCLUDE --python_out=$OUTPUT --grpc_python_out=$OUTPUT $PROTO_FILES
+
+To use as a build step in distutils-based projects, you may use the provided
+command class in your :code:`setup.py`:
+
+::
+
+ setuptools.setup(
+ # ...
+ cmdclass={
+ 'build_proto_modules': grpc.tools.command.BuildPackageProtos,
+ }
+ # ...
+ )
+
+Invocation of the command will walk the project tree and transpile every
+:code:`.proto` file into a :code:`_pb2.py` file in the same directory.
+
+Note that this particular approach requires :code:`grpcio-tools` to be
+installed on the machine before the setup script is invoked (i.e. no
+combination of :code:`setup_requires` or :code:`install_requires` will provide
+access to :code:`grpc.tools.command.BuildPackageProtos` if it isn't already
+installed). One way to work around this can be found in our
+:code:`grpcio-health-checking`
+`package <https://pypi.python.org/pypi/grpcio-health-checking>`_:
+
+::
+
+ class BuildPackageProtos(setuptools.Command):
+ """Command to generate project *_pb2.py modules from proto files."""
+ # ...
+ def run(self):
+ from grpc.tools import command
+ command.build_package_protos(self.distribution.package_dir[''])
+
+Now including :code:`grpcio-tools` in :code:`setup_requires` will provide the
+command on-setup as desired.
+
+For more information on command classes, consult :code:`distutils` and
+:code:`setuptools` documentation.
diff --git a/tools/distrib/python/grpcio_tools/grpc/tools/command.py b/tools/distrib/python/grpcio_tools/grpc/tools/command.py
index ccf38b7d56..2520099835 100644
--- a/tools/distrib/python/grpcio_tools/grpc/tools/command.py
+++ b/tools/distrib/python/grpcio_tools/grpc/tools/command.py
@@ -35,7 +35,26 @@ import setuptools
from grpc.tools import protoc
-class BuildProtoModules(setuptools.Command):
+def build_package_protos(package_root):
+ proto_files = []
+ inclusion_root = os.path.abspath(package_root)
+ for root, _, files in os.walk(inclusion_root):
+ for filename in files:
+ if filename.endswith('.proto'):
+ proto_files.append(os.path.abspath(os.path.join(root, filename)))
+
+ for proto_file in proto_files:
+ command = [
+ 'grpc.tools.protoc',
+ '--proto_path={}'.format(inclusion_root),
+ '--python_out={}'.format(inclusion_root),
+ '--grpc_python_out={}'.format(inclusion_root),
+ ] + [proto_file]
+ if protoc.main(command) != 0:
+ sys.stderr.write('warning: {} failed'.format(command))
+
+
+class BuildPackageProtos(setuptools.Command):
"""Command to generate project *_pb2.py modules from proto files."""
description = 'build grpc protobuf modules'
@@ -52,19 +71,4 @@ class BuildProtoModules(setuptools.Command):
# directory is provided as an 'include' directory. We assume it's the '' key
# to `self.distribution.package_dir` (and get a key error if it's not
# there).
- proto_files = []
- inclusion_root = os.path.abspath(self.distribution.package_dir[''])
- for root, _, files in os.walk(inclusion_root):
- for filename in files:
- if filename.endswith('.proto'):
- proto_files.append(os.path.abspath(os.path.join(root, filename)))
-
- for proto_file in proto_files:
- command = [
- 'grpc.tools.protoc',
- '--proto_path={}'.format(inclusion_root),
- '--python_out={}'.format(inclusion_root),
- '--grpc_python_out={}'.format(inclusion_root),
- ] + [proto_file]
- if protoc.main(command) != 0:
- sys.stderr.write('warning: {} failed'.format(command))
+ build_package_protos(self.distribution.package_dir[''])
diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py
index bb2c71d843..bb1f1cf085 100644
--- a/tools/distrib/python/grpcio_tools/setup.py
+++ b/tools/distrib/python/grpcio_tools/setup.py
@@ -28,11 +28,13 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from distutils import extension
+from distutils import util
import errno
import os
import os.path
import pkg_resources
import platform
+import re
import shlex
import shutil
import sys
@@ -89,6 +91,13 @@ if EXTRA_ENV_LINK_ARGS is None:
EXTRA_COMPILE_ARGS = shlex.split(EXTRA_ENV_COMPILE_ARGS)
EXTRA_LINK_ARGS = shlex.split(EXTRA_ENV_LINK_ARGS)
+CC_FILES = [
+ os.path.normpath(cc_file) for cc_file in protoc_lib_deps.CC_FILES]
+PROTO_FILES = [
+ os.path.normpath(proto_file) for proto_file in protoc_lib_deps.PROTO_FILES]
+CC_INCLUDE = os.path.normpath(protoc_lib_deps.CC_INCLUDE)
+PROTO_INCLUDE = os.path.normpath(protoc_lib_deps.PROTO_INCLUDE)
+
GRPC_PYTHON_TOOLS_PACKAGE = 'grpc.tools'
GRPC_PYTHON_PROTO_RESOURCES_NAME = '_proto'
@@ -104,14 +113,18 @@ if 'darwin' in sys.platform and PY3:
if mac_target and (pkg_resources.parse_version(mac_target) <
pkg_resources.parse_version('10.9.0')):
os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.9'
+ os.environ['_PYTHON_HOST_PLATFORM'] = re.sub(
+ r'macosx-[0-9]+\.[0-9]+-(.+)',
+ r'macosx-10.9-\1',
+ util.get_platform())
def package_data():
tools_path = GRPC_PYTHON_TOOLS_PACKAGE.replace('.', os.path.sep)
proto_resources_path = os.path.join(tools_path,
GRPC_PYTHON_PROTO_RESOURCES_NAME)
proto_files = []
- for proto_file in protoc_lib_deps.PROTO_FILES:
- source = os.path.join(protoc_lib_deps.PROTO_INCLUDE, proto_file)
+ for proto_file in PROTO_FILES:
+ source = os.path.join(PROTO_INCLUDE, proto_file)
target = os.path.join(proto_resources_path, proto_file)
relative_target = os.path.join(GRPC_PYTHON_PROTO_RESOURCES_NAME, proto_file)
try:
@@ -127,22 +140,22 @@ def package_data():
def extension_modules():
if BUILD_WITH_CYTHON:
- plugin_sources = ['grpc/tools/_protoc_compiler.pyx']
+ plugin_sources = [os.path.join('grpc', 'tools', '_protoc_compiler.pyx')]
else:
- plugin_sources = ['grpc/tools/_protoc_compiler.cpp']
+ plugin_sources = [os.path.join('grpc', 'tools', '_protoc_compiler.cpp')]
plugin_sources += [
- 'grpc/tools/main.cc',
- 'grpc_root/src/compiler/python_generator.cc'] + [
- os.path.join(protoc_lib_deps.CC_INCLUDE, cc_file)
- for cc_file in protoc_lib_deps.CC_FILES]
+ os.path.join('grpc', 'tools', 'main.cc'),
+ os.path.join('grpc_root', 'src', 'compiler', 'python_generator.cc')] + [
+ os.path.join(CC_INCLUDE, cc_file)
+ for cc_file in CC_FILES]
plugin_ext = extension.Extension(
name='grpc.tools._protoc_compiler',
sources=plugin_sources,
include_dirs=[
'.',
'grpc_root',
- 'grpc_root/include',
- protoc_lib_deps.CC_INCLUDE,
+ os.path.join('grpc_root', 'include'),
+ CC_INCLUDE,
],
language='c++',
define_macros=list(DEFINE_MACROS),
diff --git a/tools/distrib/python/make_grpcio_tools.py b/tools/distrib/python/make_grpcio_tools.py
index fd9b38b084..adf58445af 100755
--- a/tools/distrib/python/make_grpcio_tools.py
+++ b/tools/distrib/python/make_grpcio_tools.py
@@ -29,12 +29,18 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+from __future__ import print_function
+
+import errno
+import filecmp
+import glob
import os
import os.path
import shutil
import subprocess
import sys
import traceback
+import uuid
DEPS_FILE_CONTENT="""
# Copyright 2016, Google Inc.
@@ -124,20 +130,88 @@ def get_deps():
proto_include=repr(GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT))
return deps_file_content
+def long_path(path):
+ if os.name == 'nt':
+ return '\\\\?\\' + path
+ else:
+ return path
+
+def atomic_file_copy(src, dst):
+ """Based on the lock-free-whack-a-mole algorithm, depending on filesystem
+ renaming being atomic. Described at http://stackoverflow.com/a/28090883.
+ """
+ try:
+ if filecmp.cmp(src, dst):
+ return
+ except:
+ pass
+ dst_dir = os.path.abspath(os.path.dirname(dst))
+ dst_base = os.path.basename(dst)
+ this_id = str(uuid.uuid4()).replace('.', '-')
+ temporary_file = os.path.join(dst_dir, '{}.{}.tmp'.format(dst_base, this_id))
+ mole_file = os.path.join(dst_dir, '{}.{}.mole.tmp'.format(dst_base, this_id))
+ mole_pattern = os.path.join(dst_dir, '{}.*.mole.tmp'.format(dst_base))
+ src = long_path(src)
+ dst = long_path(dst)
+ temporary_file = long_path(temporary_file)
+ mole_file = long_path(mole_file)
+ mole_pattern = long_path(mole_pattern)
+ shutil.copy2(src, temporary_file)
+ try:
+ os.rename(temporary_file, mole_file)
+ except:
+ print('Error moving temporary file {} to {}'.format(temporary_file, mole_file), file=sys.stderr)
+ print('while trying to copy file {} to {}'.format(src, dst), file=sys.stderr)
+ raise
+ for other_file in glob.glob(mole_pattern):
+ other_id = other_file.split('.')[-3]
+ if this_id == other_id:
+ pass
+ elif this_id < other_id:
+ try:
+ os.remove(other_file)
+ except:
+ pass
+ else:
+ try:
+ os.remove(mole_file)
+ except:
+ pass
+ this_id = other_id
+ mole_file = other_file
+ try:
+ if filecmp.cmp(src, dst):
+ try:
+ os.remove(mole_file)
+ except:
+ pass
+ return
+ except:
+ pass
+ try:
+ os.rename(mole_file, dst)
+ except:
+ pass
+
def main():
os.chdir(GRPC_ROOT)
- for tree in [GRPC_PYTHON_PROTOBUF,
- GRPC_PYTHON_PROTOC_PLUGINS,
- GRPC_PYTHON_INCLUDE]:
- try:
- shutil.rmtree(tree)
- except Exception as _:
- pass
- shutil.copytree(GRPC_PROTOBUF, GRPC_PYTHON_PROTOBUF)
- shutil.copytree(GRPC_PROTOC_PLUGINS, GRPC_PYTHON_PROTOC_PLUGINS)
- shutil.copytree(GRPC_INCLUDE, GRPC_PYTHON_INCLUDE)
+ for source, target in [
+ (GRPC_PROTOBUF, GRPC_PYTHON_PROTOBUF),
+ (GRPC_PROTOC_PLUGINS, GRPC_PYTHON_PROTOC_PLUGINS),
+ (GRPC_INCLUDE, GRPC_PYTHON_INCLUDE)]:
+ for source_dir, _, files in os.walk(source):
+ target_dir = os.path.abspath(os.path.join(target, os.path.relpath(source_dir, source)))
+ try:
+ os.makedirs(target_dir)
+ except OSError as error:
+ if error.errno != errno.EEXIST:
+ raise
+ for relative_file in files:
+ source_file = os.path.abspath(os.path.join(source_dir, relative_file))
+ target_file = os.path.abspath(os.path.join(target_dir, relative_file))
+ atomic_file_copy(source_file, target_file)
try:
protoc_lib_deps_content = get_deps()
diff --git a/tools/dockerfile/distribtest/python_arch_x64/Dockerfile b/tools/dockerfile/distribtest/python_arch_x64/Dockerfile
index 2f79cc3017..dff72eee97 100644
--- a/tools/dockerfile/distribtest/python_arch_x64/Dockerfile
+++ b/tools/dockerfile/distribtest/python_arch_x64/Dockerfile
@@ -33,4 +33,4 @@ RUN pacman --noconfirm -Syy
RUN pacman --noconfirm -S openssl
RUN pacman --noconfirm -S python2
RUN pacman --noconfirm -S python2-pip
-
+RUN pip install virtualenv
diff --git a/tools/dockerfile/distribtest/python_centos6_x64/Dockerfile b/tools/dockerfile/distribtest/python_centos6_x64/Dockerfile
index d4f473792e..967450156c 100644
--- a/tools/dockerfile/distribtest/python_centos6_x64/Dockerfile
+++ b/tools/dockerfile/distribtest/python_centos6_x64/Dockerfile
@@ -44,3 +44,5 @@ RUN curl https://bootstrap.pypa.io/get-pip.py | python -
# "which" command required by python's run_distrib_test.sh
RUN yum install -y which
+
+RUN pip install virtualenv
diff --git a/tools/dockerfile/distribtest/python_centos7_x64/Dockerfile b/tools/dockerfile/distribtest/python_centos7_x64/Dockerfile
index ca64fa7bea..0127fe1e28 100644
--- a/tools/dockerfile/distribtest/python_centos7_x64/Dockerfile
+++ b/tools/dockerfile/distribtest/python_centos7_x64/Dockerfile
@@ -32,4 +32,4 @@ FROM centos:7
RUN yum install -y python
RUN yum install -y epel-release
RUN yum install -y python-pip
-
+RUN pip install virtualenv
diff --git a/tools/dockerfile/distribtest/python_fedora20_x64/Dockerfile b/tools/dockerfile/distribtest/python_fedora20_x64/Dockerfile
index 8b0f769c26..3d3636e43d 100644
--- a/tools/dockerfile/distribtest/python_fedora20_x64/Dockerfile
+++ b/tools/dockerfile/distribtest/python_fedora20_x64/Dockerfile
@@ -35,3 +35,5 @@ RUN yum clean all && yum update -y && yum install -y python python-pip
# Trying twice makes it work fine.
# https://github.com/docker/docker/issues/10180
RUN pip install --upgrade six || pip install --upgrade six
+
+RUN pip install virtualenv
diff --git a/tools/dockerfile/distribtest/python_fedora21_x64/Dockerfile b/tools/dockerfile/distribtest/python_fedora21_x64/Dockerfile
index fcbe053f1f..0b1b6aeb35 100644
--- a/tools/dockerfile/distribtest/python_fedora21_x64/Dockerfile
+++ b/tools/dockerfile/distribtest/python_fedora21_x64/Dockerfile
@@ -40,3 +40,5 @@ RUN yum clean all && yum update -y && yum install -y python python-pip
# Trying twice makes it work fine.
# https://github.com/docker/docker/issues/10180
RUN pip2 install --upgrade six || pip2 install --upgrade six
+
+RUN pip2 install virtualenv
diff --git a/tools/dockerfile/distribtest/python_fedora22_x64/Dockerfile b/tools/dockerfile/distribtest/python_fedora22_x64/Dockerfile
index ddcacb4257..4d75034c15 100644
--- a/tools/dockerfile/distribtest/python_fedora22_x64/Dockerfile
+++ b/tools/dockerfile/distribtest/python_fedora22_x64/Dockerfile
@@ -30,3 +30,4 @@
FROM fedora:22
RUN yum clean all && yum update -y && yum install -y python python-pip
+RUN pip install virtualenv
diff --git a/tools/dockerfile/distribtest/python_fedora23_x64/Dockerfile b/tools/dockerfile/distribtest/python_fedora23_x64/Dockerfile
index d45195e509..a1bc9ba8d6 100644
--- a/tools/dockerfile/distribtest/python_fedora23_x64/Dockerfile
+++ b/tools/dockerfile/distribtest/python_fedora23_x64/Dockerfile
@@ -30,3 +30,4 @@
FROM fedora:23
RUN yum clean all && yum update -y && yum install -y python python-pip
+RUN pip install virtualenv
diff --git a/tools/dockerfile/distribtest/python_jessie_x64/Dockerfile b/tools/dockerfile/distribtest/python_jessie_x64/Dockerfile
index 83df4ed4fa..7dc32a088e 100644
--- a/tools/dockerfile/distribtest/python_jessie_x64/Dockerfile
+++ b/tools/dockerfile/distribtest/python_jessie_x64/Dockerfile
@@ -30,3 +30,4 @@
FROM debian:jessie
RUN apt-get update && apt-get install -y python python-pip
+RUN pip install virtualenv
diff --git a/tools/dockerfile/distribtest/python_jessie_x86/Dockerfile b/tools/dockerfile/distribtest/python_jessie_x86/Dockerfile
index 19addb2912..04c1402e72 100644
--- a/tools/dockerfile/distribtest/python_jessie_x86/Dockerfile
+++ b/tools/dockerfile/distribtest/python_jessie_x86/Dockerfile
@@ -31,6 +31,8 @@ FROM 32bit/debian:jessie
RUN apt-get update && apt-get install -y python python-pip
+RUN pip install virtualenv
+
# docker is running on a 64-bit machine, so we need to
# override "uname -m" to report i686 instead of x86_64, otherwise
# python will choose a wrong binary package to install.
diff --git a/tools/dockerfile/distribtest/python_opensuse_x64/Dockerfile b/tools/dockerfile/distribtest/python_opensuse_x64/Dockerfile
index fe1406be98..27159c72e3 100644
--- a/tools/dockerfile/distribtest/python_opensuse_x64/Dockerfile
+++ b/tools/dockerfile/distribtest/python_opensuse_x64/Dockerfile
@@ -38,3 +38,5 @@ RUN zypper --non-interactive install which
# Without this, pip won't be able to connect to
# https://pypi.python.org/simple/
RUN zypper --non-interactive install ca-certificates-mozilla
+
+RUN pip install virtualenv
diff --git a/tools/dockerfile/distribtest/python_ubuntu1204_x64/Dockerfile b/tools/dockerfile/distribtest/python_ubuntu1204_x64/Dockerfile
index 4068fbe2ba..7a8c91b79b 100644
--- a/tools/dockerfile/distribtest/python_ubuntu1204_x64/Dockerfile
+++ b/tools/dockerfile/distribtest/python_ubuntu1204_x64/Dockerfile
@@ -30,3 +30,5 @@
FROM ubuntu:12.04
RUN apt-get update -y && apt-get install -y python python-pip
+
+RUN pip install virtualenv
diff --git a/tools/dockerfile/distribtest/python_ubuntu1404_x64/Dockerfile b/tools/dockerfile/distribtest/python_ubuntu1404_x64/Dockerfile
index 0858fb0c06..65189a44de 100644
--- a/tools/dockerfile/distribtest/python_ubuntu1404_x64/Dockerfile
+++ b/tools/dockerfile/distribtest/python_ubuntu1404_x64/Dockerfile
@@ -30,3 +30,5 @@
FROM ubuntu:14.04
RUN apt-get update -y && apt-get install -y python python-pip
+
+RUN pip install virtualenv
diff --git a/tools/dockerfile/distribtest/python_ubuntu1504_x64/Dockerfile b/tools/dockerfile/distribtest/python_ubuntu1504_x64/Dockerfile
index ed6ffddbec..abf36c4a24 100644
--- a/tools/dockerfile/distribtest/python_ubuntu1504_x64/Dockerfile
+++ b/tools/dockerfile/distribtest/python_ubuntu1504_x64/Dockerfile
@@ -30,3 +30,5 @@
FROM ubuntu:15.04
RUN apt-get update -y && apt-get install -y python python-pip
+
+RUN pip install virtualenv
diff --git a/tools/dockerfile/distribtest/python_ubuntu1510_x64/Dockerfile b/tools/dockerfile/distribtest/python_ubuntu1510_x64/Dockerfile
index 9e3e0c260f..6e862d203b 100644
--- a/tools/dockerfile/distribtest/python_ubuntu1510_x64/Dockerfile
+++ b/tools/dockerfile/distribtest/python_ubuntu1510_x64/Dockerfile
@@ -30,3 +30,5 @@
FROM ubuntu:15.10
RUN apt-get update -y && apt-get install -y python python-pip
+
+RUN pip install virtualenv
diff --git a/tools/dockerfile/distribtest/python_ubuntu1604_x64/Dockerfile b/tools/dockerfile/distribtest/python_ubuntu1604_x64/Dockerfile
index 5098da8a26..59f4feab55 100644
--- a/tools/dockerfile/distribtest/python_ubuntu1604_x64/Dockerfile
+++ b/tools/dockerfile/distribtest/python_ubuntu1604_x64/Dockerfile
@@ -30,3 +30,5 @@
FROM ubuntu:16.04
RUN apt-get update -y && apt-get install -y python python-pip
+
+RUN pip install virtualenv
diff --git a/tools/dockerfile/distribtest/python_wheezy_x64/Dockerfile b/tools/dockerfile/distribtest/python_wheezy_x64/Dockerfile
index 66165ee929..bc8816d305 100644
--- a/tools/dockerfile/distribtest/python_wheezy_x64/Dockerfile
+++ b/tools/dockerfile/distribtest/python_wheezy_x64/Dockerfile
@@ -30,3 +30,5 @@
FROM debian:wheezy
RUN apt-get update -y && apt-get install -y python python-pip
+
+RUN pip install virtualenv
diff --git a/tools/dockerfile/interoptest/grpc_interop_php/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_php/Dockerfile
index 65a8334269..0d6171c170 100644
--- a/tools/dockerfile/interoptest/grpc_interop_php/Dockerfile
+++ b/tools/dockerfile/interoptest/grpc_interop_php/Dockerfile
@@ -1,4 +1,4 @@
-# Copyright 2015, Google Inc.
+# Copyright 2016, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
@@ -63,21 +63,6 @@ RUN apt-get update && apt-get install -y \
# Build profiling
RUN apt-get update && apt-get install -y time && apt-get clean
-#====================
-# Python dependencies
-
-# Install dependencies
-
-RUN apt-get update && apt-get install -y \
- python-all-dev \
- python3-all-dev \
- python-pip
-
-# Install Python packages from PyPI
-RUN pip install pip --upgrade
-RUN pip install virtualenv
-RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0
-
#==================
# Ruby dependencies
@@ -98,12 +83,6 @@ RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc"
# Install dependencies
-RUN /bin/bash -l -c "echo 'deb http://packages.dotdeb.org wheezy-php55 all' \
- >> /etc/apt/sources.list.d/dotdeb.list"
-RUN /bin/bash -l -c "echo 'deb-src http://packages.dotdeb.org wheezy-php55 all' \
- >> /etc/apt/sources.list.d/dotdeb.list"
-RUN wget http://www.dotdeb.org/dotdeb.gpg -O- | apt-key add -
-
RUN apt-get update && apt-get install -y \
git php5 php5-dev phpunit unzip
@@ -128,11 +107,6 @@ RUN /bin/bash -l -c "rvm all do gem install ronn rake"
RUN curl -sS https://getcomposer.org/installer | php
RUN mv composer.phar /usr/local/bin/composer
-# As an attempt to work around #4212, try to prefetch Protobuf-PHP dependency
-# into composer cache to prevent "composer install" from cloning on each build.
-RUN git clone --mirror https://github.com/stanley-cheung/Protobuf-PHP.git \
- /root/.composer/cache/vcs/git-github.com-stanley-cheung-Protobuf-PHP.git/
-
# Download the patched PHP protobuf so that PHP gRPC clients can be generated
# from proto3 schemas.
RUN git clone https://github.com/stanley-cheung/Protobuf-PHP.git /var/local/git/protobuf-php
@@ -144,3 +118,4 @@ RUN /bin/bash -l -c "rvm use ruby-2.1 \
# Define the default command.
CMD ["bash"]
+
diff --git a/tools/dockerfile/interoptest/grpc_interop_php7/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_php7/Dockerfile
new file mode 100644
index 0000000000..be8f25f8ff
--- /dev/null
+++ b/tools/dockerfile/interoptest/grpc_interop_php7/Dockerfile
@@ -0,0 +1,125 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+FROM debian:jessie
+
+#=================
+# PHP7 dependencies
+
+# Install Git and basic packages.
+RUN apt-get update && apt-get install -y \
+ autoconf \
+ automake \
+ build-essential \
+ ccache \
+ curl \
+ git \
+ libcurl4-openssl-dev \
+ libgmp-dev \
+ libgmp3-dev \
+ libssl-dev \
+ libtool \
+ libxml2-dev \
+ pkg-config \
+ re2c \
+ time \
+ unzip \
+ wget \
+ zip && apt-get clean
+
+# Install other dependencies
+RUN ln -sf /usr/include/x86_64-linux-gnu/gmp.h /usr/include/gmp.h
+RUN wget http://ftp.gnu.org/gnu/bison/bison-2.6.4.tar.gz -O /var/local/bison-2.6.4.tar.gz
+RUN cd /var/local \
+ && tar -zxvf bison-2.6.4.tar.gz \
+ && cd /var/local/bison-2.6.4 \
+ && ./configure \
+ && make \
+ && make install
+
+# Compile PHP7 from source
+RUN git clone https://github.com/php/php-src /var/local/git/php-src
+RUN cd /var/local/git/php-src \
+ && git checkout PHP-7.0.9 \
+ && ./buildconf --force \
+ && ./configure \
+ --with-gmp \
+ --with-openssl \
+ --with-zlib \
+ && make \
+ && make install
+
+#==================
+# Ruby dependencies
+
+# Install rvm
+RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3
+RUN \curl -sSL https://get.rvm.io | bash -s stable
+
+# Install Ruby 2.1
+RUN /bin/bash -l -c "rvm install ruby-2.1"
+RUN /bin/bash -l -c "rvm use --default ruby-2.1"
+RUN /bin/bash -l -c "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc"
+RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc"
+RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.1' >> ~/.bashrc"
+RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc"
+
+# Prepare ccache
+RUN ln -s /usr/bin/ccache /usr/local/bin/gcc
+RUN ln -s /usr/bin/ccache /usr/local/bin/g++
+RUN ln -s /usr/bin/ccache /usr/local/bin/cc
+RUN ln -s /usr/bin/ccache /usr/local/bin/c++
+RUN ln -s /usr/bin/ccache /usr/local/bin/clang
+RUN ln -s /usr/bin/ccache /usr/local/bin/clang++
+
+
+RUN mkdir /var/local/jenkins
+
+# ronn: a ruby tool used to convert markdown to man pages, used during the
+# install of Protobuf extensions
+#
+# rake: a ruby version of make used to build the PHP Protobuf extension
+RUN /bin/bash -l -c "rvm all do gem install ronn rake"
+
+# Install composer
+RUN curl -sS https://getcomposer.org/installer | php
+RUN mv composer.phar /usr/local/bin/composer
+
+# Download the patched PHP protobuf so that PHP gRPC clients can be generated
+# from proto3 schemas.
+RUN git clone https://github.com/stanley-cheung/Protobuf-PHP.git /var/local/git/protobuf-php
+
+RUN /bin/bash -l -c "rvm use ruby-2.1 \
+ && cd /var/local/git/protobuf-php \
+ && rvm all do rake pear:package version=1.0 \
+ && pear install Protobuf-1.0.tgz"
+
+# Define the default command.
+CMD ["bash"]
+
diff --git a/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh
new file mode 100755
index 0000000000..261dded282
--- /dev/null
+++ b/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# Builds PHP interop server and client in a base image.
+set -ex
+
+mkdir -p /var/local/git
+git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
+
+# copy service account keys if available
+cp -r /var/local/jenkins/service_account $HOME || true
+
+cd /var/local/git/grpc
+rvm --default use ruby-2.1
+
+# gRPC core and protobuf need to be installed
+make install
+
+(cd src/php/ext/grpc && phpize && ./configure && make)
+
+(cd third_party/protobuf && make install)
+
+(cd src/php && composer install)
+
+(cd src/php && protoc-gen-php -i tests/interop/ -o tests/interop/ tests/interop/test.proto)
diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_php/Dockerfile b/tools/dockerfile/stress_test/grpc_interop_stress_php/Dockerfile
index bbbdd4a151..0716be5a9d 100644
--- a/tools/dockerfile/stress_test/grpc_interop_stress_php/Dockerfile
+++ b/tools/dockerfile/stress_test/grpc_interop_stress_php/Dockerfile
@@ -103,12 +103,6 @@ RUN pip install --upgrade google-api-python-client
# Install dependencies
-RUN /bin/bash -l -c "echo 'deb http://packages.dotdeb.org wheezy-php55 all' \
- >> /etc/apt/sources.list.d/dotdeb.list"
-RUN /bin/bash -l -c "echo 'deb-src http://packages.dotdeb.org wheezy-php55 all' \
- >> /etc/apt/sources.list.d/dotdeb.list"
-RUN wget http://www.dotdeb.org/dotdeb.gpg -O- | apt-key add -
-
RUN apt-get update && apt-get install -y \
git php5 php5-dev phpunit unzip
@@ -133,11 +127,6 @@ RUN /bin/bash -l -c "rvm all do gem install ronn rake"
RUN curl -sS https://getcomposer.org/installer | php
RUN mv composer.phar /usr/local/bin/composer
-# As an attempt to work around #4212, try to prefetch Protobuf-PHP dependency
-# into composer cache to prevent "composer install" from cloning on each build.
-RUN git clone --mirror https://github.com/stanley-cheung/Protobuf-PHP.git \
- /root/.composer/cache/vcs/git-github.com-stanley-cheung-Protobuf-PHP.git/
-
# Download the patched PHP protobuf so that PHP gRPC clients can be generated
# from proto3 schemas.
RUN git clone https://github.com/stanley-cheung/Protobuf-PHP.git /var/local/git/protobuf-php
diff --git a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile
index 92c8436851..13f7c10f92 100644
--- a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile
+++ b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile
@@ -100,12 +100,6 @@ RUN /bin/bash -l -c "nvm alias default 4"
# Install dependencies
-RUN /bin/bash -l -c "echo 'deb http://packages.dotdeb.org wheezy-php55 all' \
- >> /etc/apt/sources.list.d/dotdeb.list"
-RUN /bin/bash -l -c "echo 'deb-src http://packages.dotdeb.org wheezy-php55 all' \
- >> /etc/apt/sources.list.d/dotdeb.list"
-RUN wget http://www.dotdeb.org/dotdeb.gpg -O- | apt-key add -
-
RUN apt-get update && apt-get install -y \
git php5 php5-dev phpunit unzip
diff --git a/tools/dockerfile/test/php7_jessie_x64/Dockerfile b/tools/dockerfile/test/php7_jessie_x64/Dockerfile
new file mode 100644
index 0000000000..221338956e
--- /dev/null
+++ b/tools/dockerfile/test/php7_jessie_x64/Dockerfile
@@ -0,0 +1,105 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+FROM debian:jessie
+
+#=================
+# PHP7 dependencies
+
+# Install Git and basic packages.
+RUN apt-get update && apt-get install -y \
+ autoconf \
+ automake \
+ build-essential \
+ ccache \
+ curl \
+ git \
+ libcurl4-openssl-dev \
+ libgmp-dev \
+ libgmp3-dev \
+ libssl-dev \
+ libtool \
+ libxml2-dev \
+ pkg-config \
+ re2c \
+ time \
+ unzip \
+ wget \
+ zip && apt-get clean
+
+# Install other dependencies
+RUN ln -sf /usr/include/x86_64-linux-gnu/gmp.h /usr/include/gmp.h
+RUN wget http://ftp.gnu.org/gnu/bison/bison-2.6.4.tar.gz -O /var/local/bison-2.6.4.tar.gz
+RUN cd /var/local \
+ && tar -zxvf bison-2.6.4.tar.gz \
+ && cd /var/local/bison-2.6.4 \
+ && ./configure \
+ && make \
+ && make install
+
+# Compile PHP7 from source
+RUN git clone https://github.com/php/php-src /var/local/git/php-src
+RUN cd /var/local/git/php-src \
+ && git checkout PHP-7.0.9 \
+ && ./buildconf --force \
+ && ./configure \
+ --with-gmp \
+ --with-openssl \
+ --with-zlib \
+ && make \
+ && make install
+
+#====================
+# Python dependencies
+
+# Install dependencies
+
+RUN apt-get update && apt-get install -y \
+ python-all-dev \
+ python3-all-dev \
+ python-pip
+
+# Install Python packages from PyPI
+RUN pip install pip --upgrade
+RUN pip install virtualenv
+RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0
+
+# Prepare ccache
+RUN ln -s /usr/bin/ccache /usr/local/bin/gcc
+RUN ln -s /usr/bin/ccache /usr/local/bin/g++
+RUN ln -s /usr/bin/ccache /usr/local/bin/cc
+RUN ln -s /usr/bin/ccache /usr/local/bin/c++
+RUN ln -s /usr/bin/ccache /usr/local/bin/clang
+RUN ln -s /usr/bin/ccache /usr/local/bin/clang++
+
+
+RUN mkdir /var/local/jenkins
+
+# Define the default command.
+CMD ["bash"]
diff --git a/tools/dockerfile/test/php_jessie_x64/Dockerfile b/tools/dockerfile/test/php_jessie_x64/Dockerfile
index 2ef6e1d47f..17ea36b76c 100644
--- a/tools/dockerfile/test/php_jessie_x64/Dockerfile
+++ b/tools/dockerfile/test/php_jessie_x64/Dockerfile
@@ -83,12 +83,6 @@ RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0
# Install dependencies
-RUN /bin/bash -l -c "echo 'deb http://packages.dotdeb.org wheezy-php55 all' \
- >> /etc/apt/sources.list.d/dotdeb.list"
-RUN /bin/bash -l -c "echo 'deb-src http://packages.dotdeb.org wheezy-php55 all' \
- >> /etc/apt/sources.list.d/dotdeb.list"
-RUN wget http://www.dotdeb.org/dotdeb.gpg -O- | apt-key add -
-
RUN apt-get update && apt-get install -y \
git php5 php5-dev phpunit unzip
diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifact_targets.py
index e9267be58b..8550ee7b84 100644
--- a/tools/run_tests/artifact_targets.py
+++ b/tools/run_tests/artifact_targets.py
@@ -30,11 +30,14 @@
"""Definition of targets to build artifacts."""
+import os.path
+import sys
+
import jobset
def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={},
- flake_retries=0, timeout_retries=0):
+ flake_retries=0, timeout_retries=0, timeout_seconds=30*60):
"""Creates jobspec for a task running under docker."""
environ = environ.copy()
environ['RUN_COMMAND'] = shell_command
@@ -49,20 +52,20 @@ def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={},
cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] + docker_args,
environ=docker_env,
shortname='build_artifact.%s' % (name),
- timeout_seconds=30*60,
+ timeout_seconds=timeout_seconds,
flake_retries=flake_retries,
timeout_retries=timeout_retries)
return jobspec
def create_jobspec(name, cmdline, environ=None, shell=False,
- flake_retries=0, timeout_retries=0):
+ flake_retries=0, timeout_retries=0, timeout_seconds=30*60):
"""Creates jobspec."""
jobspec = jobset.JobSpec(
cmdline=cmdline,
environ=environ,
shortname='build_artifact.%s' % (name),
- timeout_seconds=30*60,
+ timeout_seconds=timeout_seconds,
flake_retries=flake_retries,
timeout_retries=timeout_retries,
shell=shell)
@@ -76,27 +79,30 @@ _ARCH_FLAG_MAP = {
'x64': '-m64'
}
-python_version_arch_map = {
- 'x86': 'Python27_32bits',
- 'x64': 'Python27'
+python_windows_version_arch_map = {
+ ('x86', '2.7'): 'Python27_32bits',
+ ('x64', '2.7'): 'Python27',
+ ('x86', '3.4'): 'Python34_32bits',
+ ('x64', '3.4'): 'Python34',
}
class PythonArtifact:
"""Builds Python artifacts."""
- def __init__(self, platform, arch, manylinux_build=None):
+ def __init__(self, platform, arch, python_version, manylinux_build=None):
if manylinux_build:
- self.name = 'python_%s_%s_%s' % (platform, arch, manylinux_build)
+ self.name = 'python%s_%s_%s_%s' % (python_version, platform, arch, manylinux_build)
else:
- self.name = 'python_%s_%s' % (platform, arch)
+ self.name = 'python%s_%s_%s' % (python_version, platform, arch)
self.platform = platform
self.arch = arch
- self.labels = ['artifact', 'python', platform, arch]
- self.python_version = python_version_arch_map[arch]
+ self.labels = ['artifact', 'python', python_version, platform, arch]
+ self.python_version = python_version
+ self.python_windows_prefix = python_windows_version_arch_map[arch, python_version]
self.manylinux_build = manylinux_build
def pre_build_jobspecs(self):
- return []
+ return []
def build_jobspec(self):
environ = {}
@@ -107,26 +113,27 @@ class PythonArtifact:
# special places...
environ['PYTHON'] = '/opt/python/{}/bin/python'.format(self.manylinux_build)
environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.manylinux_build)
- # Our docker image has all the prerequisites pip-installed already.
- environ['SKIP_PIP_INSTALL'] = '1'
# Platform autodetection for the manylinux1 image breaks so we set the
# defines ourselves.
# TODO(atash) get better platform-detection support in core so we don't
# need to do this manually...
environ['CFLAGS'] = '-DGPR_MANYLINUX1=1'
+ environ['BUILD_HEALTH_CHECKING'] = 'TRUE'
+ environ['BUILD_MANYLINUX_WHEEL'] = 'TRUE'
return create_docker_jobspec(self.name,
'tools/dockerfile/grpc_artifact_python_manylinux_%s' % self.arch,
'tools/run_tests/build_artifact_python.sh',
- environ=environ)
+ environ=environ,
+ timeout_seconds=60*60)
elif self.platform == 'windows':
return create_jobspec(self.name,
['tools\\run_tests\\build_artifact_python.bat',
- self.python_version,
+ self.python_windows_prefix,
'32' if self.arch == 'x86' else '64'
],
shell=True)
else:
- environ['SKIP_PIP_INSTALL'] = 'TRUE'
+ environ['PYTHON'] = 'python{}'.format(self.python_version)
return create_jobspec(self.name,
['tools/run_tests/build_artifact_python.sh'],
environ=environ)
@@ -323,13 +330,18 @@ def targets():
for Cls in (CSharpExtArtifact, NodeExtArtifact, ProtocArtifact)
for platform in ('linux', 'macos', 'windows')
for arch in ('x86', 'x64')] +
- [PythonArtifact('linux', 'x86', 'cp27-cp27m'),
- PythonArtifact('linux', 'x86', 'cp27-cp27mu'),
- PythonArtifact('linux', 'x64', 'cp27-cp27m'),
- PythonArtifact('linux', 'x64', 'cp27-cp27mu'),
- PythonArtifact('macos', 'x64'),
- PythonArtifact('windows', 'x86'),
- PythonArtifact('windows', 'x64'),
+ [PythonArtifact('linux', 'x86', '2.7', 'cp27-cp27m'),
+ PythonArtifact('linux', 'x86', '2.7', 'cp27-cp27mu'),
+ PythonArtifact('linux', 'x64', '2.7', 'cp27-cp27m'),
+ PythonArtifact('linux', 'x64', '2.7', 'cp27-cp27mu'),
+ PythonArtifact('macos', 'x64', '2.7'),
+ PythonArtifact('windows', 'x86', '2.7'),
+ PythonArtifact('windows', 'x64', '2.7'),
+ PythonArtifact('linux', 'x86', '3.4', 'cp34-cp34m'),
+ PythonArtifact('linux', 'x64', '3.4', 'cp34-cp34m'),
+ PythonArtifact('macos', 'x64', '3.4'),
+ PythonArtifact('windows', 'x86', '3.4'),
+ PythonArtifact('windows', 'x64', '3.4'),
RubyArtifact('linux', 'x86'),
RubyArtifact('linux', 'x64'),
RubyArtifact('macos', 'x64'),
diff --git a/tools/run_tests/build_artifact_python.bat b/tools/run_tests/build_artifact_python.bat
index a7b1a58284..074a3c6781 100644
--- a/tools/run_tests/build_artifact_python.bat
+++ b/tools/run_tests/build_artifact_python.bat
@@ -42,11 +42,19 @@ python tools\distrib\python\make_grpcio_tools.py
@rem Build gRPC Python extensions
python setup.py build_ext -c mingw32
-python tools\distrib\python\grpcio_tools\setup.py build_ext -c mingw32
+
+pushd tools\distrib\python\grpcio_tools
+python setup.py build_ext -c mingw32
+popd
+
@rem Build gRPC Python distributions
python setup.py bdist_wheel
-python tools\distrib\python\grpcio_tools\setup.py bdist_wheel
+
+pushd tools\distrib\python\grpcio_tools
+python setup.py bdist_wheel
+popd
+
mkdir artifacts
xcopy /Y /I /S dist\* artifacts\ || goto :error
diff --git a/tools/run_tests/build_artifact_python.sh b/tools/run_tests/build_artifact_python.sh
index 55f8eb634b..8f8330ef24 100755
--- a/tools/run_tests/build_artifact_python.sh
+++ b/tools/run_tests/build_artifact_python.sh
@@ -39,15 +39,6 @@ export PIP=${PIP:-pip}
export AUDITWHEEL=${AUDITWHEEL:-auditwheel}
-if [ "$SKIP_PIP_INSTALL" == "" ]
-then
- ${PIP} install --upgrade six
- # There's a bug in newer versions of setuptools (see
- # https://bitbucket.org/pypa/setuptools/issues/503/pkg_resources_vendorpackagingrequirementsi)
- ${PIP} pip install --upgrade 'setuptools==18'
- ${PIP} install -rrequirements.txt
-fi
-
# Build the source distribution first because MANIFEST.in cannot override
# exclusion of built shared objects among package resources (for some
# inexplicable reason).
@@ -71,15 +62,33 @@ CFLAGS="$CFLAGS -fno-wrapv" ${SETARCH_CMD} \
${PYTHON} tools/distrib/python/grpcio_tools/setup.py bdist_wheel
mkdir -p artifacts
-if command -v ${AUDITWHEEL}
+if [ "$BUILD_MANYLINUX_WHEEL" != "" ]
then
for wheel in dist/*.whl; do
${AUDITWHEEL} repair $wheel -w artifacts/
+ rm $wheel
done
for wheel in tools/distrib/python/grpcio_tools/dist/*.whl; do
${AUDITWHEEL} repair $wheel -w artifacts/
+ rm $wheel
done
fi
+# We need to use the built grpcio-tools/grpcio to compile the health proto
+# Wheels are not supported by setup_requires/dependency_links, so we
+# manually install the dependency. Note we should only do this if we
+# are in a docker image or in a virtualenv.
+if [ "$BUILD_HEALTH_CHECKING" != "" ]
+then
+ ${PIP} install -rrequirements.txt
+ ${PIP} install grpcio --no-index --find-links "file://${PWD}/artifacts/"
+ ${PIP} install grpcio-tools --no-index --find-links "file://${PWD}/artifacts/"
+
+ # Build gRPC health check source distribution
+ ${SETARCH_CMD} ${PYTHON} src/python/grpcio_health_checking/setup.py \
+ preprocess build_package_protos sdist
+ cp -r src/python/grpcio_health_checking/dist/* artifacts
+fi
+
cp -r dist/* artifacts
cp -r tools/distrib/python/grpcio_tools/dist/* artifacts
diff --git a/tools/run_tests/build_package_node.sh b/tools/run_tests/build_package_node.sh
index ff4cfdb8bf..ef4a10cca7 100755
--- a/tools/run_tests/build_package_node.sh
+++ b/tools/run_tests/build_package_node.sh
@@ -61,7 +61,7 @@ mkdir -p $output_dir
well_known_protos=( any api compiler/plugin descriptor duration empty field_mask source_context struct timestamp type wrappers )
for arch in {x86,x64}; do
- case arch in
+ case $arch in
x86)
node_arch=ia32
;;
@@ -70,7 +70,7 @@ for arch in {x86,x64}; do
;;
esac
for plat in {windows,linux,macos}; do
- case plat in
+ case $plat in
windows)
node_plat=win32
;;
diff --git a/tools/run_tests/build_python.sh b/tools/run_tests/build_python.sh
index 9cb3cb12a9..c4b5d98a53 100755
--- a/tools/run_tests/build_python.sh
+++ b/tools/run_tests/build_python.sh
@@ -164,7 +164,8 @@ pip_install_dir $ROOT/tools/distrib/python/grpcio_tools
# etc...
pip_install_dir $ROOT
$VENV_PYTHON $ROOT/src/python/grpcio_health_checking/setup.py preprocess
+$VENV_PYTHON $ROOT/src/python/grpcio_health_checking/setup.py build_package_protos
pip_install_dir $ROOT/src/python/grpcio_health_checking
$VENV_PYTHON $ROOT/src/python/grpcio_tests/setup.py preprocess
-$VENV_PYTHON $ROOT/src/python/grpcio_tests/setup.py build_proto_modules
+$VENV_PYTHON $ROOT/src/python/grpcio_tests/setup.py build_package_protos
pip_install_dir $ROOT/src/python/grpcio_tests
diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py
index 2e5a2f7721..78096b216c 100755
--- a/tools/run_tests/run_interop_tests.py
+++ b/tools/run_tests/run_interop_tests.py
@@ -268,6 +268,31 @@ class PHPLanguage:
return 'php'
+class PHP7Language:
+
+ def __init__(self):
+ self.client_cwd = None
+ self.safename = str(self)
+
+ def client_cmd(self, args):
+ return ['src/php/bin/interop_client.sh'] + args
+
+ def cloud_to_prod_env(self):
+ return {}
+
+ def global_env(self):
+ return {}
+
+ def unimplemented_test_cases(self):
+ return _SKIP_COMPRESSION
+
+ def unimplemented_test_cases_server(self):
+ return []
+
+ def __str__(self):
+ return 'php7'
+
+
class RubyLanguage:
def __init__(self):
@@ -346,6 +371,7 @@ _LANGUAGES = {
'java' : JavaLanguage(),
'node' : NodeLanguage(),
'php' : PHPLanguage(),
+ 'php7' : PHP7Language(),
'ruby' : RubyLanguage(),
'python' : PythonLanguage(),
}
@@ -409,7 +435,7 @@ def auth_options(language, test_case):
default_account_arg = '--default_service_account=830293263384-compute@developer.gserviceaccount.com'
if test_case in ['jwt_token_creds', 'per_rpc_creds', 'oauth2_auth_token']:
- if language in ['csharp', 'node', 'php', 'python', 'ruby']:
+ if language in ['csharp', 'node', 'php', 'php7', 'python', 'ruby']:
env['GOOGLE_APPLICATION_CREDENTIALS'] = key_filepath
else:
cmdargs += [key_file_arg]
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index 57fff2ec9c..542415d908 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -63,7 +63,9 @@ _ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
os.chdir(_ROOT)
-_FORCE_ENVIRON_FOR_WRAPPERS = {}
+_FORCE_ENVIRON_FOR_WRAPPERS = {
+ 'GRPC_VERBOSITY': 'DEBUG',
+}
_POLLING_STRATEGIES = {
@@ -164,7 +166,8 @@ class CLanguage(object):
for polling_strategy in polling_strategies:
env={'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH':
_ROOT + '/src/core/lib/tsi/test_creds/ca.pem',
- 'GRPC_POLL_STRATEGY': polling_strategy}
+ 'GRPC_POLL_STRATEGY': polling_strategy,
+ 'GRPC_VERBOSITY': 'DEBUG'}
shortname_ext = '' if polling_strategy=='all' else ' GRPC_POLL_STRATEGY=%s' % polling_strategy
if self.config.build_config in target['exclude_configs']:
continue
@@ -378,6 +381,42 @@ class PhpLanguage(object):
return 'php'
+class Php7Language(object):
+
+ def configure(self, config, args):
+ self.config = config
+ self.args = args
+ _check_compiler(self.args.compiler, ['default'])
+
+ def test_specs(self):
+ return [self.config.job_spec(['src/php/bin/run_tests.sh'], None,
+ environ=_FORCE_ENVIRON_FOR_WRAPPERS)]
+
+ def pre_build_steps(self):
+ return []
+
+ def make_targets(self):
+ return ['static_c', 'shared_c']
+
+ def make_options(self):
+ return []
+
+ def build_steps(self):
+ return [['tools/run_tests/build_php.sh']]
+
+ def post_tests_steps(self):
+ return [['tools/run_tests/post_tests_php.sh']]
+
+ def makefile_name(self):
+ return 'Makefile'
+
+ def dockerfile_dir(self):
+ return 'tools/dockerfile/test/php7_jessie_%s' % _docker_arch_suffix(self.args.arch)
+
+ def __str__(self):
+ return 'php7'
+
+
class PythonConfig(collections.namedtuple('PythonConfig', [
'name', 'build', 'run'])):
"""Tuple of commands (named s.t. 'what it says on the tin' applies)"""
@@ -746,6 +785,7 @@ _LANGUAGES = {
'c': CLanguage('c', 'c'),
'node': NodeLanguage(),
'php': PhpLanguage(),
+ 'php7': Php7Language(),
'python': PythonLanguage(),
'ruby': RubyLanguage(),
'csharp': CSharpLanguage(),
diff --git a/tools/run_tests/sanity/check_sources_and_headers.py b/tools/run_tests/sanity/check_sources_and_headers.py
index c028499ca6..28c1dc46d7 100755
--- a/tools/run_tests/sanity/check_sources_and_headers.py
+++ b/tools/run_tests/sanity/check_sources_and_headers.py
@@ -55,7 +55,8 @@ def target_has_header(target, name):
for dep in target['deps']:
if target_has_header(get_target(dep), name):
return True
- if name == 'src/core/lib/profiling/stap_probes.h':
+ if name in ['src/core/lib/profiling/stap_probes.h',
+ 'src/proto/grpc/reflection/v1alpha/reflection.grpc.pb.h']:
return True
return False
diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json
index 223b711fe2..d09aa418d7 100644
--- a/tools/run_tests/sources_and_headers.json
+++ b/tools/run_tests/sources_and_headers.json
@@ -4395,25 +4395,19 @@
{
"deps": [
"grpc++",
- "grpc++_codegen_proto"
+ "grpc++_reflection_proto"
],
"headers": [
"include/grpc++/ext/proto_server_reflection_plugin.h",
- "include/grpc++/ext/reflection.grpc.pb.h",
- "include/grpc++/ext/reflection.pb.h",
"src/cpp/ext/proto_server_reflection.h"
],
"language": "c++",
"name": "grpc++_reflection",
"src": [
"include/grpc++/ext/proto_server_reflection_plugin.h",
- "include/grpc++/ext/reflection.grpc.pb.h",
- "include/grpc++/ext/reflection.pb.h",
"src/cpp/ext/proto_server_reflection.cc",
"src/cpp/ext/proto_server_reflection.h",
- "src/cpp/ext/proto_server_reflection_plugin.cc",
- "src/cpp/ext/reflection.grpc.pb.cc",
- "src/cpp/ext/reflection.pb.cc"
+ "src/cpp/ext/proto_server_reflection_plugin.cc"
],
"third_party": false,
"type": "lib"
@@ -4421,6 +4415,18 @@
{
"deps": [],
"headers": [
+ "src/proto/grpc/reflection/v1alpha/reflection.grpc.pb.h",
+ "src/proto/grpc/reflection/v1alpha/reflection.pb.h"
+ ],
+ "language": "c++",
+ "name": "grpc++_reflection_codegen",
+ "src": [],
+ "third_party": false,
+ "type": "lib"
+ },
+ {
+ "deps": [],
+ "headers": [
"test/cpp/util/test_config.h"
],
"language": "c++",
@@ -6796,5 +6802,24 @@
],
"third_party": false,
"type": "filegroup"
+ },
+ {
+ "deps": [
+ "grpc++_codegen_proto"
+ ],
+ "headers": [
+ "include/grpc++/ext/reflection.grpc.pb.h",
+ "include/grpc++/ext/reflection.pb.h"
+ ],
+ "language": "c++",
+ "name": "grpc++_reflection_proto",
+ "src": [
+ "include/grpc++/ext/reflection.grpc.pb.h",
+ "include/grpc++/ext/reflection.pb.h",
+ "src/cpp/ext/reflection.grpc.pb.cc",
+ "src/cpp/ext/reflection.pb.cc"
+ ],
+ "third_party": false,
+ "type": "filegroup"
}
]