diff options
Diffstat (limited to 'tools')
47 files changed, 947 insertions, 301 deletions
diff --git a/tools/cmake/gRPCConfig.cmake.in b/tools/cmake/gRPCConfig.cmake.in new file mode 100644 index 0000000000..48f0674579 --- /dev/null +++ b/tools/cmake/gRPCConfig.cmake.in @@ -0,0 +1,7 @@ +# Depend packages
+@_gRPC_FIND_ZLIB@
+@_gRPC_FIND_PROTOBUF@
+@_gRPC_FIND_SSL@
+
+# Targets
+include(${CMAKE_CURRENT_LIST_DIR}/gRPCTargets.cmake)
diff --git a/tools/cmake/gRPCConfigVersion.cmake.in b/tools/cmake/gRPCConfigVersion.cmake.in new file mode 100644 index 0000000000..f3c19fd403 --- /dev/null +++ b/tools/cmake/gRPCConfigVersion.cmake.in @@ -0,0 +1,11 @@ +set(PACKAGE_VERSION "@PACKAGE_VERSION@")
+
+# Check whether the requested PACKAGE_FIND_VERSION is compatible
+if("${PACKAGE_VERSION}" VERSION_LESS "${PACKAGE_FIND_VERSION}")
+ set(PACKAGE_VERSION_COMPATIBLE FALSE)
+else()
+ set(PACKAGE_VERSION_COMPATIBLE TRUE)
+ if ("${PACKAGE_VERSION}" VERSION_EQUAL "${PACKAGE_FIND_VERSION}")
+ set(PACKAGE_VERSION_EXACT TRUE)
+ endif()
+endif()
diff --git a/tools/distrib/python/docgen.py b/tools/distrib/python/docgen.py index f5e89f1da6..15bd8d855f 100755 --- a/tools/distrib/python/docgen.py +++ b/tools/distrib/python/docgen.py @@ -71,6 +71,8 @@ environment.update({ subprocess_arguments_list = [ {'args': ['virtualenv', VIRTUALENV_DIR], 'env': environment}, + {'args': [VIRTUALENV_PIP_PATH, 'install', '--upgrade', 'pip'], + 'env': environment}, {'args': [VIRTUALENV_PIP_PATH, 'install', '-r', REQUIREMENTS_PATH], 'env': environment}, {'args': [VIRTUALENV_PYTHON_PATH, SETUP_PATH, 'build'], 'env': environment}, diff --git a/tools/distrib/python/grpcio_tools/README.rst b/tools/distrib/python/grpcio_tools/README.rst index 8946a1d5b3..55521d17bb 100644 --- a/tools/distrib/python/grpcio_tools/README.rst +++ b/tools/distrib/python/grpcio_tools/README.rst @@ -122,6 +122,7 @@ Help, I ... third_party/protobuf/src/google/protobuf/stubs/mathlimits.h:173:31: note: in expansion of macro 'SIGNED_INT_MAX' static const Type kPosMax = SIGNED_INT_MAX(Type); \\ ^ + And your toolchain is GCC (at the time of this writing, up through at least GCC 6.0), this is probably a bug where GCC chokes on constant expressions when the :code:`-fwrapv` flag is specified. You should consider setting your @@ -136,3 +137,42 @@ Given protobuf include directories :code:`$INCLUDE`, an output directory :: $ python -m grpc.tools.protoc -I$INCLUDE --python_out=$OUTPUT --grpc_python_out=$OUTPUT $PROTO_FILES + +To use as a build step in distutils-based projects, you may use the provided +command class in your :code:`setup.py`: + +:: + + setuptools.setup( + # ... + cmdclass={ + 'build_proto_modules': grpc.tools.command.BuildPackageProtos, + } + # ... + ) + +Invocation of the command will walk the project tree and transpile every +:code:`.proto` file into a :code:`_pb2.py` file in the same directory. + +Note that this particular approach requires :code:`grpcio-tools` to be +installed on the machine before the setup script is invoked (i.e. no +combination of :code:`setup_requires` or :code:`install_requires` will provide +access to :code:`grpc.tools.command.BuildPackageProtos` if it isn't already +installed). One way to work around this can be found in our +:code:`grpcio-health-checking` +`package <https://pypi.python.org/pypi/grpcio-health-checking>`_: + +:: + + class BuildPackageProtos(setuptools.Command): + """Command to generate project *_pb2.py modules from proto files.""" + # ... + def run(self): + from grpc.tools import command + command.build_package_protos(self.distribution.package_dir['']) + +Now including :code:`grpcio-tools` in :code:`setup_requires` will provide the +command on-setup as desired. + +For more information on command classes, consult :code:`distutils` and +:code:`setuptools` documentation. diff --git a/tools/distrib/python/grpcio_tools/grpc/tools/command.py b/tools/distrib/python/grpcio_tools/grpc/tools/command.py index ccf38b7d56..2520099835 100644 --- a/tools/distrib/python/grpcio_tools/grpc/tools/command.py +++ b/tools/distrib/python/grpcio_tools/grpc/tools/command.py @@ -35,7 +35,26 @@ import setuptools from grpc.tools import protoc -class BuildProtoModules(setuptools.Command): +def build_package_protos(package_root): + proto_files = [] + inclusion_root = os.path.abspath(package_root) + for root, _, files in os.walk(inclusion_root): + for filename in files: + if filename.endswith('.proto'): + proto_files.append(os.path.abspath(os.path.join(root, filename))) + + for proto_file in proto_files: + command = [ + 'grpc.tools.protoc', + '--proto_path={}'.format(inclusion_root), + '--python_out={}'.format(inclusion_root), + '--grpc_python_out={}'.format(inclusion_root), + ] + [proto_file] + if protoc.main(command) != 0: + sys.stderr.write('warning: {} failed'.format(command)) + + +class BuildPackageProtos(setuptools.Command): """Command to generate project *_pb2.py modules from proto files.""" description = 'build grpc protobuf modules' @@ -52,19 +71,4 @@ class BuildProtoModules(setuptools.Command): # directory is provided as an 'include' directory. We assume it's the '' key # to `self.distribution.package_dir` (and get a key error if it's not # there). - proto_files = [] - inclusion_root = os.path.abspath(self.distribution.package_dir['']) - for root, _, files in os.walk(inclusion_root): - for filename in files: - if filename.endswith('.proto'): - proto_files.append(os.path.abspath(os.path.join(root, filename))) - - for proto_file in proto_files: - command = [ - 'grpc.tools.protoc', - '--proto_path={}'.format(inclusion_root), - '--python_out={}'.format(inclusion_root), - '--grpc_python_out={}'.format(inclusion_root), - ] + [proto_file] - if protoc.main(command) != 0: - sys.stderr.write('warning: {} failed'.format(command)) + build_package_protos(self.distribution.package_dir['']) diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index d804f34fc6..bb1f1cf085 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -28,11 +28,13 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from distutils import extension +from distutils import util import errno import os import os.path import pkg_resources import platform +import re import shlex import shutil import sys @@ -51,15 +53,50 @@ import grpc_version PY3 = sys.version_info.major == 3 +# Environment variable to determine whether or not the Cython extension should +# *use* Cython or use the generated C files. Note that this requires the C files +# to have been generated by building first *with* Cython support. +BUILD_WITH_CYTHON = os.environ.get('GRPC_PYTHON_BUILD_WITH_CYTHON', False) + # There are some situations (like on Windows) where CC, CFLAGS, and LDFLAGS are # entirely ignored/dropped/forgotten by distutils and its Cygwin/MinGW support. # We use these environment variables to thus get around that without locking # ourselves in w.r.t. the multitude of operating systems this ought to build on. -# By default we assume a GCC-like compiler. -EXTRA_COMPILE_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_CFLAGS', - '-fno-wrapv -frtti -std=c++11')) -EXTRA_LINK_ARGS = shlex.split(os.environ.get('GRPC_PYTHON_LDFLAGS', - '-lpthread')) +# We can also use these variables as a way to inject environment-specific +# compiler/linker flags. We assume GCC-like compilers and/or MinGW as a +# reasonable default. +EXTRA_ENV_COMPILE_ARGS = os.environ.get('GRPC_PYTHON_CFLAGS', None) +EXTRA_ENV_LINK_ARGS = os.environ.get('GRPC_PYTHON_LDFLAGS', None) +if EXTRA_ENV_COMPILE_ARGS is None: + EXTRA_ENV_COMPILE_ARGS = '-fno-wrapv -frtti -std=c++11' + if 'win32' in sys.platform: + # We use define flags here and don't directly add to DEFINE_MACROS below to + # ensure that the expert user/builder has a way of turning it off (via the + # envvars) without adding yet more GRPC-specific envvars. + # See https://sourceforge.net/p/mingw-w64/bugs/363/ + if '32' in platform.architecture()[0]: + EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime32 -D_timeb=__timeb32 -D_ftime_s=_ftime32_s' + else: + EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime64 -D_timeb=__timeb64' +if EXTRA_ENV_LINK_ARGS is None: + EXTRA_ENV_LINK_ARGS = '-lpthread' + if 'win32' in sys.platform: + # TODO(atash) check if this is actually safe to just import and call on + # non-Windows (to avoid breaking import style) + from distutils.cygwinccompiler import get_msvcr + msvcr = get_msvcr()[0] + EXTRA_ENV_LINK_ARGS += ( + ' -static-libgcc -static-libstdc++ -mcrtdll={msvcr} ' + '-static'.format(msvcr=msvcr)) +EXTRA_COMPILE_ARGS = shlex.split(EXTRA_ENV_COMPILE_ARGS) +EXTRA_LINK_ARGS = shlex.split(EXTRA_ENV_LINK_ARGS) + +CC_FILES = [ + os.path.normpath(cc_file) for cc_file in protoc_lib_deps.CC_FILES] +PROTO_FILES = [ + os.path.normpath(proto_file) for proto_file in protoc_lib_deps.PROTO_FILES] +CC_INCLUDE = os.path.normpath(protoc_lib_deps.CC_INCLUDE) +PROTO_INCLUDE = os.path.normpath(protoc_lib_deps.PROTO_INCLUDE) GRPC_PYTHON_TOOLS_PACKAGE = 'grpc.tools' GRPC_PYTHON_PROTO_RESOURCES_NAME = '_proto' @@ -76,14 +113,18 @@ if 'darwin' in sys.platform and PY3: if mac_target and (pkg_resources.parse_version(mac_target) < pkg_resources.parse_version('10.9.0')): os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.9' + os.environ['_PYTHON_HOST_PLATFORM'] = re.sub( + r'macosx-[0-9]+\.[0-9]+-(.+)', + r'macosx-10.9-\1', + util.get_platform()) def package_data(): tools_path = GRPC_PYTHON_TOOLS_PACKAGE.replace('.', os.path.sep) proto_resources_path = os.path.join(tools_path, GRPC_PYTHON_PROTO_RESOURCES_NAME) proto_files = [] - for proto_file in protoc_lib_deps.PROTO_FILES: - source = os.path.join(protoc_lib_deps.PROTO_INCLUDE, proto_file) + for proto_file in PROTO_FILES: + source = os.path.join(PROTO_INCLUDE, proto_file) target = os.path.join(proto_resources_path, proto_file) relative_target = os.path.join(GRPC_PYTHON_PROTO_RESOURCES_NAME, proto_file) try: @@ -97,39 +138,42 @@ def package_data(): proto_files.append(relative_target) return {GRPC_PYTHON_TOOLS_PACKAGE: proto_files} -def protoc_ext_module(): - plugin_sources = [ - 'grpc/tools/main.cc', - 'grpc_root/src/compiler/python_generator.cc'] + [ - os.path.join(protoc_lib_deps.CC_INCLUDE, cc_file) - for cc_file in protoc_lib_deps.CC_FILES] +def extension_modules(): + if BUILD_WITH_CYTHON: + plugin_sources = [os.path.join('grpc', 'tools', '_protoc_compiler.pyx')] + else: + plugin_sources = [os.path.join('grpc', 'tools', '_protoc_compiler.cpp')] + plugin_sources += [ + os.path.join('grpc', 'tools', 'main.cc'), + os.path.join('grpc_root', 'src', 'compiler', 'python_generator.cc')] + [ + os.path.join(CC_INCLUDE, cc_file) + for cc_file in CC_FILES] plugin_ext = extension.Extension( name='grpc.tools._protoc_compiler', - sources=['grpc/tools/_protoc_compiler.pyx'] + plugin_sources, + sources=plugin_sources, include_dirs=[ '.', 'grpc_root', - 'grpc_root/include', - protoc_lib_deps.CC_INCLUDE, + os.path.join('grpc_root', 'include'), + CC_INCLUDE, ], language='c++', define_macros=list(DEFINE_MACROS), extra_compile_args=list(EXTRA_COMPILE_ARGS), extra_link_args=list(EXTRA_LINK_ARGS), ) - return plugin_ext - -def maybe_cythonize(exts): - from Cython import Build - return Build.cythonize(exts) + extensions = [plugin_ext] + if BUILD_WITH_CYTHON: + from Cython import Build + return Build.cythonize(extensions) + else: + return extensions setuptools.setup( name='grpcio_tools', version=grpc_version.VERSION, license='3-clause BSD', - ext_modules=maybe_cythonize([ - protoc_ext_module(), - ]), + ext_modules=extension_modules(), packages=setuptools.find_packages('.'), namespace_packages=['grpc'], install_requires=[ diff --git a/tools/distrib/python/make_grpcio_tools.py b/tools/distrib/python/make_grpcio_tools.py index fd9b38b084..adf58445af 100755 --- a/tools/distrib/python/make_grpcio_tools.py +++ b/tools/distrib/python/make_grpcio_tools.py @@ -29,12 +29,18 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from __future__ import print_function + +import errno +import filecmp +import glob import os import os.path import shutil import subprocess import sys import traceback +import uuid DEPS_FILE_CONTENT=""" # Copyright 2016, Google Inc. @@ -124,20 +130,88 @@ def get_deps(): proto_include=repr(GRPC_PYTHON_PROTOBUF_RELATIVE_ROOT)) return deps_file_content +def long_path(path): + if os.name == 'nt': + return '\\\\?\\' + path + else: + return path + +def atomic_file_copy(src, dst): + """Based on the lock-free-whack-a-mole algorithm, depending on filesystem + renaming being atomic. Described at http://stackoverflow.com/a/28090883. + """ + try: + if filecmp.cmp(src, dst): + return + except: + pass + dst_dir = os.path.abspath(os.path.dirname(dst)) + dst_base = os.path.basename(dst) + this_id = str(uuid.uuid4()).replace('.', '-') + temporary_file = os.path.join(dst_dir, '{}.{}.tmp'.format(dst_base, this_id)) + mole_file = os.path.join(dst_dir, '{}.{}.mole.tmp'.format(dst_base, this_id)) + mole_pattern = os.path.join(dst_dir, '{}.*.mole.tmp'.format(dst_base)) + src = long_path(src) + dst = long_path(dst) + temporary_file = long_path(temporary_file) + mole_file = long_path(mole_file) + mole_pattern = long_path(mole_pattern) + shutil.copy2(src, temporary_file) + try: + os.rename(temporary_file, mole_file) + except: + print('Error moving temporary file {} to {}'.format(temporary_file, mole_file), file=sys.stderr) + print('while trying to copy file {} to {}'.format(src, dst), file=sys.stderr) + raise + for other_file in glob.glob(mole_pattern): + other_id = other_file.split('.')[-3] + if this_id == other_id: + pass + elif this_id < other_id: + try: + os.remove(other_file) + except: + pass + else: + try: + os.remove(mole_file) + except: + pass + this_id = other_id + mole_file = other_file + try: + if filecmp.cmp(src, dst): + try: + os.remove(mole_file) + except: + pass + return + except: + pass + try: + os.rename(mole_file, dst) + except: + pass + def main(): os.chdir(GRPC_ROOT) - for tree in [GRPC_PYTHON_PROTOBUF, - GRPC_PYTHON_PROTOC_PLUGINS, - GRPC_PYTHON_INCLUDE]: - try: - shutil.rmtree(tree) - except Exception as _: - pass - shutil.copytree(GRPC_PROTOBUF, GRPC_PYTHON_PROTOBUF) - shutil.copytree(GRPC_PROTOC_PLUGINS, GRPC_PYTHON_PROTOC_PLUGINS) - shutil.copytree(GRPC_INCLUDE, GRPC_PYTHON_INCLUDE) + for source, target in [ + (GRPC_PROTOBUF, GRPC_PYTHON_PROTOBUF), + (GRPC_PROTOC_PLUGINS, GRPC_PYTHON_PROTOC_PLUGINS), + (GRPC_INCLUDE, GRPC_PYTHON_INCLUDE)]: + for source_dir, _, files in os.walk(source): + target_dir = os.path.abspath(os.path.join(target, os.path.relpath(source_dir, source))) + try: + os.makedirs(target_dir) + except OSError as error: + if error.errno != errno.EEXIST: + raise + for relative_file in files: + source_file = os.path.abspath(os.path.join(source_dir, relative_file)) + target_file = os.path.abspath(os.path.join(target_dir, relative_file)) + atomic_file_copy(source_file, target_file) try: protoc_lib_deps_content = get_deps() diff --git a/tools/dockerfile/distribtest/python_arch_x64/Dockerfile b/tools/dockerfile/distribtest/python_arch_x64/Dockerfile index 2f79cc3017..dff72eee97 100644 --- a/tools/dockerfile/distribtest/python_arch_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_arch_x64/Dockerfile @@ -33,4 +33,4 @@ RUN pacman --noconfirm -Syy RUN pacman --noconfirm -S openssl RUN pacman --noconfirm -S python2 RUN pacman --noconfirm -S python2-pip - +RUN pip install virtualenv diff --git a/tools/dockerfile/distribtest/python_centos6_x64/Dockerfile b/tools/dockerfile/distribtest/python_centos6_x64/Dockerfile index d4f473792e..967450156c 100644 --- a/tools/dockerfile/distribtest/python_centos6_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_centos6_x64/Dockerfile @@ -44,3 +44,5 @@ RUN curl https://bootstrap.pypa.io/get-pip.py | python - # "which" command required by python's run_distrib_test.sh RUN yum install -y which + +RUN pip install virtualenv diff --git a/tools/dockerfile/distribtest/python_centos7_x64/Dockerfile b/tools/dockerfile/distribtest/python_centos7_x64/Dockerfile index ca64fa7bea..0127fe1e28 100644 --- a/tools/dockerfile/distribtest/python_centos7_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_centos7_x64/Dockerfile @@ -32,4 +32,4 @@ FROM centos:7 RUN yum install -y python RUN yum install -y epel-release RUN yum install -y python-pip - +RUN pip install virtualenv diff --git a/tools/dockerfile/distribtest/python_fedora20_x64/Dockerfile b/tools/dockerfile/distribtest/python_fedora20_x64/Dockerfile index 8b0f769c26..3d3636e43d 100644 --- a/tools/dockerfile/distribtest/python_fedora20_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_fedora20_x64/Dockerfile @@ -35,3 +35,5 @@ RUN yum clean all && yum update -y && yum install -y python python-pip # Trying twice makes it work fine. # https://github.com/docker/docker/issues/10180 RUN pip install --upgrade six || pip install --upgrade six + +RUN pip install virtualenv diff --git a/tools/dockerfile/distribtest/python_fedora21_x64/Dockerfile b/tools/dockerfile/distribtest/python_fedora21_x64/Dockerfile index fcbe053f1f..0b1b6aeb35 100644 --- a/tools/dockerfile/distribtest/python_fedora21_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_fedora21_x64/Dockerfile @@ -40,3 +40,5 @@ RUN yum clean all && yum update -y && yum install -y python python-pip # Trying twice makes it work fine. # https://github.com/docker/docker/issues/10180 RUN pip2 install --upgrade six || pip2 install --upgrade six + +RUN pip2 install virtualenv diff --git a/tools/dockerfile/distribtest/python_fedora22_x64/Dockerfile b/tools/dockerfile/distribtest/python_fedora22_x64/Dockerfile index ddcacb4257..4d75034c15 100644 --- a/tools/dockerfile/distribtest/python_fedora22_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_fedora22_x64/Dockerfile @@ -30,3 +30,4 @@ FROM fedora:22 RUN yum clean all && yum update -y && yum install -y python python-pip +RUN pip install virtualenv diff --git a/tools/dockerfile/distribtest/python_fedora23_x64/Dockerfile b/tools/dockerfile/distribtest/python_fedora23_x64/Dockerfile index d45195e509..a1bc9ba8d6 100644 --- a/tools/dockerfile/distribtest/python_fedora23_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_fedora23_x64/Dockerfile @@ -30,3 +30,4 @@ FROM fedora:23 RUN yum clean all && yum update -y && yum install -y python python-pip +RUN pip install virtualenv diff --git a/tools/dockerfile/distribtest/python_jessie_x64/Dockerfile b/tools/dockerfile/distribtest/python_jessie_x64/Dockerfile index 83df4ed4fa..7dc32a088e 100644 --- a/tools/dockerfile/distribtest/python_jessie_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_jessie_x64/Dockerfile @@ -30,3 +30,4 @@ FROM debian:jessie RUN apt-get update && apt-get install -y python python-pip +RUN pip install virtualenv diff --git a/tools/dockerfile/distribtest/python_jessie_x86/Dockerfile b/tools/dockerfile/distribtest/python_jessie_x86/Dockerfile index 19addb2912..04c1402e72 100644 --- a/tools/dockerfile/distribtest/python_jessie_x86/Dockerfile +++ b/tools/dockerfile/distribtest/python_jessie_x86/Dockerfile @@ -31,6 +31,8 @@ FROM 32bit/debian:jessie RUN apt-get update && apt-get install -y python python-pip +RUN pip install virtualenv + # docker is running on a 64-bit machine, so we need to # override "uname -m" to report i686 instead of x86_64, otherwise # python will choose a wrong binary package to install. diff --git a/tools/dockerfile/distribtest/python_opensuse_x64/Dockerfile b/tools/dockerfile/distribtest/python_opensuse_x64/Dockerfile index fe1406be98..27159c72e3 100644 --- a/tools/dockerfile/distribtest/python_opensuse_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_opensuse_x64/Dockerfile @@ -38,3 +38,5 @@ RUN zypper --non-interactive install which # Without this, pip won't be able to connect to # https://pypi.python.org/simple/ RUN zypper --non-interactive install ca-certificates-mozilla + +RUN pip install virtualenv diff --git a/tools/dockerfile/distribtest/python_ubuntu1204_x64/Dockerfile b/tools/dockerfile/distribtest/python_ubuntu1204_x64/Dockerfile index 4068fbe2ba..7a8c91b79b 100644 --- a/tools/dockerfile/distribtest/python_ubuntu1204_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_ubuntu1204_x64/Dockerfile @@ -30,3 +30,5 @@ FROM ubuntu:12.04 RUN apt-get update -y && apt-get install -y python python-pip + +RUN pip install virtualenv diff --git a/tools/dockerfile/distribtest/python_ubuntu1404_x64/Dockerfile b/tools/dockerfile/distribtest/python_ubuntu1404_x64/Dockerfile index 0858fb0c06..65189a44de 100644 --- a/tools/dockerfile/distribtest/python_ubuntu1404_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_ubuntu1404_x64/Dockerfile @@ -30,3 +30,5 @@ FROM ubuntu:14.04 RUN apt-get update -y && apt-get install -y python python-pip + +RUN pip install virtualenv diff --git a/tools/dockerfile/distribtest/python_ubuntu1504_x64/Dockerfile b/tools/dockerfile/distribtest/python_ubuntu1504_x64/Dockerfile index ed6ffddbec..abf36c4a24 100644 --- a/tools/dockerfile/distribtest/python_ubuntu1504_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_ubuntu1504_x64/Dockerfile @@ -30,3 +30,5 @@ FROM ubuntu:15.04 RUN apt-get update -y && apt-get install -y python python-pip + +RUN pip install virtualenv diff --git a/tools/dockerfile/distribtest/python_ubuntu1510_x64/Dockerfile b/tools/dockerfile/distribtest/python_ubuntu1510_x64/Dockerfile index 9e3e0c260f..6e862d203b 100644 --- a/tools/dockerfile/distribtest/python_ubuntu1510_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_ubuntu1510_x64/Dockerfile @@ -30,3 +30,5 @@ FROM ubuntu:15.10 RUN apt-get update -y && apt-get install -y python python-pip + +RUN pip install virtualenv diff --git a/tools/dockerfile/distribtest/python_ubuntu1604_x64/Dockerfile b/tools/dockerfile/distribtest/python_ubuntu1604_x64/Dockerfile index 5098da8a26..59f4feab55 100644 --- a/tools/dockerfile/distribtest/python_ubuntu1604_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_ubuntu1604_x64/Dockerfile @@ -30,3 +30,5 @@ FROM ubuntu:16.04 RUN apt-get update -y && apt-get install -y python python-pip + +RUN pip install virtualenv diff --git a/tools/dockerfile/distribtest/python_wheezy_x64/Dockerfile b/tools/dockerfile/distribtest/python_wheezy_x64/Dockerfile index 66165ee929..bc8816d305 100644 --- a/tools/dockerfile/distribtest/python_wheezy_x64/Dockerfile +++ b/tools/dockerfile/distribtest/python_wheezy_x64/Dockerfile @@ -30,3 +30,5 @@ FROM debian:wheezy RUN apt-get update -y && apt-get install -y python python-pip + +RUN pip install virtualenv diff --git a/tools/dockerfile/interoptest/grpc_interop_php/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_php/Dockerfile index 65a8334269..0d6171c170 100644 --- a/tools/dockerfile/interoptest/grpc_interop_php/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_php/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2015, Google Inc. +# Copyright 2016, Google Inc. # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -63,21 +63,6 @@ RUN apt-get update && apt-get install -y \ # Build profiling RUN apt-get update && apt-get install -y time && apt-get clean -#==================== -# Python dependencies - -# Install dependencies - -RUN apt-get update && apt-get install -y \ - python-all-dev \ - python3-all-dev \ - python-pip - -# Install Python packages from PyPI -RUN pip install pip --upgrade -RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 - #================== # Ruby dependencies @@ -98,12 +83,6 @@ RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc" # Install dependencies -RUN /bin/bash -l -c "echo 'deb http://packages.dotdeb.org wheezy-php55 all' \ - >> /etc/apt/sources.list.d/dotdeb.list" -RUN /bin/bash -l -c "echo 'deb-src http://packages.dotdeb.org wheezy-php55 all' \ - >> /etc/apt/sources.list.d/dotdeb.list" -RUN wget http://www.dotdeb.org/dotdeb.gpg -O- | apt-key add - - RUN apt-get update && apt-get install -y \ git php5 php5-dev phpunit unzip @@ -128,11 +107,6 @@ RUN /bin/bash -l -c "rvm all do gem install ronn rake" RUN curl -sS https://getcomposer.org/installer | php RUN mv composer.phar /usr/local/bin/composer -# As an attempt to work around #4212, try to prefetch Protobuf-PHP dependency -# into composer cache to prevent "composer install" from cloning on each build. -RUN git clone --mirror https://github.com/stanley-cheung/Protobuf-PHP.git \ - /root/.composer/cache/vcs/git-github.com-stanley-cheung-Protobuf-PHP.git/ - # Download the patched PHP protobuf so that PHP gRPC clients can be generated # from proto3 schemas. RUN git clone https://github.com/stanley-cheung/Protobuf-PHP.git /var/local/git/protobuf-php @@ -144,3 +118,4 @@ RUN /bin/bash -l -c "rvm use ruby-2.1 \ # Define the default command. CMD ["bash"] + diff --git a/tools/dockerfile/interoptest/grpc_interop_php7/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_php7/Dockerfile new file mode 100644 index 0000000000..be8f25f8ff --- /dev/null +++ b/tools/dockerfile/interoptest/grpc_interop_php7/Dockerfile @@ -0,0 +1,125 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +FROM debian:jessie + +#================= +# PHP7 dependencies + +# Install Git and basic packages. +RUN apt-get update && apt-get install -y \ + autoconf \ + automake \ + build-essential \ + ccache \ + curl \ + git \ + libcurl4-openssl-dev \ + libgmp-dev \ + libgmp3-dev \ + libssl-dev \ + libtool \ + libxml2-dev \ + pkg-config \ + re2c \ + time \ + unzip \ + wget \ + zip && apt-get clean + +# Install other dependencies +RUN ln -sf /usr/include/x86_64-linux-gnu/gmp.h /usr/include/gmp.h +RUN wget http://ftp.gnu.org/gnu/bison/bison-2.6.4.tar.gz -O /var/local/bison-2.6.4.tar.gz +RUN cd /var/local \ + && tar -zxvf bison-2.6.4.tar.gz \ + && cd /var/local/bison-2.6.4 \ + && ./configure \ + && make \ + && make install + +# Compile PHP7 from source +RUN git clone https://github.com/php/php-src /var/local/git/php-src +RUN cd /var/local/git/php-src \ + && git checkout PHP-7.0.9 \ + && ./buildconf --force \ + && ./configure \ + --with-gmp \ + --with-openssl \ + --with-zlib \ + && make \ + && make install + +#================== +# Ruby dependencies + +# Install rvm +RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 +RUN \curl -sSL https://get.rvm.io | bash -s stable + +# Install Ruby 2.1 +RUN /bin/bash -l -c "rvm install ruby-2.1" +RUN /bin/bash -l -c "rvm use --default ruby-2.1" +RUN /bin/bash -l -c "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc" +RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" +RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.1' >> ~/.bashrc" +RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc" + +# Prepare ccache +RUN ln -s /usr/bin/ccache /usr/local/bin/gcc +RUN ln -s /usr/bin/ccache /usr/local/bin/g++ +RUN ln -s /usr/bin/ccache /usr/local/bin/cc +RUN ln -s /usr/bin/ccache /usr/local/bin/c++ +RUN ln -s /usr/bin/ccache /usr/local/bin/clang +RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ + + +RUN mkdir /var/local/jenkins + +# ronn: a ruby tool used to convert markdown to man pages, used during the +# install of Protobuf extensions +# +# rake: a ruby version of make used to build the PHP Protobuf extension +RUN /bin/bash -l -c "rvm all do gem install ronn rake" + +# Install composer +RUN curl -sS https://getcomposer.org/installer | php +RUN mv composer.phar /usr/local/bin/composer + +# Download the patched PHP protobuf so that PHP gRPC clients can be generated +# from proto3 schemas. +RUN git clone https://github.com/stanley-cheung/Protobuf-PHP.git /var/local/git/protobuf-php + +RUN /bin/bash -l -c "rvm use ruby-2.1 \ + && cd /var/local/git/protobuf-php \ + && rvm all do rake pear:package version=1.0 \ + && pear install Protobuf-1.0.tgz" + +# Define the default command. +CMD ["bash"] + diff --git a/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh new file mode 100755 index 0000000000..261dded282 --- /dev/null +++ b/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Builds PHP interop server and client in a base image. +set -ex + +mkdir -p /var/local/git +git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc + +# copy service account keys if available +cp -r /var/local/jenkins/service_account $HOME || true + +cd /var/local/git/grpc +rvm --default use ruby-2.1 + +# gRPC core and protobuf need to be installed +make install + +(cd src/php/ext/grpc && phpize && ./configure && make) + +(cd third_party/protobuf && make install) + +(cd src/php && composer install) + +(cd src/php && protoc-gen-php -i tests/interop/ -o tests/interop/ tests/interop/test.proto) diff --git a/tools/dockerfile/stress_test/grpc_interop_stress_php/Dockerfile b/tools/dockerfile/stress_test/grpc_interop_stress_php/Dockerfile index bbbdd4a151..0716be5a9d 100644 --- a/tools/dockerfile/stress_test/grpc_interop_stress_php/Dockerfile +++ b/tools/dockerfile/stress_test/grpc_interop_stress_php/Dockerfile @@ -103,12 +103,6 @@ RUN pip install --upgrade google-api-python-client # Install dependencies -RUN /bin/bash -l -c "echo 'deb http://packages.dotdeb.org wheezy-php55 all' \ - >> /etc/apt/sources.list.d/dotdeb.list" -RUN /bin/bash -l -c "echo 'deb-src http://packages.dotdeb.org wheezy-php55 all' \ - >> /etc/apt/sources.list.d/dotdeb.list" -RUN wget http://www.dotdeb.org/dotdeb.gpg -O- | apt-key add - - RUN apt-get update && apt-get install -y \ git php5 php5-dev phpunit unzip @@ -133,11 +127,6 @@ RUN /bin/bash -l -c "rvm all do gem install ronn rake" RUN curl -sS https://getcomposer.org/installer | php RUN mv composer.phar /usr/local/bin/composer -# As an attempt to work around #4212, try to prefetch Protobuf-PHP dependency -# into composer cache to prevent "composer install" from cloning on each build. -RUN git clone --mirror https://github.com/stanley-cheung/Protobuf-PHP.git \ - /root/.composer/cache/vcs/git-github.com-stanley-cheung-Protobuf-PHP.git/ - # Download the patched PHP protobuf so that PHP gRPC clients can be generated # from proto3 schemas. RUN git clone https://github.com/stanley-cheung/Protobuf-PHP.git /var/local/git/protobuf-php diff --git a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile index 92c8436851..13f7c10f92 100644 --- a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile @@ -100,12 +100,6 @@ RUN /bin/bash -l -c "nvm alias default 4" # Install dependencies -RUN /bin/bash -l -c "echo 'deb http://packages.dotdeb.org wheezy-php55 all' \ - >> /etc/apt/sources.list.d/dotdeb.list" -RUN /bin/bash -l -c "echo 'deb-src http://packages.dotdeb.org wheezy-php55 all' \ - >> /etc/apt/sources.list.d/dotdeb.list" -RUN wget http://www.dotdeb.org/dotdeb.gpg -O- | apt-key add - - RUN apt-get update && apt-get install -y \ git php5 php5-dev phpunit unzip diff --git a/tools/dockerfile/test/php7_jessie_x64/Dockerfile b/tools/dockerfile/test/php7_jessie_x64/Dockerfile new file mode 100644 index 0000000000..221338956e --- /dev/null +++ b/tools/dockerfile/test/php7_jessie_x64/Dockerfile @@ -0,0 +1,105 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +FROM debian:jessie + +#================= +# PHP7 dependencies + +# Install Git and basic packages. +RUN apt-get update && apt-get install -y \ + autoconf \ + automake \ + build-essential \ + ccache \ + curl \ + git \ + libcurl4-openssl-dev \ + libgmp-dev \ + libgmp3-dev \ + libssl-dev \ + libtool \ + libxml2-dev \ + pkg-config \ + re2c \ + time \ + unzip \ + wget \ + zip && apt-get clean + +# Install other dependencies +RUN ln -sf /usr/include/x86_64-linux-gnu/gmp.h /usr/include/gmp.h +RUN wget http://ftp.gnu.org/gnu/bison/bison-2.6.4.tar.gz -O /var/local/bison-2.6.4.tar.gz +RUN cd /var/local \ + && tar -zxvf bison-2.6.4.tar.gz \ + && cd /var/local/bison-2.6.4 \ + && ./configure \ + && make \ + && make install + +# Compile PHP7 from source +RUN git clone https://github.com/php/php-src /var/local/git/php-src +RUN cd /var/local/git/php-src \ + && git checkout PHP-7.0.9 \ + && ./buildconf --force \ + && ./configure \ + --with-gmp \ + --with-openssl \ + --with-zlib \ + && make \ + && make install + +#==================== +# Python dependencies + +# Install dependencies + +RUN apt-get update && apt-get install -y \ + python-all-dev \ + python3-all-dev \ + python-pip + +# Install Python packages from PyPI +RUN pip install pip --upgrade +RUN pip install virtualenv +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 + +# Prepare ccache +RUN ln -s /usr/bin/ccache /usr/local/bin/gcc +RUN ln -s /usr/bin/ccache /usr/local/bin/g++ +RUN ln -s /usr/bin/ccache /usr/local/bin/cc +RUN ln -s /usr/bin/ccache /usr/local/bin/c++ +RUN ln -s /usr/bin/ccache /usr/local/bin/clang +RUN ln -s /usr/bin/ccache /usr/local/bin/clang++ + + +RUN mkdir /var/local/jenkins + +# Define the default command. +CMD ["bash"] diff --git a/tools/dockerfile/test/php_jessie_x64/Dockerfile b/tools/dockerfile/test/php_jessie_x64/Dockerfile index 2ef6e1d47f..17ea36b76c 100644 --- a/tools/dockerfile/test/php_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/php_jessie_x64/Dockerfile @@ -83,12 +83,6 @@ RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.0.0a2 six==1.10.0 # Install dependencies -RUN /bin/bash -l -c "echo 'deb http://packages.dotdeb.org wheezy-php55 all' \ - >> /etc/apt/sources.list.d/dotdeb.list" -RUN /bin/bash -l -c "echo 'deb-src http://packages.dotdeb.org wheezy-php55 all' \ - >> /etc/apt/sources.list.d/dotdeb.list" -RUN wget http://www.dotdeb.org/dotdeb.gpg -O- | apt-key add - - RUN apt-get update && apt-get install -y \ git php5 php5-dev phpunit unzip diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 38f5fa65f4..745069480c 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -796,6 +796,7 @@ src/core/lib/channel/channel_stack_builder.h \ src/core/lib/channel/compress_filter.h \ src/core/lib/channel/connected_channel.h \ src/core/lib/channel/context.h \ +src/core/lib/channel/handshaker.h \ src/core/lib/channel/http_client_filter.h \ src/core/lib/channel/http_server_filter.h \ src/core/lib/compression/algorithm_metadata.h \ @@ -952,6 +953,7 @@ src/core/lib/channel/channel_stack.c \ src/core/lib/channel/channel_stack_builder.c \ src/core/lib/channel/compress_filter.c \ src/core/lib/channel/connected_channel.c \ +src/core/lib/channel/handshaker.c \ src/core/lib/channel/http_client_filter.c \ src/core/lib/channel/http_server_filter.c \ src/core/lib/compression/compression.c \ diff --git a/tools/gcp/stress_test/stress_test_utils.py b/tools/gcp/stress_test/stress_test_utils.py index b821fc8fcc..be50af3184 100755 --- a/tools/gcp/stress_test/stress_test_utils.py +++ b/tools/gcp/stress_test/stress_test_utils.py @@ -121,7 +121,7 @@ class BigQueryHelper: if not page['jobComplete']: print('TIMEOUT ERROR: The query %s timed out. Current timeout value is' ' %d msec. Returning False (i.e assuming there are no failures)' - ) % (query, timeoout_msec) + ) % (query, timeout_msec) return False num_failures = int(page['totalRows']) diff --git a/tools/profiling/latency_profile/profile_analyzer.py b/tools/profiling/latency_profile/profile_analyzer.py index dad0712d40..48b8e9b950 100755 --- a/tools/profiling/latency_profile/profile_analyzer.py +++ b/tools/profiling/latency_profile/profile_analyzer.py @@ -43,6 +43,7 @@ TIME_FROM_SCOPE_START = object() TIME_TO_SCOPE_END = object() TIME_FROM_STACK_START = object() TIME_TO_STACK_END = object() +TIME_FROM_LAST_IMPORTANT = object() argp = argparse.ArgumentParser(description='Process output of basic_prof builds') @@ -78,10 +79,14 @@ class ScopeBuilder(object): self.call_stack_builder.lines.append(line_item) def finish(self, line): - assert line['tag'] == self.top_line.tag, 'expected %s, got %s; thread=%s; t0=%f t1=%f' % (self.top_line.tag, line['tag'], line['thd'], self.top_line.start_time, line['t']) + assert line['tag'] == self.top_line.tag, ( + 'expected %s, got %s; thread=%s; t0=%f t1=%f' % + (self.top_line.tag, line['tag'], line['thd'], self.top_line.start_time, + line['t'])) final_time_stamp = line['t'] assert self.top_line.end_time is None self.top_line.end_time = final_time_stamp + self.top_line.important = self.top_line.important or line['imp'] assert SELF_TIME not in self.top_line.times self.top_line.times[SELF_TIME] = final_time_stamp - self.top_line.start_time for line in self.call_stack_builder.lines[self.first_child_pos:]: @@ -101,9 +106,14 @@ class CallStackBuilder(object): start_time = self.lines[0].start_time end_time = self.lines[0].end_time self.signature = self.signature.hexdigest() + last_important = start_time for line in self.lines: line.times[TIME_FROM_STACK_START] = line.start_time - start_time line.times[TIME_TO_STACK_END] = end_time - line.end_time + line.times[TIME_FROM_LAST_IMPORTANT] = line.start_time - last_important + if line.important: + last_important = line.end_time + last_important = end_time def add(self, line): line_type = line['type'] @@ -113,7 +123,9 @@ class CallStackBuilder(object): self.stk.append(ScopeBuilder(self, line)) return False elif line_type == '}': - assert self.stk, 'expected non-empty stack for closing %s; thread=%s; t=%f' % (line['tag'], line['thd'], line['t']) + assert self.stk, ( + 'expected non-empty stack for closing %s; thread=%s; t=%f' % + (line['tag'], line['thd'], line['t'])) self.stk.pop().finish(line) if not self.stk: self.finish() @@ -216,9 +228,16 @@ def time_format(idx): return '' return ent +BANNER = { + 'simple': 'Count: %(count)d', + 'html': '<h1>Count: %(count)d</h1>' +} + FORMAT = [ ('TAG', lambda line: '..'*line.indent + tidy_tag(line.tag)), ('LOC', lambda line: '%s:%d' % (line.filename[line.filename.rfind('/')+1:], line.fileline)), + ('IMP', lambda line: '*' if line.important else ''), + ('FROM_IMP', time_format(TIME_FROM_LAST_IMPORTANT)), ('FROM_STACK_START', time_format(TIME_FROM_STACK_START)), ('SELF', time_format(SELF_TIME)), ('TO_STACK_END', time_format(TIME_TO_STACK_END)), @@ -227,11 +246,6 @@ FORMAT = [ ('TO_SCOPE_END', time_format(TIME_TO_SCOPE_END)), ] -BANNER = { - 'simple': 'Count: %(count)d', - 'html': '<h1>Count: %(count)d</h1>' -} - if args.fmt == 'html': print '<html>' print '<head>' diff --git a/tools/run_tests/artifact_targets.py b/tools/run_tests/artifact_targets.py index e9267be58b..8550ee7b84 100644 --- a/tools/run_tests/artifact_targets.py +++ b/tools/run_tests/artifact_targets.py @@ -30,11 +30,14 @@ """Definition of targets to build artifacts.""" +import os.path +import sys + import jobset def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, - flake_retries=0, timeout_retries=0): + flake_retries=0, timeout_retries=0, timeout_seconds=30*60): """Creates jobspec for a task running under docker.""" environ = environ.copy() environ['RUN_COMMAND'] = shell_command @@ -49,20 +52,20 @@ def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] + docker_args, environ=docker_env, shortname='build_artifact.%s' % (name), - timeout_seconds=30*60, + timeout_seconds=timeout_seconds, flake_retries=flake_retries, timeout_retries=timeout_retries) return jobspec def create_jobspec(name, cmdline, environ=None, shell=False, - flake_retries=0, timeout_retries=0): + flake_retries=0, timeout_retries=0, timeout_seconds=30*60): """Creates jobspec.""" jobspec = jobset.JobSpec( cmdline=cmdline, environ=environ, shortname='build_artifact.%s' % (name), - timeout_seconds=30*60, + timeout_seconds=timeout_seconds, flake_retries=flake_retries, timeout_retries=timeout_retries, shell=shell) @@ -76,27 +79,30 @@ _ARCH_FLAG_MAP = { 'x64': '-m64' } -python_version_arch_map = { - 'x86': 'Python27_32bits', - 'x64': 'Python27' +python_windows_version_arch_map = { + ('x86', '2.7'): 'Python27_32bits', + ('x64', '2.7'): 'Python27', + ('x86', '3.4'): 'Python34_32bits', + ('x64', '3.4'): 'Python34', } class PythonArtifact: """Builds Python artifacts.""" - def __init__(self, platform, arch, manylinux_build=None): + def __init__(self, platform, arch, python_version, manylinux_build=None): if manylinux_build: - self.name = 'python_%s_%s_%s' % (platform, arch, manylinux_build) + self.name = 'python%s_%s_%s_%s' % (python_version, platform, arch, manylinux_build) else: - self.name = 'python_%s_%s' % (platform, arch) + self.name = 'python%s_%s_%s' % (python_version, platform, arch) self.platform = platform self.arch = arch - self.labels = ['artifact', 'python', platform, arch] - self.python_version = python_version_arch_map[arch] + self.labels = ['artifact', 'python', python_version, platform, arch] + self.python_version = python_version + self.python_windows_prefix = python_windows_version_arch_map[arch, python_version] self.manylinux_build = manylinux_build def pre_build_jobspecs(self): - return [] + return [] def build_jobspec(self): environ = {} @@ -107,26 +113,27 @@ class PythonArtifact: # special places... environ['PYTHON'] = '/opt/python/{}/bin/python'.format(self.manylinux_build) environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.manylinux_build) - # Our docker image has all the prerequisites pip-installed already. - environ['SKIP_PIP_INSTALL'] = '1' # Platform autodetection for the manylinux1 image breaks so we set the # defines ourselves. # TODO(atash) get better platform-detection support in core so we don't # need to do this manually... environ['CFLAGS'] = '-DGPR_MANYLINUX1=1' + environ['BUILD_HEALTH_CHECKING'] = 'TRUE' + environ['BUILD_MANYLINUX_WHEEL'] = 'TRUE' return create_docker_jobspec(self.name, 'tools/dockerfile/grpc_artifact_python_manylinux_%s' % self.arch, 'tools/run_tests/build_artifact_python.sh', - environ=environ) + environ=environ, + timeout_seconds=60*60) elif self.platform == 'windows': return create_jobspec(self.name, ['tools\\run_tests\\build_artifact_python.bat', - self.python_version, + self.python_windows_prefix, '32' if self.arch == 'x86' else '64' ], shell=True) else: - environ['SKIP_PIP_INSTALL'] = 'TRUE' + environ['PYTHON'] = 'python{}'.format(self.python_version) return create_jobspec(self.name, ['tools/run_tests/build_artifact_python.sh'], environ=environ) @@ -323,13 +330,18 @@ def targets(): for Cls in (CSharpExtArtifact, NodeExtArtifact, ProtocArtifact) for platform in ('linux', 'macos', 'windows') for arch in ('x86', 'x64')] + - [PythonArtifact('linux', 'x86', 'cp27-cp27m'), - PythonArtifact('linux', 'x86', 'cp27-cp27mu'), - PythonArtifact('linux', 'x64', 'cp27-cp27m'), - PythonArtifact('linux', 'x64', 'cp27-cp27mu'), - PythonArtifact('macos', 'x64'), - PythonArtifact('windows', 'x86'), - PythonArtifact('windows', 'x64'), + [PythonArtifact('linux', 'x86', '2.7', 'cp27-cp27m'), + PythonArtifact('linux', 'x86', '2.7', 'cp27-cp27mu'), + PythonArtifact('linux', 'x64', '2.7', 'cp27-cp27m'), + PythonArtifact('linux', 'x64', '2.7', 'cp27-cp27mu'), + PythonArtifact('macos', 'x64', '2.7'), + PythonArtifact('windows', 'x86', '2.7'), + PythonArtifact('windows', 'x64', '2.7'), + PythonArtifact('linux', 'x86', '3.4', 'cp34-cp34m'), + PythonArtifact('linux', 'x64', '3.4', 'cp34-cp34m'), + PythonArtifact('macos', 'x64', '3.4'), + PythonArtifact('windows', 'x86', '3.4'), + PythonArtifact('windows', 'x64', '3.4'), RubyArtifact('linux', 'x86'), RubyArtifact('linux', 'x64'), RubyArtifact('macos', 'x64'), diff --git a/tools/run_tests/build_artifact_python.bat b/tools/run_tests/build_artifact_python.bat index 7c8c2aa12d..074a3c6781 100644 --- a/tools/run_tests/build_artifact_python.bat +++ b/tools/run_tests/build_artifact_python.bat @@ -34,29 +34,6 @@ pip install --upgrade six pip install --upgrade setuptools pip install -rrequirements.txt -@rem Because this is windows and *everything seems to hate Windows* we have to -@rem set all of these flags ourselves because Python won't help us (see the -@rem setup.py of the grpcio_tools project). -set GRPC_PYTHON_CFLAGS=-fno-wrapv -frtti -std=c++11 - -@rem See https://sourceforge.net/p/mingw-w64/bugs/363/ -if %2 == 32 ( - set GRPC_PYTHON_CFLAGS=%GRPC_PYTHON_CFLAGS% -D_ftime=_ftime32 -D_timeb=__timeb32 -D_ftime_s=_ftime32_s -) else ( - set GRPC_PYTHON_CFLAGS=%GRPC_PYTHON_CFLAGS% -D_ftime=_ftime64 -D_timeb=__timeb64 -) - -@rem Further confusing things, MSYS2's mingw64 tries to dynamically link -@rem libgcc, libstdc++, and winpthreads. We have to override this or our -@rem extensions end up linking to MSYS2 DLLs, which the normal Python on -@rem Windows user won't have... and ON TOP OF THIS, there's MinGW's GCC default -@rem behavior of linking msvcrt.dll as the C runtime library, which we need to -@rem override so that Python's distutils doesn't link us against multiple C -@rem runtimes. -python -c "from distutils.cygwinccompiler import get_msvcr; print(get_msvcr()[0])" > temp.txt -set /p PYTHON_MSVCR=<temp.txt -set GRPC_PYTHON_LDFLAGS=-static-libgcc -static-libstdc++ -mcrtdll=%PYTHON_MSVCR% -static -lpthread - set GRPC_PYTHON_BUILD_WITH_CYTHON=1 @@ -65,11 +42,19 @@ python tools\distrib\python\make_grpcio_tools.py @rem Build gRPC Python extensions python setup.py build_ext -c mingw32 -python tools\distrib\python\grpcio_tools\setup.py build_ext -c mingw32 + +pushd tools\distrib\python\grpcio_tools +python setup.py build_ext -c mingw32 +popd + @rem Build gRPC Python distributions python setup.py bdist_wheel -python tools\distrib\python\grpcio_tools\setup.py bdist_wheel + +pushd tools\distrib\python\grpcio_tools +python setup.py bdist_wheel +popd + mkdir artifacts xcopy /Y /I /S dist\* artifacts\ || goto :error diff --git a/tools/run_tests/build_artifact_python.sh b/tools/run_tests/build_artifact_python.sh index 55f8eb634b..8f8330ef24 100755 --- a/tools/run_tests/build_artifact_python.sh +++ b/tools/run_tests/build_artifact_python.sh @@ -39,15 +39,6 @@ export PIP=${PIP:-pip} export AUDITWHEEL=${AUDITWHEEL:-auditwheel} -if [ "$SKIP_PIP_INSTALL" == "" ] -then - ${PIP} install --upgrade six - # There's a bug in newer versions of setuptools (see - # https://bitbucket.org/pypa/setuptools/issues/503/pkg_resources_vendorpackagingrequirementsi) - ${PIP} pip install --upgrade 'setuptools==18' - ${PIP} install -rrequirements.txt -fi - # Build the source distribution first because MANIFEST.in cannot override # exclusion of built shared objects among package resources (for some # inexplicable reason). @@ -71,15 +62,33 @@ CFLAGS="$CFLAGS -fno-wrapv" ${SETARCH_CMD} \ ${PYTHON} tools/distrib/python/grpcio_tools/setup.py bdist_wheel mkdir -p artifacts -if command -v ${AUDITWHEEL} +if [ "$BUILD_MANYLINUX_WHEEL" != "" ] then for wheel in dist/*.whl; do ${AUDITWHEEL} repair $wheel -w artifacts/ + rm $wheel done for wheel in tools/distrib/python/grpcio_tools/dist/*.whl; do ${AUDITWHEEL} repair $wheel -w artifacts/ + rm $wheel done fi +# We need to use the built grpcio-tools/grpcio to compile the health proto +# Wheels are not supported by setup_requires/dependency_links, so we +# manually install the dependency. Note we should only do this if we +# are in a docker image or in a virtualenv. +if [ "$BUILD_HEALTH_CHECKING" != "" ] +then + ${PIP} install -rrequirements.txt + ${PIP} install grpcio --no-index --find-links "file://${PWD}/artifacts/" + ${PIP} install grpcio-tools --no-index --find-links "file://${PWD}/artifacts/" + + # Build gRPC health check source distribution + ${SETARCH_CMD} ${PYTHON} src/python/grpcio_health_checking/setup.py \ + preprocess build_package_protos sdist + cp -r src/python/grpcio_health_checking/dist/* artifacts +fi + cp -r dist/* artifacts cp -r tools/distrib/python/grpcio_tools/dist/* artifacts diff --git a/tools/run_tests/build_package_node.sh b/tools/run_tests/build_package_node.sh index ff4cfdb8bf..ef4a10cca7 100755 --- a/tools/run_tests/build_package_node.sh +++ b/tools/run_tests/build_package_node.sh @@ -61,7 +61,7 @@ mkdir -p $output_dir well_known_protos=( any api compiler/plugin descriptor duration empty field_mask source_context struct timestamp type wrappers ) for arch in {x86,x64}; do - case arch in + case $arch in x86) node_arch=ia32 ;; @@ -70,7 +70,7 @@ for arch in {x86,x64}; do ;; esac for plat in {windows,linux,macos}; do - case plat in + case $plat in windows) node_plat=win32 ;; diff --git a/tools/run_tests/build_python.sh b/tools/run_tests/build_python.sh index a3fa8200d5..c4b5d98a53 100755 --- a/tools/run_tests/build_python.sh +++ b/tools/run_tests/build_python.sh @@ -127,19 +127,6 @@ if [ $(is_linux) ]; then fi fi fi -# TODO(atash) consider conceptualizing MinGW as a first-class platform and move -# these flags into our `setup.py`s -if [ $(is_mingw) ]; then - # We're on MinGW, and our CFLAGS and LDFLAGS will be eaten by the void. Use - # our work-around environment variables instead. - PYTHON_MSVCR=`$PYTHON -c "from distutils.cygwinccompiler import get_msvcr; print(get_msvcr()[0])"` - export GRPC_PYTHON_LDFLAGS="-static-libgcc -static-libstdc++ -mcrtdll=$PYTHON_MSVCR -static -lpthread" - # See https://sourceforge.net/p/mingw-w64/bugs/363/ - export GRPC_PYTHON_CFLAGS="-D_ftime=_ftime64 -D_timeb=__timeb64" - # TODO(atash) set these flags for only grpcio-tools (they don't do any harm to - # grpcio, but they result in noisy warnings). - export GRPC_PYTHON_CFLAGS="-frtti -std=c++11 $GRPC_PYTHON_CFLAGS" -fi ############################ # Perform build operations # @@ -177,7 +164,8 @@ pip_install_dir $ROOT/tools/distrib/python/grpcio_tools # etc... pip_install_dir $ROOT $VENV_PYTHON $ROOT/src/python/grpcio_health_checking/setup.py preprocess +$VENV_PYTHON $ROOT/src/python/grpcio_health_checking/setup.py build_package_protos pip_install_dir $ROOT/src/python/grpcio_health_checking $VENV_PYTHON $ROOT/src/python/grpcio_tests/setup.py preprocess -$VENV_PYTHON $ROOT/src/python/grpcio_tests/setup.py build_proto_modules +$VENV_PYTHON $ROOT/src/python/grpcio_tests/setup.py build_package_protos pip_install_dir $ROOT/src/python/grpcio_tests diff --git a/tools/run_tests/performance/bq_upload_result.py b/tools/run_tests/performance/bq_upload_result.py index fbccf3bdca..2a99499843 100755 --- a/tools/run_tests/performance/bq_upload_result.py +++ b/tools/run_tests/performance/bq_upload_result.py @@ -118,6 +118,8 @@ def _flatten_result_inplace(scenario_result): for stats in scenario_result['clientStats']: stats['latencies'] = json.dumps(stats['latencies']) scenario_result['serverCores'] = json.dumps(scenario_result['serverCores']) + scenario_result['clientSuccess'] = json.dumps(scenario_result['clientSuccess']) + scenario_result['serverSuccess'] = json.dumps(scenario_result['serverSuccess']) def _populate_metadata_inplace(scenario_result): diff --git a/tools/run_tests/performance/scenario_result_schema.json b/tools/run_tests/performance/scenario_result_schema.json index 0325414757..6bec21df39 100644 --- a/tools/run_tests/performance/scenario_result_schema.json +++ b/tools/run_tests/performance/scenario_result_schema.json @@ -198,5 +198,15 @@ "mode": "NULLABLE" } ] + }, + { + "name": "clientSuccess", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "serverSuccess", + "type": "STRING", + "mode": "NULLABLE" } ] diff --git a/tools/run_tests/pre_build_csharp.bat b/tools/run_tests/pre_build_csharp.bat index e7131d504c..580d5638fd 100644 --- a/tools/run_tests/pre_build_csharp.bat +++ b/tools/run_tests/pre_build_csharp.bat @@ -38,8 +38,61 @@ cd /d %~dp0\..\.. set NUGET=C:\nuget\nuget.exe if exist %NUGET% ( + @rem Restore Grpc packages by packages since Nuget client 3.4.4 doesnt support restore + @rem by solution + @rem Moving into each directory to let the restores work with both nuget 3.4 and 2.8 %NUGET% restore vsprojects/grpc_csharp_ext.sln || goto :error - %NUGET% restore src/csharp/Grpc.sln || goto :error + + cd src/csharp + + cd Grpc.Auth || goto :error + %NUGET% restore -PackagesDirectory ../packages || goto :error + cd .. + + cd Grpc.Core || goto :error + %NUGET% restore -PackagesDirectory ../packages || goto :error + cd .. + + cd Grpc.Core.Tests || goto :error + %NUGET% restore -PackagesDirectory ../packages || goto :error + cd .. + + cd Grpc.Examples.MathClient || goto :error + %NUGET% restore -PackagesDirectory ../packages || goto :error + cd .. + + cd Grpc.Examples.MathServer || goto :error + %NUGET% restore -PackagesDirectory ../packages || goto :error + cd .. + + cd Grpc.Examples || goto :error + %NUGET% restore -PackagesDirectory ../packages || goto :error + cd .. + + cd Grpc.HealthCheck.Tests || goto :error + %NUGET% restore -PackagesDirectory ../packages || goto :error + cd .. + + cd Grpc.HealthCheck || goto :error + %NUGET% restore -PackagesDirectory ../packages || goto :error + cd .. + + cd Grpc.IntegrationTesting.Client || goto :error + %NUGET% restore -PackagesDirectory ../packages || goto :error + cd .. + + cd Grpc.IntegrationTesting.QpsWorker || goto :error + %NUGET% restore -PackagesDirectory ../packages || goto :error + cd .. + + cd Grpc.IntegrationTesting.StressClient || goto :error + %NUGET% restore -PackagesDirectory ../packages || goto :error + cd .. + + cd Grpc.IntegrationTesting || goto :error + %NUGET% restore -PackagesDirectory ../packages || goto :error + + cd /d %~dp0\..\.. || goto :error ) endlocal diff --git a/tools/run_tests/pre_build_csharp.sh b/tools/run_tests/pre_build_csharp.sh index 3ff1a4e5a8..0fd3b92a95 100755 --- a/tools/run_tests/pre_build_csharp.sh +++ b/tools/run_tests/pre_build_csharp.sh @@ -37,5 +37,54 @@ root=`pwd` if [ -x "$(command -v nuget)" ] then - nuget restore Grpc.sln + # Restoring Nuget packages by packages rather than by solution because of + # inability to restore by solution with Nuget client 3.4.4 + # Moving into each directory to let the restores work with nuget 3.4 and 2.8 + cd Grpc.Auth + nuget restore -PackagesDirectory ../packages + cd .. + + cd Grpc.Core.Tests + nuget restore -PackagesDirectory ../packages + cd .. + + cd Grpc.Core + nuget restore -PackagesDirectory ../packages + cd .. + + cd Grpc.Examples.MathClient + nuget restore -PackagesDirectory ../packages + cd .. + + cd Grpc.Examples.MathServer + nuget restore -PackagesDirectory ../packages + cd .. + + cd Grpc.Examples + nuget restore -PackagesDirectory ../packages + cd .. + + cd Grpc.HealthCheck.Tests + nuget restore -PackagesDirectory ../packages + cd .. + + cd Grpc.HealthCheck + nuget restore -PackagesDirectory ../packages + cd .. + + cd Grpc.IntegrationTesting.Client + nuget restore -PackagesDirectory ../packages + cd .. + + cd Grpc.IntegrationTesting.QpsWorker + nuget restore -PackagesDirectory ../packages + cd .. + + cd Grpc.IntegrationTesting.StressClient + nuget restore -PackagesDirectory ../packages + cd .. + + cd Grpc.IntegrationTesting + nuget restore -PackagesDirectory ../packages + cd .. fi diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index f9065c5bfd..78096b216c 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -268,6 +268,31 @@ class PHPLanguage: return 'php' +class PHP7Language: + + def __init__(self): + self.client_cwd = None + self.safename = str(self) + + def client_cmd(self, args): + return ['src/php/bin/interop_client.sh'] + args + + def cloud_to_prod_env(self): + return {} + + def global_env(self): + return {} + + def unimplemented_test_cases(self): + return _SKIP_COMPRESSION + + def unimplemented_test_cases_server(self): + return [] + + def __str__(self): + return 'php7' + + class RubyLanguage: def __init__(self): @@ -288,7 +313,7 @@ class RubyLanguage: return {} def unimplemented_test_cases(self): - return _SKIP_ADVANCED + _SKIP_COMPRESSION + return _SKIP_ADVANCED + _SKIP_SERVER_COMPRESSION def unimplemented_test_cases_server(self): return _SKIP_ADVANCED + _SKIP_COMPRESSION @@ -346,6 +371,7 @@ _LANGUAGES = { 'java' : JavaLanguage(), 'node' : NodeLanguage(), 'php' : PHPLanguage(), + 'php7' : PHP7Language(), 'ruby' : RubyLanguage(), 'python' : PythonLanguage(), } @@ -409,7 +435,7 @@ def auth_options(language, test_case): default_account_arg = '--default_service_account=830293263384-compute@developer.gserviceaccount.com' if test_case in ['jwt_token_creds', 'per_rpc_creds', 'oauth2_auth_token']: - if language in ['csharp', 'node', 'php', 'python', 'ruby']: + if language in ['csharp', 'node', 'php', 'php7', 'python', 'ruby']: env['GOOGLE_APPLICATION_CREDENTIALS'] = key_filepath else: cmdargs += [key_file_arg] diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 59d2cad856..542415d908 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -41,6 +41,7 @@ import json import multiprocessing import os import os.path +import pipes import platform import random import re @@ -62,7 +63,9 @@ _ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) os.chdir(_ROOT) -_FORCE_ENVIRON_FOR_WRAPPERS = {} +_FORCE_ENVIRON_FOR_WRAPPERS = { + 'GRPC_VERBOSITY': 'DEBUG', +} _POLLING_STRATEGIES = { @@ -74,6 +77,9 @@ def platform_string(): return jobset.platform_string() +_DEFAULT_TIMEOUT_SECONDS = 5 * 60 + + # SimpleConfig: just compile with CONFIG=config, and run the binary to test class Config(object): @@ -86,7 +92,7 @@ class Config(object): self.tool_prefix = tool_prefix self.timeout_multiplier = timeout_multiplier - def job_spec(self, cmdline, timeout_seconds=5*60, + def job_spec(self, cmdline, timeout_seconds=_DEFAULT_TIMEOUT_SECONDS, shortname=None, environ={}, cpu_cost=1.0, flaky=False): """Construct a jobset.JobSpec for a test under this config @@ -160,8 +166,9 @@ class CLanguage(object): for polling_strategy in polling_strategies: env={'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH': _ROOT + '/src/core/lib/tsi/test_creds/ca.pem', - 'GRPC_POLL_STRATEGY': polling_strategy} - shortname_ext = '' if polling_strategy=='all' else ' polling=%s' % polling_strategy + 'GRPC_POLL_STRATEGY': polling_strategy, + 'GRPC_VERBOSITY': 'DEBUG'} + shortname_ext = '' if polling_strategy=='all' else ' GRPC_POLL_STRATEGY=%s' % polling_strategy if self.config.build_config in target['exclude_configs']: continue if self.platform == 'windows': @@ -192,28 +199,26 @@ class CLanguage(object): assert line[1] == ' ' test = base + line.strip() cmdline = [binary] + ['--gtest_filter=%s' % test] - out.append(self.config.job_spec(cmdline, [binary], - shortname='%s:%s %s' % (binary, test, shortname_ext), + out.append(self.config.job_spec(cmdline, + shortname='%s --gtest_filter=%s %s' % (binary, test, shortname_ext), cpu_cost=target['cpu_cost'], environ=env)) else: cmdline = [binary] + target['args'] - out.append(self.config.job_spec(cmdline, [binary], - shortname=' '.join(cmdline) + shortname_ext, + out.append(self.config.job_spec(cmdline, + shortname=' '.join( + pipes.quote(arg) + for arg in cmdline) + + shortname_ext, cpu_cost=target['cpu_cost'], flaky=target.get('flaky', False), + timeout_seconds=target.get('timeout_seconds', _DEFAULT_TIMEOUT_SECONDS), environ=env)) elif self.args.regex == '.*' or self.platform == 'windows': print('\nWARNING: binary not found, skipping', binary) return sorted(out) def make_targets(self): - test_regex = self.args.regex - if self.platform != 'windows' and self.args.regex != '.*': - # use the regex to minimize the number of things to build - return [os.path.basename(target['name']) - for target in get_c_tests(False, self.test_lang) - if re.search(test_regex, '/' + target['name'])] if self.platform == 'windows': # don't build tools on windows just yet return ['buildtests_%s' % self.make_target] @@ -376,6 +381,42 @@ class PhpLanguage(object): return 'php' +class Php7Language(object): + + def configure(self, config, args): + self.config = config + self.args = args + _check_compiler(self.args.compiler, ['default']) + + def test_specs(self): + return [self.config.job_spec(['src/php/bin/run_tests.sh'], None, + environ=_FORCE_ENVIRON_FOR_WRAPPERS)] + + def pre_build_steps(self): + return [] + + def make_targets(self): + return ['static_c', 'shared_c'] + + def make_options(self): + return [] + + def build_steps(self): + return [['tools/run_tests/build_php.sh']] + + def post_tests_steps(self): + return [['tools/run_tests/post_tests_php.sh']] + + def makefile_name(self): + return 'Makefile' + + def dockerfile_dir(self): + return 'tools/dockerfile/test/php7_jessie_%s' % _docker_arch_suffix(self.args.arch) + + def __str__(self): + return 'php7' + + class PythonConfig(collections.namedtuple('PythonConfig', [ 'name', 'build', 'run'])): """Tuple of commands (named s.t. 'what it says on the tin' applies)""" @@ -744,6 +785,7 @@ _LANGUAGES = { 'c': CLanguage('c', 'c'), 'node': NodeLanguage(), 'php': PhpLanguage(), + 'php7': Php7Language(), 'python': PythonLanguage(), 'ruby': RubyLanguage(), 'csharp': CSharpLanguage(), @@ -1294,8 +1336,6 @@ def _build_and_run( jobset.message( 'FLAKE', '%s [%d/%d runs flaked]' % (k, num_failures, num_runs), do_newline=True) - else: - jobset.message('PASSED', k, do_newline=True) finally: for antagonist in antagonists: antagonist.kill() diff --git a/tools/run_tests/sanity/check_sources_and_headers.py b/tools/run_tests/sanity/check_sources_and_headers.py index c028499ca6..28c1dc46d7 100755 --- a/tools/run_tests/sanity/check_sources_and_headers.py +++ b/tools/run_tests/sanity/check_sources_and_headers.py @@ -55,7 +55,8 @@ def target_has_header(target, name): for dep in target['deps']: if target_has_header(get_target(dep), name): return True - if name == 'src/core/lib/profiling/stap_probes.h': + if name in ['src/core/lib/profiling/stap_probes.h', + 'src/proto/grpc/reflection/v1alpha/reflection.grpc.pb.h']: return True return False diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index c297025d67..78ab4dbf1d 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -1823,22 +1823,6 @@ "gpr", "gpr_test_util", "grpc", - "grpc_test_util" - ], - "headers": [], - "language": "c", - "name": "workqueue_test", - "src": [ - "test/core/iomgr/workqueue_test.c" - ], - "third_party": false, - "type": "target" - }, - { - "deps": [ - "gpr", - "gpr_test_util", - "grpc", "grpc++", "grpc++_test_util", "grpc_test_util" @@ -2149,6 +2133,7 @@ "gpr_test_util", "grpc", "grpc++", + "grpc++_reflection", "grpc++_test_config", "grpc++_test_util", "grpc_cli_libs", @@ -4400,26 +4385,32 @@ { "deps": [ "grpc++", - "grpc++_codegen_proto" + "grpc++_reflection_proto" ], "headers": [ "include/grpc++/ext/proto_server_reflection_plugin.h", - "include/grpc++/ext/reflection.grpc.pb.h", - "include/grpc++/ext/reflection.pb.h", "src/cpp/ext/proto_server_reflection.h" ], "language": "c++", "name": "grpc++_reflection", "src": [ "include/grpc++/ext/proto_server_reflection_plugin.h", - "include/grpc++/ext/reflection.grpc.pb.h", - "include/grpc++/ext/reflection.pb.h", "src/cpp/ext/proto_server_reflection.cc", "src/cpp/ext/proto_server_reflection.h", - "src/cpp/ext/proto_server_reflection_plugin.cc", - "src/cpp/ext/reflection.grpc.pb.cc", - "src/cpp/ext/reflection.pb.cc" + "src/cpp/ext/proto_server_reflection_plugin.cc" + ], + "third_party": false, + "type": "lib" + }, + { + "deps": [], + "headers": [ + "src/proto/grpc/reflection/v1alpha/reflection.grpc.pb.h", + "src/proto/grpc/reflection/v1alpha/reflection.pb.h" ], + "language": "c++", + "name": "grpc++_reflection_codegen", + "src": [], "third_party": false, "type": "lib" }, @@ -4500,11 +4491,13 @@ { "deps": [ "grpc++", + "grpc++_reflection", "grpc_plugin_support" ], "headers": [ "test/cpp/util/cli_call.h", - "test/cpp/util/proto_file_parser.h" + "test/cpp/util/proto_file_parser.h", + "test/cpp/util/proto_reflection_descriptor_database.h" ], "language": "c++", "name": "grpc_cli_libs", @@ -4512,7 +4505,9 @@ "test/cpp/util/cli_call.cc", "test/cpp/util/cli_call.h", "test/cpp/util/proto_file_parser.cc", - "test/cpp/util/proto_file_parser.h" + "test/cpp/util/proto_file_parser.h", + "test/cpp/util/proto_reflection_descriptor_database.cc", + "test/cpp/util/proto_reflection_descriptor_database.h" ], "third_party": false, "type": "lib" @@ -5736,6 +5731,7 @@ "src/core/lib/channel/compress_filter.h", "src/core/lib/channel/connected_channel.h", "src/core/lib/channel/context.h", + "src/core/lib/channel/handshaker.h", "src/core/lib/channel/http_client_filter.h", "src/core/lib/channel/http_server_filter.h", "src/core/lib/compression/algorithm_metadata.h", @@ -5829,6 +5825,8 @@ "src/core/lib/channel/connected_channel.c", "src/core/lib/channel/connected_channel.h", "src/core/lib/channel/context.h", + "src/core/lib/channel/handshaker.c", + "src/core/lib/channel/handshaker.h", "src/core/lib/channel/http_client_filter.c", "src/core/lib/channel/http_client_filter.h", "src/core/lib/channel/http_server_filter.c", @@ -6794,5 +6792,24 @@ ], "third_party": false, "type": "filegroup" + }, + { + "deps": [ + "grpc++_codegen_proto" + ], + "headers": [ + "include/grpc++/ext/reflection.grpc.pb.h", + "include/grpc++/ext/reflection.pb.h" + ], + "language": "c++", + "name": "grpc++_reflection_proto", + "src": [ + "include/grpc++/ext/reflection.grpc.pb.h", + "include/grpc++/ext/reflection.pb.h", + "src/cpp/ext/reflection.grpc.pb.cc", + "src/cpp/ext/reflection.pb.cc" + ], + "third_party": false, + "type": "filegroup" } ] diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index 6abfb8770f..22f69e0170 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -1940,25 +1940,6 @@ "ci_platforms": [ "linux", "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "flaky": false, - "gtest": false, - "language": "c", - "name": "workqueue_test", - "platforms": [ - "linux", - "mac", - "posix" - ] - }, - { - "args": [], - "ci_platforms": [ - "linux", - "mac", "posix", "windows" ], @@ -27174,8 +27155,8 @@ }, { "args": [ - "--scenario_json", - "'{\"name\": \"cpp_generic_async_streaming_ping_pong_secure\", \"warmup_seconds\": 1, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'" + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_ping_pong_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}]}" ], "boringssl": true, "ci_platforms": [ @@ -27196,12 +27177,13 @@ "posix", "windows" ], - "shortname": "json_run_localhost:cpp_generic_async_streaming_ping_pong_secure" + "shortname": "json_run_localhost:cpp_generic_async_streaming_ping_pong_secure", + "timeout_seconds": 180 }, { "args": [ - "--scenario_json", - "'{\"name\": \"cpp_protobuf_async_streaming_ping_pong_secure\", \"warmup_seconds\": 1, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'" + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_ping_pong_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}]}" ], "boringssl": true, "ci_platforms": [ @@ -27222,12 +27204,13 @@ "posix", "windows" ], - "shortname": "json_run_localhost:cpp_protobuf_async_streaming_ping_pong_secure" + "shortname": "json_run_localhost:cpp_protobuf_async_streaming_ping_pong_secure", + "timeout_seconds": 180 }, { "args": [ - "--scenario_json", - "'{\"name\": \"cpp_protobuf_async_unary_ping_pong_secure\", \"warmup_seconds\": 1, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'" + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_ping_pong_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}]}" ], "boringssl": true, "ci_platforms": [ @@ -27248,12 +27231,13 @@ "posix", "windows" ], - "shortname": "json_run_localhost:cpp_protobuf_async_unary_ping_pong_secure" + "shortname": "json_run_localhost:cpp_protobuf_async_unary_ping_pong_secure", + "timeout_seconds": 180 }, { "args": [ - "--scenario_json", - "'{\"name\": \"cpp_protobuf_sync_unary_ping_pong_secure\", \"warmup_seconds\": 1, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'" + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_ping_pong_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}]}" ], "boringssl": true, "ci_platforms": [ @@ -27274,12 +27258,13 @@ "posix", "windows" ], - "shortname": "json_run_localhost:cpp_protobuf_sync_unary_ping_pong_secure" + "shortname": "json_run_localhost:cpp_protobuf_sync_unary_ping_pong_secure", + "timeout_seconds": 180 }, { "args": [ - "--scenario_json", - "'{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure\", \"warmup_seconds\": 1, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'" + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}" ], "boringssl": true, "ci_platforms": [ @@ -27300,12 +27285,13 @@ "posix", "windows" ], - "shortname": "json_run_localhost:cpp_protobuf_async_unary_qps_unconstrained_secure" + "shortname": "json_run_localhost:cpp_protobuf_async_unary_qps_unconstrained_secure", + "timeout_seconds": 180 }, { "args": [ - "--scenario_json", - "'{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 1, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'" + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}" ], "boringssl": true, "ci_platforms": [ @@ -27326,12 +27312,13 @@ "posix", "windows" ], - "shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_secure" + "shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_secure", + "timeout_seconds": 180 }, { "args": [ - "--scenario_json", - "'{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 1, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'" + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}" ], "boringssl": true, "ci_platforms": [ @@ -27352,12 +27339,13 @@ "posix", "windows" ], - "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_unconstrained_secure" + "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_unconstrained_secure", + "timeout_seconds": 180 }, { "args": [ - "--scenario_json", - "'{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_secure\", \"warmup_seconds\": 1, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'" + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_secure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}" ], "boringssl": true, "ci_platforms": [ @@ -27378,12 +27366,13 @@ "posix", "windows" ], - "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_one_server_core_secure" + "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_one_server_core_secure", + "timeout_seconds": 180 }, { "args": [ - "--scenario_json", - "'{\"name\": \"cpp_generic_async_streaming_ping_pong_insecure\", \"warmup_seconds\": 1, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'" + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_ping_pong_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}]}" ], "boringssl": true, "ci_platforms": [ @@ -27404,12 +27393,13 @@ "posix", "windows" ], - "shortname": "json_run_localhost:cpp_generic_async_streaming_ping_pong_insecure" + "shortname": "json_run_localhost:cpp_generic_async_streaming_ping_pong_insecure", + "timeout_seconds": 180 }, { "args": [ - "--scenario_json", - "'{\"name\": \"cpp_protobuf_async_streaming_ping_pong_insecure\", \"warmup_seconds\": 1, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'" + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_ping_pong_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}]}" ], "boringssl": true, "ci_platforms": [ @@ -27430,12 +27420,13 @@ "posix", "windows" ], - "shortname": "json_run_localhost:cpp_protobuf_async_streaming_ping_pong_insecure" + "shortname": "json_run_localhost:cpp_protobuf_async_streaming_ping_pong_insecure", + "timeout_seconds": 180 }, { "args": [ - "--scenario_json", - "'{\"name\": \"cpp_protobuf_async_unary_ping_pong_insecure\", \"warmup_seconds\": 1, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'" + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_ping_pong_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}]}" ], "boringssl": true, "ci_platforms": [ @@ -27456,12 +27447,13 @@ "posix", "windows" ], - "shortname": "json_run_localhost:cpp_protobuf_async_unary_ping_pong_insecure" + "shortname": "json_run_localhost:cpp_protobuf_async_unary_ping_pong_insecure", + "timeout_seconds": 180 }, { "args": [ - "--scenario_json", - "'{\"name\": \"cpp_protobuf_sync_unary_ping_pong_insecure\", \"warmup_seconds\": 1, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'" + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_protobuf_sync_unary_ping_pong_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}]}" ], "boringssl": true, "ci_platforms": [ @@ -27482,12 +27474,13 @@ "posix", "windows" ], - "shortname": "json_run_localhost:cpp_protobuf_sync_unary_ping_pong_insecure" + "shortname": "json_run_localhost:cpp_protobuf_sync_unary_ping_pong_insecure", + "timeout_seconds": 180 }, { "args": [ - "--scenario_json", - "'{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 1, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'" + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}" ], "boringssl": true, "ci_platforms": [ @@ -27508,12 +27501,13 @@ "posix", "windows" ], - "shortname": "json_run_localhost:cpp_protobuf_async_unary_qps_unconstrained_insecure" + "shortname": "json_run_localhost:cpp_protobuf_async_unary_qps_unconstrained_insecure", + "timeout_seconds": 180 }, { "args": [ - "--scenario_json", - "'{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 1, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'" + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}" ], "boringssl": true, "ci_platforms": [ @@ -27534,12 +27528,13 @@ "posix", "windows" ], - "shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_insecure" + "shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_insecure", + "timeout_seconds": 180 }, { "args": [ - "--scenario_json", - "'{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 1, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'" + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}" ], "boringssl": true, "ci_platforms": [ @@ -27560,12 +27555,13 @@ "posix", "windows" ], - "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_unconstrained_insecure" + "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_unconstrained_insecure", + "timeout_seconds": 180 }, { "args": [ - "--scenario_json", - "'{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_insecure\", \"warmup_seconds\": 1, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'" + "--scenarios_json", + "{\"scenarios\": [{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_insecure\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}]}" ], "boringssl": true, "ci_platforms": [ @@ -27586,7 +27582,8 @@ "posix", "windows" ], - "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_one_server_core_insecure" + "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_one_server_core_insecure", + "timeout_seconds": 180 }, { "args": [ |