diff options
Diffstat (limited to 'tools')
196 files changed, 4289 insertions, 2371 deletions
diff --git a/tools/bazel.rc b/tools/bazel.rc index 39f8071535..75be7c13df 100644 --- a/tools/bazel.rc +++ b/tools/bazel.rc @@ -1,51 +1,58 @@ +# bazelrc file +# bazel >= 0.18 looks for %workspace%/.bazelrc (which redirects here) +# Older bazel versions look for %workspace%/tools/bazel.rc (this file) +# See https://github.com/bazelbuild/bazel/issues/6319 + build --client_env=CC=clang -build --copt -DGRPC_BAZEL_BUILD +build --copt=-DGRPC_BAZEL_BUILD + +build:opt --compilation_mode=opt +build:opt --copt=-Wframe-larger-than=16384 -build:opt --copt -Wframe-larger-than=16384 +build:dbg --compilation_mode=dbg build:asan --strip=never -build:asan --copt -fsanitize-coverage=edge -build:asan --copt -fsanitize=address -build:asan --copt -O0 -build:asan --copt -fno-omit-frame-pointer -build:asan --copt -DGPR_NO_DIRECT_SYSCALLS -build:asan --linkopt -fsanitize=address +build:asan --copt=-fsanitize=address +build:asan --copt=-O0 +build:asan --copt=-fno-omit-frame-pointer +build:asan --copt=-DGPR_NO_DIRECT_SYSCALLS +build:asan --copt=-DADDRESS_SANITIZER # used by absl +build:asan --linkopt=-fsanitize=address build:asan --action_env=ASAN_OPTIONS=detect_leaks=1:color=always build:asan --action_env=LSAN_OPTIONS=suppressions=test/core/util/lsan_suppressions.txt:report_objects=1 build:msan --strip=never -build:msan --copt -fsanitize-coverage=edge -build:msan --copt -fsanitize=memory -build:msan --copt -O0 -build:msan --copt -fsanitize-memory-track-origins -build:msan --copt -fsanitize-memory-use-after-dtor -build:msan --copt -fno-omit-frame-pointer -build:msan --copt -fPIE -build:msan --copt -DGPR_NO_DIRECT_SYSCALLS -build:msan --linkopt -fsanitize=memory -build:msan --linkopt -fPIE +build:msan --copt=-fsanitize=memory +build:msan --copt=-O0 +build:msan --copt=-fsanitize-memory-track-origins +build:msan --copt=-fsanitize-memory-use-after-dtor +build:msan --copt=-fno-omit-frame-pointer +build:msan --copt=-DGPR_NO_DIRECT_SYSCALLS +build:msan --copt=-DMEMORY_SANITIZER # used by absl +build:msan --linkopt=-fsanitize=memory build:msan --action_env=MSAN_OPTIONS=poison_in_dtor=1 build:tsan --strip=never -build:tsan --copt -fsanitize=thread -build:tsan --copt -fno-omit-frame-pointer -build:tsan --copt -DGPR_NO_DIRECT_SYSCALLS -build:tsan --copt -DGRPC_TSAN -build:tsan --linkopt -fsanitize=thread +build:tsan --copt=-fsanitize=thread +build:tsan --copt=-fno-omit-frame-pointer +build:tsan --copt=-DGPR_NO_DIRECT_SYSCALLS +build:tsan --copt=-DGRPC_TSAN +build:tsan --copt=-DTHREAD_SANITIZER # used by absl +build:tsan --linkopt=-fsanitize=thread build:tsan --action_env=TSAN_OPTIONS=suppressions=test/core/util/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1 build:ubsan --strip=never -build:ubsan --copt -fsanitize-coverage=edge -build:ubsan --copt -fsanitize=undefined -build:ubsan --copt -fno-omit-frame-pointer -build:ubsan --copt -DGRPC_UBSAN -build:ubsan --copt -DNDEBUG -build:ubsan --copt -fno-sanitize=function,vptr -build:ubsan --linkopt -fsanitize=undefined +build:ubsan --copt=-fsanitize=undefined +build:ubsan --copt=-fno-omit-frame-pointer +build:ubsan --copt=-DGRPC_UBSAN +build:ubsan --copt=-DNDEBUG +build:ubsan --copt=-DUNDEFINED_BEHAVIOR_SANITIZER # used by absl +build:ubsan --copt=-fno-sanitize=function,vptr +build:ubsan --linkopt=-fsanitize=undefined build:ubsan --action_env=UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1:suppressions=test/core/util/ubsan_suppressions.txt build:basicprof --strip=never -build:basicprof --copt -DNDEBUG -build:basicprof --copt -O2 -build:basicprof --copt -DGRPC_BASIC_PROFILER -build:basicprof --copt -DGRPC_TIMERS_RDTSC +build:basicprof --copt=-DNDEBUG +build:basicprof --copt=-O2 +build:basicprof --copt=-DGRPC_BASIC_PROFILER +build:basicprof --copt=-DGRPC_TIMERS_RDTSC diff --git a/tools/buildgen/generate_build_additions.sh b/tools/buildgen/generate_build_additions.sh index 693c02fdb2..5a1f4a598a 100755 --- a/tools/buildgen/generate_build_additions.sh +++ b/tools/buildgen/generate_build_additions.sh @@ -25,7 +25,8 @@ gen_build_yaml_dirs=" \ test/core/bad_ssl \ test/core/end2end \ test/cpp/naming \ - test/cpp/qps" + test/cpp/qps \ + tools/run_tests/lb_interop_tests" gen_build_files="" for gen_build_yaml in $gen_build_yaml_dirs do diff --git a/tools/codegen/core/gen_static_metadata.py b/tools/codegen/core/gen_static_metadata.py index 25da3fdd5f..adfd4a24f9 100755 --- a/tools/codegen/core/gen_static_metadata.py +++ b/tools/codegen/core/gen_static_metadata.py @@ -23,10 +23,14 @@ import subprocess import re import perfection -# configuration: a list of either strings or 2-tuples of strings -# a single string represents a static grpc_mdstr -# a 2-tuple represents a static grpc_mdelem (and appropriate grpc_mdstrs will -# also be created) +# Configuration: a list of either strings or 2-tuples of strings. +# A single string represents a static grpc_mdstr. +# A 2-tuple represents a static grpc_mdelem (and appropriate grpc_mdstrs will +# also be created). +# The list of 2-tuples must begin with the static hpack table elements as +# defined by RFC 7541 and be in the same order because of an hpack encoding +# performance optimization that relies on this. If you want to change this, then +# you must change the implementation of the encoding optimization as well. CONFIG = [ # metadata strings @@ -59,37 +63,28 @@ CONFIG = [ 'grpc.max_response_message_bytes', # well known method names '/grpc.lb.v1.LoadBalancer/BalanceLoad', + '/grpc.health.v1.Health/Watch', # compression algorithm names 'deflate', 'gzip', 'stream/gzip', # metadata elements - ('grpc-status', '0'), - ('grpc-status', '1'), - ('grpc-status', '2'), - ('grpc-encoding', 'identity'), - ('grpc-encoding', 'gzip'), - ('grpc-encoding', 'deflate'), - ('te', 'trailers'), - ('content-type', 'application/grpc'), - (':method', 'POST'), - (':status', '200'), - (':status', '404'), - (':scheme', 'http'), - (':scheme', 'https'), - (':scheme', 'grpc'), + # begin hpack static elements (':authority', ''), (':method', 'GET'), - (':method', 'PUT'), + (':method', 'POST'), (':path', '/'), (':path', '/index.html'), + (':scheme', 'http'), + (':scheme', 'https'), + (':status', '200'), (':status', '204'), (':status', '206'), (':status', '304'), (':status', '400'), + (':status', '404'), (':status', '500'), ('accept-charset', ''), - ('accept-encoding', ''), ('accept-encoding', 'gzip, deflate'), ('accept-language', ''), ('accept-ranges', ''), @@ -100,8 +95,6 @@ CONFIG = [ ('authorization', ''), ('cache-control', ''), ('content-disposition', ''), - ('content-encoding', 'identity'), - ('content-encoding', 'gzip'), ('content-encoding', ''), ('content-language', ''), ('content-length', ''), @@ -121,8 +114,6 @@ CONFIG = [ ('if-range', ''), ('if-unmodified-since', ''), ('last-modified', ''), - ('lb-token', ''), - ('lb-cost-bin', ''), ('link', ''), ('location', ''), ('max-forwards', ''), @@ -140,37 +131,52 @@ CONFIG = [ ('vary', ''), ('via', ''), ('www-authenticate', ''), + # end hpack static elements + ('grpc-status', '0'), + ('grpc-status', '1'), + ('grpc-status', '2'), + ('grpc-encoding', 'identity'), + ('grpc-encoding', 'gzip'), + ('grpc-encoding', 'deflate'), + ('te', 'trailers'), + ('content-type', 'application/grpc'), + (':scheme', 'grpc'), + (':method', 'PUT'), + ('accept-encoding', ''), + ('content-encoding', 'identity'), + ('content-encoding', 'gzip'), + ('lb-token', ''), + ('lb-cost-bin', ''), ] -# Entries marked with is_default=True are ignored when counting -# non-default initial metadata that prevents the chttp2 server from -# sending a Trailers-Only response. +# All entries here are ignored when counting non-default initial metadata that +# prevents the chttp2 server from sending a Trailers-Only response. METADATA_BATCH_CALLOUTS = [ - # (name, is_default) - (':path', True), - (':method', True), - (':status', True), - (':authority', True), - (':scheme', True), - ('te', True), - ('grpc-message', True), - ('grpc-status', True), - ('grpc-payload-bin', True), - ('grpc-encoding', True), - ('grpc-accept-encoding', True), - ('grpc-server-stats-bin', True), - ('grpc-tags-bin', True), - ('grpc-trace-bin', True), - ('content-type', True), - ('content-encoding', True), - ('accept-encoding', True), - ('grpc-internal-encoding-request', True), - ('grpc-internal-stream-encoding-request', True), - ('user-agent', True), - ('host', True), - ('lb-token', True), - ('grpc-previous-rpc-attempts', True), - ('grpc-retry-pushback-ms', True), + # (name) + (':path'), + (':method'), + (':status'), + (':authority'), + (':scheme'), + ('te'), + ('grpc-message'), + ('grpc-status'), + ('grpc-payload-bin'), + ('grpc-encoding'), + ('grpc-accept-encoding'), + ('grpc-server-stats-bin'), + ('grpc-tags-bin'), + ('grpc-trace-bin'), + ('content-type'), + ('content-encoding'), + ('accept-encoding'), + ('grpc-internal-encoding-request'), + ('grpc-internal-stream-encoding-request'), + ('user-agent'), + ('host'), + ('lb-token'), + ('grpc-previous-rpc-attempts'), + ('grpc-retry-pushback-ms'), ] COMPRESSION_ALGORITHMS = [ @@ -252,7 +258,7 @@ all_elems = list() static_userdata = {} # put metadata batch callouts first, to make the check of if a static metadata # string is a callout trivial -for elem, _ in METADATA_BATCH_CALLOUTS: +for elem in METADATA_BATCH_CALLOUTS: if elem not in all_strs: all_strs.append(elem) for elem in CONFIG: @@ -367,9 +373,12 @@ an explanation of what's going on. print >> H, '#ifndef GRPC_CORE_LIB_TRANSPORT_STATIC_METADATA_H' print >> H, '#define GRPC_CORE_LIB_TRANSPORT_STATIC_METADATA_H' print >> H +print >> H, '#include <grpc/support/port_platform.h>' +print >> H print >> H, '#include "src/core/lib/transport/metadata.h"' print >> H - +print >> C, '#include <grpc/support/port_platform.h>' +print >> C print >> C, '#include "src/core/lib/transport/static_metadata.h"' print >> C print >> C, '#include "src/core/lib/slice/slice_internal.h"' @@ -388,7 +397,7 @@ def slice_def(i): # validate configuration -for elem, _ in METADATA_BATCH_CALLOUTS: +for elem in METADATA_BATCH_CALLOUTS: assert elem in all_strs print >> H, '#define GRPC_STATIC_MDSTR_COUNT %d' % len(all_strs) @@ -454,6 +463,7 @@ for i, elem in enumerate(all_elems): print >> H, ('#define %s (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[%d], ' 'GRPC_MDELEM_STORAGE_STATIC))') % (mangle(elem).upper(), i) print >> H + print >> C, ('uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT] ' '= {') print >> C, ' %s' % ','.join( @@ -551,7 +561,7 @@ for a, b in all_elems: print >> C, '};' print >> H, 'typedef enum {' -for elem, _ in METADATA_BATCH_CALLOUTS: +for elem in METADATA_BATCH_CALLOUTS: print >> H, ' %s,' % mangle(elem, 'batch').upper() print >> H, ' GRPC_BATCH_CALLOUTS_COUNT' print >> H, '} grpc_metadata_batch_callouts_index;' @@ -559,7 +569,7 @@ print >> H print >> H, 'typedef union {' print >> H, ' struct grpc_linked_mdelem *array[GRPC_BATCH_CALLOUTS_COUNT];' print >> H, ' struct {' -for elem, _ in METADATA_BATCH_CALLOUTS: +for elem in METADATA_BATCH_CALLOUTS: print >> H, ' struct grpc_linked_mdelem *%s;' % mangle(elem, '').lower() print >> H, ' } named;' print >> H, '} grpc_metadata_batch_callouts;' @@ -567,14 +577,6 @@ print >> H print >> H, '#define GRPC_BATCH_INDEX_OF(slice) \\' print >> H, ' (GRPC_IS_STATIC_METADATA_STRING((slice)) ? (grpc_metadata_batch_callouts_index)GPR_CLAMP(GRPC_STATIC_METADATA_INDEX((slice)), 0, GRPC_BATCH_CALLOUTS_COUNT) : GRPC_BATCH_CALLOUTS_COUNT)' print >> H -print >> H, ('extern bool grpc_static_callout_is_default[' - 'GRPC_BATCH_CALLOUTS_COUNT];') -print >> H -print >> C, 'bool grpc_static_callout_is_default[GRPC_BATCH_CALLOUTS_COUNT] = {' -for elem, is_default in METADATA_BATCH_CALLOUTS: - print >> C, ' %s, // %s' % (str(is_default).lower(), elem) -print >> C, '};' -print >> C print >> H, 'extern const uint8_t grpc_static_accept_encoding_metadata[%d];' % ( 1 << len(COMPRESSION_ALGORITHMS)) diff --git a/tools/distrib/check_copyright.py b/tools/distrib/check_copyright.py index 77f602ad1f..787bef1778 100755 --- a/tools/distrib/check_copyright.py +++ b/tools/distrib/check_copyright.py @@ -75,6 +75,8 @@ _EXEMPT = frozenset(( 'examples/python/multiplex/route_guide_pb2_grpc.py', 'examples/python/route_guide/route_guide_pb2.py', 'examples/python/route_guide/route_guide_pb2_grpc.py', + 'src/core/ext/filters/client_channel/health/health.pb.h', + 'src/core/ext/filters/client_channel/health/health.pb.c', 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c', 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h', @@ -87,8 +89,6 @@ _EXEMPT = frozenset(( 'src/core/tsi/alts/handshaker/handshaker.pb.c', 'src/core/tsi/alts/handshaker/transport_security_common.pb.h', 'src/core/tsi/alts/handshaker/transport_security_common.pb.c', - 'src/cpp/server/health/health.pb.h', - 'src/cpp/server/health/health.pb.c', # An older file originally from outside gRPC. 'src/php/tests/bootstrap.php', diff --git a/tools/distrib/check_include_guards.py b/tools/distrib/check_include_guards.py index 56f48af56a..b8d530cce0 100755 --- a/tools/distrib/check_include_guards.py +++ b/tools/distrib/check_include_guards.py @@ -156,6 +156,7 @@ argp.add_argument('--precommit', default=False, action='store_true') args = argp.parse_args() KNOWN_BAD = set([ + 'src/core/ext/filters/client_channel/health/health.pb.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.h', diff --git a/tools/distrib/check_nanopb_output.sh b/tools/distrib/check_nanopb_output.sh index 6b98619c32..018cbb7b66 100755 --- a/tools/distrib/check_nanopb_output.sh +++ b/tools/distrib/check_nanopb_output.sh @@ -16,6 +16,7 @@ set -ex readonly NANOPB_ALTS_TMP_OUTPUT="$(mktemp -d)" +readonly NANOPB_HEALTH_TMP_OUTPUT="$(mktemp -d)" readonly NANOPB_TMP_OUTPUT="$(mktemp -d)" readonly PROTOBUF_INSTALL_PREFIX="$(mktemp -d)" @@ -68,6 +69,23 @@ if ! diff -r "$NANOPB_TMP_OUTPUT" src/core/ext/filters/client_channel/lb_policy/ fi # +# checks for health.proto +# +readonly HEALTH_GRPC_OUTPUT_PATH='src/core/ext/filters/client_channel/health' +# nanopb-compile the proto to a temp location +./tools/codegen/core/gen_nano_proto.sh \ + src/proto/grpc/health/v1/health.proto \ + "$NANOPB_HEALTH_TMP_OUTPUT" \ + "$HEALTH_GRPC_OUTPUT_PATH" +# compare outputs to checked compiled code +for NANOPB_OUTPUT_FILE in $NANOPB_HEALTH_TMP_OUTPUT/*.pb.*; do + if ! diff "$NANOPB_OUTPUT_FILE" "${HEALTH_GRPC_OUTPUT_PATH}/$(basename $NANOPB_OUTPUT_FILE)"; then + echo "Outputs differ: $NANOPB_HEALTH_TMP_OUTPUT vs $HEALTH_GRPC_OUTPUT_PATH" + exit 2 + fi +done + +# # Checks for handshaker.proto and transport_security_common.proto # readonly HANDSHAKER_GRPC_OUTPUT_PATH='src/core/tsi/alts/handshaker' diff --git a/tools/distrib/python/grpcio_tools/MANIFEST.in b/tools/distrib/python/grpcio_tools/MANIFEST.in index 11ce367747..4943751879 100644 --- a/tools/distrib/python/grpcio_tools/MANIFEST.in +++ b/tools/distrib/python/grpcio_tools/MANIFEST.in @@ -1,3 +1,4 @@ +include _parallel_compile_patch.py include grpc_version.py include protoc_deps.py include protoc_lib_deps.py diff --git a/tools/distrib/python/grpcio_tools/_parallel_compile_patch.py b/tools/distrib/python/grpcio_tools/_parallel_compile_patch.py new file mode 100644 index 0000000000..4d03ef49ba --- /dev/null +++ b/tools/distrib/python/grpcio_tools/_parallel_compile_patch.py @@ -0,0 +1,63 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Patches the compile() to allow enable parallel compilation of C/C++. + +build_ext has lots of C/C++ files and normally them one by one. +Enabling parallel build helps a lot. +""" + +import distutils.ccompiler +import os + +try: + BUILD_EXT_COMPILER_JOBS = int( + os.environ.get('GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS', '1')) +except ValueError: + BUILD_EXT_COMPILER_JOBS = 1 + + +# monkey-patch for parallel compilation +def _parallel_compile(self, + sources, + output_dir=None, + macros=None, + include_dirs=None, + debug=0, + extra_preargs=None, + extra_postargs=None, + depends=None): + # setup the same way as distutils.ccompiler.CCompiler + # https://github.com/python/cpython/blob/31368a4f0e531c19affe2a1becd25fc316bc7501/Lib/distutils/ccompiler.py#L564 + macros, objects, extra_postargs, pp_opts, build = self._setup_compile( + output_dir, macros, include_dirs, sources, depends, extra_postargs) + cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) + + def _compile_single_file(obj): + try: + src, ext = build[obj] + except KeyError: + return + self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) + + # run compilation of individual files in parallel + import multiprocessing.pool + multiprocessing.pool.ThreadPool(BUILD_EXT_COMPILER_JOBS).map( + _compile_single_file, objects) + return objects + + +def monkeypatch_compile_maybe(): + """Monkeypatching is dumb, but the build speed gain is worth it.""" + if BUILD_EXT_COMPILER_JOBS > 1: + distutils.ccompiler.CCompiler.compile = _parallel_compile diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index 581dab3b4e..4b775e667e 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION = '1.16.0.dev0' +VERSION = '1.17.0.dev0' diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index c13dfe9ade..64c468cbf7 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -34,9 +34,12 @@ from setuptools.command import build_ext os.chdir(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.abspath('.')) +import _parallel_compile_patch import protoc_lib_deps import grpc_version +_parallel_compile_patch.monkeypatch_compile_maybe() + CLASSIFIERS = [ 'Development Status :: 5 - Production/Stable', 'Programming Language :: Python', diff --git a/tools/dockerfile/OWNERS b/tools/dockerfile/OWNERS index 5f0ad58d61..8ad09b7e83 100644 --- a/tools/dockerfile/OWNERS +++ b/tools/dockerfile/OWNERS @@ -7,5 +7,5 @@ set noparent # for kokoro to be able to access the pre-built images. @jtattermusch -@mehrdada +@apolcyn @nicolasnoble diff --git a/tools/dockerfile/grpc_artifact_linux_x64/Dockerfile b/tools/dockerfile/grpc_artifact_linux_x64/Dockerfile index 0251b2b392..7ec061ebe5 100644 --- a/tools/dockerfile/grpc_artifact_linux_x64/Dockerfile +++ b/tools/dockerfile/grpc_artifact_linux_x64/Dockerfile @@ -16,10 +16,8 @@ FROM debian:jessie -RUN apt-get update && apt-get install debian-keyring && apt-key update - # Install Git and basic packages. -RUN apt-get update && apt-key update && apt-get install -y \ +RUN apt-get update && apt-get install -y \ autoconf \ autotools-dev \ build-essential \ @@ -46,11 +44,6 @@ RUN apt-get update && apt-key update && apt-get install -y \ wget \ zip && apt-get clean -# Install Node dependencies -RUN touch .profile -RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.25.4/install.sh | bash -RUN /bin/bash -l -c "nvm install 8 && npm install -g node-pre-gyp" - ################## # Ruby dependencies @@ -72,15 +65,15 @@ RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc" # PHP dependencies RUN apt-get update && apt-get install -y \ - php5 php5-dev php-pear phpunit + php5 php5-dev php-pear phpunit && apt-get clean -################## -# Install cross compiler for ARM -RUN echo 'deb http://emdebian.org/tools/debian/ jessie main' | tee -a /etc/apt/sources.list.d/crosstools.list && \ - curl http://emdebian.org/tools/debian/emdebian-toolchain-archive.key | apt-key add - +################## +# C# dependencies (needed to build grpc_csharp_ext) -RUN dpkg --add-architecture armhf && apt-get update && apt-get install -y crossbuild-essential-armhf +# Use cmake 3.6 from jessie-backports +RUN echo "deb http://ftp.debian.org/debian jessie-backports main" | tee /etc/apt/sources.list.d/jessie-backports.list +RUN apt-get update && apt-get install -t jessie-backports -y cmake && apt-get clean RUN mkdir /var/local/jenkins diff --git a/tools/dockerfile/grpc_artifact_linux_x86/Dockerfile b/tools/dockerfile/grpc_artifact_linux_x86/Dockerfile index 2d179c8c45..f81d8e5ba0 100644 --- a/tools/dockerfile/grpc_artifact_linux_x86/Dockerfile +++ b/tools/dockerfile/grpc_artifact_linux_x86/Dockerfile @@ -16,10 +16,8 @@ FROM 32bit/debian:jessie -RUN apt-get update && apt-get install debian-keyring && apt-key update - # Install Git and basic packages. -RUN apt-get update && apt-key update && apt-get install -y \ +RUN apt-get update && apt-get install -y \ autoconf \ autotools-dev \ build-essential \ @@ -46,11 +44,6 @@ RUN apt-get update && apt-key update && apt-get install -y \ wget \ zip && apt-get clean -# Install Node dependencies -RUN touch .profile -RUN curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.25.4/install.sh | bash -RUN /bin/bash -l -c "nvm install 8 && npm install -g node-pre-gyp" - ################## # Ruby dependencies @@ -67,6 +60,13 @@ RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.1' >> ~/.bashrc" RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc" +################## +# C# dependencies (needed to build grpc_csharp_ext) + +# Use cmake 3.6 from jessie-backports +RUN echo "deb http://ftp.debian.org/debian jessie-backports main" | tee /etc/apt/sources.list.d/jessie-backports.list +RUN apt-get update && apt-get install -t jessie-backports -y cmake && apt-get clean + RUN mkdir /var/local/jenkins # Define the default command. diff --git a/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile index 511e2932d6..b2216c79d4 100644 --- a/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile @@ -82,6 +82,13 @@ RUN apt-get update && apt-get -y dist-upgrade && apt-get install -y \ RUN nuget update -self +#================= +# Use cmake 3.6 from jessie-backports +# needed to build grpc_csharp_ext with cmake + +RUN echo "deb http://ftp.debian.org/debian jessie-backports main" | tee /etc/apt/sources.list.d/jessie-backports.list +RUN apt-get update && apt-get install -t jessie-backports -y cmake && apt-get clean + # Install dotnet SDK based on https://www.microsoft.com/net/core#debian RUN apt-get update && apt-get install -y curl libunwind8 gettext # dotnet-dev-1.0.0-preview2-003131 diff --git a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile index 511e2932d6..b2216c79d4 100644 --- a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile @@ -82,6 +82,13 @@ RUN apt-get update && apt-get -y dist-upgrade && apt-get install -y \ RUN nuget update -self +#================= +# Use cmake 3.6 from jessie-backports +# needed to build grpc_csharp_ext with cmake + +RUN echo "deb http://ftp.debian.org/debian jessie-backports main" | tee /etc/apt/sources.list.d/jessie-backports.list +RUN apt-get update && apt-get install -t jessie-backports -y cmake && apt-get clean + # Install dotnet SDK based on https://www.microsoft.com/net/core#debian RUN apt-get update && apt-get install -y curl libunwind8 gettext # dotnet-dev-1.0.0-preview2-003131 diff --git a/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh index 2f31bea69b..8b8c6be8f1 100755 --- a/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh +++ b/tools/dockerfile/interoptest/grpc_interop_cxx/build_interop.sh @@ -28,10 +28,10 @@ cp -r /var/local/jenkins/service_account $HOME || true cd /var/local/git/grpc -make install-certs +make install-certs -j4 # build C++ interop client & server -make interop_client interop_server +make interop_client interop_server -j4 # build C++ http2 client -make http2_client +make http2_client -j4 diff --git a/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile index b136259ce9..e5e68943a4 100644 --- a/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM golang:latest +FROM golang:1.11 # Using login shell removes Go from path, so we add it. RUN ln -s /usr/local/go/bin/go /usr/local/bin diff --git a/tools/dockerfile/interoptest/grpc_interop_java/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_java/build_interop.sh index b651ac5b88..0e36c193c9 100644 --- a/tools/dockerfile/interoptest/grpc_interop_java/build_interop.sh +++ b/tools/dockerfile/interoptest/grpc_interop_java/build_interop.sh @@ -26,3 +26,11 @@ cd /var/local/git/grpc-java ./gradlew :grpc-interop-testing:installDist -PskipCodegen=true +# enable extra java logging +mkdir -p /var/local/grpc_java_logging +echo "handlers = java.util.logging.ConsoleHandler +java.util.logging.ConsoleHandler.level = ALL +.level = FINE +io.grpc.netty.NettyClientHandler = ALL +io.grpc.netty.NettyServerHandler = ALL" > /var/local/grpc_java_logging/logconf.txt + diff --git a/tools/dockerfile/interoptest/grpc_interop_java_oracle8/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_java_oracle8/build_interop.sh index 521111acaa..4c5ba4b7a3 100644 --- a/tools/dockerfile/interoptest/grpc_interop_java_oracle8/build_interop.sh +++ b/tools/dockerfile/interoptest/grpc_interop_java_oracle8/build_interop.sh @@ -25,4 +25,12 @@ cp -r /var/local/jenkins/service_account $HOME || true cd /var/local/git/grpc-java ./gradlew :grpc-interop-testing:installDist -PskipCodegen=true + +# enable extra java logging +mkdir -p /var/local/grpc_java_logging +echo "handlers = java.util.logging.ConsoleHandler +java.util.logging.ConsoleHandler.level = ALL +.level = FINE +io.grpc.netty.NettyClientHandler = ALL +io.grpc.netty.NettyServerHandler = ALL" > /var/local/grpc_java_logging/logconf.txt diff --git a/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh index 999976d15d..fc5c22083a 100755 --- a/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh +++ b/tools/dockerfile/interoptest/grpc_interop_php/build_interop.sh @@ -28,12 +28,13 @@ cp -r /var/local/jenkins/service_account $HOME || true cd /var/local/git/grpc -# gRPC core and protobuf need to be installed -make install +# Install gRPC C core and build codegen plugins +make -j4 install_c plugins -(cd src/php/ext/grpc && phpize && ./configure && make) +(cd src/php/ext/grpc && phpize && ./configure && make -j4) -(cd third_party/protobuf && make install) +# Install protobuf (need access to protoc) +(cd third_party/protobuf && make -j4 install) (cd src/php && php -d extension=ext/grpc/modules/grpc.so /usr/local/bin/composer install) diff --git a/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh b/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh index efa97530c8..248a8f680b 100755 --- a/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh +++ b/tools/dockerfile/interoptest/grpc_interop_php7/build_interop.sh @@ -28,12 +28,13 @@ cp -r /var/local/jenkins/service_account $HOME || true cd /var/local/git/grpc -# gRPC core and protobuf need to be installed -make install +# Install gRPC C core and build codegen plugins +make -j4 install_c plugins -(cd src/php/ext/grpc && phpize && ./configure && make) +(cd src/php/ext/grpc && phpize && ./configure && make -j4) -(cd third_party/protobuf && make install) +# Install protobuf (need access to protoc) +(cd third_party/protobuf && make -j4 install) (cd src/php && php -d extension=ext/grpc/modules/grpc.so /usr/local/bin/composer install) diff --git a/tools/dockerfile/interoptest/lb_interop_fake_servers/Dockerfile b/tools/dockerfile/interoptest/lb_interop_fake_servers/Dockerfile new file mode 100644 index 0000000000..22963f7f83 --- /dev/null +++ b/tools/dockerfile/interoptest/lb_interop_fake_servers/Dockerfile @@ -0,0 +1,34 @@ +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM golang:1.10 + +RUN apt-get update && apt-get install -y \ + dnsutils \ + git \ + vim \ + curl \ + python-pip \ + python-yaml \ + make && apt-get clean + +RUN ln -s /usr/local/go/bin/go /usr/local/bin + +# Install Python packages from PyPI +RUN pip install --upgrade pip==10.0.1 +RUN pip install virtualenv +RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.10.0 twisted==17.5.0 + +# Define the default command. +CMD ["bash"] diff --git a/tools/dockerfile/interoptest/lb_interop_fake_servers/build_interop.sh b/tools/dockerfile/interoptest/lb_interop_fake_servers/build_interop.sh new file mode 100644 index 0000000000..1846d51753 --- /dev/null +++ b/tools/dockerfile/interoptest/lb_interop_fake_servers/build_interop.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Gets a built Go interop server, fake balancer server, and python +# DNS server into a base image. +set -e + +# Clone just the grpc-go source code without any dependencies. +# We are cloning from a local git repo that contains the right revision +# to test instead of using "go get" to download from Github directly. +git clone --recursive /var/local/jenkins/grpc-go src/google.golang.org/grpc + +# Get all gRPC Go dependencies +(cd src/google.golang.org/grpc && make deps && make testdeps) + +# Build the interop server and fake balancer +(cd src/google.golang.org/grpc/interop/server && go install) +(cd src/google.golang.org/grpc/interop/fake_grpclb && go install) + +# Clone the grpc/grpc repo to get the python DNS server. +# Hack: we don't need to init submodules for the scripts we need. +mkdir -p /var/local/git/grpc +git clone /var/local/jenkins/grpc /var/local/git/grpc diff --git a/tools/dockerfile/test/bazel/Dockerfile b/tools/dockerfile/test/bazel/Dockerfile index 4f913dc396..0aa6209f4f 100644 --- a/tools/dockerfile/test/bazel/Dockerfile +++ b/tools/dockerfile/test/bazel/Dockerfile @@ -44,9 +44,10 @@ RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.10.0 t #======================== # Bazel installation -RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" > /etc/apt/sources.list.d/bazel.list -RUN curl https://bazel.build/bazel-release.pub.gpg | apt-key add - -RUN apt-get -y update && apt-get -y install bazel=0.15.0 && apt-get clean + +RUN apt-get update && apt-get install -y wget && apt-get clean +RUN wget -q https://github.com/bazelbuild/bazel/releases/download/0.17.1/bazel-0.17.1-linux-x86_64 -O /usr/local/bin/bazel +RUN chmod 755 /usr/local/bin/bazel RUN mkdir -p /var/local/jenkins diff --git a/tools/dockerfile/test/csharp_jessie_x64/Dockerfile b/tools/dockerfile/test/csharp_jessie_x64/Dockerfile index 56bfb89925..030d301a40 100644 --- a/tools/dockerfile/test/csharp_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/csharp_jessie_x64/Dockerfile @@ -51,7 +51,7 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client +RUN pip install --upgrade google-api-python-client oauth2client #==================== # Python dependencies @@ -86,6 +86,13 @@ RUN apt-get update && apt-get -y dist-upgrade && apt-get install -y \ RUN nuget update -self +#================= +# Use cmake 3.6 from jessie-backports +# needed to build grpc_csharp_ext with cmake + +RUN echo "deb http://ftp.debian.org/debian jessie-backports main" | tee /etc/apt/sources.list.d/jessie-backports.list +RUN apt-get update && apt-get install -t jessie-backports -y cmake && apt-get clean + # Install dotnet SDK based on https://www.microsoft.com/net/core#debian RUN apt-get update && apt-get install -y curl libunwind8 gettext # dotnet-dev-1.0.0-preview2-003131 diff --git a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile index c4f959fc1d..f9dc8f20d7 100644 --- a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile @@ -51,7 +51,7 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client +RUN pip install --upgrade google-api-python-client oauth2client #==================== # Python dependencies diff --git a/tools/dockerfile/test/cxx_jessie_x86/Dockerfile b/tools/dockerfile/test/cxx_jessie_x86/Dockerfile index d07ea9a9b0..76015c8c42 100644 --- a/tools/dockerfile/test/cxx_jessie_x86/Dockerfile +++ b/tools/dockerfile/test/cxx_jessie_x86/Dockerfile @@ -51,7 +51,7 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client +RUN pip install --upgrade google-api-python-client oauth2client #==================== # Python dependencies diff --git a/tools/dockerfile/test/cxx_sanitizers_jessie_x64/Dockerfile b/tools/dockerfile/test/cxx_sanitizers_jessie_x64/Dockerfile index f60f67665d..77aac3cdec 100644 --- a/tools/dockerfile/test/cxx_sanitizers_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_sanitizers_jessie_x64/Dockerfile @@ -52,7 +52,7 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client +RUN pip install --upgrade google-api-python-client oauth2client #==================== # Python dependencies diff --git a/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile index b0d9261af2..09479ba2f7 100644 --- a/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile @@ -51,7 +51,7 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client +RUN pip install --upgrade google-api-python-client oauth2client #==================== # Python dependencies diff --git a/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile index 439baadc2c..3a87639794 100644 --- a/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile @@ -51,7 +51,7 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client +RUN pip install --upgrade google-api-python-client oauth2client #==================== # Python dependencies diff --git a/tools/dockerfile/test/cxx_ubuntu1710_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1710_x64/Dockerfile index b80249d2d8..d4117f83c8 100644 --- a/tools/dockerfile/test/cxx_ubuntu1710_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_ubuntu1710_x64/Dockerfile @@ -51,7 +51,7 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client +RUN pip install --upgrade google-api-python-client oauth2client #==================== # Python dependencies diff --git a/tools/dockerfile/test/fuzzer/Dockerfile b/tools/dockerfile/test/fuzzer/Dockerfile index 986cfcbada..7f871f2f62 100644 --- a/tools/dockerfile/test/fuzzer/Dockerfile +++ b/tools/dockerfile/test/fuzzer/Dockerfile @@ -51,7 +51,7 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client +RUN pip install --upgrade google-api-python-client oauth2client #==================== # Python dependencies diff --git a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile index a82e7050fc..3c95554b02 100644 --- a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile @@ -51,7 +51,7 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client +RUN pip install --upgrade google-api-python-client oauth2client #================ # C# dependencies @@ -71,6 +71,13 @@ RUN apt-get update && apt-get -y dist-upgrade && apt-get install -y \ RUN nuget update -self +#================= +# Use cmake 3.6 from jessie-backports +# needed to build grpc_csharp_ext with cmake + +RUN echo "deb http://ftp.debian.org/debian jessie-backports main" | tee /etc/apt/sources.list.d/jessie-backports.list +RUN apt-get update && apt-get install -t jessie-backports -y cmake && apt-get clean + # Install dotnet SDK based on https://www.microsoft.com/net/core#debian RUN apt-get update && apt-get install -y curl libunwind8 gettext # dotnet-dev-1.0.0-preview2-003131 diff --git a/tools/dockerfile/test/node_jessie_x64/Dockerfile b/tools/dockerfile/test/node_jessie_x64/Dockerfile index 1fe44b6dd7..7c36fb3734 100644 --- a/tools/dockerfile/test/node_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/node_jessie_x64/Dockerfile @@ -51,7 +51,7 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client +RUN pip install --upgrade google-api-python-client oauth2client # Install Electron apt dependencies diff --git a/tools/dockerfile/test/php7_jessie_x64/Dockerfile b/tools/dockerfile/test/php7_jessie_x64/Dockerfile index 53ef7b3443..0dff839904 100644 --- a/tools/dockerfile/test/php7_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/php7_jessie_x64/Dockerfile @@ -62,7 +62,7 @@ RUN cd /var/local/git/php-src \ # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client +RUN pip install --upgrade google-api-python-client oauth2client #==================== # Python dependencies diff --git a/tools/dockerfile/test/php_jessie_x64/Dockerfile b/tools/dockerfile/test/php_jessie_x64/Dockerfile index e884572640..ed59e56995 100644 --- a/tools/dockerfile/test/php_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/php_jessie_x64/Dockerfile @@ -51,7 +51,7 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client +RUN pip install --upgrade google-api-python-client oauth2client #==================== # Python dependencies diff --git a/tools/dockerfile/test/python_jessie_x64/Dockerfile b/tools/dockerfile/test/python_jessie_x64/Dockerfile index c2b4c1845b..a4c3a9f91e 100644 --- a/tools/dockerfile/test/python_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/python_jessie_x64/Dockerfile @@ -51,7 +51,7 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client +RUN pip install --upgrade google-api-python-client oauth2client #==================== # Python dependencies diff --git a/tools/dockerfile/test/python_stretch_2.7_x64/Dockerfile b/tools/dockerfile/test/python_stretch_2.7_x64/Dockerfile new file mode 100644 index 0000000000..a7a8174db4 --- /dev/null +++ b/tools/dockerfile/test/python_stretch_2.7_x64/Dockerfile @@ -0,0 +1,69 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM debian:stretch + +# Install Git and basic packages. +RUN apt-get update && apt-get install -y \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + ccache \ + curl \ + dnsutils \ + gcc \ + gcc-multilib \ + git \ + golang \ + gyp \ + lcov \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libgtest-dev \ + libtool \ + make \ + perl \ + strace \ + python-dev \ + python-setuptools \ + python-yaml \ + telnet \ + unzip \ + wget \ + zip && apt-get clean + +#================ +# Build profiling +RUN apt-get update && apt-get install -y time && apt-get clean + +# Google Cloud platform API libraries +RUN apt-get update && apt-get install -y python-pip && apt-get clean +RUN pip install --upgrade google-api-python-client oauth2client + +# Install Python 2.7 +RUN apt-get update && apt-get install -y python2.7 python-all-dev +RUN curl https://bootstrap.pypa.io/get-pip.py | python2.7 + +# Add Debian 'testing' repository +RUN echo 'deb http://ftp.de.debian.org/debian testing main' >> /etc/apt/sources.list +RUN echo 'APT::Default-Release "stable";' | tee -a /etc/apt/apt.conf.d/00local + + +RUN mkdir /var/local/jenkins + +# Define the default command. +CMD ["bash"] + diff --git a/tools/dockerfile/test/python_pyenv_x64/Dockerfile b/tools/dockerfile/test/python_stretch_3.5_x64/Dockerfile index c23e67c904..0e97e77e2f 100644 --- a/tools/dockerfile/test/python_pyenv_x64/Dockerfile +++ b/tools/dockerfile/test/python_stretch_3.5_x64/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2016 gRPC authors. +# Copyright 2018 The gRPC Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,7 +13,7 @@ # limitations under the License. FROM debian:stretch - + # Install Git and basic packages. RUN apt-get update && apt-get install -y \ autoconf \ @@ -51,53 +51,22 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client - -#==================== -# Python dependencies - -# Install dependencies - -RUN apt-get update && apt-get install -y \ - python-all-dev \ - python3-all-dev \ - python-pip - -# Install Python packages from PyPI -RUN pip install --upgrade pip==10.0.1 -RUN pip install virtualenv -RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.10.0 twisted==17.5.0 +RUN pip install --upgrade google-api-python-client oauth2client -# Install dependencies for pyenv -RUN apt-get update && apt-get install -y \ - libbz2-dev \ - libncurses5-dev \ - libncursesw5-dev \ - libreadline-dev \ - libsqlite3-dev \ - libssl-dev \ - llvm \ - mercurial \ - zlib1g-dev && apt-get clean - -# Install Pyenv and dev Python versions 3.{5,6,7} -RUN curl -L https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash -ENV PATH /root/.pyenv/bin:$PATH -RUN eval "$(pyenv init -)" -RUN eval "$(pyenv virtualenv-init -)" -RUN pyenv update -RUN pyenv install 3.5-dev -RUN pyenv install 3.6-dev -RUN pyenv install 3.7-dev -RUN pyenv install pypy-5.3.1 -RUN pyenv local 3.5-dev 3.6-dev 3.7-dev pypy-5.3.1 +# Install Python 2.7 +RUN apt-get update && apt-get install -y python2.7 python-all-dev +RUN curl https://bootstrap.pypa.io/get-pip.py | python2.7 -# Install pip and virtualenv for Python 3.5 -RUN curl https://bootstrap.pypa.io/get-pip.py | python3.5 -RUN python3.5 -m pip install virtualenv +# Add Debian 'testing' repository +RUN echo 'deb http://ftp.de.debian.org/debian testing main' >> /etc/apt/sources.list +RUN echo 'APT::Default-Release "stable";' | tee -a /etc/apt/apt.conf.d/00local RUN mkdir /var/local/jenkins # Define the default command. CMD ["bash"] + + +RUN apt-get update && apt-get install -y python3.5 python3-all-dev +RUN curl https://bootstrap.pypa.io/get-pip.py | python3.5 diff --git a/tools/dockerfile/test/python_stretch_3.6_x64/Dockerfile b/tools/dockerfile/test/python_stretch_3.6_x64/Dockerfile new file mode 100644 index 0000000000..9b16b2d3a1 --- /dev/null +++ b/tools/dockerfile/test/python_stretch_3.6_x64/Dockerfile @@ -0,0 +1,72 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM debian:stretch + +# Install Git and basic packages. +RUN apt-get update && apt-get install -y \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + ccache \ + curl \ + dnsutils \ + gcc \ + gcc-multilib \ + git \ + golang \ + gyp \ + lcov \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libgtest-dev \ + libtool \ + make \ + perl \ + strace \ + python-dev \ + python-setuptools \ + python-yaml \ + telnet \ + unzip \ + wget \ + zip && apt-get clean + +#================ +# Build profiling +RUN apt-get update && apt-get install -y time && apt-get clean + +# Google Cloud platform API libraries +RUN apt-get update && apt-get install -y python-pip && apt-get clean +RUN pip install --upgrade google-api-python-client oauth2client + +# Install Python 2.7 +RUN apt-get update && apt-get install -y python2.7 python-all-dev +RUN curl https://bootstrap.pypa.io/get-pip.py | python2.7 + +# Add Debian 'testing' repository +RUN echo 'deb http://ftp.de.debian.org/debian testing main' >> /etc/apt/sources.list +RUN echo 'APT::Default-Release "stable";' | tee -a /etc/apt/apt.conf.d/00local + + +RUN mkdir /var/local/jenkins + +# Define the default command. +CMD ["bash"] + + +RUN apt-get update && apt-get -t testing install -y python3.6 python3-all-dev +RUN curl https://bootstrap.pypa.io/get-pip.py | python3.6 diff --git a/tools/dockerfile/test/python_stretch_3.7_x64/Dockerfile b/tools/dockerfile/test/python_stretch_3.7_x64/Dockerfile new file mode 100644 index 0000000000..add1cc509d --- /dev/null +++ b/tools/dockerfile/test/python_stretch_3.7_x64/Dockerfile @@ -0,0 +1,72 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM debian:stretch + +# Install Git and basic packages. +RUN apt-get update && apt-get install -y \ + autoconf \ + autotools-dev \ + build-essential \ + bzip2 \ + ccache \ + curl \ + dnsutils \ + gcc \ + gcc-multilib \ + git \ + golang \ + gyp \ + lcov \ + libc6 \ + libc6-dbg \ + libc6-dev \ + libgtest-dev \ + libtool \ + make \ + perl \ + strace \ + python-dev \ + python-setuptools \ + python-yaml \ + telnet \ + unzip \ + wget \ + zip && apt-get clean + +#================ +# Build profiling +RUN apt-get update && apt-get install -y time && apt-get clean + +# Google Cloud platform API libraries +RUN apt-get update && apt-get install -y python-pip && apt-get clean +RUN pip install --upgrade google-api-python-client oauth2client + +# Install Python 2.7 +RUN apt-get update && apt-get install -y python2.7 python-all-dev +RUN curl https://bootstrap.pypa.io/get-pip.py | python2.7 + +# Add Debian 'testing' repository +RUN echo 'deb http://ftp.de.debian.org/debian testing main' >> /etc/apt/sources.list +RUN echo 'APT::Default-Release "stable";' | tee -a /etc/apt/apt.conf.d/00local + + +RUN mkdir /var/local/jenkins + +# Define the default command. +CMD ["bash"] + + +RUN apt-get update && apt-get -t testing install -y python3.7 python3-all-dev +RUN curl https://bootstrap.pypa.io/get-pip.py | python3.7 diff --git a/tools/dockerfile/test/ruby_jessie_x64/Dockerfile b/tools/dockerfile/test/ruby_jessie_x64/Dockerfile index d6f7459685..321b501de2 100644 --- a/tools/dockerfile/test/ruby_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/ruby_jessie_x64/Dockerfile @@ -51,7 +51,7 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client +RUN pip install --upgrade google-api-python-client oauth2client #==================== # Python dependencies diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile index cb153012ea..e6bdb4ee03 100644 --- a/tools/dockerfile/test/sanity/Dockerfile +++ b/tools/dockerfile/test/sanity/Dockerfile @@ -51,7 +51,7 @@ RUN apt-get update && apt-get install -y time && apt-get clean # Google Cloud platform API libraries RUN apt-get update && apt-get install -y python-pip && apt-get clean -RUN pip install --upgrade google-api-python-client +RUN pip install --upgrade google-api-python-client oauth2client #==================== # Python dependencies diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 3b7fd1fa8e..392113c284 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.16.0-dev +PROJECT_NUMBER = 1.17.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -764,7 +764,6 @@ INPUT = doc/PROTOCOL-HTTP2.md \ doc/PROTOCOL-WEB.md \ doc/binary-logging.md \ doc/c-style-guide.md \ -doc/combiner-explainer.md \ doc/command_line_tool.md \ doc/compression.md \ doc/compression_cookbook.md \ @@ -775,7 +774,6 @@ doc/cpp-style-guide.md \ doc/cpp/pending_api_cleanups.md \ doc/cpp/perf_notes.md \ doc/environment_variables.md \ -doc/epoll-polling-engine.md \ doc/fail_fast.md \ doc/fork_support.md \ doc/g_stands_for.md \ @@ -945,10 +943,13 @@ include/grpcpp/impl/codegen/async_unary_call.h \ include/grpcpp/impl/codegen/byte_buffer.h \ include/grpcpp/impl/codegen/call.h \ include/grpcpp/impl/codegen/call_hook.h \ +include/grpcpp/impl/codegen/call_op_set.h \ +include/grpcpp/impl/codegen/call_op_set_interface.h \ include/grpcpp/impl/codegen/callback_common.h \ include/grpcpp/impl/codegen/channel_interface.h \ include/grpcpp/impl/codegen/client_callback.h \ include/grpcpp/impl/codegen/client_context.h \ +include/grpcpp/impl/codegen/client_interceptor.h \ include/grpcpp/impl/codegen/client_unary_call.h \ include/grpcpp/impl/codegen/completion_queue.h \ include/grpcpp/impl/codegen/completion_queue_tag.h \ @@ -958,6 +959,9 @@ include/grpcpp/impl/codegen/core_codegen.h \ include/grpcpp/impl/codegen/core_codegen_interface.h \ include/grpcpp/impl/codegen/create_auth_context.h \ include/grpcpp/impl/codegen/grpc_library.h \ +include/grpcpp/impl/codegen/intercepted_channel.h \ +include/grpcpp/impl/codegen/interceptor.h \ +include/grpcpp/impl/codegen/interceptor_common.h \ include/grpcpp/impl/codegen/metadata_map.h \ include/grpcpp/impl/codegen/method_handler_impl.h \ include/grpcpp/impl/codegen/proto_buffer_reader.h \ @@ -967,7 +971,9 @@ include/grpcpp/impl/codegen/rpc_method.h \ include/grpcpp/impl/codegen/rpc_service_method.h \ include/grpcpp/impl/codegen/security/auth_context.h \ include/grpcpp/impl/codegen/serialization_traits.h \ +include/grpcpp/impl/codegen/server_callback.h \ include/grpcpp/impl/codegen/server_context.h \ +include/grpcpp/impl/codegen/server_interceptor.h \ include/grpcpp/impl/codegen/server_interface.h \ include/grpcpp/impl/codegen/service_type.h \ include/grpcpp/impl/codegen/slice.h \ @@ -1003,6 +1009,7 @@ include/grpcpp/support/client_callback.h \ include/grpcpp/support/config.h \ include/grpcpp/support/proto_buffer_reader.h \ include/grpcpp/support/proto_buffer_writer.h \ +include/grpcpp/support/server_callback.h \ include/grpcpp/support/slice.h \ include/grpcpp/support/status.h \ include/grpcpp/support/status_code_enum.h \ diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index a72390d9f8..a96683883c 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.16.0-dev +PROJECT_NUMBER = 1.17.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -764,7 +764,6 @@ INPUT = doc/PROTOCOL-HTTP2.md \ doc/PROTOCOL-WEB.md \ doc/binary-logging.md \ doc/c-style-guide.md \ -doc/combiner-explainer.md \ doc/command_line_tool.md \ doc/compression.md \ doc/compression_cookbook.md \ @@ -775,7 +774,6 @@ doc/cpp-style-guide.md \ doc/cpp/pending_api_cleanups.md \ doc/cpp/perf_notes.md \ doc/environment_variables.md \ -doc/epoll-polling-engine.md \ doc/fail_fast.md \ doc/fork_support.md \ doc/g_stands_for.md \ @@ -946,10 +944,13 @@ include/grpcpp/impl/codegen/async_unary_call.h \ include/grpcpp/impl/codegen/byte_buffer.h \ include/grpcpp/impl/codegen/call.h \ include/grpcpp/impl/codegen/call_hook.h \ +include/grpcpp/impl/codegen/call_op_set.h \ +include/grpcpp/impl/codegen/call_op_set_interface.h \ include/grpcpp/impl/codegen/callback_common.h \ include/grpcpp/impl/codegen/channel_interface.h \ include/grpcpp/impl/codegen/client_callback.h \ include/grpcpp/impl/codegen/client_context.h \ +include/grpcpp/impl/codegen/client_interceptor.h \ include/grpcpp/impl/codegen/client_unary_call.h \ include/grpcpp/impl/codegen/completion_queue.h \ include/grpcpp/impl/codegen/completion_queue_tag.h \ @@ -960,6 +961,9 @@ include/grpcpp/impl/codegen/core_codegen.h \ include/grpcpp/impl/codegen/core_codegen_interface.h \ include/grpcpp/impl/codegen/create_auth_context.h \ include/grpcpp/impl/codegen/grpc_library.h \ +include/grpcpp/impl/codegen/intercepted_channel.h \ +include/grpcpp/impl/codegen/interceptor.h \ +include/grpcpp/impl/codegen/interceptor_common.h \ include/grpcpp/impl/codegen/metadata_map.h \ include/grpcpp/impl/codegen/method_handler_impl.h \ include/grpcpp/impl/codegen/proto_buffer_reader.h \ @@ -969,7 +973,9 @@ include/grpcpp/impl/codegen/rpc_method.h \ include/grpcpp/impl/codegen/rpc_service_method.h \ include/grpcpp/impl/codegen/security/auth_context.h \ include/grpcpp/impl/codegen/serialization_traits.h \ +include/grpcpp/impl/codegen/server_callback.h \ include/grpcpp/impl/codegen/server_context.h \ +include/grpcpp/impl/codegen/server_interceptor.h \ include/grpcpp/impl/codegen/server_interface.h \ include/grpcpp/impl/codegen/service_type.h \ include/grpcpp/impl/codegen/slice.h \ @@ -1005,6 +1011,7 @@ include/grpcpp/support/client_callback.h \ include/grpcpp/support/config.h \ include/grpcpp/support/proto_buffer_reader.h \ include/grpcpp/support/proto_buffer_writer.h \ +include/grpcpp/support/server_callback.h \ include/grpcpp/support/slice.h \ include/grpcpp/support/status.h \ include/grpcpp/support/status_code_enum.h \ @@ -1012,6 +1019,8 @@ include/grpcpp/support/string_ref.h \ include/grpcpp/support/stub_options.h \ include/grpcpp/support/sync_stream.h \ include/grpcpp/support/time.h \ +src/core/ext/filters/client_channel/health/health.pb.c \ +src/core/ext/filters/client_channel/health/health.pb.h \ src/core/ext/transport/inproc/inproc_transport.h \ src/core/lib/avl/avl.h \ src/core/lib/backoff/backoff.h \ @@ -1080,7 +1089,6 @@ src/core/lib/iomgr/error.h \ src/core/lib/iomgr/error_internal.h \ src/core/lib/iomgr/ev_epoll1_linux.h \ src/core/lib/iomgr/ev_epollex_linux.h \ -src/core/lib/iomgr/ev_epollsig_linux.h \ src/core/lib/iomgr/ev_poll_posix.h \ src/core/lib/iomgr/ev_posix.h \ src/core/lib/iomgr/exec_ctx.h \ @@ -1175,9 +1183,11 @@ src/core/lib/transport/status_metadata.h \ src/core/lib/transport/timeout_encoding.h \ src/core/lib/transport/transport.h \ src/core/lib/transport/transport_impl.h \ +src/core/lib/uri/uri_parser.h \ src/cpp/README.md \ src/cpp/client/channel_cc.cc \ src/cpp/client/client_context.cc \ +src/cpp/client/client_interceptor.cc \ src/cpp/client/create_channel.cc \ src/cpp/client/create_channel_internal.cc \ src/cpp/client/create_channel_internal.h \ @@ -1190,7 +1200,6 @@ src/cpp/client/secure_credentials.h \ src/cpp/codegen/codegen_init.cc \ src/cpp/common/alarm.cc \ src/cpp/common/auth_property_iterator.cc \ -src/cpp/common/callback_common.cc \ src/cpp/common/channel_arguments.cc \ src/cpp/common/channel_filter.cc \ src/cpp/common/channel_filter.h \ @@ -1210,8 +1219,6 @@ src/cpp/server/dynamic_thread_pool.cc \ src/cpp/server/dynamic_thread_pool.h \ src/cpp/server/health/default_health_check_service.cc \ src/cpp/server/health/default_health_check_service.h \ -src/cpp/server/health/health.pb.c \ -src/cpp/server/health/health.pb.h \ src/cpp/server/health/health_check_service.cc \ src/cpp/server/health/health_check_service_server_builder_option.cc \ src/cpp/server/insecure_server_credentials.cc \ @@ -1230,8 +1237,11 @@ src/cpp/util/status.cc \ src/cpp/util/string_ref.cc \ src/cpp/util/time_cc.cc \ third_party/nanopb/pb.h \ +third_party/nanopb/pb_common.c \ third_party/nanopb/pb_common.h \ +third_party/nanopb/pb_decode.c \ third_party/nanopb/pb_decode.h \ +third_party/nanopb/pb_encode.c \ third_party/nanopb/pb_encode.h # This tag can be used to specify the character encoding of the source files diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index aa75bc6828..b78fb607ad 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Core" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 6.0.0-dev +PROJECT_NUMBER = 7.0.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -764,20 +764,23 @@ INPUT = doc/PROTOCOL-HTTP2.md \ doc/PROTOCOL-WEB.md \ doc/binary-logging.md \ doc/c-style-guide.md \ -doc/combiner-explainer.md \ doc/command_line_tool.md \ doc/compression.md \ doc/compression_cookbook.md \ doc/connection-backoff-interop-test-description.md \ doc/connection-backoff.md \ doc/connectivity-semantics-and-api.md \ +doc/core/combiner-explainer.md \ +doc/core/epoll-polling-engine.md \ +doc/core/grpc-client-server-polling-engine-usage.md \ +doc/core/grpc-cq.md \ doc/core/grpc-error.md \ +doc/core/grpc-polling-engines.md \ doc/core/moving-to-c++.md \ doc/core/pending_api_cleanups.md \ doc/core/transport_explainer.md \ doc/cpp-style-guide.md \ doc/environment_variables.md \ -doc/epoll-polling-engine.md \ doc/fail_fast.md \ doc/fork_support.md \ doc/g_stands_for.md \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 7cd1dc7bf3..791318bf36 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC Core" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 6.0.0-dev +PROJECT_NUMBER = 7.0.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -764,20 +764,23 @@ INPUT = doc/PROTOCOL-HTTP2.md \ doc/PROTOCOL-WEB.md \ doc/binary-logging.md \ doc/c-style-guide.md \ -doc/combiner-explainer.md \ doc/command_line_tool.md \ doc/compression.md \ doc/compression_cookbook.md \ doc/connection-backoff-interop-test-description.md \ doc/connection-backoff.md \ doc/connectivity-semantics-and-api.md \ +doc/core/combiner-explainer.md \ +doc/core/epoll-polling-engine.md \ +doc/core/grpc-client-server-polling-engine-usage.md \ +doc/core/grpc-cq.md \ doc/core/grpc-error.md \ +doc/core/grpc-polling-engines.md \ doc/core/moving-to-c++.md \ doc/core/pending_api_cleanups.md \ doc/core/transport_explainer.md \ doc/cpp-style-guide.md \ doc/environment_variables.md \ -doc/epoll-polling-engine.md \ doc/fail_fast.md \ doc/fork_support.md \ doc/g_stands_for.md \ @@ -882,6 +885,10 @@ src/core/ext/filters/client_channel/client_channel_factory.h \ src/core/ext/filters/client_channel/client_channel_plugin.cc \ src/core/ext/filters/client_channel/connector.cc \ src/core/ext/filters/client_channel/connector.h \ +src/core/ext/filters/client_channel/health/health.pb.c \ +src/core/ext/filters/client_channel/health/health.pb.h \ +src/core/ext/filters/client_channel/health/health_check_client.cc \ +src/core/ext/filters/client_channel/health/health_check_client.h \ src/core/ext/filters/client_channel/http_connect_handshaker.cc \ src/core/ext/filters/client_channel/http_connect_handshaker.h \ src/core/ext/filters/client_channel/http_proxy.cc \ @@ -907,6 +914,14 @@ src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balan src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc \ src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc \ src/core/ext/filters/client_channel/lb_policy/subchannel_list.h \ +src/core/ext/filters/client_channel/lb_policy/xds/xds.cc \ +src/core/ext/filters/client_channel/lb_policy/xds/xds.h \ +src/core/ext/filters/client_channel/lb_policy/xds/xds_channel.h \ +src/core/ext/filters/client_channel/lb_policy/xds/xds_channel_secure.cc \ +src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.cc \ +src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.h \ +src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.cc \ +src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.h \ src/core/ext/filters/client_channel/lb_policy_factory.cc \ src/core/ext/filters/client_channel/lb_policy_factory.h \ src/core/ext/filters/client_channel/lb_policy_registry.cc \ @@ -947,8 +962,6 @@ src/core/ext/filters/client_channel/subchannel.cc \ src/core/ext/filters/client_channel/subchannel.h \ src/core/ext/filters/client_channel/subchannel_index.cc \ src/core/ext/filters/client_channel/subchannel_index.h \ -src/core/ext/filters/client_channel/uri_parser.cc \ -src/core/ext/filters/client_channel/uri_parser.h \ src/core/ext/filters/deadline/deadline_filter.cc \ src/core/ext/filters/deadline/deadline_filter.h \ src/core/ext/filters/http/client/http_client_filter.cc \ @@ -1179,8 +1192,6 @@ src/core/lib/iomgr/ev_epoll1_linux.cc \ src/core/lib/iomgr/ev_epoll1_linux.h \ src/core/lib/iomgr/ev_epollex_linux.cc \ src/core/lib/iomgr/ev_epollex_linux.h \ -src/core/lib/iomgr/ev_epollsig_linux.cc \ -src/core/lib/iomgr/ev_epollsig_linux.h \ src/core/lib/iomgr/ev_poll_posix.cc \ src/core/lib/iomgr/ev_poll_posix.h \ src/core/lib/iomgr/ev_posix.cc \ @@ -1360,16 +1371,22 @@ src/core/lib/security/credentials/plugin/plugin_credentials.cc \ src/core/lib/security/credentials/plugin/plugin_credentials.h \ src/core/lib/security/credentials/ssl/ssl_credentials.cc \ src/core/lib/security/credentials/ssl/ssl_credentials.h \ -src/core/lib/security/security_connector/alts_security_connector.cc \ -src/core/lib/security/security_connector/alts_security_connector.h \ +src/core/lib/security/security_connector/alts/alts_security_connector.cc \ +src/core/lib/security/security_connector/alts/alts_security_connector.h \ +src/core/lib/security/security_connector/fake/fake_security_connector.cc \ +src/core/lib/security/security_connector/fake/fake_security_connector.h \ src/core/lib/security/security_connector/load_system_roots.h \ src/core/lib/security/security_connector/load_system_roots_fallback.cc \ src/core/lib/security/security_connector/load_system_roots_linux.cc \ src/core/lib/security/security_connector/load_system_roots_linux.h \ -src/core/lib/security/security_connector/local_security_connector.cc \ -src/core/lib/security/security_connector/local_security_connector.h \ +src/core/lib/security/security_connector/local/local_security_connector.cc \ +src/core/lib/security/security_connector/local/local_security_connector.h \ src/core/lib/security/security_connector/security_connector.cc \ src/core/lib/security/security_connector/security_connector.h \ +src/core/lib/security/security_connector/ssl/ssl_security_connector.cc \ +src/core/lib/security/security_connector/ssl/ssl_security_connector.h \ +src/core/lib/security/security_connector/ssl_utils.cc \ +src/core/lib/security/security_connector/ssl_utils.h \ src/core/lib/security/transport/auth_filters.h \ src/core/lib/security/transport/client_auth_filter.cc \ src/core/lib/security/transport/secure_endpoint.cc \ @@ -1459,6 +1476,8 @@ src/core/lib/transport/transport.cc \ src/core/lib/transport/transport.h \ src/core/lib/transport/transport_impl.h \ src/core/lib/transport/transport_op_string.cc \ +src/core/lib/uri/uri_parser.cc \ +src/core/lib/uri/uri_parser.h \ src/core/plugin_registry/grpc_plugin_registry.cc \ src/core/tsi/README.md \ src/core/tsi/alts/crypt/aes_gcm.cc \ @@ -1482,8 +1501,8 @@ src/core/tsi/alts/handshaker/alts_handshaker_service_api.cc \ src/core/tsi/alts/handshaker/alts_handshaker_service_api.h \ src/core/tsi/alts/handshaker/alts_handshaker_service_api_util.cc \ src/core/tsi/alts/handshaker/alts_handshaker_service_api_util.h \ -src/core/tsi/alts/handshaker/alts_tsi_event.cc \ -src/core/tsi/alts/handshaker/alts_tsi_event.h \ +src/core/tsi/alts/handshaker/alts_shared_resource.cc \ +src/core/tsi/alts/handshaker/alts_shared_resource.h \ src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc \ src/core/tsi/alts/handshaker/alts_tsi_handshaker.h \ src/core/tsi/alts/handshaker/alts_tsi_handshaker_private.h \ diff --git a/tools/gce/create_interop_worker.sh b/tools/gce/create_interop_worker.sh deleted file mode 100755 index 205c0bf8c5..0000000000 --- a/tools/gce/create_interop_worker.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash -# Copyright 2015 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Creates an interop worker on GCE. -# IMPORTANT: After this script finishes, there are still some manual -# steps needed there are hard to automatize. -# See go/grpc-jenkins-setup for followup instructions. - -set -ex - -cd "$(dirname "$0")" - -CLOUD_PROJECT=grpc-testing -ZONE=us-east1-a # canary gateway is reachable from this zone - -INSTANCE_NAME="${1:-grpc-canary-interop2}" - -gcloud compute instances create "$INSTANCE_NAME" \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - --machine-type n1-standard-16 \ - --image ubuntu-15-10 \ - --boot-disk-size 1000 \ - --scopes https://www.googleapis.com/auth/xapi.zoo \ - --tags=allow-ssh - -echo 'Created GCE instance, waiting 60 seconds for it to come online.' -sleep 60 - -gcloud compute copy-files \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - jenkins_master.pub linux_worker_init.sh "${INSTANCE_NAME}":~ - -gcloud compute ssh \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - "$INSTANCE_NAME" --command "./linux_worker_init.sh" diff --git a/tools/gce/create_linux_kokoro_performance_worker.sh b/tools/gce/create_linux_kokoro_performance_worker.sh index d08a1aa2c6..63d54a48bf 100755 --- a/tools/gce/create_linux_kokoro_performance_worker.sh +++ b/tools/gce/create_linux_kokoro_performance_worker.sh @@ -15,6 +15,13 @@ # Creates a performance worker on GCE to be used on Kokoro. +# IMPORTANT: Instructions for updating +# If the VM configuration / installed software is updated, +# - all existing performance worker VMs need to be updated to reflect the changes +# - a new GCE image named "grpc-performance-kokoro-v1" needs to be created, +# incrementing the version number. +# - kokoro jobs need to be reconfigured to use the new image version + set -ex cd "$(dirname "$0")" @@ -30,7 +37,7 @@ gcloud compute instances create "$INSTANCE_NAME" \ --zone "$ZONE" \ --machine-type $MACHINE_TYPE \ --image-project ubuntu-os-cloud \ - --image-family ubuntu-1710 \ + --image-family ubuntu-1804-lts \ --boot-disk-size 300 \ --scopes https://www.googleapis.com/auth/bigquery \ --tags=allow-ssh diff --git a/tools/gce/create_linux_worker.sh b/tools/gce/create_linux_kokoro_performance_worker_from_image.sh index a93d8c5e83..0f7939be4c 100755 --- a/tools/gce/create_linux_worker.sh +++ b/tools/gce/create_linux_kokoro_performance_worker_from_image.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2015 gRPC authors. +# Copyright 2018 The gRPC Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,36 +13,26 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Creates a standard jenkins worker on GCE. +# Creates a performance worker on GCE from an image that's used for kokoro +# perf workers. set -ex cd "$(dirname "$0")" CLOUD_PROJECT=grpc-testing -ZONE=us-central1-a +ZONE=us-central1-b # this zone allows 32core machines +LATEST_PERF_WORKER_IMAGE=grpc-performance-kokoro-v2 # update if newer image exists -INSTANCE_NAME="${1:-grpc-jenkins-worker1}" +INSTANCE_NAME="${1:-grpc-kokoro-performance-server}" +MACHINE_TYPE="${2:-n1-standard-32}" gcloud compute instances create "$INSTANCE_NAME" \ --project="$CLOUD_PROJECT" \ --zone "$ZONE" \ - --machine-type n1-standard-16 \ - --image=ubuntu-1510 \ - --image-project=grpc-testing \ - --boot-disk-size 1000 \ + --machine-type "$MACHINE_TYPE" \ + --image-project "$CLOUD_PROJECT" \ + --image "$LATEST_PERF_WORKER_IMAGE" \ + --boot-disk-size 300 \ --scopes https://www.googleapis.com/auth/bigquery \ --tags=allow-ssh - -echo 'Created GCE instance, waiting 60 seconds for it to come online.' -sleep 60 - -gcloud compute copy-files \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - jenkins_master.pub linux_worker_init.sh "${INSTANCE_NAME}":~ - -gcloud compute ssh \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - "$INSTANCE_NAME" --command "./linux_worker_init.sh" diff --git a/tools/gce/create_linux_performance_worker.sh b/tools/gce/create_linux_performance_worker.sh deleted file mode 100755 index e9033ec443..0000000000 --- a/tools/gce/create_linux_performance_worker.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -# Copyright 2015 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Creates a performance worker on GCE. -# IMPORTANT: After creating the worker, one needs to manually add the pubkey -# of jenkins@the-machine-where-jenkins-starts-perf-tests -# to ~/.ssh/authorized_keys so that multi-machine scenarios can work. -# See tools/run_tests/run_performance_tests.py for details. - -set -ex - -cd "$(dirname "$0")" - -CLOUD_PROJECT=grpc-testing -ZONE=us-central1-b # this zone allows 32core machines - -INSTANCE_NAME="${1:-grpc-performance-server1}" -MACHINE_TYPE=n1-standard-32 - -gcloud compute instances create "$INSTANCE_NAME" \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - --machine-type $MACHINE_TYPE \ - --image-project ubuntu-os-cloud \ - --image-family ubuntu-1710 \ - --boot-disk-size 300 \ - --scopes https://www.googleapis.com/auth/bigquery \ - --tags=allow-ssh - -echo 'Created GCE instance, waiting 60 seconds for it to come online.' -sleep 60 - -gcloud compute copy-files \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - jenkins_master.pub linux_performance_worker_init.sh "jenkins@${INSTANCE_NAME}":~ - -gcloud compute ssh \ - --project="$CLOUD_PROJECT" \ - --zone "$ZONE" \ - "jenkins@${INSTANCE_NAME}" --command "./linux_performance_worker_init.sh" diff --git a/tools/gce/jenkins_master.pub b/tools/gce/jenkins_master.pub deleted file mode 100644 index e9853224e1..0000000000 --- a/tools/gce/jenkins_master.pub +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDzj9l7Tp4yKnMV8sSMNvm5Q9v/F2F187xF93niJFY8lz6ig4bhusqvNbAxPoeypds9NYjLDK6kONN9teemgv2+IcmmlAI4wkCkkWcL/kzdNNH0h5J7+YbPiUGFAu0hZNHg5jzwrZ3VFKwv6d/7dUdPOYmPaOG1JOEcxXcBvm1hMIe474jpUTTiG4/gMDJ1GhMg5T3cuCm2l0gCiv7ybRAgwaZ2EKEEWLy9qAL/pnr3umBjQvzAUGcOgXJyG0mbr977YdJo9kb+EELRTVN2q8mKZJEZ1BJAylkaI9783K2+cGaM8hPtKFcX4ImEYEkWgfOyGNolGDquWtvusGGzQXwF jenkins@grpc-jenkins-master diff --git a/tools/gce/kokoro_performance.pub b/tools/gce/kokoro_performance.pub index 1154debe78..4eca916b8a 100644 --- a/tools/gce/kokoro_performance.pub +++ b/tools/gce/kokoro_performance.pub @@ -1 +1,4 @@ +# Enable Kokoro CI to SSH to the VM (Added by linux_kokoro_performance_worker_init.sh) +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDg7L/ZaEauETWrPklUTky3kvxqQfe2Ax/2CsSqhNIGNMnK/8d79CHlmY9+dE1FFQ/RzKNCaltgy7XcN/fCYiCZr5jm2ZtnLuGNOTzupMNhaYiPL419qmL+5rZXt4/dWTrsHbFRACxT8j51PcRMO5wgbL0Bg2XXimbx8kDFaurL2gqduQYqlu4lxWCaJqOL71WogcimeL63Nq/yeH5PJPWpqE4P9VUQSwAzBWFK/hLeds/AiP3MgVS65qHBnhq0JsHy8JQsqjZbG7Iidt/Ll0+gqzEbi62gDIcczG4KC0iOVzDDP/1BxDtt1lKeA23ll769Fcm3rJyoBMYxjvdw1TDx sabujp@trigger.mtv.corp.google.com +# Enable kokoro multi-machine benchmark driver VM to SSH to the VM (Added by linux_kokoro_performance_worker_init.sh) ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKQ5UEX4AFefec9BKICupFS7x9Hoq4ZyLKy+QX0J31I49ew9mG2AJlr3sp8ql15eX+A2Ml9MKJkmgZGHpJtw+SfvmI94SmomSyiCLAK92sQ85NMzaRdo4b9e30E9nhXnAvAaemvIEQbgCMYFvzk0C8AtXj6+htCrN4jFaLqTCPISJhX3ETc4TgX1qaHQHyl31tdaXHYlITvBDsfokcGcZQnhmCUDtD8wyaSC8GFk9gZbXshkfaYCuuLPPA0vwWGBw+YPbonHsFCsOog1IYSzYPCkIjq8dt6evsusK6Kaoyw/Z+l2kYty2FKTj+wU3l06QMoxwcfNT4WxdhcnVbY71r kbuilder@kokoro-performance-driver diff --git a/tools/gce/linux_kokoro_performance_worker_init.sh b/tools/gce/linux_kokoro_performance_worker_init.sh index 4a1e3e608b..b78695d802 100755 --- a/tools/gce/linux_kokoro_performance_worker_init.sh +++ b/tools/gce/linux_kokoro_performance_worker_init.sh @@ -47,7 +47,6 @@ sudo apt-get install -y \ libtool \ make \ strace \ - pypy \ python-dev \ python-pip \ python-setuptools \ @@ -68,30 +67,34 @@ sudo apt-get install -y google-perftools libgoogle-perftools-dev # netperf sudo apt-get install -y netperf +# required to run kokoro_log_reader.py +sudo apt-get install -y python-psutil python3-psutil + +# gcloud tools, including gsutil +sudo apt-get install -y google-cloud-sdk + # C++ dependencies sudo apt-get install -y libgflags-dev libgtest-dev libc++-dev clang # Python dependencies sudo pip install --upgrade pip==10.0.1 sudo pip install tabulate -sudo pip install google-api-python-client +sudo pip install google-api-python-client oauth2client sudo pip install virtualenv -# Building gRPC Python depends on python3.4 being installed, but python3.4 -# is not available on Ubuntu 16.10, so install from source -curl -O https://www.python.org/ftp/python/3.4.6/Python-3.4.6.tgz -tar xzvf Python-3.4.6.tgz -( -cd Python-3.4.6 || exit -./configure --enable-shared --prefix=/usr/local LDFLAGS="-Wl,--rpath=/usr/local/lib" -sudo make altinstall -) -rm Python-3.4.6.tgz - +# pypy is used instead of python for postprocessing benchmark outputs +# because some reports are huge and pypy is much faster. +# TODO(jtattermusch): get rid of pypy once possible, it's hard to +# keep track of all the installed variants of python. +sudo apt-get install -y pypy pypy-dev curl -O https://bootstrap.pypa.io/get-pip.py sudo pypy get-pip.py sudo pypy -m pip install tabulate -sudo pip install google-api-python-client +sudo pypy -m pip install google-api-python-client oauth2client +# TODO(jtattermusch): for some reason, we need psutil installed +# in pypy for kokoro_log_reader.py (strange, because the comand is +# "python kokoro_log_reader.py" and pypy is not the system default) +sudo pypy -m pip install psutil # Node dependencies (nvm has to be installed under user kbuilder) touch .profile @@ -104,31 +107,31 @@ nvm install 4 && npm config set cache /tmp/npm-cache nvm install 5 && npm config set cache /tmp/npm-cache nvm alias default 4 +# C# dependencies +sudo apt-get install -y cmake + # C# mono dependencies (http://www.mono-project.com/docs/getting-started/install/linux/#debian-ubuntu-and-derivatives) sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -echo "deb http://download.mono-project.com/repo/debian wheezy main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list +echo "deb https://download.mono-project.com/repo/ubuntu stable-bionic main" | sudo tee /etc/apt/sources.list.d/mono-official-stable.list sudo apt-get update -sudo apt-get install -y mono-devel nuget +sudo apt-get install -y mono-devel -# C# .NET Core dependencies (https://www.microsoft.com/net/core#ubuntu) -sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet-release/ yakkety main" > /etc/apt/sources.list.d/dotnetdev.list' -sudo apt-key adv --keyserver apt-mo.trafficmanager.net --recv-keys 417A0893 +# C# .NET Core dependencies (https://www.microsoft.com/net/download) +wget -q https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb +sudo dpkg -i packages-microsoft-prod.deb + +sudo apt-get install -y apt-transport-https sudo apt-get update -sudo apt-get install -y dotnet-dev-1.0.0-preview2.1-003155 -sudo apt-get install -y dotnet-dev-1.0.1 - -# C# 1.0.4 SDK -curl -O https://download.microsoft.com/download/2/4/A/24A06858-E8AC-469B-8AE6-D0CEC9BA982A/dotnet-ubuntu.16.04-x64.1.0.5.tar.gz -sudo mkdir -p /opt/dotnet -sudo tar zxf dotnet-ubuntu.16.04-x64.1.0.5.tar.gz -C /opt/dotnet -sudo ln -s /opt/dotnet/dotnet /usr/local/bin - -# C# .NET dependencies -wget http://security.ubuntu.com/ubuntu/pool/main/i/icu/libicu52_52.1-8ubuntu0.2_amd64.deb -sudo dpkg -i libicu52_52.1-8ubuntu0.2_amd64.deb -wget http://security.ubuntu.com/ubuntu/pool/main/i/icu/libicu55_55.1-7ubuntu0.3_amd64.deb -sudo dpkg -i libicu55_55.1-7ubuntu0.3_amd64.deb -sudo apt-get update && sudo apt-get install -y libicu55 +sudo apt-get install -y dotnet-sdk-2.1 + +# Install .NET Core 1.0.5 Runtime (required to run netcoreapp1.0) +wget -q https://download.microsoft.com/download/2/4/A/24A06858-E8AC-469B-8AE6-D0CEC9BA982A/dotnet-ubuntu.16.04-x64.1.0.5.tar.gz +mkdir -p dotnet105_download +tar zxf dotnet-ubuntu.16.04-x64.1.0.5.tar.gz -C dotnet105_download +sudo cp -r dotnet105_download/shared/Microsoft.NETCore.App/1.0.5/ /usr/share/dotnet/shared/Microsoft.NETCore.App/ +# To prevent "Failed to initialize CoreCLR, HRESULT: 0x80131500" with .NET Core 1.0.5 runtime +wget -q http://security.ubuntu.com/ubuntu/pool/main/i/icu/libicu55_55.1-7ubuntu0.4_amd64.deb +sudo dpkg -i libicu55_55.1-7ubuntu0.4_amd64.deb # Ruby dependencies gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 @@ -163,7 +166,7 @@ sudo mv composer.phar /usr/local/bin/composer # Significant performance improvements with grpc-go have been observed after # upgrading from go 1.5 to a later version, so a later go version is preferred. # Following go install instructions from https://golang.org/doc/install -GO_VERSION=1.8 +GO_VERSION=1.10 OS=linux ARCH=amd64 curl -O https://storage.googleapis.com/golang/go${GO_VERSION}.${OS}-${ARCH}.tar.gz @@ -190,11 +193,22 @@ git clone -v https://github.com/brendangregg/FlameGraph ~/FlameGraph # Install scipy and numpy for benchmarking scripts sudo apt-get install -y python-scipy python-numpy +# Install docker +curl -sSL https://get.docker.com/ | sh +# Enable kbuilder to use docker without sudo: +sudo usermod -aG docker kbuilder + # Add pubkey of Kokoro driver VM to allow SSH # silence false-positive shellcheck warning ("< redirect does not affect sudo") # shellcheck disable=SC2024 sudo tee --append ~kbuilder/.ssh/authorized_keys < kokoro_performance.pub +# Kokoro requires /tmpfs/READY file to exist the directory and file itself should +# be owned by kbuilder. +sudo mkdir /tmpfs +sudo chown kbuilder /tmpfs +touch /tmpfs/READY + # Restart for VM to pick up kernel update echo 'Successfully initialized the linux worker, going for reboot in 10 seconds' sleep 10 diff --git a/tools/gce/linux_performance_worker_init.sh b/tools/gce/linux_performance_worker_init.sh deleted file mode 100755 index 7222cef9a2..0000000000 --- a/tools/gce/linux_performance_worker_init.sh +++ /dev/null @@ -1,184 +0,0 @@ -#!/bin/bash -# Copyright 2015 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Initializes a fresh GCE VM to become a jenkins linux performance worker. -# You shouldn't run this script on your own, -# use create_linux_performance_worker.sh instead. - -set -ex - -sudo apt-get update - -# Install Java 8 JDK (to build gRPC Java) -sudo apt-get install -y openjdk-8-jdk -sudo apt-get install -y unzip lsof - -sudo apt-get install -y \ - autoconf \ - autotools-dev \ - build-essential \ - bzip2 \ - ccache \ - curl \ - gcc \ - gcc-multilib \ - git \ - gyp \ - lcov \ - libc6 \ - libc6-dbg \ - libc6-dev \ - libcurl4-openssl-dev \ - libgtest-dev \ - libreadline-dev \ - libssl-dev \ - libtool \ - make \ - strace \ - pypy \ - python-dev \ - python-pip \ - python-setuptools \ - python-yaml \ - python3-dev \ - python3-pip \ - python3-setuptools \ - python3-yaml \ - telnet \ - unzip \ - wget \ - zip \ - zlib1g-dev - -# perftools -sudo apt-get install -y google-perftools libgoogle-perftools-dev - -# netperf -sudo apt-get install -y netperf - -# C++ dependencies -sudo apt-get install -y libgflags-dev libgtest-dev libc++-dev clang - -# Python dependencies -sudo pip install --upgrade pip==10.0.1 -sudo pip install tabulate -sudo pip install google-api-python-client -sudo pip install virtualenv - -# Building gRPC Python depends on python3.4 being installed, but python3.4 -# is not available on Ubuntu 16.10, so install from source -curl -O https://www.python.org/ftp/python/3.4.6/Python-3.4.6.tgz -tar xzvf Python-3.4.6.tgz -( -cd Python-3.4.6 || exit -./configure --enable-shared --prefix=/usr/local LDFLAGS="-Wl,--rpath=/usr/local/lib" -sudo make altinstall -) -rm Python-3.4.6.tgz - -curl -O https://bootstrap.pypa.io/get-pip.py -sudo pypy get-pip.py -sudo pypy -m pip install tabulate -sudo pip install google-api-python-client - -# Node dependencies (nvm has to be installed under user jenkins) -touch .profile -curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.25.4/install.sh | bash -# silence shellcheck warning as it cannot follow the `source` path statically: -# shellcheck disable=SC1090 -source ~/.nvm/nvm.sh -nvm install 0.12 && npm config set cache /tmp/npm-cache -nvm install 4 && npm config set cache /tmp/npm-cache -nvm install 5 && npm config set cache /tmp/npm-cache -nvm alias default 4 - -# C# mono dependencies (http://www.mono-project.com/docs/getting-started/install/linux/#debian-ubuntu-and-derivatives) -sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF -echo "deb http://download.mono-project.com/repo/debian wheezy main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list -sudo apt-get update -sudo apt-get install -y mono-devel nuget - -# C# .NET Core dependencies (https://www.microsoft.com/net/core#ubuntu) -sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet-release/ yakkety main" > /etc/apt/sources.list.d/dotnetdev.list' -sudo apt-key adv --keyserver apt-mo.trafficmanager.net --recv-keys 417A0893 -sudo apt-get update -sudo apt-get install -y dotnet-dev-1.0.0-preview2.1-003155 -sudo apt-get install -y dotnet-dev-1.0.1 - -# Ruby dependencies -git clone https://github.com/rbenv/rbenv.git ~/.rbenv -export PATH="$HOME/.rbenv/bin:$PATH" -eval "$(rbenv init -)" - -git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build -export PATH="$HOME/.rbenv/plugins/ruby-build/bin:$PATH" - -rbenv install 2.4.0 -rbenv global 2.4.0 -ruby -v - -# Install bundler (prerequisite for gRPC Ruby) -gem install bundler - -# PHP dependencies -sudo apt-get install -y php php-dev phpunit php-pear unzip zlib1g-dev -curl -sS https://getcomposer.org/installer | php -sudo mv composer.phar /usr/local/bin/composer - -# Java dependencies - nothing as we already have Java JDK 8 - -# Go dependencies -# Currently, the golang package available via apt-get doesn't have the latest go. -# Significant performance improvements with grpc-go have been observed after -# upgrading from go 1.5 to a later version, so a later go version is preferred. -# Following go install instructions from https://golang.org/doc/install -GO_VERSION=1.8 -OS=linux -ARCH=amd64 -curl -O https://storage.googleapis.com/golang/go${GO_VERSION}.${OS}-${ARCH}.tar.gz -sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz -# Put go on the PATH, keep the usual installation dir -sudo ln -s /usr/local/go/bin/go /usr/bin/go -rm go$GO_VERSION.$OS-$ARCH.tar.gz - -# Install perf, to profile benchmarks. (need to get the right linux-tools-<> for kernel version) -sudo apt-get install -y linux-tools-common linux-tools-generic "linux-tools-$(uname -r)" -# see http://unix.stackexchange.com/questions/14227/do-i-need-root-admin-permissions-to-run-userspace-perf-tool-perf-events-ar -echo 0 | sudo tee /proc/sys/kernel/perf_event_paranoid -# see http://stackoverflow.com/questions/21284906/perf-couldnt-record-kernel-reference-relocation-symbol -echo 0 | sudo tee /proc/sys/kernel/kptr_restrict - -# qps workers under perf appear to need a lot of mmap pages under certain scenarios and perf args in -# order to not lose perf events or time out -echo 4096 | sudo tee /proc/sys/kernel/perf_event_mlock_kb - -# Fetch scripts to generate flame graphs from perf data collected -# on benchmarks -git clone -v https://github.com/brendangregg/FlameGraph ~/FlameGraph - -# Install scipy and numpy for benchmarking scripts -sudo apt-get install -y python-scipy python-numpy - -# Add pubkey of jenkins@grpc-jenkins-master to authorized keys of jenkins@ -# This needs to happen as the last step to prevent Jenkins master from connecting -# to a machine that hasn't been properly setup yet. -# silence false-positive shellcheck warning ("< redirect does not affect sudo") -# shellcheck disable=SC2024 -sudo tee --append ~jenkins/.ssh/authorized_keys < jenkins_master.pub - -# Restart for VM to pick up kernel update -echo 'Successfully initialized the linux worker, going for reboot in 10 seconds' -sleep 10 -sudo reboot diff --git a/tools/gce/linux_worker_init.sh b/tools/gce/linux_worker_init.sh deleted file mode 100755 index 05855354ff..0000000000 --- a/tools/gce/linux_worker_init.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash -# Copyright 2015 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Initializes a fresh GCE VM to become a jenkins linux worker. -# You shouldn't run this script on your own, use create_linux_worker.sh -# instead. - -set -ex - -# Create some swap space -sudo dd if=/dev/zero of=/swap bs=1024 count=10485760 -sudo chmod 600 /swap -sudo mkswap /swap -sudo sed -i '$ a\/swap none swap sw 0 0' /etc/fstab -sudo swapon -a - -# Typical apt-get maintenance -sudo apt-get update - -# Install JRE -sudo apt-get install -y openjdk-8-jre -sudo apt-get install -y unzip lsof - -# Install Docker -curl -sSL https://get.docker.com/ | sh - -# Setup jenkins user (or the user will already exist bcuz magic) -sudo adduser jenkins --disabled-password || true - -# Enable jenkins to use docker without sudo: -sudo usermod -aG docker jenkins - -# Use "overlay" storage driver for docker -# see https://github.com/grpc/grpc/issues/4988 -printf "{\n\t\"storage-driver\": \"overlay\"\n}" | sudo tee /etc/docker/daemon.json - -# Install pip and Google API library to enable using GCP services -sudo apt-get install -y python-pip -sudo pip install google-api-python-client - -# Install RVM -# TODO(jtattermusch): why is RVM needed? -gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 -curl -sSL https://get.rvm.io | bash -s stable --ruby - -# Upgrade Linux kernel to 4.9 -wget \ - kernel.ubuntu.com/~kernel-ppa/mainline/v4.9.20/linux-headers-4.9.20-040920_4.9.20-040920.201703310531_all.deb \ - kernel.ubuntu.com/~kernel-ppa/mainline/v4.9.20/linux-headers-4.9.20-040920-generic_4.9.20-040920.201703310531_amd64.deb \ - kernel.ubuntu.com/~kernel-ppa/mainline/v4.9.20/linux-image-4.9.20-040920-generic_4.9.20-040920.201703310531_amd64.deb -sudo dpkg -i linux-headers-4.9*.deb linux-image-4.9*.deb -rm linux-* - -# Add pubkey of jenkins@grpc-jenkins-master to authorized keys of jenkins@ -# This needs to happen as the last step to prevent Jenkins master from connecting -# to a machine that hasn't been properly setup yet. - -# disable superfluous warning by shellcheck: -# shellcheck disable=SC2024 -sudo tee --append ~jenkins/.ssh/authorized_keys < jenkins_master.pub - -# Restart for docker to pick up the config changes. -echo 'Successfully initialized the linux worker, going for reboot in 10 seconds' -sleep 10 - -sudo reboot diff --git a/tools/gcp/utils/big_query_utils.py b/tools/gcp/utils/big_query_utils.py index 3e811ca2bf..6e9cda376b 100755 --- a/tools/gcp/utils/big_query_utils.py +++ b/tools/gcp/utils/big_query_utils.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import argparse import json import uuid @@ -50,11 +52,11 @@ def create_dataset(biq_query, project_id, dataset_id): dataset_req.execute(num_retries=NUM_RETRIES) except HttpError as http_error: if http_error.resp.status == 409: - print 'Warning: The dataset %s already exists' % dataset_id + print('Warning: The dataset %s already exists' % dataset_id) else: # Note: For more debugging info, print "http_error.content" - print 'Error in creating dataset: %s. Err: %s' % (dataset_id, - http_error) + print('Error in creating dataset: %s. Err: %s' % (dataset_id, + http_error)) is_success = False return is_success @@ -122,13 +124,13 @@ def create_table2(big_query, table_req = big_query.tables().insert( projectId=project_id, datasetId=dataset_id, body=body) res = table_req.execute(num_retries=NUM_RETRIES) - print 'Successfully created %s "%s"' % (res['kind'], res['id']) + print('Successfully created %s "%s"' % (res['kind'], res['id'])) except HttpError as http_error: if http_error.resp.status == 409: - print 'Warning: Table %s already exists' % table_id + print('Warning: Table %s already exists' % table_id) else: - print 'Error in creating table: %s. Err: %s' % (table_id, - http_error) + print('Error in creating table: %s. Err: %s' % (table_id, + http_error)) is_success = False return is_success @@ -154,9 +156,9 @@ def patch_table(big_query, project_id, dataset_id, table_id, fields_schema): tableId=table_id, body=body) res = table_req.execute(num_retries=NUM_RETRIES) - print 'Successfully patched %s "%s"' % (res['kind'], res['id']) + print('Successfully patched %s "%s"' % (res['kind'], res['id'])) except HttpError as http_error: - print 'Error in creating table: %s. Err: %s' % (table_id, http_error) + print('Error in creating table: %s. Err: %s' % (table_id, http_error)) is_success = False return is_success @@ -172,10 +174,10 @@ def insert_rows(big_query, project_id, dataset_id, table_id, rows_list): body=body) res = insert_req.execute(num_retries=NUM_RETRIES) if res.get('insertErrors', None): - print 'Error inserting rows! Response: %s' % res + print('Error inserting rows! Response: %s' % res) is_success = False except HttpError as http_error: - print 'Error inserting rows to the table %s' % table_id + print('Error inserting rows to the table %s' % table_id) is_success = False return is_success @@ -189,8 +191,8 @@ def sync_query_job(big_query, project_id, query, timeout=5000): projectId=project_id, body=query_data).execute(num_retries=NUM_RETRIES) except HttpError as http_error: - print 'Query execute job failed with error: %s' % http_error - print http_error.content + print('Query execute job failed with error: %s' % http_error) + print(http_error.content) return query_job diff --git a/tools/internal_ci/README.md b/tools/internal_ci/README.md index e80e342dcd..af582c471e 100644 --- a/tools/internal_ci/README.md +++ b/tools/internal_ci/README.md @@ -1,6 +1,7 @@ -# Internal continuous integration +# Kokoro CI job configurations and testing scripts -gRPC's externally facing testing is managed by Jenkins CI (see `tools/jenkins` -directory). Nevertheless, some of the tests are better suited for being run -on internal infrastructure and using an internal CI system. Configuration for -such tests is under this directory. +gRPC uses a continous integration tool called "Kokoro" (a.k.a "internal CI") +for running majority of its open source tests. +This directory contains the external part of kokoro test job configurations +(the actual job definitions live in an internal repository) and the shell +scripts that act as entry points to exectute the actual tests. diff --git a/tools/internal_ci/helper_scripts/delete_nonartifacts.sh b/tools/internal_ci/helper_scripts/delete_nonartifacts.sh index c7d6ba6d44..01e9427e1c 100755 --- a/tools/internal_ci/helper_scripts/delete_nonartifacts.sh +++ b/tools/internal_ci/helper_scripts/delete_nonartifacts.sh @@ -24,4 +24,4 @@ cd "$(dirname "$0")/../../.." # after finishing each build. We only leave files we want to keep: # - reports and artifacts # - directory containing the kokoro scripts to prevent deleting a script while being executed. -time find . -type f -not -iname "*sponge_log.xml" -not -path "./reports/*" -not -path "./artifacts/*" -not -path "./tools/internal_ci/*" -exec rm -f {} + +time find . -type f -not -iname "*sponge_log.*" -not -path "./reports/*" -not -path "./artifacts/*" -not -path "./tools/internal_ci/*" -exec rm -f {} + diff --git a/tools/internal_ci/helper_scripts/prepare_build_grpclb_interop_rc b/tools/internal_ci/helper_scripts/prepare_build_grpclb_interop_rc new file mode 100644 index 0000000000..a8e350be58 --- /dev/null +++ b/tools/internal_ci/helper_scripts/prepare_build_grpclb_interop_rc @@ -0,0 +1,33 @@ +#!/bin/bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Source this rc script to prepare the environment for interop builds +# This rc script must be used in the root directory of gRPC + +export LANG=en_US.UTF-8 + +# Download Docker images from DockerHub +export DOCKERHUB_ORGANIZATION=grpctesting + +git submodule update --init + +# Set up gRPC-Go and gRPC-Java to test +git clone --recursive https://github.com/grpc/grpc-go ./../grpc-go +git clone --recursive https://github.com/grpc/grpc-java ./../grpc-java + +# TODO(apolcyn): move to kokoro image? +virtualenv env +source env/bin/activate +pip install twisted diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index 3212e80854..2362b370d5 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -57,25 +57,25 @@ source $HOME/.rvm/scripts/rvm set -e # rvm commands are very verbose time rvm install 2.5.0 rvm use 2.5.0 --default -gem install bundler --no-ri --no-doc -gem install cocoapods --version 1.3.1 --no-ri --no-doc -gem install rake-compiler --no-ri --no-doc +time gem install bundler --no-ri --no-doc +time gem install cocoapods --version 1.3.1 --no-ri --no-doc +time gem install rake-compiler --no-ri --no-doc rvm osx-ssl-certs status all rvm osx-ssl-certs update all set -ex # cocoapods export LANG=en_US.UTF-8 -pod repo update # needed by python +time pod repo update # needed by python # python -pip install virtualenv --user python -pip install -U Mako six tox setuptools twisted pyyaml --user python +time pip install virtualenv --user python +time pip install -U Mako six tox setuptools twisted pyyaml --user python export PYTHONPATH=/Library/Python/3.4/site-packages # Install Python 3.7 -curl -O https://www.python.org/ftp/python/3.7.0/python-3.7.0-macosx10.9.pkg -sudo installer -pkg ./python-3.7.0-macosx10.9.pkg -target / +time curl -O https://www.python.org/ftp/python/3.7.0/python-3.7.0-macosx10.9.pkg +time sudo installer -pkg ./python-3.7.0-macosx10.9.pkg -target / # set xcode version for Obj-C tests sudo xcode-select -switch /Applications/Xcode_9.2.app/Contents/Developer/ @@ -88,11 +88,12 @@ export DOTNET_CLI_TELEMETRY_OPTOUT=true # TODO(jtattermusch): better debugging of clock skew, remove once not needed date -git submodule update --init +time git submodule update --init # Store intermediate build files of ObjC tests into /tmpfs mkdir /tmpfs/Build-ios-binary-size ln -s /tmpfs/Build-ios-binary-size src/objective-c/examples/Sample/Build -mkdir /tmpfs/DerivedData +mkdir -p /tmpfs/DerivedData rm -rf ~/Library/Developer/Xcode/DerivedData +mkdir -p ~/Library/Developer/Xcode ln -s /tmpfs/DerivedData ~/Library/Developer/Xcode/DerivedData diff --git a/tools/internal_ci/linux/grpc_asan_on_foundry.sh b/tools/internal_ci/linux/grpc_asan_on_foundry.sh index dfef004a60..87ec60c7c0 100755 --- a/tools/internal_ci/linux/grpc_asan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_asan_on_foundry.sh @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -ex + export UPLOAD_TEST_RESULTS=true -EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=address --linkopt=-fsanitize=address --test_timeout=3600 --cache_test_results=no" -EXCLUDE_TESTS="--test_tag_filters=-qps_json_driver,-json_run_localhost" -github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" "${EXCLUDE_TESTS}" +github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh --config=asan --cache_test_results=no diff --git a/tools/internal_ci/linux/grpc_basictests_c_cpp_dbg.cfg b/tools/internal_ci/linux/grpc_basictests_c_cpp_dbg.cfg index 4a0badf43b..07f7f0c659 100644 --- a/tools/internal_ci/linux/grpc_basictests_c_cpp_dbg.cfg +++ b/tools/internal_ci/linux/grpc_basictests_c_cpp_dbg.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_basictests_c_cpp_opt.cfg b/tools/internal_ci/linux/grpc_basictests_c_cpp_opt.cfg index a2cfe021e1..8f2813febf 100644 --- a/tools/internal_ci/linux/grpc_basictests_c_cpp_opt.cfg +++ b/tools/internal_ci/linux/grpc_basictests_c_cpp_opt.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_basictests_multilang.cfg b/tools/internal_ci/linux/grpc_basictests_multilang.cfg index 4433d14cd7..f8a5a4aea5 100644 --- a/tools/internal_ci/linux/grpc_basictests_multilang.cfg +++ b/tools/internal_ci/linux/grpc_basictests_multilang.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh index bb2a85138c..74778d9d29 100755 --- a/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh +++ b/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh @@ -21,45 +21,28 @@ set -ex mkdir -p ${KOKORO_KEYSTORE_DIR} cp ${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json ${KOKORO_KEYSTORE_DIR}/4321_grpc-testing-service -temp_dir=$(mktemp -d) -ln -f "${KOKORO_GFILE_DIR}/bazel-latest-release" ${temp_dir}/bazel -chmod 755 "${KOKORO_GFILE_DIR}/bazel-latest-release" +# Download bazel +temp_dir="$(mktemp -d)" +wget -q https://github.com/bazelbuild/bazel/releases/download/0.17.1/bazel-0.17.1-linux-x86_64 -O "${temp_dir}/bazel" +chmod 755 "${temp_dir}/bazel" export PATH="${temp_dir}:${PATH}" # This should show ${temp_dir}/bazel which bazel -chmod +x "${KOKORO_GFILE_DIR}/bazel_wrapper.py" # change to grpc repo root cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc -export KOKORO_FOUNDRY_PROJECT_ID="projects/grpc-testing/instances/default_instance" +# to get "bazel" link for kokoro build, we need to generate +# invocation UUID, set an env var for bazel to pick it up +# and upload "bazel_invocation_ids" file as artifact. +export BAZEL_INTERNAL_INVOCATION_ID="$(uuidgen)" +echo "${BAZEL_INTERNAL_INVOCATION_ID}" >"${KOKORO_ARTIFACTS_DIR}/bazel_invocation_ids" -# TODO(adelez): implement size for test targets and change test_timeout back -"${KOKORO_GFILE_DIR}/bazel_wrapper.py" \ - --host_jvm_args=-Dbazel.DigestFunction=SHA256 \ - test --jobs="200" \ - --test_output=errors \ - --verbose_failures=true \ - --keep_going \ - --remote_accept_cached=true \ - --spawn_strategy=remote \ - --remote_local_fallback=false \ - --remote_timeout=3600 \ - --strategy=Javac=remote \ - --strategy=Closure=remote \ - --genrule_strategy=remote \ - --experimental_strict_action_env=true \ - --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.16.1/default:toolchain \ - --define GRPC_PORT_ISOLATED_RUNTIME=1 \ - --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 \ - --extra_toolchains=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.16.1/cpp:cc-toolchain-clang-x86_64-default \ - --extra_execution_platforms=//third_party/toolchains:rbe_ubuntu1604 \ - --host_platform=//third_party/toolchains:rbe_ubuntu1604 \ - --platforms=//third_party/toolchains:rbe_ubuntu1604 \ - --test_env=GRPC_VERBOSITY=debug \ - --remote_instance_name=projects/grpc-testing/instances/default_instance \ +bazel \ + --bazelrc=tools/remote_build/kokoro.bazelrc \ + test \ $@ \ -- //test/... || FAILED="true" diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh index 51cb66f5b8..fdd5b2e4cd 100644 --- a/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh +++ b/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -ex + export UPLOAD_TEST_RESULTS=true -EXTRA_FLAGS="-c dbg --test_timeout=300,450,1200,3600 --cache_test_results=no" +EXTRA_FLAGS="--config=dbg --test_timeout=300,450,1200,3600 --cache_test_results=no" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh index cbba9067ad..30b2b17a67 100644 --- a/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh +++ b/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -ex + export UPLOAD_TEST_RESULTS=true -EXTRA_FLAGS="-c opt --test_timeout=300,450,1200,3600 --cache_test_results=no" +EXTRA_FLAGS="--config=opt --test_timeout=300,450,1200,3600 --cache_test_results=no" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" diff --git a/tools/internal_ci/linux/grpc_build_artifacts.cfg b/tools/internal_ci/linux/grpc_build_artifacts.cfg index 88fc6b7b35..1e04a1e788 100644 --- a/tools/internal_ci/linux/grpc_build_artifacts.cfg +++ b/tools/internal_ci/linux/grpc_build_artifacts.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_build_artifacts.sh" timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/linux/grpc_build_artifacts_extra.cfg b/tools/internal_ci/linux/grpc_build_artifacts_extra.cfg index 619e3ea3a9..2737e2f345 100644 --- a/tools/internal_ci/linux/grpc_build_artifacts_extra.cfg +++ b/tools/internal_ci/linux/grpc_build_artifacts_extra.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_build_artifacts_extra.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/linux/grpc_build_artifacts_extra_release.cfg b/tools/internal_ci/linux/grpc_build_artifacts_extra_release.cfg index 619e3ea3a9..2737e2f345 100644 --- a/tools/internal_ci/linux/grpc_build_artifacts_extra_release.cfg +++ b/tools/internal_ci/linux/grpc_build_artifacts_extra_release.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_build_artifacts_extra.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/linux/grpc_build_boringssl_at_head.cfg b/tools/internal_ci/linux/grpc_build_boringssl_at_head.cfg index 11c211fd2b..9a430db0f9 100644 --- a/tools/internal_ci/linux/grpc_build_boringssl_at_head.cfg +++ b/tools/internal_ci/linux/grpc_build_boringssl_at_head.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_build_submodule_at_head.sh" timeout_mins: 180 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_build_packages.cfg b/tools/internal_ci/linux/grpc_build_packages.cfg index 6a4a163dfc..23a676cf72 100644 --- a/tools/internal_ci/linux/grpc_build_packages.cfg +++ b/tools/internal_ci/linux/grpc_build_packages.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_build_packages.sh" timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/linux/grpc_build_protobuf_at_head.cfg b/tools/internal_ci/linux/grpc_build_protobuf_at_head.cfg index 2f08e15e63..aef6a7a1dc 100644 --- a/tools/internal_ci/linux/grpc_build_protobuf_at_head.cfg +++ b/tools/internal_ci/linux/grpc_build_protobuf_at_head.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_build_submodule_at_head.sh" timeout_mins: 180 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_coverage.cfg b/tools/internal_ci/linux/grpc_coverage.cfg index 794a51d3f1..6eb37b7dec 100644 --- a/tools/internal_ci/linux/grpc_coverage.cfg +++ b/tools/internal_ci/linux/grpc_coverage.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_coverage.sh" timeout_mins: 420 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_distribtests.cfg b/tools/internal_ci/linux/grpc_distribtests.cfg index 0f1d79355a..848d571333 100644 --- a/tools/internal_ci/linux/grpc_distribtests.cfg +++ b/tools/internal_ci/linux/grpc_distribtests.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_distribtests.sh" timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/linux/grpc_distribtests_standalone.cfg b/tools/internal_ci/linux/grpc_distribtests_standalone.cfg index bc6c8e8f80..734bdfd78a 100644 --- a/tools/internal_ci/linux/grpc_distribtests_standalone.cfg +++ b/tools/internal_ci/linux/grpc_distribtests_standalone.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_standalone.sh" timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/linux/grpc_e2e_performance_singlevm.cfg b/tools/internal_ci/linux/grpc_e2e_performance_singlevm.cfg new file mode 100644 index 0000000000..c27baad3d7 --- /dev/null +++ b/tools/internal_ci/linux/grpc_e2e_performance_singlevm.cfg @@ -0,0 +1,25 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_e2e_performance_singlevm.sh" +timeout_mins: 360 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "**/perf_reports/**" + } +} diff --git a/tools/internal_ci/linux/grpc_e2e_performance_singlevm.sh b/tools/internal_ci/linux/grpc_e2e_performance_singlevm.sh new file mode 100755 index 0000000000..21f9d48ac4 --- /dev/null +++ b/tools/internal_ci/linux/grpc_e2e_performance_singlevm.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -ex + +# Enter the gRPC repo root +cd $(dirname $0)/../../.. + +source tools/internal_ci/helper_scripts/prepare_build_linux_perf_multilang_rc + +# "smoketest" scenarios on a single VM (=no remote VM for running qps_workers) +tools/run_tests/run_performance_tests.py \ + -l c++ csharp ruby java python go php7 php7_protobuf_c node \ + --netperf \ + --category smoketest \ + -u kbuilder \ + --bq_result_table performance_test.performance_experiment_singlevm \ + --xml_report reports/singlemachine/sponge_log.xml diff --git a/tools/internal_ci/linux/grpc_full_performance_master.cfg b/tools/internal_ci/linux/grpc_full_performance_master.cfg index 8852130a13..8ad93ee052 100644 --- a/tools/internal_ci/linux/grpc_full_performance_master.cfg +++ b/tools/internal_ci/linux/grpc_full_performance_master.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_full_performance_master.sh" timeout_mins: 600 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "**/perf_reports/**" } } diff --git a/tools/internal_ci/linux/grpc_full_performance_master.sh b/tools/internal_ci/linux/grpc_full_performance_master.sh index 24ee71edd1..05a014b88a 100755 --- a/tools/internal_ci/linux/grpc_full_performance_master.sh +++ b/tools/internal_ci/linux/grpc_full_performance_master.sh @@ -21,7 +21,7 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_perf_multilang_rc # run 8core client vs 8core server tools/run_tests/run_performance_tests.py \ - -l c++ csharp ruby java python go php7 php7_protobuf_c node node_purejs \ + -l c++ csharp ruby java python go php7 php7_protobuf_c node \ --netperf \ --category scalable \ --remote_worker_host grpc-kokoro-performance-server-8core grpc-kokoro-performance-client-8core grpc-kokoro-performance-client2-8core \ diff --git a/tools/internal_ci/linux/grpc_full_performance_release.cfg b/tools/internal_ci/linux/grpc_full_performance_release.cfg index e9a4bcdcaf..11a95e889f 100644 --- a/tools/internal_ci/linux/grpc_full_performance_release.cfg +++ b/tools/internal_ci/linux/grpc_full_performance_release.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_full_performance_release.sh" timeout_mins: 600 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "**/perf_reports/**" } } diff --git a/tools/internal_ci/linux/grpc_interop_alts.cfg b/tools/internal_ci/linux/grpc_interop_alts.cfg index bda76faf44..4684aba96b 100644 --- a/tools/internal_ci/linux/grpc_interop_alts.cfg +++ b/tools/internal_ci/linux/grpc_interop_alts.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_interop_tests.sh" timeout_mins: 60 action { define_artifacts { - regex: "**/sponge_log.xml" + regex: "**/sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_interop_matrix.cfg b/tools/internal_ci/linux/grpc_interop_matrix.cfg index ae59e930f7..e13c26e8ba 100644 --- a/tools/internal_ci/linux/grpc_interop_matrix.cfg +++ b/tools/internal_ci/linux/grpc_interop_matrix.cfg @@ -16,11 +16,15 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/grpc_interop_matrix.sh" -# grpc_interop tests can take 1 hours to complete. timeout_mins: 300 action { define_artifacts { - regex: "**/sponge_log.xml" + regex: "**/sponge_log.*" regex: "github/grpc/reports/**" } } + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "--language=all --release=all --allow_flakes --report_file=sponge_log.xml --bq_result_table interop_results" +} diff --git a/tools/internal_ci/linux/grpc_interop_matrix.sh b/tools/internal_ci/linux/grpc_interop_matrix.sh index 4c24c43488..a5220ea087 100755 --- a/tools/internal_ci/linux/grpc_interop_matrix.sh +++ b/tools/internal_ci/linux/grpc_interop_matrix.sh @@ -22,4 +22,4 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc -tools/interop_matrix/run_interop_matrix_tests.py --language=all --release=all --allow_flakes --report_file=sponge_log.xml --bq_result_table interop_results $@ +tools/interop_matrix/run_interop_matrix_tests.py $RUN_TESTS_FLAGS diff --git a/tools/internal_ci/linux/grpc_interop_tocloud.cfg b/tools/internal_ci/linux/grpc_interop_tocloud.cfg index 81f2fe9dec..9b35adcece 100644 --- a/tools/internal_ci/linux/grpc_interop_tocloud.cfg +++ b/tools/internal_ci/linux/grpc_interop_tocloud.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_interop_tests.sh" timeout_mins: 60 action { define_artifacts { - regex: "**/sponge_log.xml" + regex: "**/sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_interop_toprod.cfg b/tools/internal_ci/linux/grpc_interop_toprod.cfg index 8dfc529947..de4db81e6b 100644 --- a/tools/internal_ci/linux/grpc_interop_toprod.cfg +++ b/tools/internal_ci/linux/grpc_interop_toprod.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_interop_tests.sh" timeout_mins: 60 action { define_artifacts { - regex: "**/sponge_log.xml" + regex: "**/sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_msan_on_foundry.sh b/tools/internal_ci/linux/grpc_msan_on_foundry.sh index 1ef13ef0d4..c7e4b1fa1a 100644 --- a/tools/internal_ci/linux/grpc_msan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_msan_on_foundry.sh @@ -15,67 +15,5 @@ set -ex -# A temporary solution to give Kokoro credentials. -# The file name 4321_grpc-testing-service needs to match auth_credential in -# the build config. -# TODO: Use keystore. -mkdir -p ${KOKORO_KEYSTORE_DIR} -cp ${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json ${KOKORO_KEYSTORE_DIR}/4321_grpc-testing-service - -temp_dir=$(mktemp -d) -ln -f "${KOKORO_GFILE_DIR}/bazel-latest-release" ${temp_dir}/bazel -chmod 755 "${KOKORO_GFILE_DIR}/bazel-latest-release" -export PATH="${temp_dir}:${PATH}" -# This should show ${temp_dir}/bazel -which bazel -chmod +x "${KOKORO_GFILE_DIR}/bazel_wrapper.py" - -# change to grpc repo root -cd $(dirname $0)/../../.. - -source tools/internal_ci/helper_scripts/prepare_build_linux_rc - -export KOKORO_FOUNDRY_PROJECT_ID="projects/grpc-testing/instances/default_instance" - -"${KOKORO_GFILE_DIR}/bazel_wrapper.py" \ - --host_jvm_args=-Dbazel.DigestFunction=SHA256 \ - test --jobs="200" \ - --test_timeout="3600,3600,3600,3600" \ - --test_output=errors \ - --verbose_failures=true \ - --keep_going \ - --remote_accept_cached=true \ - --spawn_strategy=remote \ - --remote_local_fallback=false \ - --remote_timeout=3600 \ - --strategy=Javac=remote \ - --strategy=Closure=remote \ - --genrule_strategy=remote \ - --experimental_strict_action_env=true \ - --define GRPC_PORT_ISOLATED_RUNTIME=1 \ - --copt=-gmlt \ - --strip=never \ - --cxxopt=--stdlib=libc++ \ - --copt=-fsanitize=memory \ - --linkopt=-fsanitize=memory \ - --copt=-fsanitize-memory-track-origins \ - --action_env=LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH \ - --host_crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.16.1/default:toolchain \ - --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.16.1/msan:toolchain \ - --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 \ - --extra_toolchains=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.16.1/cpp:cc-toolchain-clang-x86_64-default \ - --extra_execution_platforms=//third_party/toolchains:rbe_ubuntu1604 \ - --host_platform=//third_party/toolchains:rbe_ubuntu1604 \ - --platforms=//third_party/toolchains:rbe_ubuntu1604 \ - --test_env=GRPC_VERBOSITY=debug \ - --remote_instance_name=projects/grpc-testing/instances/default_instance \ - -- //test/... || FAILED="true" - -# Sleep to let ResultStore finish writing results before querying -sleep 60 -python ./tools/run_tests/python_utils/upload_rbe_results.py - -if [ "$FAILED" != "" ] -then - exit 1 -fi +export UPLOAD_TEST_RESULTS=true +github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh --config=msan --cache_test_results=no diff --git a/tools/internal_ci/linux/grpc_performance_profile_daily.sh b/tools/internal_ci/linux/grpc_performance_profile_daily.sh index 34d41bc04c..0e08699923 100755 --- a/tools/internal_ci/linux/grpc_performance_profile_daily.sh +++ b/tools/internal_ci/linux/grpc_performance_profile_daily.sh @@ -24,8 +24,8 @@ CPUS=`python -c 'import multiprocessing; print multiprocessing.cpu_count()'` ./tools/run_tests/start_port_server.py || true -make CONFIG=opt memory_profile_test memory_profile_client memory_profile_server -j $CPUS -bins/opt/memory_profile_test +make CONFIG=opt memory_usage_test memory_usage_client memory_usage_server -j $CPUS +bins/opt/memory_usage_test bq load microbenchmarks.memory memory_usage.csv tools/run_tests/run_microbenchmark.py --collect summary --bigquery_upload || FAILED="true" diff --git a/tools/internal_ci/linux/grpc_portability.cfg b/tools/internal_ci/linux/grpc_portability.cfg index 76e5028477..f417f24bb1 100644 --- a/tools/internal_ci/linux/grpc_portability.cfg +++ b/tools/internal_ci/linux/grpc_portability.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 1440 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_portability_build_only.cfg b/tools/internal_ci/linux/grpc_portability_build_only.cfg index 4acd9353fb..fab9dde3e5 100644 --- a/tools/internal_ci/linux/grpc_portability_build_only.cfg +++ b/tools/internal_ci/linux/grpc_portability_build_only.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 180 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_publish_packages.cfg b/tools/internal_ci/linux/grpc_publish_packages.cfg index 82d571d642..dc9fe7d0a7 100644 --- a/tools/internal_ci/linux/grpc_publish_packages.cfg +++ b/tools/internal_ci/linux/grpc_publish_packages.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_publish_packages.sh" timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/linux/grpc_pull_request_sanity.cfg b/tools/internal_ci/linux/grpc_pull_request_sanity.cfg index b20d2ffba8..704d5c6cbc 100644 --- a/tools/internal_ci/linux/grpc_pull_request_sanity.cfg +++ b/tools/internal_ci/linux/grpc_pull_request_sanity.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 30 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh b/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh index 4f98d0a93a..156d65955a 100755 --- a/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh +++ b/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh @@ -24,4 +24,4 @@ git clone /var/local/jenkins/grpc /var/local/git/grpc && git submodule update --init --reference /var/local/jenkins/grpc/${name} \ ${name}') cd /var/local/git/grpc/test -bazel test --spawn_strategy=standalone --genrule_strategy=standalone //src/python/... +bazel test --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //src/python/... diff --git a/tools/internal_ci/linux/grpc_run_grpclb_interop_tests.sh b/tools/internal_ci/linux/grpc_run_grpclb_interop_tests.sh new file mode 100755 index 0000000000..806b5c947e --- /dev/null +++ b/tools/internal_ci/linux/grpc_run_grpclb_interop_tests.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +export LANG=en_US.UTF-8 + +# Enter the gRPC repo root +cd $(dirname $0)/../../.. + +source tools/internal_ci/helper_scripts/prepare_build_linux_rc +source tools/internal_ci/helper_scripts/prepare_build_grpclb_interop_rc + +tools/run_tests/run_grpclb_interop_tests.py -l all --scenarios_file=tools/run_tests/generated/lb_interop_test_scenarios.json diff --git a/tools/internal_ci/linux/grpc_sanity.cfg b/tools/internal_ci/linux/grpc_sanity.cfg index 9f65918e23..341471bbb5 100644 --- a/tools/internal_ci/linux/grpc_sanity.cfg +++ b/tools/internal_ci/linux/grpc_sanity.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 40 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_tsan_on_foundry.sh b/tools/internal_ci/linux/grpc_tsan_on_foundry.sh index 366b5cbe34..fcf3095d48 100644 --- a/tools/internal_ci/linux/grpc_tsan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_tsan_on_foundry.sh @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -ex + export UPLOAD_TEST_RESULTS=true -EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=thread --linkopt=-fsanitize=thread --test_timeout=3600 --action_env=TSAN_OPTIONS=suppressions=test/core/util/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1 --cache_test_results=no" -EXCLUDE_TESTS="--test_tag_filters=-qps_json_driver,-json_run_localhost" -github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" "${EXCLUDE_TESTS}" +github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh --config=tsan --cache_test_results=no diff --git a/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh b/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh index e0ae9103c4..f45be4d1c9 100644 --- a/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh @@ -15,64 +15,5 @@ set -ex -# A temporary solution to give Kokoro credentials. -# The file name 4321_grpc-testing-service needs to match auth_credential in -# the build config. -# TODO: Use keystore. -mkdir -p ${KOKORO_KEYSTORE_DIR} -cp ${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json ${KOKORO_KEYSTORE_DIR}/4321_grpc-testing-service - -temp_dir=$(mktemp -d) -ln -f "${KOKORO_GFILE_DIR}/bazel-latest-release" ${temp_dir}/bazel -chmod 755 "${KOKORO_GFILE_DIR}/bazel-latest-release" -export PATH="${temp_dir}:${PATH}" -# This should show ${temp_dir}/bazel -which bazel -chmod +x "${KOKORO_GFILE_DIR}/bazel_wrapper.py" - -# change to grpc repo root -cd $(dirname $0)/../../.. - -source tools/internal_ci/helper_scripts/prepare_build_linux_rc - -export KOKORO_FOUNDRY_PROJECT_ID="projects/grpc-testing/instances/default_instance" - -"${KOKORO_GFILE_DIR}/bazel_wrapper.py" \ - --host_jvm_args=-Dbazel.DigestFunction=SHA256 \ - test --jobs="200" \ - --test_timeout="3600,3600,3600,3600" \ - --test_output=errors \ - --verbose_failures=true \ - --keep_going \ - --remote_accept_cached=true \ - --spawn_strategy=remote \ - --remote_local_fallback=false \ - --remote_timeout=3600 \ - --strategy=Javac=remote \ - --strategy=Closure=remote \ - --genrule_strategy=remote \ - --experimental_strict_action_env=true \ - --define GRPC_PORT_ISOLATED_RUNTIME=1 \ - --copt=-gmlt \ - --strip=never \ - --copt=-fsanitize=undefined \ - --linkopt=-fsanitize=undefined \ - --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/experimental/ubuntu16_04_clang/1.0/bazel_0.15.0/ubsan:toolchain \ - --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 \ - --extra_toolchains=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.16.1/cpp:cc-toolchain-clang-x86_64-default \ - --extra_execution_platforms=//third_party/toolchains:rbe_ubuntu1604 \ - --host_platform=//third_party/toolchains:rbe_ubuntu1604 \ - --platforms=//third_party/toolchains:rbe_ubuntu1604 \ - --cache_test_results=no \ - --test_env=GRPC_VERBOSITY=debug \ - --remote_instance_name=projects/grpc-testing/instances/default_instance \ - -- //test/... || FAILED="true" - -# Sleep to let ResultStore finish writing results before querying -sleep 60 -python ./tools/run_tests/python_utils/upload_rbe_results.py - -if [ "$FAILED" != "" ] -then - exit 1 -fi +export UPLOAD_TEST_RESULTS=true +github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh --config=ubsan --cache_test_results=no diff --git a/tools/internal_ci/linux/grpclb_in_dns_interop.cfg b/tools/internal_ci/linux/grpclb_in_dns_interop.cfg new file mode 100644 index 0000000000..6cd5f2e21a --- /dev/null +++ b/tools/internal_ci/linux/grpclb_in_dns_interop.cfg @@ -0,0 +1,25 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_run_grpclb_interop_tests.sh" +timeout_mins: 60 +action { + define_artifacts { + regex: "**/sponge_log.xml" + regex: "github/grpc/reports/**" + } +} diff --git a/tools/internal_ci/linux/pull_request/grpc_asan_on_foundry.sh b/tools/internal_ci/linux/pull_request/grpc_asan_on_foundry.sh index 39c991f291..00f92921de 100644 --- a/tools/internal_ci/linux/pull_request/grpc_asan_on_foundry.sh +++ b/tools/internal_ci/linux/pull_request/grpc_asan_on_foundry.sh @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=address --linkopt=-fsanitize=address --test_timeout=3600" -EXCLUDE_TESTS="--test_tag_filters=-qps_json_driver,-json_run_localhost" -github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" "${EXCLUDE_TESTS}" +set -ex + +github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh --config=asan diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_dbg.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_dbg.cfg index 8124f5c1b3..8a67d28ce4 100644 --- a/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_dbg.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_dbg.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_opt.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_opt.cfg index ecedc73e44..a681978b6e 100644 --- a/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_opt.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_c_cpp_opt.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_c_dbg.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_c_dbg.cfg index 577cb28ae5..249ecc99ce 100644 --- a/tools/internal_ci/linux/pull_request/grpc_basictests_c_dbg.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_c_dbg.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_c_opt.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_c_opt.cfg index 9e0b724b2e..665d7f3d2c 100644 --- a/tools/internal_ci/linux/pull_request/grpc_basictests_c_opt.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_c_opt.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_dbg.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_dbg.cfg index 0fda74cf44..163274d4dc 100644 --- a/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_dbg.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_dbg.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_opt.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_opt.cfg index 199a8905d9..b65cd3e05c 100644 --- a/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_opt.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_cpp_opt.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_basictests_multilang.cfg b/tools/internal_ci/linux/pull_request/grpc_basictests_multilang.cfg index f7e8d8ad33..59f38f0d1b 100644 --- a/tools/internal_ci/linux/pull_request/grpc_basictests_multilang.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_basictests_multilang.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_bazel_on_foundry_dbg.sh b/tools/internal_ci/linux/pull_request/grpc_bazel_on_foundry_dbg.sh index 8e2aaebaee..f1e6588517 100644 --- a/tools/internal_ci/linux/pull_request/grpc_bazel_on_foundry_dbg.sh +++ b/tools/internal_ci/linux/pull_request/grpc_bazel_on_foundry_dbg.sh @@ -13,5 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -EXTRA_FLAGS="-c dbg --test_timeout=300,450,1200,3600" +set -ex + +EXTRA_FLAGS="--config=dbg --test_timeout=300,450,1200,3600" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" diff --git a/tools/internal_ci/linux/pull_request/grpc_bazel_on_foundry_opt.sh b/tools/internal_ci/linux/pull_request/grpc_bazel_on_foundry_opt.sh index ded0d368a5..77744de49f 100644 --- a/tools/internal_ci/linux/pull_request/grpc_bazel_on_foundry_opt.sh +++ b/tools/internal_ci/linux/pull_request/grpc_bazel_on_foundry_opt.sh @@ -13,5 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -EXTRA_FLAGS="-c opt --test_timeout=300,450,1200,3600" +set -ex + +EXTRA_FLAGS="--config=opt --test_timeout=300,450,1200,3600" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" diff --git a/tools/internal_ci/linux/pull_request/grpc_interop_alts.cfg b/tools/internal_ci/linux/pull_request/grpc_interop_alts.cfg index c1253b30f7..e91a612878 100644 --- a/tools/internal_ci/linux/pull_request/grpc_interop_alts.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_interop_alts.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_interop_tests.sh" timeout_mins: 60 action { define_artifacts { - regex: "**/sponge_log.xml" + regex: "**/sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_interop_matrix_adhoc.cfg b/tools/internal_ci/linux/pull_request/grpc_interop_matrix_adhoc.cfg new file mode 100644 index 0000000000..6726384f18 --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_interop_matrix_adhoc.cfg @@ -0,0 +1,30 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_interop_matrix.sh" +timeout_mins: 300 +action { + define_artifacts { + regex: "**/sponge_log.xml" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "--language=all --release=all --allow_flakes --report_file=sponge_log.xml" +} diff --git a/tools/internal_ci/linux/pull_request/grpc_interop_tocloud.cfg b/tools/internal_ci/linux/pull_request/grpc_interop_tocloud.cfg index cb18e8e868..b1eb575605 100644 --- a/tools/internal_ci/linux/pull_request/grpc_interop_tocloud.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_interop_tocloud.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_interop_tests.sh" timeout_mins: 60 action { define_artifacts { - regex: "**/sponge_log.xml" + regex: "**/sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_interop_toprod.cfg b/tools/internal_ci/linux/pull_request/grpc_interop_toprod.cfg index d14c79a1f6..7321effc12 100644 --- a/tools/internal_ci/linux/pull_request/grpc_interop_toprod.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_interop_toprod.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_interop_tests.sh" timeout_mins: 60 action { define_artifacts { - regex: "**/sponge_log.xml" + regex: "**/sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_microbenchmark_diff.cfg b/tools/internal_ci/linux/pull_request/grpc_microbenchmark_diff.cfg index 9269c345f0..47301d6141 100644 --- a/tools/internal_ci/linux/pull_request/grpc_microbenchmark_diff.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_microbenchmark_diff.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_microbenchmark_diff.sh" timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_msan_on_foundry.sh b/tools/internal_ci/linux/pull_request/grpc_msan_on_foundry.sh new file mode 100644 index 0000000000..c85a837a44 --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_msan_on_foundry.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh --config=msan diff --git a/tools/internal_ci/linux/pull_request/grpc_sanity.cfg b/tools/internal_ci/linux/pull_request/grpc_sanity.cfg index 0f83299aab..276c34f0cf 100644 --- a/tools/internal_ci/linux/pull_request/grpc_sanity.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_sanity.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 40 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_trickle_diff.cfg b/tools/internal_ci/linux/pull_request/grpc_trickle_diff.cfg index e86b3ab475..78358eac28 100644 --- a/tools/internal_ci/linux/pull_request/grpc_trickle_diff.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_trickle_diff.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_trickle_diff.sh" timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/pull_request/grpc_tsan_on_foundry.sh b/tools/internal_ci/linux/pull_request/grpc_tsan_on_foundry.sh index 3dee115300..f3e98e63da 100644 --- a/tools/internal_ci/linux/pull_request/grpc_tsan_on_foundry.sh +++ b/tools/internal_ci/linux/pull_request/grpc_tsan_on_foundry.sh @@ -13,6 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=thread --linkopt=-fsanitize=thread --test_timeout=3600 --action_env=TSAN_OPTIONS=suppressions=test/core/util/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1" -EXCLUDE_TESTS="--test_tag_filters=-qps_json_driver,-json_run_localhost" -github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" "${EXCLUDE_TESTS}" +set -ex + +github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh --config=tsan diff --git a/tools/internal_ci/linux/pull_request/grpc_ubsan_on_foundry.sh b/tools/internal_ci/linux/pull_request/grpc_ubsan_on_foundry.sh index 8547fa4d93..b94935eab1 100644 --- a/tools/internal_ci/linux/pull_request/grpc_ubsan_on_foundry.sh +++ b/tools/internal_ci/linux/pull_request/grpc_ubsan_on_foundry.sh @@ -15,59 +15,4 @@ set -ex -# A temporary solution to give Kokoro credentials. -# The file name 4321_grpc-testing-service needs to match auth_credential in -# the build config. -# TODO: Use keystore. -mkdir -p ${KOKORO_KEYSTORE_DIR} -cp ${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json ${KOKORO_KEYSTORE_DIR}/4321_grpc-testing-service - -temp_dir=$(mktemp -d) -ln -f "${KOKORO_GFILE_DIR}/bazel-latest-release" ${temp_dir}/bazel -chmod 755 "${KOKORO_GFILE_DIR}/bazel-latest-release" -export PATH="${temp_dir}:${PATH}" -# This should show ${temp_dir}/bazel -which bazel -chmod +x "${KOKORO_GFILE_DIR}/bazel_wrapper.py" - -# change to grpc repo root -cd $(dirname $0)/../../../.. - -source tools/internal_ci/helper_scripts/prepare_build_linux_rc - -export KOKORO_FOUNDRY_PROJECT_ID="projects/grpc-testing/instances/default_instance" - -"${KOKORO_GFILE_DIR}/bazel_wrapper.py" \ - --host_jvm_args=-Dbazel.DigestFunction=SHA256 \ - test --jobs="200" \ - --test_timeout="3600,3600,3600,3600" \ - --test_output=errors \ - --verbose_failures=true \ - --keep_going \ - --remote_accept_cached=true \ - --spawn_strategy=remote \ - --remote_local_fallback=false \ - --remote_timeout=3600 \ - --strategy=Javac=remote \ - --strategy=Closure=remote \ - --genrule_strategy=remote \ - --experimental_strict_action_env=true \ - --define GRPC_PORT_ISOLATED_RUNTIME=1 \ - --copt=-gmlt \ - --strip=never \ - --copt=-fsanitize=undefined \ - --linkopt=-fsanitize=undefined \ - --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/experimental/ubuntu16_04_clang/1.0/bazel_0.15.0/ubsan:toolchain \ - --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 \ - --extra_toolchains=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.16.1/cpp:cc-toolchain-clang-x86_64-default \ - --extra_execution_platforms=//third_party/toolchains:rbe_ubuntu1604 \ - --host_platform=//third_party/toolchains:rbe_ubuntu1604 \ - --platforms=//third_party/toolchains:rbe_ubuntu1604 \ - --test_env=GRPC_VERBOSITY=debug \ - --remote_instance_name=projects/grpc-testing/instances/default_instance \ - -- //test/... || FAILED="true" - -if [ "$FAILED" != "" ] -then - exit 1 -fi +github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh --config=ubsan diff --git a/tools/internal_ci/linux/run_performance_profile_hourly.sh b/tools/internal_ci/linux/run_performance_profile_hourly.sh index edf85c2e2c..2be9edf756 100755 --- a/tools/internal_ci/linux/run_performance_profile_hourly.sh +++ b/tools/internal_ci/linux/run_performance_profile_hourly.sh @@ -21,8 +21,8 @@ cd $(dirname $0)/../../.. CPUS=`python -c 'import multiprocessing; print multiprocessing.cpu_count()'` -make CONFIG=opt memory_profile_test memory_profile_client memory_profile_server -j $CPUS -bins/opt/memory_profile_test +make CONFIG=opt memory_usage_test memory_usage_client memory_usage_server -j $CPUS +bins/opt/memory_usage_test bq load microbenchmarks.memory memory_usage.csv tools/run_tests/run_microbenchmark.py --collect summary --bigquery_upload diff --git a/tools/internal_ci/linux/sanitizer/grpc_c_asan.cfg b/tools/internal_ci/linux/sanitizer/grpc_c_asan.cfg index 06a4372ce2..d6d70677dd 100644 --- a/tools/internal_ci/linux/sanitizer/grpc_c_asan.cfg +++ b/tools/internal_ci/linux/sanitizer/grpc_c_asan.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 1440 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/grpc_c_msan.cfg b/tools/internal_ci/linux/sanitizer/grpc_c_msan.cfg index f875327c1b..7b22c6afef 100644 --- a/tools/internal_ci/linux/sanitizer/grpc_c_msan.cfg +++ b/tools/internal_ci/linux/sanitizer/grpc_c_msan.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 1440 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/grpc_c_tsan.cfg b/tools/internal_ci/linux/sanitizer/grpc_c_tsan.cfg index 6658a804d8..6c9dd6ef8d 100644 --- a/tools/internal_ci/linux/sanitizer/grpc_c_tsan.cfg +++ b/tools/internal_ci/linux/sanitizer/grpc_c_tsan.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 1440 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/grpc_c_ubsan.cfg b/tools/internal_ci/linux/sanitizer/grpc_c_ubsan.cfg index 957a91ef2b..8700c74c8d 100644 --- a/tools/internal_ci/linux/sanitizer/grpc_c_ubsan.cfg +++ b/tools/internal_ci/linux/sanitizer/grpc_c_ubsan.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 1440 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/grpc_cpp_asan.cfg b/tools/internal_ci/linux/sanitizer/grpc_cpp_asan.cfg index dbbfce90cb..02162d860b 100644 --- a/tools/internal_ci/linux/sanitizer/grpc_cpp_asan.cfg +++ b/tools/internal_ci/linux/sanitizer/grpc_cpp_asan.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 1440 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/grpc_cpp_tsan.cfg b/tools/internal_ci/linux/sanitizer/grpc_cpp_tsan.cfg index fb0cefa160..95582184ed 100644 --- a/tools/internal_ci/linux/sanitizer/grpc_cpp_tsan.cfg +++ b/tools/internal_ci/linux/sanitizer/grpc_cpp_tsan.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 1440 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_asan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_asan.cfg index 1daf7a514e..d444596a81 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_asan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_asan.cfg @@ -20,7 +20,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_msan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_msan.cfg index a8503b7bcb..3891cc37e8 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_msan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_msan.cfg @@ -20,7 +20,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_tsan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_tsan.cfg index 12af4581eb..91ce1627af 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_tsan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_tsan.cfg @@ -20,7 +20,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_ubsan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_ubsan.cfg index 0d3803bf23..fcdc2de37b 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_ubsan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_c_ubsan.cfg @@ -20,7 +20,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_asan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_asan.cfg index 557561810b..de2a74051b 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_asan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_asan.cfg @@ -20,7 +20,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_tsan.cfg b/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_tsan.cfg index cb15ca34fd..51c291f7df 100644 --- a/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_tsan.cfg +++ b/tools/internal_ci/linux/sanitizer/pull_request/grpc_cpp_tsan.cfg @@ -20,7 +20,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" timeout_mins: 1440 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/macos/grpc_basictests_dbg.cfg b/tools/internal_ci/macos/grpc_basictests_dbg.cfg index 53bda1ff0a..1c7ed26d64 100644 --- a/tools/internal_ci/macos/grpc_basictests_dbg.cfg +++ b/tools/internal_ci/macos/grpc_basictests_dbg.cfg @@ -20,12 +20,12 @@ gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0e timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } env_vars { key: "RUN_TESTS_FLAGS" - value: "-f basictests macos dbg --internal_ci -j 1 --inner_jobs 4 --bq_result_table aggregate_results" + value: "-f basictests macos dbg --exclude objc --internal_ci -j 1 --inner_jobs 4 --bq_result_table aggregate_results" } diff --git a/tools/internal_ci/macos/grpc_basictests_objc_dbg.cfg b/tools/internal_ci/macos/grpc_basictests_objc_dbg.cfg new file mode 100644 index 0000000000..068961234b --- /dev/null +++ b/tools/internal_ci/macos/grpc_basictests_objc_dbg.cfg @@ -0,0 +1,31 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix.sh" +gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json" +timeout_mins: 120 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "-f basictests macos objc dbg --internal_ci -j 1 --inner_jobs 4 --bq_result_table aggregate_results" +} diff --git a/tools/internal_ci/macos/grpc_basictests_objc_opt.cfg b/tools/internal_ci/macos/grpc_basictests_objc_opt.cfg new file mode 100644 index 0000000000..927fa50deb --- /dev/null +++ b/tools/internal_ci/macos/grpc_basictests_objc_opt.cfg @@ -0,0 +1,31 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix.sh" +gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json" +timeout_mins: 120 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "-f basictests macos objc opt --internal_ci -j 1 --inner_jobs 4 --bq_result_table aggregate_results" +} diff --git a/tools/internal_ci/macos/grpc_basictests_opt.cfg b/tools/internal_ci/macos/grpc_basictests_opt.cfg index d359eb601a..f46ebaa786 100644 --- a/tools/internal_ci/macos/grpc_basictests_opt.cfg +++ b/tools/internal_ci/macos/grpc_basictests_opt.cfg @@ -20,12 +20,12 @@ gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0e timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } env_vars { key: "RUN_TESTS_FLAGS" - value: "-f basictests macos opt --internal_ci -j 1 --inner_jobs 4 --bq_result_table aggregate_results" + value: "-f basictests macos opt --exclude objc --internal_ci -j 1 --inner_jobs 4 --bq_result_table aggregate_results" } diff --git a/tools/internal_ci/macos/grpc_build_artifacts.cfg b/tools/internal_ci/macos/grpc_build_artifacts.cfg index 4da61faed3..c73cf4359f 100644 --- a/tools/internal_ci/macos/grpc_build_artifacts.cfg +++ b/tools/internal_ci/macos/grpc_build_artifacts.cfg @@ -20,7 +20,7 @@ gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0e timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/macos/grpc_distribtests.cfg b/tools/internal_ci/macos/grpc_distribtests.cfg index ae88f39b90..156ec6fe33 100644 --- a/tools/internal_ci/macos/grpc_distribtests.cfg +++ b/tools/internal_ci/macos/grpc_distribtests.cfg @@ -20,7 +20,7 @@ gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0e timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/macos/grpc_interop.cfg b/tools/internal_ci/macos/grpc_interop.cfg index b4b1b15cb4..434ecd19c4 100644 --- a/tools/internal_ci/macos/grpc_interop.cfg +++ b/tools/internal_ci/macos/grpc_interop.cfg @@ -20,7 +20,7 @@ gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0e timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/macos/grpc_interop_toprod.cfg b/tools/internal_ci/macos/grpc_interop_toprod.cfg index c92c397daa..2cfc8a2d6d 100644 --- a/tools/internal_ci/macos/grpc_interop_toprod.cfg +++ b/tools/internal_ci/macos/grpc_interop_toprod.cfg @@ -21,7 +21,7 @@ gfile_resources: "/bigstore/grpc-testing-secrets/interop/service_account/GrpcTes timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/macos/pull_request/grpc_basictests_dbg.cfg b/tools/internal_ci/macos/pull_request/grpc_basictests_dbg.cfg index 30c01d3e2f..c91c033916 100644 --- a/tools/internal_ci/macos/pull_request/grpc_basictests_dbg.cfg +++ b/tools/internal_ci/macos/pull_request/grpc_basictests_dbg.cfg @@ -20,12 +20,12 @@ gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0e timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } env_vars { key: "RUN_TESTS_FLAGS" - value: "-f basictests macos dbg --internal_ci -j 1 --inner_jobs 4 --max_time=3600" + value: "-f basictests macos dbg --exclude objc --internal_ci -j 1 --inner_jobs 4 --max_time=3600" } diff --git a/tools/internal_ci/macos/pull_request/grpc_basictests_objc_dbg.cfg b/tools/internal_ci/macos/pull_request/grpc_basictests_objc_dbg.cfg new file mode 100644 index 0000000000..775fd355a5 --- /dev/null +++ b/tools/internal_ci/macos/pull_request/grpc_basictests_objc_dbg.cfg @@ -0,0 +1,31 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix.sh" +gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json" +timeout_mins: 120 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "-f basictests macos objc dbg --internal_ci -j 1 --inner_jobs 4 --max_time=3600" +} diff --git a/tools/internal_ci/macos/pull_request/grpc_basictests_objc_opt.cfg b/tools/internal_ci/macos/pull_request/grpc_basictests_objc_opt.cfg new file mode 100644 index 0000000000..652ef1bb77 --- /dev/null +++ b/tools/internal_ci/macos/pull_request/grpc_basictests_objc_opt.cfg @@ -0,0 +1,31 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix.sh" +gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json" +timeout_mins: 120 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "-f basictests macos objc opt --internal_ci -j 1 --inner_jobs 4 --max_time=3600" +} diff --git a/tools/internal_ci/macos/pull_request/grpc_basictests_opt.cfg b/tools/internal_ci/macos/pull_request/grpc_basictests_opt.cfg index b63ee713bc..532234fa84 100644 --- a/tools/internal_ci/macos/pull_request/grpc_basictests_opt.cfg +++ b/tools/internal_ci/macos/pull_request/grpc_basictests_opt.cfg @@ -20,12 +20,12 @@ gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0e timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } env_vars { key: "RUN_TESTS_FLAGS" - value: "-f basictests macos opt --internal_ci -j 1 --inner_jobs 4 --max_time=3600" + value: "-f basictests macos opt --exclude objc --internal_ci -j 1 --inner_jobs 4 --max_time=3600" } diff --git a/tools/internal_ci/macos/pull_request/grpc_interop.cfg b/tools/internal_ci/macos/pull_request/grpc_interop.cfg index b4b1b15cb4..434ecd19c4 100644 --- a/tools/internal_ci/macos/pull_request/grpc_interop.cfg +++ b/tools/internal_ci/macos/pull_request/grpc_interop.cfg @@ -20,7 +20,7 @@ gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0e timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/macos/pull_request/grpc_ios_binary_size.cfg b/tools/internal_ci/macos/pull_request/grpc_ios_binary_size.cfg index 8942bc7ba7..fb215bdf99 100644 --- a/tools/internal_ci/macos/pull_request/grpc_ios_binary_size.cfg +++ b/tools/internal_ci/macos/pull_request/grpc_ios_binary_size.cfg @@ -20,7 +20,7 @@ timeout_mins: 60 gfile_resources: "/bigstore/grpc-testing-secrets/github_credentials/oauth_token.txt" action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/windows/grpc_basictests.cfg b/tools/internal_ci/windows/grpc_basictests.cfg index 8e644e4c5e..fcf5237bf3 100644 --- a/tools/internal_ci/windows/grpc_basictests.cfg +++ b/tools/internal_ci/windows/grpc_basictests.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/windows/grpc_basictests_dbg.cfg b/tools/internal_ci/windows/grpc_basictests_dbg.cfg index 28d53cdc7b..4e5e7b6545 100644 --- a/tools/internal_ci/windows/grpc_basictests_dbg.cfg +++ b/tools/internal_ci/windows/grpc_basictests_dbg.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/windows/grpc_basictests_opt.cfg b/tools/internal_ci/windows/grpc_basictests_opt.cfg index 4b7a965977..f5db6a9897 100644 --- a/tools/internal_ci/windows/grpc_basictests_opt.cfg +++ b/tools/internal_ci/windows/grpc_basictests_opt.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/windows/grpc_build_artifacts.cfg b/tools/internal_ci/windows/grpc_build_artifacts.cfg index 38b0abd519..f45cfda121 100644 --- a/tools/internal_ci/windows/grpc_build_artifacts.cfg +++ b/tools/internal_ci/windows/grpc_build_artifacts.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_build_artifacts.bat" timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/windows/grpc_build_packages.cfg b/tools/internal_ci/windows/grpc_build_packages.cfg index 65a8b1eef3..b351bbb11b 100644 --- a/tools/internal_ci/windows/grpc_build_packages.cfg +++ b/tools/internal_ci/windows/grpc_build_packages.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_build_packages.bat" timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/windows/grpc_distribtests.cfg b/tools/internal_ci/windows/grpc_distribtests.cfg index 1766e601e5..e12d4bf98d 100644 --- a/tools/internal_ci/windows/grpc_distribtests.cfg +++ b/tools/internal_ci/windows/grpc_distribtests.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_distribtests.bat" timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/windows/grpc_distribtests_standalone.cfg b/tools/internal_ci/windows/grpc_distribtests_standalone.cfg index 33a50fdc45..7f32ba4c70 100644 --- a/tools/internal_ci/windows/grpc_distribtests_standalone.cfg +++ b/tools/internal_ci/windows/grpc_distribtests_standalone.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_distribtests_standalone.bat" timeout_mins: 120 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" regex: "github/grpc/artifacts/**" } diff --git a/tools/internal_ci/windows/grpc_portability.cfg b/tools/internal_ci/windows/grpc_portability.cfg index 94e71753ef..aa86c06632 100644 --- a/tools/internal_ci/windows/grpc_portability.cfg +++ b/tools/internal_ci/windows/grpc_portability.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/windows/grpc_portability_build_only.cfg b/tools/internal_ci/windows/grpc_portability_build_only.cfg index 3bc27f1f24..3dace627eb 100644 --- a/tools/internal_ci/windows/grpc_portability_build_only.cfg +++ b/tools/internal_ci/windows/grpc_portability_build_only.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/windows/pull_request/grpc_basictests.cfg b/tools/internal_ci/windows/pull_request/grpc_basictests.cfg index 91777cd7cb..a8e6f0675b 100644 --- a/tools/internal_ci/windows/pull_request/grpc_basictests.cfg +++ b/tools/internal_ci/windows/pull_request/grpc_basictests.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/windows/pull_request/grpc_basictests_dbg.cfg b/tools/internal_ci/windows/pull_request/grpc_basictests_dbg.cfg index 81a1e9d3c8..ce957232fd 100644 --- a/tools/internal_ci/windows/pull_request/grpc_basictests_dbg.cfg +++ b/tools/internal_ci/windows/pull_request/grpc_basictests_dbg.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/windows/pull_request/grpc_basictests_opt.cfg b/tools/internal_ci/windows/pull_request/grpc_basictests_opt.cfg index 3bb6510ca7..7d2abbfb19 100644 --- a/tools/internal_ci/windows/pull_request/grpc_basictests_opt.cfg +++ b/tools/internal_ci/windows/pull_request/grpc_basictests_opt.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/windows/pull_request/grpc_portability.cfg b/tools/internal_ci/windows/pull_request/grpc_portability.cfg index 2bda487629..6f332416fe 100644 --- a/tools/internal_ci/windows/pull_request/grpc_portability.cfg +++ b/tools/internal_ci/windows/pull_request/grpc_portability.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/windows/grpc_run_tests_matrix.bat" timeout_mins: 240 action { define_artifacts { - regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.*" regex: "github/grpc/reports/**" } } diff --git a/tools/interop_matrix/client_matrix.py b/tools/interop_matrix/client_matrix.py index 15b53d1716..ff3344cd95 100644 --- a/tools/interop_matrix/client_matrix.py +++ b/tools/interop_matrix/client_matrix.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Dictionaries used for client matrix testing. +# Defines languages, runtimes and releases for backward compatibility testing def get_github_repo(lang): @@ -53,8 +53,7 @@ LANG_RUNTIME_MATRIX = { 'csharp': ['csharp', 'csharpcoreclr'], } -# Dictionary of releases per language. For each language, we need to provide -# a release tag pointing to the latest build of the branch. +# Dictionary of known releases for given language. LANG_RELEASE_MATRIX = { 'cxx': [ { @@ -102,6 +101,9 @@ LANG_RELEASE_MATRIX = { { 'v1.15.0': None }, + { + 'v1.16.0': None + }, ], 'go': [ { @@ -138,7 +140,19 @@ LANG_RELEASE_MATRIX = { 'v1.11.3': None }, { - 'v1.12.0': None + 'v1.12.2': None + }, + { + 'v1.13.0': None + }, + { + 'v1.14.0': None + }, + { + 'v1.15.0': None + }, + { + 'v1.16.0': None }, ], 'java': [ @@ -190,6 +204,9 @@ LANG_RELEASE_MATRIX = { { 'v1.15.0': None }, + { + 'v1.16.1': None + }, ], 'python': [ { @@ -237,6 +254,9 @@ LANG_RELEASE_MATRIX = { { 'v1.15.0': None }, + { + 'v1.16.0': None + }, ], 'node': [ { @@ -328,6 +348,9 @@ LANG_RELEASE_MATRIX = { { 'v1.15.0': None }, + { + 'v1.16.0': None + }, ], 'php': [ { @@ -375,6 +398,9 @@ LANG_RELEASE_MATRIX = { { 'v1.15.0': None }, + { + 'v1.16.0': None + }, ], 'csharp': [ { @@ -427,6 +453,9 @@ LANG_RELEASE_MATRIX = { { 'v1.15.0': None }, + { + 'v1.16.0': None + }, ], } diff --git a/tools/interop_matrix/run_interop_matrix_tests.py b/tools/interop_matrix/run_interop_matrix_tests.py index 6cd6f43167..6cf2a9b036 100755 --- a/tools/interop_matrix/run_interop_matrix_tests.py +++ b/tools/interop_matrix/run_interop_matrix_tests.py @@ -26,7 +26,7 @@ import subprocess import sys import uuid -# Langauage Runtime Matrix +# Language Runtime Matrix import client_matrix python_util_dir = os.path.abspath( @@ -37,6 +37,9 @@ import jobset import report_utils import upload_test_results +_TEST_TIMEOUT_SECONDS = 60 +_PULL_IMAGE_TIMEOUT_SECONDS = 15 * 60 +_MAX_PARALLEL_DOWNLOADS = 6 _LANGUAGES = client_matrix.LANG_RUNTIME_MATRIX.keys() # All gRPC release tags, flattened, deduped and sorted. _RELEASES = sorted( @@ -45,7 +48,6 @@ _RELEASES = sorted( client_matrix.get_release_tag_name(info) for lang in client_matrix.LANG_RELEASE_MATRIX.values() for info in lang))) -_TEST_TIMEOUT = 60 argp = argparse.ArgumentParser(description='Run interop tests.') argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int) @@ -56,7 +58,7 @@ argp.add_argument( argp.add_argument( '--release', default='all', - choices=['all', 'master'] + _RELEASES, + choices=['all'] + _RELEASES, help='Release tags to test. When testing all ' 'releases defined in client_matrix.py, use "all".') argp.add_argument( @@ -92,136 +94,154 @@ argp.add_argument( nargs='?', help='The gateway to backend services.') -args = argp.parse_args() - -print(str(args)) - -def find_all_images_for_lang(lang): +def _get_test_images_for_lang(lang, release_arg, image_path_prefix): """Find docker images for a language across releases and runtimes. Returns dictionary of list of (<tag>, <image-full-path>) keyed by runtime. """ - # Find all defined releases. - if args.release == 'all': - releases = ['master'] + client_matrix.get_release_tags(lang) + if release_arg == 'all': + # Use all defined releases for given language + releases = client_matrix.get_release_tags(lang) else: # Look for a particular release. - if args.release not in ['master' - ] + client_matrix.get_release_tags(lang): + if release_arg not in client_matrix.get_release_tags(lang): jobset.message( 'SKIPPED', - '%s for %s is not defined' % (args.release, lang), + 'release %s for %s is not defined' % (release_arg, lang), do_newline=True) return {} - releases = [args.release] + releases = [release_arg] - # TODO(jtattermusch): why do we need to query the existing images/tags? - # From LANG_RUNTIME_MATRIX and LANG_RELEASE_MATRIX it should be obvious - # which tags we want to test - and it should be an error if they are - # missing. # Images tuples keyed by runtime. images = {} for runtime in client_matrix.LANG_RUNTIME_MATRIX[lang]: - image_path = '%s/grpc_interop_%s' % (args.gcr_path, runtime) - output = subprocess.check_output([ - 'gcloud', 'beta', 'container', 'images', 'list-tags', - '--format=json', image_path - ]) - docker_image_list = json.loads(output) - # All images should have a single tag or no tag. - # TODO(adelez): Remove tagless images. - tags = [i['tags'][0] for i in docker_image_list if i['tags']] - jobset.message( - 'START', - 'Found images for %s: %s' % (image_path, tags), - do_newline=True) - skipped = len(docker_image_list) - len(tags) - jobset.message( - 'SKIPPED', - 'Skipped images (no-tag/unknown-tag): %d' % skipped, - do_newline=True) - # Filter tags based on the releases. - images[runtime] = [(tag, '%s:%s' % (image_path, tag)) - for tag in tags - if tag in releases] + image_path = '%s/grpc_interop_%s' % (image_path_prefix, runtime) + images[runtime] = [ + (tag, '%s:%s' % (image_path, tag)) for tag in releases + ] return images -# caches test cases (list of JobSpec) loaded from file. Keyed by lang and runtime. -def find_test_cases(lang, runtime, release, suite_name): - """Returns the list of test cases from testcase files per lang/release.""" +def _read_test_cases_file(lang, runtime, release): + """Read test cases from a bash-like file and return a list of commands""" testcase_dir = os.path.join(os.path.dirname(__file__), 'testcases') filename_prefix = lang if lang == 'csharp': + # TODO(jtattermusch): remove this odd specialcase filename_prefix = runtime # Check to see if we need to use a particular version of test cases. lang_version = '%s_%s' % (filename_prefix, release) if lang_version in client_matrix.TESTCASES_VERSION_MATRIX: - testcases = os.path.join( + testcase_file = os.path.join( testcase_dir, client_matrix.TESTCASES_VERSION_MATRIX[lang_version]) else: - testcases = os.path.join(testcase_dir, '%s__master' % filename_prefix) + # TODO(jtattermusch): remove the double-underscore, it is pointless + testcase_file = os.path.join(testcase_dir, + '%s__master' % filename_prefix) + + lines = [] + with open(testcase_file) as f: + for line in f.readlines(): + line = re.sub('\\#.*$', '', line) # remove hash comments + line = line.strip() + if line and not line.startswith('echo'): + # Each non-empty line is a treated as a test case command + lines.append(line) + return lines + + +def _cleanup_docker_image(image): + jobset.message('START', 'Cleanup docker image %s' % image, do_newline=True) + dockerjob.remove_image(image, skip_nonexistent=True) + + +args = argp.parse_args() + + +# caches test cases (list of JobSpec) loaded from file. Keyed by lang and runtime. +def _generate_test_case_jobspecs(lang, runtime, release, suite_name): + """Returns the list of test cases from testcase files per lang/release.""" + testcase_lines = _read_test_cases_file(lang, runtime, release) job_spec_list = [] - try: - with open(testcases) as f: - # Only line start with 'docker run' are test cases. - for line in f.readlines(): - if line.startswith('docker run'): - m = re.search('--test_case=(.*)"', line) - shortname = m.group(1) if m else 'unknown_test' - m = re.search( - '--server_host_override=(.*).sandbox.googleapis.com', - line) - server = m.group(1) if m else 'unknown_server' - - # If server_host arg is not None, replace the original - # server_host with the one provided or append to the end of - # the command if server_host does not appear originally. - if args.server_host: - if line.find('--server_host=') > -1: - line = re.sub('--server_host=[^ ]*', - '--server_host=%s' % args.server_host, - line) - else: - line = '%s --server_host=%s"' % (line[:-1], - args.server_host) - print(line) - - spec = jobset.JobSpec( - cmdline=line, - shortname='%s:%s:%s:%s' % (suite_name, lang, server, - shortname), - timeout_seconds=_TEST_TIMEOUT, - shell=True, - flake_retries=5 if args.allow_flakes else 0) - job_spec_list.append(spec) - jobset.message( - 'START', - 'Loaded %s tests from %s' % (len(job_spec_list), testcases), - do_newline=True) - except IOError as err: - jobset.message('FAILED', err, do_newline=True) + for line in testcase_lines: + # TODO(jtattermusch): revisit the logic for updating test case commands + # what it currently being done seems fragile. + m = re.search('--test_case=(.*)"', line) + shortname = m.group(1) if m else 'unknown_test' + m = re.search('--server_host_override=(.*).sandbox.googleapis.com', + line) + server = m.group(1) if m else 'unknown_server' + + # If server_host arg is not None, replace the original + # server_host with the one provided or append to the end of + # the command if server_host does not appear originally. + if args.server_host: + if line.find('--server_host=') > -1: + line = re.sub('--server_host=[^ ]*', + '--server_host=%s' % args.server_host, line) + else: + line = '%s --server_host=%s"' % (line[:-1], args.server_host) + + spec = jobset.JobSpec( + cmdline=line, + shortname='%s:%s:%s:%s' % (suite_name, lang, server, shortname), + timeout_seconds=_TEST_TIMEOUT_SECONDS, + shell=True, + flake_retries=5 if args.allow_flakes else 0) + job_spec_list.append(spec) return job_spec_list -_xml_report_tree = report_utils.new_junit_xml_tree() +def _pull_images_for_lang(lang, images): + """Pull all images for given lang from container registry.""" + jobset.message( + 'START', 'Downloading images for language "%s"' % lang, do_newline=True) + download_specs = [] + for release, image in images: + # Pull the image and warm it up. + # First time we use an image with "docker run", it takes time to unpack + # the image and later this delay would fail our test cases. + cmdline = [ + 'time gcloud docker -- pull %s && time docker run --rm=true %s /bin/true' + % (image, image) + ] + spec = jobset.JobSpec( + cmdline=cmdline, + shortname='pull_image_%s' % (image), + timeout_seconds=_PULL_IMAGE_TIMEOUT_SECONDS, + shell=True) + download_specs.append(spec) + # too many image downloads at once tend to get stuck + max_pull_jobs = min(args.jobs, _MAX_PARALLEL_DOWNLOADS) + num_failures, resultset = jobset.run( + download_specs, newline_on_success=True, maxjobs=max_pull_jobs) + if num_failures: + jobset.message( + 'FAILED', 'Failed to download some images', do_newline=True) + return False + else: + jobset.message( + 'SUCCESS', 'All images downloaded successfully.', do_newline=True) + return True -def run_tests_for_lang(lang, runtime, images): +def _run_tests_for_lang(lang, runtime, images, xml_report_tree): """Find and run all test cases for a language. images is a list of (<release-tag>, <image-full-path>) tuple. """ + if not _pull_images_for_lang(lang, images): + jobset.message( + 'FAILED', 'Image download failed. Exiting.', do_newline=True) + return 1 + total_num_failures = 0 - for image_tuple in images: - release, image = image_tuple - jobset.message('START', 'Testing %s' % image, do_newline=True) - # Download the docker image before running each test case. - subprocess.check_call(['gcloud', 'docker', '--', 'pull', image]) + for release, image in images: suite_name = '%s__%s_%s' % (lang, runtime, release) - job_spec_list = find_test_cases(lang, runtime, release, suite_name) + job_spec_list = _generate_test_case_jobspecs(lang, runtime, release, + suite_name) if not job_spec_list: jobset.message( @@ -242,28 +262,24 @@ def run_tests_for_lang(lang, runtime, images): else: jobset.message('SUCCESS', 'All tests passed', do_newline=True) - report_utils.append_junit_xml_results(_xml_report_tree, resultset, + report_utils.append_junit_xml_results(xml_report_tree, resultset, 'grpc_interop_matrix', suite_name, str(uuid.uuid4())) if not args.keep: - cleanup(image) + _cleanup_docker_image(image) return total_num_failures -def cleanup(image): - jobset.message('START', 'Cleanup docker image %s' % image, do_newline=True) - dockerjob.remove_image(image, skip_nonexistent=True) - - languages = args.language if args.language != ['all'] else _LANGUAGES total_num_failures = 0 +_xml_report_tree = report_utils.new_junit_xml_tree() for lang in languages: - docker_images = find_all_images_for_lang(lang) + docker_images = _get_test_images_for_lang(lang, args.release, args.gcr_path) for runtime in sorted(docker_images.keys()): - total_num_failures += run_tests_for_lang(lang, runtime, - docker_images[runtime]) + total_num_failures += _run_tests_for_lang( + lang, runtime, docker_images[runtime], _xml_report_tree) report_utils.create_xml_report_file(_xml_report_tree, args.report_file) diff --git a/tools/remote_build/README.md b/tools/remote_build/README.md new file mode 100644 index 0000000000..19739e9ee1 --- /dev/null +++ b/tools/remote_build/README.md @@ -0,0 +1,33 @@ +# Running Remote Builds with bazel + +This allows you to spawn gRPC C/C++ remote build and tests from your workstation with +configuration that's very similar to what's used by our CI Kokoro. + +Note that this will only work for gRPC team members (it requires access to the +remote build and execution cluster), others will need to rely on local test runs +and tests run by Kokoro CI. + + +## Prerequisites + +- See [Installing Bazel](https://docs.bazel.build/versions/master/install.html) for instructions how to install bazel on your system. + +- Setup application default credentials for running remote builds by following [RBE Credentials Setup](https://cloud.google.com/remote-build-execution/docs/getting-started#set_credentials) + + +## Running remote build manually from dev workstation + +Run from repository root (opt, dbg): +``` +# manual run of bazel tests remotely on Foundry +bazel --bazelrc=tools/remote_build/manual.bazelrc test --config=opt //test/... +``` + +Sanitizer runs (asan, msan, tsan, ubsan): +``` +# manual run of bazel tests remotely on Foundry with given sanitizer +bazel --bazelrc=tools/remote_build/manual.bazelrc test --config=asan //test/... +``` + +Available command line options can be found in +[Bazel command line reference](https://docs.bazel.build/versions/master/command-line-reference.html) diff --git a/tools/remote_build/kokoro.bazelrc b/tools/remote_build/kokoro.bazelrc new file mode 100644 index 0000000000..11462bd301 --- /dev/null +++ b/tools/remote_build/kokoro.bazelrc @@ -0,0 +1,38 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# bazelrc file for running gRPC tests on Kokoro using Foundry + +import %workspace%/tools/remote_build/rbe_common.bazelrc + +build --remote_cache=remotebuildexecution.googleapis.com +build --remote_executor=remotebuildexecution.googleapis.com +build --tls_enabled=true + +build --auth_enabled=true +# magic location where kokoro script puts the credentials +build --auth_credentials=/tmpfs/src/keystore/4321_grpc-testing-service +build --auth_scope=https://www.googleapis.com/auth/cloud-source-tools + +build --bes_backend=buildeventservice.googleapis.com +build --bes_best_effort=false +build --bes_timeout=600s +build --project_id=grpc-testing + +# required by kokoro for some reason +build --test_env=USER=anon + +build --jobs=200 +build --test_output=errors +build --keep_going=true diff --git a/tools/remote_build/manual.bazelrc b/tools/remote_build/manual.bazelrc new file mode 100644 index 0000000000..b4fdc70637 --- /dev/null +++ b/tools/remote_build/manual.bazelrc @@ -0,0 +1,45 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# bazelrc file for running gRPC tests with Foundry (remote build execution) +# manually from developer's workstation + +import %workspace%/tools/remote_build/rbe_common.bazelrc + +build --remote_cache=remotebuildexecution.googleapis.com +build --remote_executor=remotebuildexecution.googleapis.com +build --tls_enabled=true + +# Enable authentication. This will pick up application default credentials by +# default. You can use --auth_credentials=some_file.json to use a service +# account credential instead. +# How to setup credentials: +# See https://cloud.google.com/remote-build-execution/docs/getting-started#set_credentials +build --auth_enabled=true + +# Set flags for uploading to BES in order to view results in the Bazel Build +# Results UI. +build --bes_backend="buildeventservice.googleapis.com" +build --bes_timeout=60s +build --bes_results_url="https://source.cloud.google.com/results/invocations/" +build --project_id=grpc-testing + +build --jobs=100 + +# TODO(jtattermusch): this should be part of the common config +# but currently sanitizers use different test_timeout values +build --test_timeout=300,450,1200,3600 + +# print output for tests that fail (default is "summary") +build --test_output=errors diff --git a/tools/remote_build/rbe_common.bazelrc b/tools/remote_build/rbe_common.bazelrc new file mode 100644 index 0000000000..daa1635265 --- /dev/null +++ b/tools/remote_build/rbe_common.bazelrc @@ -0,0 +1,85 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# bazelrc with Foundry setting common to both manual run and runs started by Kokoro +# see https://github.com/bazelbuild/bazel-toolchains/tree/master/bazelrc +# for examples and more documentation + +startup --host_jvm_args=-Dbazel.DigestFunction=SHA256 + +build --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.16.1/default:toolchain +build --extra_toolchains=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.16.1/cpp:cc-toolchain-clang-x86_64-default +# Use custom execution platforms defined in third_party/toolchains +build --extra_execution_platforms=//third_party/toolchains:rbe_ubuntu1604 +build --host_platform=//third_party/toolchains:rbe_ubuntu1604 +build --platforms=//third_party/toolchains:rbe_ubuntu1604 + +build --spawn_strategy=remote +build --strategy=Javac=remote +build --strategy=Closure=remote +build --genrule_strategy=remote +build --remote_timeout=3600 + +build --remote_instance_name=projects/grpc-testing/instances/default_instance + +build --verbose_failures=true + +build --experimental_strict_action_env=true +build --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 + +# don't use port server +build --define GRPC_PORT_ISOLATED_RUNTIME=1 +# without verbose gRPC logs the test outputs are not very useful +test --test_env=GRPC_VERBOSITY=debug + +# address sanitizer: most settings are already in %workspace%/.bazelrc +# we only need a few additional ones that are Foundry specific +build:asan --copt=-gmlt +# TODO(jtattermusch): use more reasonable test timeout +build:asan --test_timeout=3600 +build:asan --test_tag_filters=-qps_json_driver,-json_run_localhost + +# memory sanitizer: most settings are already in %workspace%/.bazelrc +# we only need a few additional ones that are Foundry specific +build:msan --copt=-gmlt +# TODO(jtattermusch): use more reasonable test timeout +build:msan --test_timeout=3600 +# TODO(jtattermusch): revisit the disabled tests +build:msan --test_tag_filters=-json_run_localhost +build:msan --cxxopt=--stdlib=libc++ +# setting LD_LIBRARY_PATH is necessary +# to avoid "libc++.so.1: cannot open shared object file" +build:msan --action_env=LD_LIBRARY_PATH=/usr/local/lib +build:msan --host_crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.16.1/default:toolchain +# override the config-agnostic crosstool_top +build:msan --crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/ubuntu16_04_clang/1.0/bazel_0.16.1/msan:toolchain + +# thread sanitizer: most settings are already in %workspace%/.bazelrc +# we only need a few additional ones that are Foundry specific +build:tsan --copt=-gmlt +# TODO(jtattermusch): use more reasonable test timeout +build:tsan --test_timeout=3600 +build:tsan --test_tag_filters=-qps_json_driver,-json_run_localhost + +# undefined behavior sanitizer: most settings are already in %workspace%/.bazelrc +# we only need a few additional ones that are Foundry specific +build:ubsan --copt=-gmlt +# TODO(jtattermusch): use more reasonable test timeout +build:ubsan --test_timeout=3600 +# override the config-agnostic crosstool_top +--crosstool_top=@com_github_bazelbuild_bazeltoolchains//configs/experimental/ubuntu16_04_clang/1.0/bazel_0.16.1/ubsan:toolchain +# TODO(jtattermusch): remove this once Foundry adds the env to the docker image. +# ubsan needs symbolizer to work properly, otherwise the suppression file doesn't work +# and we get test failures. +build:ubsan --action_env=UBSAN_SYMBOLIZER_PATH=/usr/local/bin/llvm-symbolizer diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index bdeb258e1f..d18ea2aca1 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -240,37 +240,38 @@ class CSharpExtArtifact: ['tools/run_tests/artifacts/build_artifact_csharp_ios.sh'], use_workspace=True) elif self.platform == 'windows': - cmake_arch_option = 'Win32' if self.arch == 'x86' else self.arch return create_jobspec( self.name, [ 'tools\\run_tests\\artifacts\\build_artifact_csharp.bat', - cmake_arch_option + self.arch ], use_workspace=True) else: - environ = { - 'CONFIG': 'opt', - 'EMBED_OPENSSL': 'true', - 'EMBED_ZLIB': 'true', - 'CFLAGS': '-DGPR_BACKWARDS_COMPATIBILITY_MODE', - 'CXXFLAGS': '-DGPR_BACKWARDS_COMPATIBILITY_MODE', - 'LDFLAGS': '' - } if self.platform == 'linux': + cmake_arch_option = '' # x64 is the default architecture + if self.arch == 'x86': + # TODO(jtattermusch): more work needed to enable + # boringssl assembly optimizations for 32-bit linux. + # Problem: currently we are building the artifact under + # 32-bit docker image, but CMAKE_SYSTEM_PROCESSOR is still + # set to x86_64, so the resulting boringssl binary + # would have undefined symbols. + cmake_arch_option = '-DOPENSSL_NO_ASM=ON' return create_docker_jobspec( self.name, 'tools/dockerfile/grpc_artifact_linux_%s' % self.arch, 'tools/run_tests/artifacts/build_artifact_csharp.sh', - environ=environ) + environ={ + 'CMAKE_ARCH_OPTION': cmake_arch_option + }) else: - archflag = _ARCH_FLAG_MAP[self.arch] - environ['CFLAGS'] += ' %s %s' % (archflag, _MACOS_COMPAT_FLAG) - environ['CXXFLAGS'] += ' %s %s' % (archflag, _MACOS_COMPAT_FLAG) - environ['LDFLAGS'] += ' %s' % archflag + cmake_arch_option = '' # x64 is the default architecture + if self.arch == 'x86': + cmake_arch_option = '-DCMAKE_OSX_ARCHITECTURES=i386' return create_jobspec( self.name, ['tools/run_tests/artifacts/build_artifact_csharp.sh'], - environ=environ, + environ={'CMAKE_ARCH_OPTION': cmake_arch_option}, use_workspace=True) def __str__(self): diff --git a/tools/run_tests/artifacts/build_artifact_csharp.bat b/tools/run_tests/artifacts/build_artifact_csharp.bat index ac2c92b716..713e480f72 100644 --- a/tools/run_tests/artifacts/build_artifact_csharp.bat +++ b/tools/run_tests/artifacts/build_artifact_csharp.bat @@ -15,16 +15,33 @@ @rem Builds C# artifacts on Windows set ARCHITECTURE=%1 -set GRPC_SKIP_DOTNET_RESTORE=true -@call tools\run_tests\helper_scripts\pre_build_csharp.bat %ARCHITECTURE% || goto :error -cd cmake\build\%ARCHITECTURE% -cmake --build . --target grpc_csharp_ext --config RelWithDebInfo +@rem enter repo root +cd /d %~dp0\..\..\.. + +mkdir cmake +cd cmake +mkdir build +cd build +mkdir %ARCHITECTURE% +cd %ARCHITECTURE% + +@rem TODO(jtattermusch): is there a better way to force using MSVC? +@rem select the MSVC compiler explicitly to avoid using gcc from mingw or cygwin +@rem (both are on path) +set "MSVC_COMPILER=C:/Program Files (x86)/Microsoft Visual Studio 14.0/VC/bin/cl.exe" +if "%ARCHITECTURE%" == "x64" ( + set "MSVC_COMPILER=C:/Program Files (x86)/Microsoft Visual Studio 14.0/VC/bin/amd64/cl.exe" +) + +call "%VS140COMNTOOLS%..\..\VC\vcvarsall.bat" %ARCHITECTURE% +cmake -G Ninja -DCMAKE_C_COMPILER="%MSVC_COMPILER%" -DCMAKE_CXX_COMPILER="%MSVC_COMPILER%" -DCMAKE_BUILD_TYPE=RelWithDebInfo -DgRPC_BUILD_TESTS=OFF -DgRPC_MSVC_STATIC_RUNTIME=ON ../../.. || goto :error +cmake --build . --target grpc_csharp_ext cd ..\..\.. mkdir -p %ARTIFACTS_OUT% -copy /Y cmake\build\Win32\RelWithDebInfo\grpc_csharp_ext.dll %ARTIFACTS_OUT% || copy /Y cmake\build\x64\RelWithDebInfo\grpc_csharp_ext.dll %ARTIFACTS_OUT% || goto :error -copy /Y cmake\build\Win32\RelWithDebInfo\grpc_csharp_ext.pdb %ARTIFACTS_OUT% || copy /Y cmake\build\x64\RelWithDebInfo\grpc_csharp_ext.pdb %ARTIFACTS_OUT% || goto :error +copy /Y cmake\build\%ARCHITECTURE%\grpc_csharp_ext.dll %ARTIFACTS_OUT% || goto :error +copy /Y cmake\build\%ARCHITECTURE%\grpc_csharp_ext.pdb %ARTIFACTS_OUT% || goto :error goto :EOF diff --git a/tools/run_tests/artifacts/build_artifact_csharp.sh b/tools/run_tests/artifacts/build_artifact_csharp.sh index d65340261d..bb8a91b520 100755 --- a/tools/run_tests/artifacts/build_artifact_csharp.sh +++ b/tools/run_tests/artifacts/build_artifact_csharp.sh @@ -17,7 +17,17 @@ set -ex cd "$(dirname "$0")/../../.." -make grpc_csharp_ext +mkdir -p cmake/build +cd cmake/build + +cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DgRPC_BACKWARDS_COMPATIBILITY_MODE=ON \ + -DgRPC_BUILD_TESTS=OFF \ + "${CMAKE_ARCH_OPTION}" \ + ../.. + +make grpc_csharp_ext -j2 +cd ../.. mkdir -p "${ARTIFACTS_OUT}" -cp libs/opt/libgrpc_csharp_ext.so "${ARTIFACTS_OUT}" || cp libs/opt/libgrpc_csharp_ext.dylib "${ARTIFACTS_OUT}" +cp cmake/build/libgrpc_csharp_ext.so "${ARTIFACTS_OUT}" || cp cmake/build/libgrpc_csharp_ext.dylib "${ARTIFACTS_OUT}" diff --git a/tools/run_tests/artifacts/build_artifact_python.bat b/tools/run_tests/artifacts/build_artifact_python.bat index d277668c94..795e80dc40 100644 --- a/tools/run_tests/artifacts/build_artifact_python.bat +++ b/tools/run_tests/artifacts/build_artifact_python.bat @@ -22,6 +22,10 @@ pip install -rrequirements.txt set GRPC_PYTHON_BUILD_WITH_CYTHON=1 +@rem Allow build_ext to build C/C++ files in parallel +@rem by enabling a monkeypatch. It speeds up the build a lot. +set GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS=2 + mkdir -p %ARTIFACTS_OUT% set ARTIFACT_DIR=%cd%\%ARTIFACTS_OUT% diff --git a/tools/run_tests/artifacts/build_artifact_python.sh b/tools/run_tests/artifacts/build_artifact_python.sh index 2878005bb2..9a2e0f739f 100755 --- a/tools/run_tests/artifacts/build_artifact_python.sh +++ b/tools/run_tests/artifacts/build_artifact_python.sh @@ -22,6 +22,10 @@ export PYTHON=${PYTHON:-python} export PIP=${PIP:-pip} export AUDITWHEEL=${AUDITWHEEL:-auditwheel} +# Allow build_ext to build C/C++ files in parallel +# by enabling a monkeypatch. It speeds up the build a lot. +export GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS=2 + mkdir -p "${ARTIFACTS_OUT}" ARTIFACT_DIR="$PWD/${ARTIFACTS_OUT}" diff --git a/tools/run_tests/artifacts/build_package_python.sh b/tools/run_tests/artifacts/build_package_python.sh index 29801a5b86..d93e8979fc 100755 --- a/tools/run_tests/artifacts/build_package_python.sh +++ b/tools/run_tests/artifacts/build_package_python.sh @@ -19,10 +19,20 @@ cd "$(dirname "$0")/../../.." mkdir -p artifacts/ -# All the python packages have been built in the artifact phase already -# and we only collect them here to deliver them to the distribtest phase. cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/python_*/* artifacts/ || true +strip_binary_wheel() { + TEMP_WHEEL_DIR=$(mktemp -d) + unzip "$1" -d "$TEMP_WHEEL_DIR" + find "$TEMP_WHEEL_DIR" -name "_protoc_compiler*.so" -exec strip --strip-debug {} ";" + find "$TEMP_WHEEL_DIR" -name "cygrpc*.so" -exec strip --strip-debug {} ";" + (cd "$TEMP_WHEEL_DIR" && zip -r - .) > "$1" +} + +for wheel in artifacts/*.whl; do + strip_binary_wheel "$wheel" +done + # TODO: all the artifact builder configurations generate a grpcio-VERSION.tar.gz # source distribution package, and only one of them will end up # in the artifacts/ directory. They should be all equivalent though. diff --git a/tools/run_tests/generated/lb_interop_test_scenarios.json b/tools/run_tests/generated/lb_interop_test_scenarios.json new file mode 100644 index 0000000000..4f956c568a --- /dev/null +++ b/tools/run_tests/generated/lb_interop_test_scenarios.json @@ -0,0 +1,1167 @@ + +[ + { + "backend_configs": [], + "balancer_configs": [], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [ + { + "transport_sec": "insecure" + } + ], + "name": "no_balancer_because_lb_a_record_returns_nx_domain_insecure", + "skip_langs": [], + "transport_sec": "insecure" + }, + { + "backend_configs": [], + "balancer_configs": [], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [ + { + "transport_sec": "alts" + } + ], + "name": "no_balancer_because_lb_a_record_returns_nx_domain_alts", + "skip_langs": [], + "transport_sec": "alts" + }, + { + "backend_configs": [], + "balancer_configs": [], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [ + { + "transport_sec": "tls" + } + ], + "name": "no_balancer_because_lb_a_record_returns_nx_domain_tls", + "skip_langs": [], + "transport_sec": "tls" + }, + { + "backend_configs": [], + "balancer_configs": [], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [ + { + "transport_sec": "tls" + } + ], + "name": "no_balancer_because_lb_a_record_returns_nx_domain_google_default_credentials", + "skip_langs": [], + "transport_sec": "google_default_credentials" + }, + { + "backend_configs": [], + "balancer_configs": [], + "cause_no_error_no_data_for_balancer_a_record": true, + "fallback_configs": [ + { + "transport_sec": "insecure" + } + ], + "name": "no_balancer_because_lb_a_record_returns_no_data_insecure", + "skip_langs": [], + "transport_sec": "insecure" + }, + { + "backend_configs": [], + "balancer_configs": [], + "cause_no_error_no_data_for_balancer_a_record": true, + "fallback_configs": [ + { + "transport_sec": "alts" + } + ], + "name": "no_balancer_because_lb_a_record_returns_no_data_alts", + "skip_langs": [], + "transport_sec": "alts" + }, + { + "backend_configs": [], + "balancer_configs": [], + "cause_no_error_no_data_for_balancer_a_record": true, + "fallback_configs": [ + { + "transport_sec": "tls" + } + ], + "name": "no_balancer_because_lb_a_record_returns_no_data_tls", + "skip_langs": [], + "transport_sec": "tls" + }, + { + "backend_configs": [], + "balancer_configs": [], + "cause_no_error_no_data_for_balancer_a_record": true, + "fallback_configs": [ + { + "transport_sec": "tls" + } + ], + "name": "no_balancer_because_lb_a_record_returns_no_data_google_default_credentials", + "skip_langs": [], + "transport_sec": "google_default_credentials" + }, + { + "backend_configs": [ + { + "transport_sec": "insecure" + } + ], + "balancer_configs": [ + { + "short_stream": true, + "transport_sec": "insecure" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [], + "name": "client_referred_to_backend_insecure_short_stream_True", + "skip_langs": [ + "java" + ], + "transport_sec": "insecure" + }, + { + "backend_configs": [ + { + "transport_sec": "alts" + } + ], + "balancer_configs": [ + { + "short_stream": true, + "transport_sec": "alts" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [], + "name": "client_referred_to_backend_alts_short_stream_True", + "skip_langs": [ + "java" + ], + "transport_sec": "alts" + }, + { + "backend_configs": [ + { + "transport_sec": "tls" + } + ], + "balancer_configs": [ + { + "short_stream": true, + "transport_sec": "tls" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [], + "name": "client_referred_to_backend_tls_short_stream_True", + "skip_langs": [ + "java", + "java" + ], + "transport_sec": "tls" + }, + { + "backend_configs": [ + { + "transport_sec": "alts" + } + ], + "balancer_configs": [ + { + "short_stream": true, + "transport_sec": "alts" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [], + "name": "client_referred_to_backend_google_default_credentials_short_stream_True", + "skip_langs": [ + "java" + ], + "transport_sec": "google_default_credentials" + }, + { + "backend_configs": [ + { + "transport_sec": "insecure" + } + ], + "balancer_configs": [ + { + "short_stream": false, + "transport_sec": "insecure" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [], + "name": "client_referred_to_backend_insecure_short_stream_False", + "skip_langs": [], + "transport_sec": "insecure" + }, + { + "backend_configs": [ + { + "transport_sec": "alts" + } + ], + "balancer_configs": [ + { + "short_stream": false, + "transport_sec": "alts" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [], + "name": "client_referred_to_backend_alts_short_stream_False", + "skip_langs": [], + "transport_sec": "alts" + }, + { + "backend_configs": [ + { + "transport_sec": "tls" + } + ], + "balancer_configs": [ + { + "short_stream": false, + "transport_sec": "tls" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [], + "name": "client_referred_to_backend_tls_short_stream_False", + "skip_langs": [ + "java" + ], + "transport_sec": "tls" + }, + { + "backend_configs": [ + { + "transport_sec": "alts" + } + ], + "balancer_configs": [ + { + "short_stream": false, + "transport_sec": "alts" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [], + "name": "client_referred_to_backend_google_default_credentials_short_stream_False", + "skip_langs": [], + "transport_sec": "google_default_credentials" + }, + { + "backend_configs": [ + { + "transport_sec": "alts" + } + ], + "balancer_configs": [ + { + "short_stream": true, + "transport_sec": "alts" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [ + { + "transport_sec": "insecure" + } + ], + "name": "client_referred_to_backend_fallback_broken_alts_short_stream_True", + "skip_langs": [ + "java" + ], + "transport_sec": "alts" + }, + { + "backend_configs": [ + { + "transport_sec": "tls" + } + ], + "balancer_configs": [ + { + "short_stream": true, + "transport_sec": "tls" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [ + { + "transport_sec": "insecure" + } + ], + "name": "client_referred_to_backend_fallback_broken_tls_short_stream_True", + "skip_langs": [ + "java", + "java" + ], + "transport_sec": "tls" + }, + { + "backend_configs": [ + { + "transport_sec": "alts" + } + ], + "balancer_configs": [ + { + "short_stream": true, + "transport_sec": "alts" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [ + { + "transport_sec": "insecure" + } + ], + "name": "client_referred_to_backend_fallback_broken_google_default_credentials_short_stream_True", + "skip_langs": [ + "java" + ], + "transport_sec": "google_default_credentials" + }, + { + "backend_configs": [ + { + "transport_sec": "alts" + } + ], + "balancer_configs": [ + { + "short_stream": false, + "transport_sec": "alts" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [ + { + "transport_sec": "insecure" + } + ], + "name": "client_referred_to_backend_fallback_broken_alts_short_stream_False", + "skip_langs": [], + "transport_sec": "alts" + }, + { + "backend_configs": [ + { + "transport_sec": "tls" + } + ], + "balancer_configs": [ + { + "short_stream": false, + "transport_sec": "tls" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [ + { + "transport_sec": "insecure" + } + ], + "name": "client_referred_to_backend_fallback_broken_tls_short_stream_False", + "skip_langs": [ + "java" + ], + "transport_sec": "tls" + }, + { + "backend_configs": [ + { + "transport_sec": "alts" + } + ], + "balancer_configs": [ + { + "short_stream": false, + "transport_sec": "alts" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [ + { + "transport_sec": "insecure" + } + ], + "name": "client_referred_to_backend_fallback_broken_google_default_credentials_short_stream_False", + "skip_langs": [], + "transport_sec": "google_default_credentials" + }, + { + "backend_configs": [ + { + "transport_sec": "insecure" + }, + { + "transport_sec": "insecure" + }, + { + "transport_sec": "insecure" + }, + { + "transport_sec": "insecure" + }, + { + "transport_sec": "insecure" + } + ], + "balancer_configs": [ + { + "short_stream": true, + "transport_sec": "insecure" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [], + "name": "client_referred_to_backend_multiple_backends_insecure_short_stream_True", + "skip_langs": [ + "java" + ], + "transport_sec": "insecure" + }, + { + "backend_configs": [ + { + "transport_sec": "alts" + }, + { + "transport_sec": "alts" + }, + { + "transport_sec": "alts" + }, + { + "transport_sec": "alts" + }, + { + "transport_sec": "alts" + } + ], + "balancer_configs": [ + { + "short_stream": true, + "transport_sec": "alts" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [], + "name": "client_referred_to_backend_multiple_backends_alts_short_stream_True", + "skip_langs": [ + "java" + ], + "transport_sec": "alts" + }, + { + "backend_configs": [ + { + "transport_sec": "tls" + }, + { + "transport_sec": "tls" + }, + { + "transport_sec": "tls" + }, + { + "transport_sec": "tls" + }, + { + "transport_sec": "tls" + } + ], + "balancer_configs": [ + { + "short_stream": true, + "transport_sec": "tls" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [], + "name": "client_referred_to_backend_multiple_backends_tls_short_stream_True", + "skip_langs": [ + "java", + "java" + ], + "transport_sec": "tls" + }, + { + "backend_configs": [ + { + "transport_sec": "alts" + }, + { + "transport_sec": "alts" + }, + { + "transport_sec": "alts" + }, + { + "transport_sec": "alts" + }, + { + "transport_sec": "alts" + } + ], + "balancer_configs": [ + { + "short_stream": true, + "transport_sec": "alts" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [], + "name": "client_referred_to_backend_multiple_backends_google_default_credentials_short_stream_True", + "skip_langs": [ + "java" + ], + "transport_sec": "google_default_credentials" + }, + { + "backend_configs": [ + { + "transport_sec": "insecure" + }, + { + "transport_sec": "insecure" + }, + { + "transport_sec": "insecure" + }, + { + "transport_sec": "insecure" + }, + { + "transport_sec": "insecure" + } + ], + "balancer_configs": [ + { + "short_stream": false, + "transport_sec": "insecure" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [], + "name": "client_referred_to_backend_multiple_backends_insecure_short_stream_False", + "skip_langs": [], + "transport_sec": "insecure" + }, + { + "backend_configs": [ + { + "transport_sec": "alts" + }, + { + "transport_sec": "alts" + }, + { + "transport_sec": "alts" + }, + { + "transport_sec": "alts" + }, + { + "transport_sec": "alts" + } + ], + "balancer_configs": [ + { + "short_stream": false, + "transport_sec": "alts" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [], + "name": "client_referred_to_backend_multiple_backends_alts_short_stream_False", + "skip_langs": [], + "transport_sec": "alts" + }, + { + "backend_configs": [ + { + "transport_sec": "tls" + }, + { + "transport_sec": "tls" + }, + { + "transport_sec": "tls" + }, + { + "transport_sec": "tls" + }, + { + "transport_sec": "tls" + } + ], + "balancer_configs": [ + { + "short_stream": false, + "transport_sec": "tls" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [], + "name": "client_referred_to_backend_multiple_backends_tls_short_stream_False", + "skip_langs": [ + "java" + ], + "transport_sec": "tls" + }, + { + "backend_configs": [ + { + "transport_sec": "alts" + }, + { + "transport_sec": "alts" + }, + { + "transport_sec": "alts" + }, + { + "transport_sec": "alts" + }, + { + "transport_sec": "alts" + } + ], + "balancer_configs": [ + { + "short_stream": false, + "transport_sec": "alts" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [], + "name": "client_referred_to_backend_multiple_backends_google_default_credentials_short_stream_False", + "skip_langs": [], + "transport_sec": "google_default_credentials" + }, + { + "backend_configs": [], + "balancer_configs": [ + { + "short_stream": true, + "transport_sec": "insecure" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [ + { + "transport_sec": "insecure" + } + ], + "name": "client_falls_back_because_no_backends_insecure_short_stream_True", + "skip_langs": [ + "go", + "java", + "java" + ], + "transport_sec": "insecure" + }, + { + "backend_configs": [], + "balancer_configs": [ + { + "short_stream": true, + "transport_sec": "alts" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [ + { + "transport_sec": "alts" + } + ], + "name": "client_falls_back_because_no_backends_alts_short_stream_True", + "skip_langs": [ + "go", + "java", + "java" + ], + "transport_sec": "alts" + }, + { + "backend_configs": [], + "balancer_configs": [ + { + "short_stream": true, + "transport_sec": "tls" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [ + { + "transport_sec": "tls" + } + ], + "name": "client_falls_back_because_no_backends_tls_short_stream_True", + "skip_langs": [ + "go", + "java", + "java", + "java" + ], + "transport_sec": "tls" + }, + { + "backend_configs": [], + "balancer_configs": [ + { + "short_stream": true, + "transport_sec": "alts" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [ + { + "transport_sec": "tls" + } + ], + "name": "client_falls_back_because_no_backends_google_default_credentials_short_stream_True", + "skip_langs": [ + "go", + "java", + "java" + ], + "transport_sec": "google_default_credentials" + }, + { + "backend_configs": [], + "balancer_configs": [ + { + "short_stream": false, + "transport_sec": "insecure" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [ + { + "transport_sec": "insecure" + } + ], + "name": "client_falls_back_because_no_backends_insecure_short_stream_False", + "skip_langs": [ + "go", + "java" + ], + "transport_sec": "insecure" + }, + { + "backend_configs": [], + "balancer_configs": [ + { + "short_stream": false, + "transport_sec": "alts" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [ + { + "transport_sec": "alts" + } + ], + "name": "client_falls_back_because_no_backends_alts_short_stream_False", + "skip_langs": [ + "go", + "java" + ], + "transport_sec": "alts" + }, + { + "backend_configs": [], + "balancer_configs": [ + { + "short_stream": false, + "transport_sec": "tls" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [ + { + "transport_sec": "tls" + } + ], + "name": "client_falls_back_because_no_backends_tls_short_stream_False", + "skip_langs": [ + "go", + "java", + "java" + ], + "transport_sec": "tls" + }, + { + "backend_configs": [], + "balancer_configs": [ + { + "short_stream": false, + "transport_sec": "alts" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [ + { + "transport_sec": "tls" + } + ], + "name": "client_falls_back_because_no_backends_google_default_credentials_short_stream_False", + "skip_langs": [ + "go", + "java" + ], + "transport_sec": "google_default_credentials" + }, + { + "backend_configs": [], + "balancer_configs": [ + { + "short_stream": false, + "transport_sec": "insecure" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [ + { + "transport_sec": "alts" + } + ], + "name": "client_falls_back_because_balancer_connection_broken_alts", + "skip_langs": [], + "transport_sec": "alts" + }, + { + "backend_configs": [], + "balancer_configs": [ + { + "short_stream": false, + "transport_sec": "insecure" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [ + { + "transport_sec": "tls" + } + ], + "name": "client_falls_back_because_balancer_connection_broken_tls", + "skip_langs": [ + "java" + ], + "transport_sec": "tls" + }, + { + "backend_configs": [], + "balancer_configs": [ + { + "short_stream": false, + "transport_sec": "insecure" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [ + { + "transport_sec": "tls" + } + ], + "name": "client_falls_back_because_balancer_connection_broken_google_default_credentials", + "skip_langs": [], + "transport_sec": "google_default_credentials" + }, + { + "backend_configs": [ + { + "transport_sec": "insecure" + } + ], + "balancer_configs": [ + { + "short_stream": true, + "transport_sec": "insecure" + }, + { + "short_stream": true, + "transport_sec": "insecure" + }, + { + "short_stream": true, + "transport_sec": "insecure" + }, + { + "short_stream": true, + "transport_sec": "insecure" + }, + { + "short_stream": true, + "transport_sec": "insecure" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [], + "name": "client_referred_to_backend_multiple_balancers_insecure_short_stream_True", + "skip_langs": [ + "java" + ], + "transport_sec": "insecure" + }, + { + "backend_configs": [ + { + "transport_sec": "alts" + } + ], + "balancer_configs": [ + { + "short_stream": true, + "transport_sec": "alts" + }, + { + "short_stream": true, + "transport_sec": "alts" + }, + { + "short_stream": true, + "transport_sec": "alts" + }, + { + "short_stream": true, + "transport_sec": "alts" + }, + { + "short_stream": true, + "transport_sec": "alts" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [], + "name": "client_referred_to_backend_multiple_balancers_alts_short_stream_True", + "skip_langs": [ + "java" + ], + "transport_sec": "alts" + }, + { + "backend_configs": [ + { + "transport_sec": "tls" + } + ], + "balancer_configs": [ + { + "short_stream": true, + "transport_sec": "tls" + }, + { + "short_stream": true, + "transport_sec": "tls" + }, + { + "short_stream": true, + "transport_sec": "tls" + }, + { + "short_stream": true, + "transport_sec": "tls" + }, + { + "short_stream": true, + "transport_sec": "tls" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [], + "name": "client_referred_to_backend_multiple_balancers_tls_short_stream_True", + "skip_langs": [ + "java", + "java" + ], + "transport_sec": "tls" + }, + { + "backend_configs": [ + { + "transport_sec": "alts" + } + ], + "balancer_configs": [ + { + "short_stream": true, + "transport_sec": "alts" + }, + { + "short_stream": true, + "transport_sec": "alts" + }, + { + "short_stream": true, + "transport_sec": "alts" + }, + { + "short_stream": true, + "transport_sec": "alts" + }, + { + "short_stream": true, + "transport_sec": "alts" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [], + "name": "client_referred_to_backend_multiple_balancers_google_default_credentials_short_stream_True", + "skip_langs": [ + "java" + ], + "transport_sec": "google_default_credentials" + }, + { + "backend_configs": [ + { + "transport_sec": "insecure" + } + ], + "balancer_configs": [ + { + "short_stream": false, + "transport_sec": "insecure" + }, + { + "short_stream": false, + "transport_sec": "insecure" + }, + { + "short_stream": false, + "transport_sec": "insecure" + }, + { + "short_stream": false, + "transport_sec": "insecure" + }, + { + "short_stream": false, + "transport_sec": "insecure" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [], + "name": "client_referred_to_backend_multiple_balancers_insecure_short_stream_False", + "skip_langs": [], + "transport_sec": "insecure" + }, + { + "backend_configs": [ + { + "transport_sec": "alts" + } + ], + "balancer_configs": [ + { + "short_stream": false, + "transport_sec": "alts" + }, + { + "short_stream": false, + "transport_sec": "alts" + }, + { + "short_stream": false, + "transport_sec": "alts" + }, + { + "short_stream": false, + "transport_sec": "alts" + }, + { + "short_stream": false, + "transport_sec": "alts" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [], + "name": "client_referred_to_backend_multiple_balancers_alts_short_stream_False", + "skip_langs": [], + "transport_sec": "alts" + }, + { + "backend_configs": [ + { + "transport_sec": "tls" + } + ], + "balancer_configs": [ + { + "short_stream": false, + "transport_sec": "tls" + }, + { + "short_stream": false, + "transport_sec": "tls" + }, + { + "short_stream": false, + "transport_sec": "tls" + }, + { + "short_stream": false, + "transport_sec": "tls" + }, + { + "short_stream": false, + "transport_sec": "tls" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [], + "name": "client_referred_to_backend_multiple_balancers_tls_short_stream_False", + "skip_langs": [ + "java" + ], + "transport_sec": "tls" + }, + { + "backend_configs": [ + { + "transport_sec": "alts" + } + ], + "balancer_configs": [ + { + "short_stream": false, + "transport_sec": "alts" + }, + { + "short_stream": false, + "transport_sec": "alts" + }, + { + "short_stream": false, + "transport_sec": "alts" + }, + { + "short_stream": false, + "transport_sec": "alts" + }, + { + "short_stream": false, + "transport_sec": "alts" + } + ], + "cause_no_error_no_data_for_balancer_a_record": false, + "fallback_configs": [], + "name": "client_referred_to_backend_multiple_balancers_google_default_credentials_short_stream_False", + "skip_langs": [], + "transport_sec": "google_default_credentials" + } +] diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index f3e93a0874..47bc525f0d 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -493,23 +493,6 @@ "headers": [], "is_filegroup": false, "language": "c", - "name": "ev_epollsig_linux_test", - "src": [ - "test/core/iomgr/ev_epollsig_linux_test.cc" - ], - "third_party": false, - "type": "target" - }, - { - "deps": [ - "gpr", - "gpr_test_util", - "grpc", - "grpc_test_util" - ], - "headers": [], - "is_filegroup": false, - "language": "c", "name": "fake_resolver_test", "src": [ "test/core/client_channel/resolvers/fake_resolver_test.cc" @@ -1176,7 +1159,7 @@ "headers": [], "is_filegroup": false, "language": "c", - "name": "handshake_client", + "name": "handshake_client_ssl", "src": [ "test/core/handshake/client_ssl.cc" ], @@ -1195,7 +1178,7 @@ ], "is_filegroup": false, "language": "c", - "name": "handshake_server", + "name": "handshake_server_ssl", "src": [ "test/core/handshake/server_ssl.cc", "test/core/handshake/server_ssl_common.cc", @@ -1610,7 +1593,7 @@ "headers": [], "is_filegroup": false, "language": "c", - "name": "memory_profile_client", + "name": "memory_usage_client", "src": [ "test/core/memory_usage/client.cc" ], @@ -1627,7 +1610,7 @@ "headers": [], "is_filegroup": false, "language": "c", - "name": "memory_profile_server", + "name": "memory_usage_server", "src": [ "test/core/memory_usage/server.cc" ], @@ -1644,7 +1627,7 @@ "headers": [], "is_filegroup": false, "language": "c", - "name": "memory_profile_test", + "name": "memory_usage_test", "src": [ "test/core/memory_usage/memory_usage_test.cc" ], @@ -1863,23 +1846,6 @@ "headers": [], "is_filegroup": false, "language": "c", - "name": "pollset_set_test", - "src": [ - "test/core/iomgr/pollset_set_test.cc" - ], - "third_party": false, - "type": "target" - }, - { - "deps": [ - "gpr", - "gpr_test_util", - "grpc", - "grpc_test_util" - ], - "headers": [], - "is_filegroup": false, - "language": "c", "name": "resolve_address_posix_test", "src": [ "test/core/iomgr/resolve_address_posix_test.cc" @@ -3428,6 +3394,29 @@ "grpc++_test_util", "grpc_test_util" ], + "headers": [ + "test/cpp/end2end/interceptors_util.h" + ], + "is_filegroup": false, + "language": "c++", + "name": "client_interceptors_end2end_test", + "src": [ + "test/cpp/end2end/client_interceptors_end2end_test.cc", + "test/cpp/end2end/interceptors_util.cc", + "test/cpp/end2end/interceptors_util.h" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_util", + "grpc_test_util" + ], "headers": [], "is_filegroup": false, "language": "c++", @@ -3612,12 +3601,16 @@ "grpc++_test_util", "grpc_test_util" ], - "headers": [], + "headers": [ + "test/cpp/end2end/interceptors_util.h" + ], "is_filegroup": false, "language": "c++", "name": "end2end_test", "src": [ - "test/cpp/end2end/end2end_test.cc" + "test/cpp/end2end/end2end_test.cc", + "test/cpp/end2end/interceptors_util.cc", + "test/cpp/end2end/interceptors_util.h" ], "third_party": false, "type": "target" @@ -4736,6 +4729,29 @@ "deps": [ "gpr", "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test_util", + "grpc_test_util" + ], + "headers": [ + "test/cpp/end2end/interceptors_util.h" + ], + "is_filegroup": false, + "language": "c++", + "name": "server_interceptors_end2end_test", + "src": [ + "test/cpp/end2end/interceptors_util.cc", + "test/cpp/end2end/interceptors_util.h", + "test/cpp/end2end/server_interceptors_end2end_test.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", "grpc++_test_util_unsecure", "grpc++_unsecure", "grpc_test_util_unsecure", @@ -6591,24 +6607,6 @@ }, { "deps": [ - "end2end_nosec_tests", - "gpr", - "gpr_test_util", - "grpc_test_util_unsecure", - "grpc_unsecure" - ], - "headers": [], - "is_filegroup": false, - "language": "c", - "name": "inproc_nosec_test", - "src": [ - "test/core/end2end/fixtures/inproc.cc" - ], - "third_party": false, - "type": "target" - }, - { - "deps": [ "gpr", "gpr_test_util", "grpc++_test_config", @@ -7081,6 +7079,7 @@ "grpc_lb_policy_grpclb_secure", "grpc_lb_policy_pick_first", "grpc_lb_policy_round_robin", + "grpc_lb_policy_xds_secure", "grpc_max_age_filter", "grpc_message_size_filter", "grpc_resolver_dns_ares", @@ -7126,19 +7125,6 @@ { "deps": [ "gpr", - "grpc" - ], - "headers": [], - "is_filegroup": false, - "language": "c", - "name": "grpc_dll", - "src": [], - "third_party": false, - "type": "lib" - }, - { - "deps": [ - "gpr", "gpr_test_util", "grpc", "grpc_test_util_base" @@ -7187,6 +7173,7 @@ "grpc_lb_policy_grpclb", "grpc_lb_policy_pick_first", "grpc_lb_policy_round_robin", + "grpc_lb_policy_xds", "grpc_max_age_filter", "grpc_message_size_filter", "grpc_resolver_dns_ares", @@ -7883,6 +7870,7 @@ "test/cpp/qps/benchmark_config.h", "test/cpp/qps/client.h", "test/cpp/qps/client_async.cc", + "test/cpp/qps/client_callback.cc", "test/cpp/qps/client_sync.cc", "test/cpp/qps/driver.cc", "test/cpp/qps/driver.h", @@ -7898,6 +7886,7 @@ "test/cpp/qps/report.h", "test/cpp/qps/server.h", "test/cpp/qps/server_async.cc", + "test/cpp/qps/server_callback.cc", "test/cpp/qps/server_sync.cc", "test/cpp/qps/stats.h", "test/cpp/qps/usage_timer.cc", @@ -9154,7 +9143,7 @@ "src/core/tsi/alts/frame_protector/alts_record_protocol_crypter_common.h", "src/core/tsi/alts/frame_protector/frame_handler.h", "src/core/tsi/alts/handshaker/alts_handshaker_client.h", - "src/core/tsi/alts/handshaker/alts_tsi_event.h", + "src/core/tsi/alts/handshaker/alts_shared_resource.h", "src/core/tsi/alts/handshaker/alts_tsi_handshaker.h", "src/core/tsi/alts/handshaker/alts_tsi_handshaker_private.h", "src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_integrity_only_record_protocol.h", @@ -9185,8 +9174,8 @@ "src/core/tsi/alts/frame_protector/frame_handler.h", "src/core/tsi/alts/handshaker/alts_handshaker_client.cc", "src/core/tsi/alts/handshaker/alts_handshaker_client.h", - "src/core/tsi/alts/handshaker/alts_tsi_event.cc", - "src/core/tsi/alts/handshaker/alts_tsi_event.h", + "src/core/tsi/alts/handshaker/alts_shared_resource.cc", + "src/core/tsi/alts/handshaker/alts_shared_resource.h", "src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc", "src/core/tsi/alts/handshaker/alts_tsi_handshaker.h", "src/core/tsi/alts/handshaker/alts_tsi_handshaker_private.h", @@ -9551,7 +9540,6 @@ "src/core/lib/iomgr/error.cc", "src/core/lib/iomgr/ev_epoll1_linux.cc", "src/core/lib/iomgr/ev_epollex_linux.cc", - "src/core/lib/iomgr/ev_epollsig_linux.cc", "src/core/lib/iomgr/ev_poll_posix.cc", "src/core/lib/iomgr/ev_posix.cc", "src/core/lib/iomgr/ev_windows.cc", @@ -9667,7 +9655,8 @@ "src/core/lib/transport/status_metadata.cc", "src/core/lib/transport/timeout_encoding.cc", "src/core/lib/transport/transport.cc", - "src/core/lib/transport/transport_op_string.cc" + "src/core/lib/transport/transport_op_string.cc", + "src/core/lib/uri/uri_parser.cc" ], "third_party": false, "type": "filegroup" @@ -9732,7 +9721,6 @@ "src/core/lib/iomgr/error_internal.h", "src/core/lib/iomgr/ev_epoll1_linux.h", "src/core/lib/iomgr/ev_epollex_linux.h", - "src/core/lib/iomgr/ev_epollsig_linux.h", "src/core/lib/iomgr/ev_poll_posix.h", "src/core/lib/iomgr/ev_posix.h", "src/core/lib/iomgr/exec_ctx.h", @@ -9825,7 +9813,8 @@ "src/core/lib/transport/status_metadata.h", "src/core/lib/transport/timeout_encoding.h", "src/core/lib/transport/transport.h", - "src/core/lib/transport/transport_impl.h" + "src/core/lib/transport/transport_impl.h", + "src/core/lib/uri/uri_parser.h" ], "is_filegroup": true, "language": "c", @@ -9884,7 +9873,6 @@ "src/core/lib/iomgr/error_internal.h", "src/core/lib/iomgr/ev_epoll1_linux.h", "src/core/lib/iomgr/ev_epollex_linux.h", - "src/core/lib/iomgr/ev_epollsig_linux.h", "src/core/lib/iomgr/ev_poll_posix.h", "src/core/lib/iomgr/ev_posix.h", "src/core/lib/iomgr/exec_ctx.h", @@ -9977,7 +9965,8 @@ "src/core/lib/transport/status_metadata.h", "src/core/lib/transport/timeout_encoding.h", "src/core/lib/transport/transport.h", - "src/core/lib/transport/transport_impl.h" + "src/core/lib/transport/transport_impl.h", + "src/core/lib/uri/uri_parser.h" ], "third_party": false, "type": "filegroup" @@ -10031,7 +10020,8 @@ "deps": [ "gpr", "grpc_base", - "grpc_deadline_filter" + "grpc_deadline_filter", + "health_proto" ], "headers": [ "src/core/ext/filters/client_channel/backup_poller.h", @@ -10039,6 +10029,7 @@ "src/core/ext/filters/client_channel/client_channel_channelz.h", "src/core/ext/filters/client_channel/client_channel_factory.h", "src/core/ext/filters/client_channel/connector.h", + "src/core/ext/filters/client_channel/health/health_check_client.h", "src/core/ext/filters/client_channel/http_connect_handshaker.h", "src/core/ext/filters/client_channel/http_proxy.h", "src/core/ext/filters/client_channel/lb_policy.h", @@ -10053,8 +10044,7 @@ "src/core/ext/filters/client_channel/resolver_registry.h", "src/core/ext/filters/client_channel/retry_throttle.h", "src/core/ext/filters/client_channel/subchannel.h", - "src/core/ext/filters/client_channel/subchannel_index.h", - "src/core/ext/filters/client_channel/uri_parser.h" + "src/core/ext/filters/client_channel/subchannel_index.h" ], "is_filegroup": true, "language": "c", @@ -10072,6 +10062,8 @@ "src/core/ext/filters/client_channel/client_channel_plugin.cc", "src/core/ext/filters/client_channel/connector.cc", "src/core/ext/filters/client_channel/connector.h", + "src/core/ext/filters/client_channel/health/health_check_client.cc", + "src/core/ext/filters/client_channel/health/health_check_client.h", "src/core/ext/filters/client_channel/http_connect_handshaker.cc", "src/core/ext/filters/client_channel/http_connect_handshaker.h", "src/core/ext/filters/client_channel/http_proxy.cc", @@ -10100,9 +10092,7 @@ "src/core/ext/filters/client_channel/subchannel.cc", "src/core/ext/filters/client_channel/subchannel.h", "src/core/ext/filters/client_channel/subchannel_index.cc", - "src/core/ext/filters/client_channel/subchannel_index.h", - "src/core/ext/filters/client_channel/uri_parser.cc", - "src/core/ext/filters/client_channel/uri_parser.h" + "src/core/ext/filters/client_channel/subchannel_index.h" ], "third_party": false, "type": "filegroup" @@ -10186,6 +10176,7 @@ "grpc_base", "grpc_client_channel", "grpc_resolver_fake", + "grpclb_proto", "nanopb" ], "headers": [ @@ -10193,10 +10184,7 @@ "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h", "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.h", "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h" + "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h" ], "is_filegroup": true, "language": "c", @@ -10211,13 +10199,7 @@ "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc", "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h", "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc", - "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.c", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.c", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h" + "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h" ], "third_party": false, "type": "filegroup" @@ -10229,6 +10211,7 @@ "grpc_client_channel", "grpc_resolver_fake", "grpc_secure", + "grpclb_proto", "nanopb" ], "headers": [ @@ -10236,10 +10219,7 @@ "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h", "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.h", "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h" + "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h" ], "is_filegroup": true, "language": "c", @@ -10254,13 +10234,7 @@ "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc", "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h", "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc", - "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.c", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.c", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.h", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c", - "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h" + "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h" ], "third_party": false, "type": "filegroup" @@ -10303,6 +10277,69 @@ "deps": [ "gpr", "grpc_base", + "grpc_client_channel", + "grpc_resolver_fake", + "grpclb_proto", + "nanopb" + ], + "headers": [ + "src/core/ext/filters/client_channel/lb_policy/xds/xds.h", + "src/core/ext/filters/client_channel/lb_policy/xds/xds_channel.h", + "src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.h", + "src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.h" + ], + "is_filegroup": true, + "language": "c", + "name": "grpc_lb_policy_xds", + "src": [ + "src/core/ext/filters/client_channel/lb_policy/xds/xds.cc", + "src/core/ext/filters/client_channel/lb_policy/xds/xds.h", + "src/core/ext/filters/client_channel/lb_policy/xds/xds_channel.cc", + "src/core/ext/filters/client_channel/lb_policy/xds/xds_channel.h", + "src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.cc", + "src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.h", + "src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.cc", + "src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.h" + ], + "third_party": false, + "type": "filegroup" + }, + { + "deps": [ + "gpr", + "grpc_base", + "grpc_client_channel", + "grpc_resolver_fake", + "grpc_secure", + "grpclb_proto", + "nanopb" + ], + "headers": [ + "src/core/ext/filters/client_channel/lb_policy/xds/xds.h", + "src/core/ext/filters/client_channel/lb_policy/xds/xds_channel.h", + "src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.h", + "src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.h" + ], + "is_filegroup": true, + "language": "c", + "name": "grpc_lb_policy_xds_secure", + "src": [ + "src/core/ext/filters/client_channel/lb_policy/xds/xds.cc", + "src/core/ext/filters/client_channel/lb_policy/xds/xds.h", + "src/core/ext/filters/client_channel/lb_policy/xds/xds_channel.h", + "src/core/ext/filters/client_channel/lb_policy/xds/xds_channel_secure.cc", + "src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.cc", + "src/core/ext/filters/client_channel/lb_policy/xds/xds_client_stats.h", + "src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.cc", + "src/core/ext/filters/client_channel/lb_policy/xds/xds_load_balancer_api.h" + ], + "third_party": false, + "type": "filegroup" + }, + { + "deps": [ + "gpr", + "grpc_base", "grpc_client_channel" ], "headers": [ @@ -10444,6 +10481,7 @@ "headers": [ "include/grpc/grpc_security.h", "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h", + "src/core/ext/filters/client_channel/lb_policy/xds/xds.h", "src/core/lib/security/context/security_context.h", "src/core/lib/security/credentials/alts/alts_credentials.h", "src/core/lib/security/credentials/composite/composite_credentials.h", @@ -10458,11 +10496,14 @@ "src/core/lib/security/credentials/oauth2/oauth2_credentials.h", "src/core/lib/security/credentials/plugin/plugin_credentials.h", "src/core/lib/security/credentials/ssl/ssl_credentials.h", - "src/core/lib/security/security_connector/alts_security_connector.h", + "src/core/lib/security/security_connector/alts/alts_security_connector.h", + "src/core/lib/security/security_connector/fake/fake_security_connector.h", "src/core/lib/security/security_connector/load_system_roots.h", "src/core/lib/security/security_connector/load_system_roots_linux.h", - "src/core/lib/security/security_connector/local_security_connector.h", + "src/core/lib/security/security_connector/local/local_security_connector.h", "src/core/lib/security/security_connector/security_connector.h", + "src/core/lib/security/security_connector/ssl/ssl_security_connector.h", + "src/core/lib/security/security_connector/ssl_utils.h", "src/core/lib/security/transport/auth_filters.h", "src/core/lib/security/transport/secure_endpoint.h", "src/core/lib/security/transport/security_handshaker.h", @@ -10476,6 +10517,7 @@ "src": [ "include/grpc/grpc_security.h", "src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h", + "src/core/ext/filters/client_channel/lb_policy/xds/xds.h", "src/core/lib/http/httpcli_security_connector.cc", "src/core/lib/security/context/security_context.cc", "src/core/lib/security/context/security_context.h", @@ -10507,16 +10549,22 @@ "src/core/lib/security/credentials/plugin/plugin_credentials.h", "src/core/lib/security/credentials/ssl/ssl_credentials.cc", "src/core/lib/security/credentials/ssl/ssl_credentials.h", - "src/core/lib/security/security_connector/alts_security_connector.cc", - "src/core/lib/security/security_connector/alts_security_connector.h", + "src/core/lib/security/security_connector/alts/alts_security_connector.cc", + "src/core/lib/security/security_connector/alts/alts_security_connector.h", + "src/core/lib/security/security_connector/fake/fake_security_connector.cc", + "src/core/lib/security/security_connector/fake/fake_security_connector.h", "src/core/lib/security/security_connector/load_system_roots.h", "src/core/lib/security/security_connector/load_system_roots_fallback.cc", "src/core/lib/security/security_connector/load_system_roots_linux.cc", "src/core/lib/security/security_connector/load_system_roots_linux.h", - "src/core/lib/security/security_connector/local_security_connector.cc", - "src/core/lib/security/security_connector/local_security_connector.h", + "src/core/lib/security/security_connector/local/local_security_connector.cc", + "src/core/lib/security/security_connector/local/local_security_connector.h", "src/core/lib/security/security_connector/security_connector.cc", "src/core/lib/security/security_connector/security_connector.h", + "src/core/lib/security/security_connector/ssl/ssl_security_connector.cc", + "src/core/lib/security/security_connector/ssl/ssl_security_connector.h", + "src/core/lib/security/security_connector/ssl_utils.cc", + "src/core/lib/security/security_connector/ssl_utils.h", "src/core/lib/security/transport/auth_filters.h", "src/core/lib/security/transport/client_auth_filter.cc", "src/core/lib/security/transport/secure_endpoint.cc", @@ -10972,6 +11020,46 @@ }, { "deps": [ + "nanopb" + ], + "headers": [ + "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h", + "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.h", + "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h" + ], + "is_filegroup": true, + "language": "c", + "name": "grpclb_proto", + "src": [ + "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.c", + "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/duration.pb.h", + "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.c", + "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/google/protobuf/timestamp.pb.h", + "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c", + "src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h" + ], + "third_party": false, + "type": "filegroup" + }, + { + "deps": [ + "nanopb" + ], + "headers": [ + "src/core/ext/filters/client_channel/health/health.pb.h" + ], + "is_filegroup": true, + "language": "c", + "name": "health_proto", + "src": [ + "src/core/ext/filters/client_channel/health/health.pb.c", + "src/core/ext/filters/client_channel/health/health.pb.h" + ], + "third_party": false, + "type": "filegroup" + }, + { + "deps": [ "nanopb_headers" ], "headers": [], @@ -11117,10 +11205,13 @@ "include/grpcpp/impl/codegen/byte_buffer.h", "include/grpcpp/impl/codegen/call.h", "include/grpcpp/impl/codegen/call_hook.h", + "include/grpcpp/impl/codegen/call_op_set.h", + "include/grpcpp/impl/codegen/call_op_set_interface.h", "include/grpcpp/impl/codegen/callback_common.h", "include/grpcpp/impl/codegen/channel_interface.h", "include/grpcpp/impl/codegen/client_callback.h", "include/grpcpp/impl/codegen/client_context.h", + "include/grpcpp/impl/codegen/client_interceptor.h", "include/grpcpp/impl/codegen/client_unary_call.h", "include/grpcpp/impl/codegen/completion_queue.h", "include/grpcpp/impl/codegen/completion_queue_tag.h", @@ -11128,13 +11219,18 @@ "include/grpcpp/impl/codegen/core_codegen_interface.h", "include/grpcpp/impl/codegen/create_auth_context.h", "include/grpcpp/impl/codegen/grpc_library.h", + "include/grpcpp/impl/codegen/intercepted_channel.h", + "include/grpcpp/impl/codegen/interceptor.h", + "include/grpcpp/impl/codegen/interceptor_common.h", "include/grpcpp/impl/codegen/metadata_map.h", "include/grpcpp/impl/codegen/method_handler_impl.h", "include/grpcpp/impl/codegen/rpc_method.h", "include/grpcpp/impl/codegen/rpc_service_method.h", "include/grpcpp/impl/codegen/security/auth_context.h", "include/grpcpp/impl/codegen/serialization_traits.h", + "include/grpcpp/impl/codegen/server_callback.h", "include/grpcpp/impl/codegen/server_context.h", + "include/grpcpp/impl/codegen/server_interceptor.h", "include/grpcpp/impl/codegen/server_interface.h", "include/grpcpp/impl/codegen/service_type.h", "include/grpcpp/impl/codegen/slice.h", @@ -11185,10 +11281,13 @@ "include/grpcpp/impl/codegen/byte_buffer.h", "include/grpcpp/impl/codegen/call.h", "include/grpcpp/impl/codegen/call_hook.h", + "include/grpcpp/impl/codegen/call_op_set.h", + "include/grpcpp/impl/codegen/call_op_set_interface.h", "include/grpcpp/impl/codegen/callback_common.h", "include/grpcpp/impl/codegen/channel_interface.h", "include/grpcpp/impl/codegen/client_callback.h", "include/grpcpp/impl/codegen/client_context.h", + "include/grpcpp/impl/codegen/client_interceptor.h", "include/grpcpp/impl/codegen/client_unary_call.h", "include/grpcpp/impl/codegen/completion_queue.h", "include/grpcpp/impl/codegen/completion_queue_tag.h", @@ -11196,13 +11295,18 @@ "include/grpcpp/impl/codegen/core_codegen_interface.h", "include/grpcpp/impl/codegen/create_auth_context.h", "include/grpcpp/impl/codegen/grpc_library.h", + "include/grpcpp/impl/codegen/intercepted_channel.h", + "include/grpcpp/impl/codegen/interceptor.h", + "include/grpcpp/impl/codegen/interceptor_common.h", "include/grpcpp/impl/codegen/metadata_map.h", "include/grpcpp/impl/codegen/method_handler_impl.h", "include/grpcpp/impl/codegen/rpc_method.h", "include/grpcpp/impl/codegen/rpc_service_method.h", "include/grpcpp/impl/codegen/security/auth_context.h", "include/grpcpp/impl/codegen/serialization_traits.h", + "include/grpcpp/impl/codegen/server_callback.h", "include/grpcpp/impl/codegen/server_context.h", + "include/grpcpp/impl/codegen/server_interceptor.h", "include/grpcpp/impl/codegen/server_interface.h", "include/grpcpp/impl/codegen/service_type.h", "include/grpcpp/impl/codegen/slice.h", @@ -11260,6 +11364,7 @@ "grpc++_codegen_base", "grpc_base_headers", "grpc_transport_inproc_headers", + "health_proto", "nanopb_headers" ], "headers": [ @@ -11349,6 +11454,7 @@ "include/grpcpp/support/config.h", "include/grpcpp/support/proto_buffer_reader.h", "include/grpcpp/support/proto_buffer_writer.h", + "include/grpcpp/support/server_callback.h", "include/grpcpp/support/slice.h", "include/grpcpp/support/status.h", "include/grpcpp/support/status_code_enum.h", @@ -11360,7 +11466,6 @@ "src/cpp/common/channel_filter.h", "src/cpp/server/dynamic_thread_pool.h", "src/cpp/server/health/default_health_check_service.h", - "src/cpp/server/health/health.pb.h", "src/cpp/server/thread_pool_interface.h", "src/cpp/thread_manager/thread_manager.h" ], @@ -11454,6 +11559,7 @@ "include/grpcpp/support/config.h", "include/grpcpp/support/proto_buffer_reader.h", "include/grpcpp/support/proto_buffer_writer.h", + "include/grpcpp/support/server_callback.h", "include/grpcpp/support/slice.h", "include/grpcpp/support/status.h", "include/grpcpp/support/status_code_enum.h", @@ -11463,6 +11569,7 @@ "include/grpcpp/support/time.h", "src/cpp/client/channel_cc.cc", "src/cpp/client/client_context.cc", + "src/cpp/client/client_interceptor.cc", "src/cpp/client/create_channel.cc", "src/cpp/client/create_channel_internal.cc", "src/cpp/client/create_channel_internal.h", @@ -11470,7 +11577,6 @@ "src/cpp/client/credentials_cc.cc", "src/cpp/client/generic_stub.cc", "src/cpp/common/alarm.cc", - "src/cpp/common/callback_common.cc", "src/cpp/common/channel_arguments.cc", "src/cpp/common/channel_filter.cc", "src/cpp/common/channel_filter.h", @@ -11486,8 +11592,6 @@ "src/cpp/server/dynamic_thread_pool.h", "src/cpp/server/health/default_health_check_service.cc", "src/cpp/server/health/default_health_check_service.h", - "src/cpp/server/health/health.pb.c", - "src/cpp/server/health/health.pb.h", "src/cpp/server/health/health_check_service.cc", "src/cpp/server/health/health_check_service_server_builder_option.cc", "src/cpp/server/server_builder.cc", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index b3c07d9215..ef34cd6556 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -605,26 +605,6 @@ "args": [], "benchmark": false, "ci_platforms": [ - "linux" - ], - "cpu_cost": 3, - "exclude_configs": [], - "exclude_iomgrs": [ - "uv" - ], - "flaky": false, - "gtest": false, - "language": "c", - "name": "ev_epollsig_linux_test", - "platforms": [ - "linux" - ], - "uses_polling": true - }, - { - "args": [], - "benchmark": false, - "ci_platforms": [ "linux", "mac", "posix", @@ -1467,7 +1447,7 @@ "flaky": false, "gtest": false, "language": "c", - "name": "handshake_client", + "name": "handshake_client_ssl", "platforms": [ "linux" ], @@ -1487,7 +1467,7 @@ "flaky": false, "gtest": false, "language": "c", - "name": "handshake_server", + "name": "handshake_server_ssl", "platforms": [ "linux" ], @@ -1899,7 +1879,7 @@ "flaky": false, "gtest": false, "language": "c", - "name": "memory_profile_test", + "name": "memory_usage_test", "platforms": [ "linux", "mac", @@ -2105,26 +2085,6 @@ "args": [], "benchmark": false, "ci_platforms": [ - "linux" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [ - "uv" - ], - "flaky": false, - "gtest": false, - "language": "c", - "name": "pollset_set_test", - "platforms": [ - "linux" - ], - "uses_polling": true - }, - { - "args": [], - "benchmark": false, - "ci_platforms": [ "linux", "mac", "posix" @@ -4076,6 +4036,30 @@ "posix", "windows" ], + "cpu_cost": 0.5, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "client_interceptors_end2end_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "uses_polling": true + }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], "cpu_cost": 1.0, "exclude_configs": [], "exclude_iomgrs": [], @@ -5166,6 +5150,30 @@ "posix", "windows" ], + "cpu_cost": 0.5, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "server_interceptors_end2end_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "uses_polling": true + }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], "cpu_cost": 1.0, "exclude_configs": [], "exclude_iomgrs": [], @@ -56306,949 +56314,6 @@ }, { "args": [ - "authority_not_supported" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "binary_metadata" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 0.1, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "cancel_after_accept" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 0.1, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "cancel_after_client_done" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 0.1, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "cancel_after_invoke" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 0.1, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "cancel_after_round_trip" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 0.1, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "cancel_before_invoke" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 0.1, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "cancel_in_a_vacuum" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 0.1, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "cancel_with_status" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 0.1, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "channelz" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "empty_batch" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 0.1, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "filter_call_init_fails" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "filter_causes_close" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 0.1, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "filter_latency" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 0.1, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "filter_status_code" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 0.1, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "high_initial_seqno" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 0.1, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "hpack_size" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 0.1, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "idempotent_request" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "invoke_large_request" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "large_metadata" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "max_message_length" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 0.1, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "negative_deadline" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "network_status_change" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 0.1, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "no_error_on_hotpath" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "no_logging" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "no_op" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "payload" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "ping_pong_streaming" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 0.1, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "registered_call" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "request_with_flags" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 0.1, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "request_with_payload" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 0.1, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "resource_quota_server" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "server_finishes_request" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 0.1, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "shutdown_finishes_calls" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 0.1, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "shutdown_finishes_tags" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 0.1, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "simple_cacheable_request" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 0.1, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "simple_metadata" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "simple_request" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "streaming_error_response" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 0.1, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "trailing_metadata" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ - "workaround_cronet_compression" - ], - "ci_platforms": [ - "windows", - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "language": "c", - "name": "inproc_nosec_test", - "platforms": [ - "windows", - "linux", - "mac", - "posix" - ] - }, - { - "args": [ "--scenarios_json", "{\"scenarios\": [{\"name\": \"cpp_protobuf_async_unary_1channel_100rpcs_1MB\", \"warmup_seconds\": 0, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"security_params\": null, \"threads_per_cq\": 0, \"server_type\": \"ASYNC_SERVER\"}, \"num_clients\": 1, \"client_config\": {\"security_params\": null, \"channel_args\": [{\"str_value\": \"throughput\", \"name\": \"grpc.optimization_target\"}], \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"payload_config\": {\"simple_params\": {\"resp_size\": 1048576, \"req_size\": 1048576}}, \"client_channels\": 1, \"threads_per_cq\": 0, \"load_params\": {\"closed_loop\": {}}, \"client_type\": \"ASYNC_CLIENT\", \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}}]}" ], diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh index 6990244e51..8394f07e51 100755 --- a/tools/run_tests/helper_scripts/build_python.sh +++ b/tools/run_tests/helper_scripts/build_python.sh @@ -56,6 +56,12 @@ function is_linux() { fi } +function inside_venv() { + if [[ -n "${VIRTUAL_ENV}" ]]; then + echo true + fi +} + # Associated virtual environment name for the given python command. function venv() { $1 -c "import sys; print('py{}{}'.format(*sys.version_info[:2]))" @@ -80,6 +86,8 @@ function toolchain() { fi } +# TODO(jtattermusch): this adds dependency on grealpath on mac +# (brew install coreutils) for little reason. # Command to invoke the linux command `realpath` or equivalent. function script_realpath() { # Find `realpath` @@ -112,6 +120,10 @@ export CFLAGS="-I$ROOT/include -std=gnu99 -fno-wrapv $CFLAGS" export GRPC_PYTHON_BUILD_WITH_CYTHON=1 export LANG=en_US.UTF-8 +# Allow build_ext to build C/C++ files in parallel +# by enabling a monkeypatch. It speeds up the build a lot. +export GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS=4 + # If ccache is available on Linux, use it. if [ "$(is_linux)" ]; then # We're not on Darwin (Mac OS X) @@ -128,10 +140,14 @@ fi # Perform build operations # ############################ -# Instantiate the virtualenv from the Python version passed in. -$PYTHON -m pip install --user virtualenv -$PYTHON -m virtualenv "$VENV" -VENV_PYTHON=$(script_realpath "$VENV/$VENV_RELATIVE_PYTHON") +if [[ "$(inside_venv)" ]]; then + VENV_PYTHON="$PYTHON" +else + # Instantiate the virtualenv from the Python version passed in. + $PYTHON -m pip install --user virtualenv + $PYTHON -m virtualenv "$VENV" + VENV_PYTHON=$(script_realpath "$VENV/$VENV_RELATIVE_PYTHON") +fi # See https://github.com/grpc/grpc/issues/14815 for more context. We cannot rely # on pip to upgrade itself because if pip is too old, it may not have the required @@ -149,10 +165,13 @@ pip_install_dir() { } case "$VENV" in - *gevent*) + *py35_gevent*) # TODO(https://github.com/grpc/grpc/issues/15411) unpin this $VENV_PYTHON -m pip install gevent==1.3.b1 ;; + *gevent*) + $VENV_PYTHON -m pip install -U gevent + ;; esac $VENV_PYTHON -m pip install --upgrade pip==10.0.1 diff --git a/tools/run_tests/helper_scripts/pre_build_csharp.bat b/tools/run_tests/helper_scripts/pre_build_csharp.bat index 2ae870ebb1..05c6cf0f61 100644 --- a/tools/run_tests/helper_scripts/pre_build_csharp.bat +++ b/tools/run_tests/helper_scripts/pre_build_csharp.bat @@ -32,9 +32,7 @@ cmake -G "Visual Studio 14 2015" -A %ARCHITECTURE% -DgRPC_BUILD_TESTS=OFF -DgRPC cd ..\..\..\src\csharp -if NOT DEFINED GRPC_SKIP_DOTNET_RESTORE ( - dotnet restore Grpc.sln || goto :error -) +dotnet restore Grpc.sln || goto :error endlocal diff --git a/tools/run_tests/helper_scripts/pre_build_csharp.sh b/tools/run_tests/helper_scripts/pre_build_csharp.sh index f9f5440a61..7d83986f90 100755 --- a/tools/run_tests/helper_scripts/pre_build_csharp.sh +++ b/tools/run_tests/helper_scripts/pre_build_csharp.sh @@ -15,7 +15,14 @@ set -ex -# cd to gRPC csharp directory -cd "$(dirname "$0")/../../../src/csharp" +# cd to repository root +cd "$(dirname "$0")/../../.." + +mkdir -p cmake/build +cd cmake/build + +cmake -DgRPC_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE="${MSBUILD_CONFIG}" ../.. + +cd ../../src/csharp dotnet restore Grpc.sln diff --git a/tools/run_tests/lb_interop_tests/gen_build_yaml.py b/tools/run_tests/lb_interop_tests/gen_build_yaml.py new file mode 100755 index 0000000000..b7d655b75b --- /dev/null +++ b/tools/run_tests/lb_interop_tests/gen_build_yaml.py @@ -0,0 +1,347 @@ +#!/usr/bin/env python2.7 +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Generates the appropriate JSON data for LB interop test scenarios.""" + +import json +import os +import yaml + +all_scenarios = [] + +# TODO(https://github.com/grpc/grpc-go/issues/2347): enable +# client_falls_back_because_no_backends_* scenarios for Java/Go. + +# TODO(https://github.com/grpc/grpc-java/issues/4887): enable +# *short_stream* scenarios for Java. + +# TODO(https://github.com/grpc/grpc-java/issues/4912): enable +# Java TLS tests involving TLS to the balancer. + + +def server_sec(transport_sec): + if transport_sec == 'google_default_credentials': + return 'alts', 'alts', 'tls' + return transport_sec, transport_sec, transport_sec + + +def generate_no_balancer_because_lb_a_record_returns_nx_domain(): + all_configs = [] + for transport_sec in [ + 'insecure', 'alts', 'tls', 'google_default_credentials' + ]: + balancer_sec, backend_sec, fallback_sec = server_sec(transport_sec) + config = { + 'name': + 'no_balancer_because_lb_a_record_returns_nx_domain_%s' % + transport_sec, + 'skip_langs': [], + 'transport_sec': + transport_sec, + 'balancer_configs': [], + 'backend_configs': [], + 'fallback_configs': [{ + 'transport_sec': fallback_sec, + }], + 'cause_no_error_no_data_for_balancer_a_record': + False, + } + all_configs.append(config) + return all_configs + + +all_scenarios += generate_no_balancer_because_lb_a_record_returns_nx_domain() + + +def generate_no_balancer_because_lb_a_record_returns_no_data(): + all_configs = [] + for transport_sec in [ + 'insecure', 'alts', 'tls', 'google_default_credentials' + ]: + balancer_sec, backend_sec, fallback_sec = server_sec(transport_sec) + config = { + 'name': + 'no_balancer_because_lb_a_record_returns_no_data_%s' % + transport_sec, + 'skip_langs': [], + 'transport_sec': + transport_sec, + 'balancer_configs': [], + 'backend_configs': [], + 'fallback_configs': [{ + 'transport_sec': fallback_sec, + }], + 'cause_no_error_no_data_for_balancer_a_record': + True, + } + all_configs.append(config) + return all_configs + + +all_scenarios += generate_no_balancer_because_lb_a_record_returns_no_data() + + +def generate_client_referred_to_backend(): + all_configs = [] + for balancer_short_stream in [True, False]: + for transport_sec in [ + 'insecure', 'alts', 'tls', 'google_default_credentials' + ]: + balancer_sec, backend_sec, fallback_sec = server_sec(transport_sec) + skip_langs = [] + if transport_sec == 'tls': + skip_langs += ['java'] + if balancer_short_stream: + skip_langs += ['java'] + config = { + 'name': + 'client_referred_to_backend_%s_short_stream_%s' % + (transport_sec, balancer_short_stream), + 'skip_langs': + skip_langs, + 'transport_sec': + transport_sec, + 'balancer_configs': [{ + 'transport_sec': balancer_sec, + 'short_stream': balancer_short_stream, + }], + 'backend_configs': [{ + 'transport_sec': backend_sec, + }], + 'fallback_configs': [], + 'cause_no_error_no_data_for_balancer_a_record': + False, + } + all_configs.append(config) + return all_configs + + +all_scenarios += generate_client_referred_to_backend() + + +def generate_client_referred_to_backend_fallback_broken(): + all_configs = [] + for balancer_short_stream in [True, False]: + for transport_sec in ['alts', 'tls', 'google_default_credentials']: + balancer_sec, backend_sec, fallback_sec = server_sec(transport_sec) + skip_langs = [] + if transport_sec == 'tls': + skip_langs += ['java'] + if balancer_short_stream: + skip_langs += ['java'] + config = { + 'name': + 'client_referred_to_backend_fallback_broken_%s_short_stream_%s' + % (transport_sec, balancer_short_stream), + 'skip_langs': + skip_langs, + 'transport_sec': + transport_sec, + 'balancer_configs': [{ + 'transport_sec': balancer_sec, + 'short_stream': balancer_short_stream, + }], + 'backend_configs': [{ + 'transport_sec': backend_sec, + }], + 'fallback_configs': [{ + 'transport_sec': 'insecure', + }], + 'cause_no_error_no_data_for_balancer_a_record': + False, + } + all_configs.append(config) + return all_configs + + +all_scenarios += generate_client_referred_to_backend_fallback_broken() + + +def generate_client_referred_to_backend_multiple_backends(): + all_configs = [] + for balancer_short_stream in [True, False]: + for transport_sec in [ + 'insecure', 'alts', 'tls', 'google_default_credentials' + ]: + balancer_sec, backend_sec, fallback_sec = server_sec(transport_sec) + skip_langs = [] + if transport_sec == 'tls': + skip_langs += ['java'] + if balancer_short_stream: + skip_langs += ['java'] + config = { + 'name': + 'client_referred_to_backend_multiple_backends_%s_short_stream_%s' + % (transport_sec, balancer_short_stream), + 'skip_langs': + skip_langs, + 'transport_sec': + transport_sec, + 'balancer_configs': [{ + 'transport_sec': balancer_sec, + 'short_stream': balancer_short_stream, + }], + 'backend_configs': [{ + 'transport_sec': backend_sec, + }, { + 'transport_sec': backend_sec, + }, { + 'transport_sec': backend_sec, + }, { + 'transport_sec': backend_sec, + }, { + 'transport_sec': backend_sec, + }], + 'fallback_configs': [], + 'cause_no_error_no_data_for_balancer_a_record': + False, + } + all_configs.append(config) + return all_configs + + +all_scenarios += generate_client_referred_to_backend_multiple_backends() + + +def generate_client_falls_back_because_no_backends(): + all_configs = [] + for balancer_short_stream in [True, False]: + for transport_sec in [ + 'insecure', 'alts', 'tls', 'google_default_credentials' + ]: + balancer_sec, backend_sec, fallback_sec = server_sec(transport_sec) + skip_langs = ['go', 'java'] + if transport_sec == 'tls': + skip_langs += ['java'] + if balancer_short_stream: + skip_langs += ['java'] + config = { + 'name': + 'client_falls_back_because_no_backends_%s_short_stream_%s' % + (transport_sec, balancer_short_stream), + 'skip_langs': + skip_langs, + 'transport_sec': + transport_sec, + 'balancer_configs': [{ + 'transport_sec': balancer_sec, + 'short_stream': balancer_short_stream, + }], + 'backend_configs': [], + 'fallback_configs': [{ + 'transport_sec': fallback_sec, + }], + 'cause_no_error_no_data_for_balancer_a_record': + False, + } + all_configs.append(config) + return all_configs + + +all_scenarios += generate_client_falls_back_because_no_backends() + + +def generate_client_falls_back_because_balancer_connection_broken(): + all_configs = [] + for transport_sec in ['alts', 'tls', 'google_default_credentials']: + balancer_sec, backend_sec, fallback_sec = server_sec(transport_sec) + skip_langs = [] + if transport_sec == 'tls': + skip_langs = ['java'] + config = { + 'name': + 'client_falls_back_because_balancer_connection_broken_%s' % + transport_sec, + 'skip_langs': + skip_langs, + 'transport_sec': + transport_sec, + 'balancer_configs': [{ + 'transport_sec': 'insecure', + 'short_stream': False, + }], + 'backend_configs': [], + 'fallback_configs': [{ + 'transport_sec': fallback_sec, + }], + 'cause_no_error_no_data_for_balancer_a_record': + False, + } + all_configs.append(config) + return all_configs + + +all_scenarios += generate_client_falls_back_because_balancer_connection_broken() + + +def generate_client_referred_to_backend_multiple_balancers(): + all_configs = [] + for balancer_short_stream in [True, False]: + for transport_sec in [ + 'insecure', 'alts', 'tls', 'google_default_credentials' + ]: + balancer_sec, backend_sec, fallback_sec = server_sec(transport_sec) + skip_langs = [] + if transport_sec == 'tls': + skip_langs += ['java'] + if balancer_short_stream: + skip_langs += ['java'] + config = { + 'name': + 'client_referred_to_backend_multiple_balancers_%s_short_stream_%s' + % (transport_sec, balancer_short_stream), + 'skip_langs': + skip_langs, + 'transport_sec': + transport_sec, + 'balancer_configs': [ + { + 'transport_sec': balancer_sec, + 'short_stream': balancer_short_stream, + }, + { + 'transport_sec': balancer_sec, + 'short_stream': balancer_short_stream, + }, + { + 'transport_sec': balancer_sec, + 'short_stream': balancer_short_stream, + }, + { + 'transport_sec': balancer_sec, + 'short_stream': balancer_short_stream, + }, + { + 'transport_sec': balancer_sec, + 'short_stream': balancer_short_stream, + }, + ], + 'backend_configs': [ + { + 'transport_sec': backend_sec, + }, + ], + 'fallback_configs': [], + 'cause_no_error_no_data_for_balancer_a_record': + False, + } + all_configs.append(config) + return all_configs + + +all_scenarios += generate_client_referred_to_backend_multiple_balancers() + +print(yaml.dump({ + 'lb_interop_test_scenarios': all_scenarios, +})) diff --git a/tools/run_tests/performance/build_performance.sh b/tools/run_tests/performance/build_performance.sh index 35d9e90598..ab6bffdc34 100755 --- a/tools/run_tests/performance/build_performance.sh +++ b/tools/run_tests/performance/build_performance.sh @@ -25,10 +25,16 @@ CONFIG=${CONFIG:-opt} # TODO(jtattermusch): C++ worker and driver are not buildable on Windows yet if [ "$OSTYPE" != "msys" ] then - # TODO(jtattermusch): not embedding OpenSSL breaks the C# build because - # grpc_csharp_ext needs OpenSSL embedded and some intermediate files from - # this build will be reused. - make CONFIG="${CONFIG}" EMBED_OPENSSL=true EMBED_ZLIB=true qps_worker qps_json_driver -j8 + # build C++ with cmake as building with "make" disables boringssl assembly + # optimizations that can have huge impact on secure channel throughput. + mkdir -p cmake/build + cd cmake/build + cmake -DgRPC_BUILD_TESTS=ON -DCMAKE_BUILD_TYPE=Release ../.. + make qps_worker qps_json_driver -j8 + cd ../.. + # unbreak subsequent make builds by restoring zconf.h (previously renamed by cmake build) + # See https://github.com/grpc/grpc/issues/11581 + (cd third_party/zlib; git checkout zconf.h) fi PHP_ALREADY_BUILT="" @@ -53,11 +59,18 @@ do fi ;; "csharp") - python tools/run_tests/run_tests.py -l "$language" -c "$CONFIG" --build_only -j 8 --compiler coreclr + python tools/run_tests/run_tests.py -l "$language" -c "$CONFIG" --build_only -j 8 + # unbreak subsequent make builds by restoring zconf.h (previously renamed by cmake portion of C#'s build) + # See https://github.com/grpc/grpc/issues/11581 + (cd third_party/zlib; git checkout zconf.h) ;; "node"|"node_purejs") tools/run_tests/performance/build_performance_node.sh ;; + "python") + # python workers are only run with python2.7 and building with multiple python versions is costly + python tools/run_tests/run_tests.py -l "$language" -c "$CONFIG" --compiler python2.7 --build_only -j 8 + ;; *) python tools/run_tests/run_tests.py -l "$language" -c "$CONFIG" --build_only -j 8 ;; diff --git a/tools/run_tests/performance/remote_host_prepare.sh b/tools/run_tests/performance/remote_host_prepare.sh index d69c85b40e..685c4357c9 100755 --- a/tools/run_tests/performance/remote_host_prepare.sh +++ b/tools/run_tests/performance/remote_host_prepare.sh @@ -38,3 +38,6 @@ ssh "${USER_AT_HOST}" "tar -xf ~/performance_workspace/grpc.tar -C ~/performance # For consistency with local run, invoke the kill_workers script remotely. # shellcheck disable=SC2088 ssh "${USER_AT_HOST}" "~/performance_workspace/grpc/tools/run_tests/performance/kill_workers.sh" + +# make sure the port server is running (required by C++ qps_worker) +ssh "${USER_AT_HOST}" "cd ~/performance_workspace/grpc/ && python tools/run_tests/start_port_server.py" diff --git a/tools/run_tests/performance/run_qps_driver.sh b/tools/run_tests/performance/run_qps_driver.sh index 2d9e310dec..47a03db026 100755 --- a/tools/run_tests/performance/run_qps_driver.sh +++ b/tools/run_tests/performance/run_qps_driver.sh @@ -17,7 +17,7 @@ set -ex cd "$(dirname "$0")/../../.." -bins/opt/qps_json_driver "$@" +cmake/build/qps_json_driver "$@" if [ "$BQ_RESULT_TABLE" != "" ] then diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 2e78bd07fb..481918c52e 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -231,7 +231,7 @@ class CXXLanguage: self.safename = 'cxx' def worker_cmdline(self): - return ['bins/opt/qps_worker'] + return ['cmake/build/qps_worker'] def worker_port_offset(self): return 0 @@ -250,7 +250,7 @@ class CXXLanguage: channels=1, num_clients=1, secure=False, - categories=[SMOKETEST] + [INPROC] + [SCALABLE]) + categories=[INPROC] + [SCALABLE]) yield _ping_pong_scenario( 'cpp_protobuf_async_streaming_from_client_1channel_1MB', @@ -280,12 +280,12 @@ class CXXLanguage: secure=False, async_server_threads=16, server_threads_per_cq=1, - categories=[SMOKETEST] + [SCALABLE]) + categories=[SCALABLE]) for secure in [True, False]: secstr = 'secure' if secure else 'insecure' - smoketest_categories = ([SMOKETEST] - if secure else [INPROC]) + [SCALABLE] + smoketest_categories = ([SMOKETEST] if secure else []) + inproc_categories = ([INPROC] if not secure else []) yield _ping_pong_scenario( 'cpp_generic_async_streaming_ping_pong_%s' % secstr, @@ -295,7 +295,8 @@ class CXXLanguage: use_generic_payload=True, async_server_threads=1, secure=secure, - categories=smoketest_categories) + categories=smoketest_categories + inproc_categories + + [SCALABLE]) yield _ping_pong_scenario( 'cpp_generic_async_streaming_qps_unconstrained_%s' % secstr, @@ -306,7 +307,8 @@ class CXXLanguage: use_generic_payload=True, secure=secure, minimal_stack=not secure, - categories=smoketest_categories + [SCALABLE]) + categories=smoketest_categories + inproc_categories + + [SCALABLE]) for mps in geometric_progression(1, 20, 10): yield _ping_pong_scenario( @@ -320,7 +322,8 @@ class CXXLanguage: secure=secure, messages_per_stream=mps, minimal_stack=not secure, - categories=smoketest_categories + [SCALABLE]) + categories=smoketest_categories + inproc_categories + + [SCALABLE]) for mps in geometric_progression(1, 200, math.sqrt(10)): yield _ping_pong_scenario( @@ -347,7 +350,7 @@ class CXXLanguage: use_generic_payload=True, secure=secure, minimal_stack=not secure, - categories=smoketest_categories + [SCALABLE], + categories=inproc_categories + [SCALABLE], channels=1, outstanding=100) @@ -363,7 +366,7 @@ class CXXLanguage: use_generic_payload=True, secure=secure, minimal_stack=not secure, - categories=smoketest_categories + [SCALABLE]) + categories=inproc_categories + [SCALABLE]) yield _ping_pong_scenario( 'cpp_generic_async_streaming_qps_unconstrained_1cq_%s' % secstr, @@ -375,7 +378,8 @@ class CXXLanguage: secure=secure, client_threads_per_cq=1000000, server_threads_per_cq=1000000, - categories=smoketest_categories + [SCALABLE]) + categories=smoketest_categories + inproc_categories + + [SCALABLE]) yield _ping_pong_scenario( 'cpp_generic_async_streaming_qps_unconstrained_2waysharedcq_%s' @@ -388,7 +392,7 @@ class CXXLanguage: secure=secure, client_threads_per_cq=2, server_threads_per_cq=2, - categories=smoketest_categories + [SCALABLE]) + categories=inproc_categories + [SCALABLE]) yield _ping_pong_scenario( 'cpp_protobuf_async_streaming_qps_unconstrained_1cq_%s' % @@ -400,7 +404,7 @@ class CXXLanguage: secure=secure, client_threads_per_cq=1000000, server_threads_per_cq=1000000, - categories=smoketest_categories + [SCALABLE]) + categories=inproc_categories + [SCALABLE]) yield _ping_pong_scenario( 'cpp_protobuf_async_streaming_qps_unconstrained_2waysharedcq_%s' @@ -412,7 +416,7 @@ class CXXLanguage: secure=secure, client_threads_per_cq=2, server_threads_per_cq=2, - categories=smoketest_categories + [SCALABLE]) + categories=inproc_categories + [SCALABLE]) yield _ping_pong_scenario( 'cpp_protobuf_async_unary_qps_unconstrained_1cq_%s' % secstr, @@ -423,7 +427,8 @@ class CXXLanguage: secure=secure, client_threads_per_cq=1000000, server_threads_per_cq=1000000, - categories=smoketest_categories + [SCALABLE]) + categories=smoketest_categories + inproc_categories + + [SCALABLE]) yield _ping_pong_scenario( 'cpp_protobuf_async_unary_qps_unconstrained_2waysharedcq_%s' % @@ -435,7 +440,7 @@ class CXXLanguage: secure=secure, client_threads_per_cq=2, server_threads_per_cq=2, - categories=smoketest_categories + [SCALABLE]) + categories=inproc_categories + [SCALABLE]) yield _ping_pong_scenario( 'cpp_generic_async_streaming_qps_one_server_core_%s' % secstr, @@ -457,7 +462,8 @@ class CXXLanguage: unconstrained_client='async', secure=secure, minimal_stack=not secure, - categories=smoketest_categories + [SCALABLE], + categories=smoketest_categories + inproc_categories + + [SCALABLE], excluded_poll_engines=['poll-cv']) yield _ping_pong_scenario( @@ -472,7 +478,7 @@ class CXXLanguage: resp_size=8 * 1024 * 1024, secure=secure, minimal_stack=not secure, - categories=smoketest_categories + [SCALABLE]) + categories=inproc_categories + [SCALABLE]) yield _ping_pong_scenario( 'cpp_protobuf_async_client_sync_server_streaming_qps_unconstrained_%s' @@ -483,7 +489,8 @@ class CXXLanguage: unconstrained_client='async', secure=secure, minimal_stack=not secure, - categories=smoketest_categories + [SCALABLE], + categories=smoketest_categories + inproc_categories + + [SCALABLE], excluded_poll_engines=['poll-cv']) yield _ping_pong_scenario( @@ -495,7 +502,8 @@ class CXXLanguage: resp_size=1024 * 1024, secure=secure, minimal_stack=not secure, - categories=smoketest_categories + [SCALABLE]) + categories=smoketest_categories + inproc_categories + + [SCALABLE]) for rpc_type in [ 'unary', 'streaming', 'streaming_from_client', @@ -538,7 +546,7 @@ class CXXLanguage: minimal_stack=not secure, server_threads_per_cq=3, client_threads_per_cq=3, - categories=smoketest_categories + [SCALABLE]) + categories=inproc_categories + [SCALABLE]) # TODO(vjpai): Re-enable this test. It has a lot of timeouts # and hasn't yet been conclusively identified as a test failure @@ -565,7 +573,7 @@ class CXXLanguage: secure=secure, messages_per_stream=mps, minimal_stack=not secure, - categories=smoketest_categories + [SCALABLE]) + categories=inproc_categories + [SCALABLE]) for mps in geometric_progression(1, 200, math.sqrt(10)): yield _ping_pong_scenario( diff --git a/tools/run_tests/python_utils/dockerjob.py b/tools/run_tests/python_utils/dockerjob.py index 2d22dc13a0..5260f7b44b 100755 --- a/tools/run_tests/python_utils/dockerjob.py +++ b/tools/run_tests/python_utils/dockerjob.py @@ -20,6 +20,7 @@ import time import uuid import os import subprocess +import json import jobset @@ -54,6 +55,25 @@ def docker_mapped_port(cid, port, timeout_seconds=15): cid)) +def docker_ip_address(cid, timeout_seconds=15): + """Get port mapped to internal given internal port for given container.""" + started = time.time() + while time.time() - started < timeout_seconds: + cmd = 'docker inspect %s' % cid + try: + output = subprocess.check_output(cmd, stderr=_DEVNULL, shell=True) + json_info = json.loads(output) + assert len(json_info) == 1 + out = json_info[0]['NetworkSettings']['IPAddress'] + if not out: + continue + return out + except subprocess.CalledProcessError as e: + pass + raise Exception( + 'Non-retryable error: Failed to get ip address of container %s.' % cid) + + def wait_for_healthy(cid, shortname, timeout_seconds): """Wait timeout_seconds for the container to become healthy""" started = time.time() @@ -74,10 +94,10 @@ def wait_for_healthy(cid, shortname, timeout_seconds): (shortname, cid)) -def finish_jobs(jobs): +def finish_jobs(jobs, suppress_failure=True): """Kills given docker containers and waits for corresponding jobs to finish""" for job in jobs: - job.kill(suppress_failure=True) + job.kill(suppress_failure=suppress_failure) while any(job.is_running() for job in jobs): time.sleep(1) @@ -120,6 +140,9 @@ class DockerJob: def mapped_port(self, port): return docker_mapped_port(self._container_name, port) + def ip_address(self): + return docker_ip_address(self._container_name) + def wait_for_healthy(self, timeout_seconds): wait_for_healthy(self._container_name, self._spec.shortname, timeout_seconds) diff --git a/tools/run_tests/python_utils/jobset.py b/tools/run_tests/python_utils/jobset.py index 561f453da7..578712c393 100755 --- a/tools/run_tests/python_utils/jobset.py +++ b/tools/run_tests/python_utils/jobset.py @@ -13,8 +13,6 @@ # limitations under the License. """Run a group of subprocesses and then finish.""" -from __future__ import print_function - import logging import multiprocessing import os @@ -118,7 +116,7 @@ def eintr_be_gone(fn): while True: try: return fn() - except IOError, e: + except IOError as e: if e.errno != errno.EINTR: raise @@ -144,7 +142,7 @@ def message(tag, msg, explanatory_text=None, do_newline=False): if do_newline or explanatory_text is not None else '')) sys.stdout.flush() return - except IOError, e: + except IOError as e: if e.errno != errno.EINTR: raise @@ -176,13 +174,15 @@ class JobSpec(object): timeout_retries=0, kill_handler=None, cpu_cost=1.0, - verbose_success=False): + verbose_success=False, + logfilename=None): """ Arguments: cmdline: a list of arguments to pass as the command line environ: a dictionary of environment variables to set in the child process kill_handler: a handler that will be called whenever job.kill() is invoked cpu_cost: number of cores per second this job needs + logfilename: use given file to store job's output, rather than using a temporary file """ if environ is None: environ = {} @@ -197,6 +197,11 @@ class JobSpec(object): self.kill_handler = kill_handler self.cpu_cost = cpu_cost self.verbose_success = verbose_success + self.logfilename = logfilename + if self.logfilename and self.flake_retries != 0 and self.timeout_retries != 0: + # Forbidden to avoid overwriting the test log when retrying. + raise Exception( + 'Cannot use custom logfile when retries are enabled') def identity(self): return '%r %r' % (self.cmdline, self.environ) @@ -261,7 +266,15 @@ class Job(object): return self._spec def start(self): - self._tempfile = tempfile.TemporaryFile() + if self._spec.logfilename: + # make sure the log directory exists + logfile_dir = os.path.dirname( + os.path.abspath(self._spec.logfilename)) + if not os.path.exists(logfile_dir): + os.makedirs(logfile_dir) + self._logfile = open(self._spec.logfilename, 'w+') + else: + self._logfile = tempfile.TemporaryFile() env = dict(os.environ) env.update(self._spec.environ) env.update(self._add_env) @@ -277,7 +290,7 @@ class Job(object): measure_cpu_costs = False try_start = lambda: subprocess.Popen(args=cmdline, stderr=subprocess.STDOUT, - stdout=self._tempfile, + stdout=self._logfile, cwd=self._spec.cwd, shell=self._spec.shell, env=env) @@ -300,7 +313,7 @@ class Job(object): """Poll current state of the job. Prints messages at completion.""" def stdout(self=self): - stdout = read_from_start(self._tempfile) + stdout = read_from_start(self._logfile) self.result.message = stdout[-_MAX_RESULT_SIZE:] return stdout diff --git a/tools/run_tests/python_utils/port_server.py b/tools/run_tests/python_utils/port_server.py index 83e09c09d0..243eef6827 100755 --- a/tools/run_tests/python_utils/port_server.py +++ b/tools/run_tests/python_utils/port_server.py @@ -14,15 +14,17 @@ # limitations under the License. """Manage TCP ports for unit tests; started by run_tests.py""" +from __future__ import print_function + import argparse -from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler +from six.moves.BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler +from six.moves.socketserver import ThreadingMixIn import hashlib import os import socket import sys import time import random -from SocketServer import ThreadingMixIn import threading import platform @@ -32,7 +34,7 @@ import platform _MY_VERSION = 20 if len(sys.argv) == 2 and sys.argv[1] == 'dump_version': - print _MY_VERSION + print(_MY_VERSION) sys.exit(0) argp = argparse.ArgumentParser(description='Server for httpcli_test') @@ -47,7 +49,7 @@ if args.logfile is not None: sys.stderr = open(args.logfile, 'w') sys.stdout = sys.stderr -print 'port server running on port %d' % args.port +print('port server running on port %d' % args.port) pool = [] in_use = {} @@ -74,7 +76,7 @@ def can_connect(port): try: s.connect(('localhost', port)) return True - except socket.error, e: + except socket.error as e: return False finally: s.close() @@ -86,7 +88,7 @@ def can_bind(port, proto): try: s.bind(('localhost', port)) return True - except socket.error, e: + except socket.error as e: return False finally: s.close() @@ -95,7 +97,7 @@ def can_bind(port, proto): def refill_pool(max_timeout, req): """Scan for ports not marked for being in use""" chk = [ - port for port in list(range(1025, 32766)) + port for port in range(1025, 32766) if port not in cronet_restricted_ports ] random.shuffle(chk) diff --git a/tools/run_tests/python_utils/report_utils.py b/tools/run_tests/python_utils/report_utils.py index b2a256ce29..8d8dedb929 100644 --- a/tools/run_tests/python_utils/report_utils.py +++ b/tools/run_tests/python_utils/report_utils.py @@ -13,8 +13,6 @@ # limitations under the License. """Generate XML and HTML test reports.""" -from __future__ import print_function - try: from mako.runtime import Context from mako.template import Template diff --git a/tools/run_tests/python_utils/start_port_server.py b/tools/run_tests/python_utils/start_port_server.py index 37995acbdf..0a32bf4418 100644 --- a/tools/run_tests/python_utils/start_port_server.py +++ b/tools/run_tests/python_utils/start_port_server.py @@ -12,8 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import urllib -import jobset +from __future__ import print_function + +from . import jobset + +import six.moves.urllib.request as request import logging import os import socket @@ -33,8 +36,8 @@ def start_port_server(): # otherwise, leave it up try: version = int( - urllib.urlopen('http://localhost:%d/version_number' % - _PORT_SERVER_PORT).read()) + request.urlopen('http://localhost:%d/version_number' % + _PORT_SERVER_PORT).read()) logging.info('detected port server running version %d', version) running = True except Exception as e: @@ -51,7 +54,7 @@ def start_port_server(): running = (version >= current_version) if not running: logging.info('port_server version mismatch: killing the old one') - urllib.urlopen( + request.urlopen( 'http://localhost:%d/quitquitquit' % _PORT_SERVER_PORT).read() time.sleep(1) if not running: @@ -92,7 +95,7 @@ def start_port_server(): # try one final time: maybe another build managed to start one time.sleep(1) try: - urllib.urlopen( + request.urlopen( 'http://localhost:%d/get' % _PORT_SERVER_PORT).read() logging.info( 'last ditch attempt to contact port server succeeded') @@ -101,11 +104,11 @@ def start_port_server(): logging.exception( 'final attempt to contact port server failed') port_log = open(logfile, 'r').read() - print port_log + print(port_log) sys.exit(1) try: port_server_url = 'http://localhost:%d/get' % _PORT_SERVER_PORT - urllib.urlopen(port_server_url).read() + request.urlopen(port_server_url).read() logging.info('port server is up and ready') break except socket.timeout: diff --git a/tools/run_tests/python_utils/upload_rbe_results.py b/tools/run_tests/python_utils/upload_rbe_results.py index 74f329048f..3f3bd382bb 100644 --- a/tools/run_tests/python_utils/upload_rbe_results.py +++ b/tools/run_tests/python_utils/upload_rbe_results.py @@ -62,7 +62,7 @@ def _get_invocation_id(): bazel_id_directory = os.getenv('KOKORO_ARTIFACTS_DIR') bazel_id_file = os.path.join(bazel_id_directory, 'bazel_invocation_ids') assert os.path.isfile(bazel_id_file), 'bazel_invocation_ids file, written ' \ - 'by bazel_wrapper.py, expected but not found.' + 'by RBE initialization script, expected but not found.' with open(bazel_id_file, 'r') as f: return f.read().replace('\n', '') diff --git a/tools/run_tests/python_utils/watch_dirs.py b/tools/run_tests/python_utils/watch_dirs.py index d2ad303a07..f2f1c006df 100755 --- a/tools/run_tests/python_utils/watch_dirs.py +++ b/tools/run_tests/python_utils/watch_dirs.py @@ -15,13 +15,14 @@ import os import time +from six import string_types class DirWatcher(object): """Helper to watch a (set) of directories for modifications.""" def __init__(self, paths): - if isinstance(paths, basestring): + if isinstance(paths, string_types): paths = [paths] self._done = False self.paths = list(paths) diff --git a/tools/run_tests/run_grpclb_interop_tests.py b/tools/run_tests/run_grpclb_interop_tests.py new file mode 100755 index 0000000000..3bfbcecf06 --- /dev/null +++ b/tools/run_tests/run_grpclb_interop_tests.py @@ -0,0 +1,609 @@ +#!/usr/bin/env python +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Run interop (cross-language) tests in parallel.""" + +from __future__ import print_function + +import argparse +import atexit +import itertools +import json +import multiprocessing +import os +import re +import subprocess +import sys +import tempfile +import time +import uuid +import six +import traceback + +import python_utils.dockerjob as dockerjob +import python_utils.jobset as jobset +import python_utils.report_utils as report_utils + +# Docker doesn't clean up after itself, so we do it on exit. +atexit.register(lambda: subprocess.call(['stty', 'echo'])) + +ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..')) +os.chdir(ROOT) + +_FALLBACK_SERVER_PORT = 443 +_BALANCER_SERVER_PORT = 12000 +_BACKEND_SERVER_PORT = 8080 + +_TEST_TIMEOUT = 30 + +_FAKE_SERVERS_SAFENAME = 'fake_servers' + +# Use a name that's verified by the test certs +_SERVICE_NAME = 'server.test.google.fr' + + +class CXXLanguage: + + def __init__(self): + self.client_cwd = '/var/local/git/grpc' + self.safename = 'cxx' + + def client_cmd(self, args): + return ['bins/opt/interop_client'] + args + + def global_env(self): + # 1) Set c-ares as the resolver, to + # enable grpclb. + # 2) Turn on verbose logging. + # 3) Set the ROOTS_PATH env variable + # to the test CA in order for + # GoogleDefaultCredentials to be + # able to use the test CA. + return { + 'GRPC_DNS_RESOLVER': + 'ares', + 'GRPC_VERBOSITY': + 'DEBUG', + 'GRPC_TRACE': + 'client_channel,glb', + 'GRPC_DEFAULT_SSL_ROOTS_FILE_PATH': + '/var/local/git/grpc/src/core/tsi/test_creds/ca.pem', + } + + def __str__(self): + return 'c++' + + +class JavaLanguage: + + def __init__(self): + self.client_cwd = '/var/local/git/grpc-java' + self.safename = str(self) + + def client_cmd(self, args): + # Take necessary steps to import our test CA into + # the set of test CA's that the Java runtime of the + # docker container will pick up, so that + # Java GoogleDefaultCreds can use it. + pem_to_der_cmd = ('openssl x509 -outform der ' + '-in /external_mount/src/core/tsi/test_creds/ca.pem ' + '-out /tmp/test_ca.der') + keystore_import_cmd = ( + 'keytool -import ' + '-keystore /usr/lib/jvm/java-8-oracle/jre/lib/security/cacerts ' + '-file /tmp/test_ca.der ' + '-deststorepass changeit ' + '-noprompt') + return [ + 'bash', '-c', ('{pem_to_der_cmd} && ' + '{keystore_import_cmd} && ' + './run-test-client.sh {java_client_args}').format( + pem_to_der_cmd=pem_to_der_cmd, + keystore_import_cmd=keystore_import_cmd, + java_client_args=' '.join(args)) + ] + + def global_env(self): + # 1) Enable grpclb + # 2) Enable verbose logging + return { + 'JAVA_OPTS': + ('-Dio.grpc.internal.DnsNameResolverProvider.enable_grpclb=true ' + '-Djava.util.logging.config.file=/var/local/grpc_java_logging/logconf.txt' + ) + } + + def __str__(self): + return 'java' + + +class GoLanguage: + + def __init__(self): + self.client_cwd = '/go/src/google.golang.org/grpc/interop/client' + self.safename = str(self) + + def client_cmd(self, args): + # Copy the test CA file into the path that + # the Go runtime in the docker container will use, so + # that Go's GoogleDefaultCredentials can use it. + # See https://golang.org/src/crypto/x509/root_linux.go. + return [ + 'bash', '-c', ('cp /external_mount/src/core/tsi/test_creds/ca.pem ' + '/etc/ssl/certs/ca-certificates.crt && ' + '/go/bin/client {go_client_args}' + ).format(go_client_args=' '.join(args)) + ] + + def global_env(self): + return { + 'GRPC_GO_LOG_VERBOSITY_LEVEL': '3', + 'GRPC_GO_LOG_SEVERITY_LEVEL': 'INFO' + } + + def __str__(self): + return 'go' + + +_LANGUAGES = { + 'c++': CXXLanguage(), + 'go': GoLanguage(), + 'java': JavaLanguage(), +} + + +def docker_run_cmdline(cmdline, image, docker_args, cwd, environ=None): + """Wraps given cmdline array to create 'docker run' cmdline from it.""" + # turn environ into -e docker args + docker_cmdline = 'docker run -i --rm=true'.split() + if environ: + for k, v in environ.items(): + docker_cmdline += ['-e', '%s=%s' % (k, v)] + return docker_cmdline + ['-w', cwd] + docker_args + [image] + cmdline + + +def _job_kill_handler(job): + assert job._spec.container_name + dockerjob.docker_kill(job._spec.container_name) + + +def transport_security_to_args(transport_security): + args = [] + if transport_security == 'tls': + args += ['--use_tls=true'] + elif transport_security == 'alts': + args += ['--use_tls=false', '--use_alts=true'] + elif transport_security == 'insecure': + args += ['--use_tls=false'] + elif transport_security == 'google_default_credentials': + args += ['--custom_credentials_type=google_default_credentials'] + else: + print('Invalid transport security option.') + sys.exit(1) + return args + + +def lb_client_interop_jobspec(language, + dns_server_ip, + docker_image, + transport_security='tls'): + """Runs a gRPC client under test in a docker container""" + interop_only_options = [ + '--server_host=%s' % _SERVICE_NAME, + '--server_port=%d' % _FALLBACK_SERVER_PORT + ] + transport_security_to_args(transport_security) + # Don't set the server host override in any client; + # Go and Java default to no override. + # We're using a DNS server so there's no need. + if language.safename == 'c++': + interop_only_options += ['--server_host_override=""'] + # Don't set --use_test_ca; we're configuring + # clients to use test CA's via alternate means. + interop_only_options += ['--use_test_ca=false'] + client_args = language.client_cmd(interop_only_options) + container_name = dockerjob.random_name( + 'lb_interop_client_%s' % language.safename) + docker_cmdline = docker_run_cmdline( + client_args, + environ=language.global_env(), + image=docker_image, + cwd=language.client_cwd, + docker_args=[ + '--dns=%s' % dns_server_ip, + '--net=host', + '--name=%s' % container_name, + '-v', + '{grpc_grpc_root_dir}:/external_mount:ro'.format( + grpc_grpc_root_dir=ROOT), + ]) + jobset.message( + 'IDLE', + 'docker_cmdline:\b|%s|' % ' '.join(docker_cmdline), + do_newline=True) + test_job = jobset.JobSpec( + cmdline=docker_cmdline, + shortname=('lb_interop_client:%s' % language), + timeout_seconds=_TEST_TIMEOUT, + kill_handler=_job_kill_handler) + test_job.container_name = container_name + return test_job + + +def fallback_server_jobspec(transport_security, shortname): + """Create jobspec for running a fallback server""" + cmdline = [ + 'bin/server', + '--port=%d' % _FALLBACK_SERVER_PORT, + ] + transport_security_to_args(transport_security) + return grpc_server_in_docker_jobspec( + server_cmdline=cmdline, shortname=shortname) + + +def backend_server_jobspec(transport_security, shortname): + """Create jobspec for running a backend server""" + cmdline = [ + 'bin/server', + '--port=%d' % _BACKEND_SERVER_PORT, + ] + transport_security_to_args(transport_security) + return grpc_server_in_docker_jobspec( + server_cmdline=cmdline, shortname=shortname) + + +def grpclb_jobspec(transport_security, short_stream, backend_addrs, shortname): + """Create jobspec for running a balancer server""" + cmdline = [ + 'bin/fake_grpclb', + '--backend_addrs=%s' % ','.join(backend_addrs), + '--port=%d' % _BALANCER_SERVER_PORT, + '--short_stream=%s' % short_stream, + '--service_name=%s' % _SERVICE_NAME, + ] + transport_security_to_args(transport_security) + return grpc_server_in_docker_jobspec( + server_cmdline=cmdline, shortname=shortname) + + +def grpc_server_in_docker_jobspec(server_cmdline, shortname): + container_name = dockerjob.random_name(shortname) + environ = { + 'GRPC_GO_LOG_VERBOSITY_LEVEL': '3', + 'GRPC_GO_LOG_SEVERITY_LEVEL': 'INFO ', + } + docker_cmdline = docker_run_cmdline( + server_cmdline, + cwd='/go', + image=docker_images.get(_FAKE_SERVERS_SAFENAME), + environ=environ, + docker_args=['--name=%s' % container_name]) + jobset.message( + 'IDLE', + 'docker_cmdline:\b|%s|' % ' '.join(docker_cmdline), + do_newline=True) + server_job = jobset.JobSpec( + cmdline=docker_cmdline, shortname=shortname, timeout_seconds=30 * 60) + server_job.container_name = container_name + return server_job + + +def dns_server_in_docker_jobspec(grpclb_ips, fallback_ips, shortname, + cause_no_error_no_data_for_balancer_a_record): + container_name = dockerjob.random_name(shortname) + run_dns_server_cmdline = [ + 'python', + 'test/cpp/naming/utils/run_dns_server_for_lb_interop_tests.py', + '--grpclb_ips=%s' % ','.join(grpclb_ips), + '--fallback_ips=%s' % ','.join(fallback_ips), + ] + if cause_no_error_no_data_for_balancer_a_record: + run_dns_server_cmdline.append( + '--cause_no_error_no_data_for_balancer_a_record') + docker_cmdline = docker_run_cmdline( + run_dns_server_cmdline, + cwd='/var/local/git/grpc', + image=docker_images.get(_FAKE_SERVERS_SAFENAME), + docker_args=['--name=%s' % container_name]) + jobset.message( + 'IDLE', + 'docker_cmdline:\b|%s|' % ' '.join(docker_cmdline), + do_newline=True) + server_job = jobset.JobSpec( + cmdline=docker_cmdline, shortname=shortname, timeout_seconds=30 * 60) + server_job.container_name = container_name + return server_job + + +def build_interop_image_jobspec(lang_safename, basename_prefix='grpc_interop'): + """Creates jobspec for building interop docker image for a language""" + tag = '%s_%s:%s' % (basename_prefix, lang_safename, uuid.uuid4()) + env = { + 'INTEROP_IMAGE': tag, + 'BASE_NAME': '%s_%s' % (basename_prefix, lang_safename), + } + build_job = jobset.JobSpec( + cmdline=['tools/run_tests/dockerize/build_interop_image.sh'], + environ=env, + shortname='build_docker_%s' % lang_safename, + timeout_seconds=30 * 60) + build_job.tag = tag + return build_job + + +argp = argparse.ArgumentParser(description='Run interop tests.') +argp.add_argument( + '-l', + '--language', + choices=['all'] + sorted(_LANGUAGES), + nargs='+', + default=['all'], + help='Clients to run.') +argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int) +argp.add_argument( + '-s', + '--scenarios_file', + default=None, + type=str, + help='File containing test scenarios as JSON configs.') +argp.add_argument( + '-n', + '--scenario_name', + default=None, + type=str, + help=( + 'Useful for manual runs: specify the name of ' + 'the scenario to run from scenarios_file. Run all scenarios if unset.')) +argp.add_argument( + '--cxx_image_tag', + default=None, + type=str, + help=('Setting this skips the clients docker image ' + 'build step and runs the client from the named ' + 'image. Only supports running a one client language.')) +argp.add_argument( + '--go_image_tag', + default=None, + type=str, + help=('Setting this skips the clients docker image build ' + 'step and runs the client from the named image. Only ' + 'supports running a one client language.')) +argp.add_argument( + '--java_image_tag', + default=None, + type=str, + help=('Setting this skips the clients docker image build ' + 'step and runs the client from the named image. Only ' + 'supports running a one client language.')) +argp.add_argument( + '--servers_image_tag', + default=None, + type=str, + help=('Setting this skips the fake servers docker image ' + 'build step and runs the servers from the named image.')) +argp.add_argument( + '--no_skips', + default=False, + type=bool, + nargs='?', + const=True, + help=('Useful for manual runs. Setting this overrides test ' + '"skips" configured in test scenarios.')) +argp.add_argument( + '--verbose', + default=False, + type=bool, + nargs='?', + const=True, + help='Increase logging.') +args = argp.parse_args() + +docker_images = {} + +build_jobs = [] +if len(args.language) and args.language[0] == 'all': + languages = _LANGUAGES.keys() +else: + languages = args.language +for lang_name in languages: + l = _LANGUAGES[lang_name] + # First check if a pre-built image was supplied, and avoid + # rebuilding the particular docker image if so. + if lang_name == 'c++' and args.cxx_image_tag: + docker_images[str(l.safename)] = args.cxx_image_tag + elif lang_name == 'go' and args.go_image_tag: + docker_images[str(l.safename)] = args.go_image_tag + elif lang_name == 'java' and args.java_image_tag: + docker_images[str(l.safename)] = args.java_image_tag + else: + # Build the test client in docker and save the fully + # built image. + job = build_interop_image_jobspec(l.safename) + build_jobs.append(job) + docker_images[str(l.safename)] = job.tag + +# First check if a pre-built image was supplied. +if args.servers_image_tag: + docker_images[_FAKE_SERVERS_SAFENAME] = args.servers_image_tag +else: + # Build the test servers in docker and save the fully + # built image. + job = build_interop_image_jobspec( + _FAKE_SERVERS_SAFENAME, basename_prefix='lb_interop') + build_jobs.append(job) + docker_images[_FAKE_SERVERS_SAFENAME] = job.tag + +if build_jobs: + jobset.message('START', 'Building interop docker images.', do_newline=True) + print('Jobs to run: \n%s\n' % '\n'.join(str(j) for j in build_jobs)) + num_failures, _ = jobset.run( + build_jobs, newline_on_success=True, maxjobs=args.jobs) + if num_failures == 0: + jobset.message( + 'SUCCESS', 'All docker images built successfully.', do_newline=True) + else: + jobset.message( + 'FAILED', 'Failed to build interop docker images.', do_newline=True) + sys.exit(1) + + +def wait_until_dns_server_is_up(dns_server_ip): + """Probes the DNS server until it's running and safe for tests.""" + for i in range(0, 30): + print('Health check: attempt to connect to DNS server over TCP.') + tcp_connect_subprocess = subprocess.Popen([ + os.path.join(os.getcwd(), 'test/cpp/naming/utils/tcp_connect.py'), + '--server_host', dns_server_ip, '--server_port', + str(53), '--timeout', + str(1) + ]) + tcp_connect_subprocess.communicate() + if tcp_connect_subprocess.returncode == 0: + print(('Health check: attempt to make an A-record ' + 'query to DNS server.')) + dns_resolver_subprocess = subprocess.Popen( + [ + os.path.join(os.getcwd(), + 'test/cpp/naming/utils/dns_resolver.py'), + '--qname', ('health-check-local-dns-server-is-alive.' + 'resolver-tests.grpctestingexp'), + '--server_host', dns_server_ip, '--server_port', + str(53) + ], + stdout=subprocess.PIPE) + dns_resolver_stdout, _ = dns_resolver_subprocess.communicate() + if dns_resolver_subprocess.returncode == 0: + if '123.123.123.123' in dns_resolver_stdout: + print(('DNS server is up! ' + 'Successfully reached it over UDP and TCP.')) + return + time.sleep(0.1) + raise Exception(('Failed to reach DNS server over TCP and/or UDP. ' + 'Exitting without running tests.')) + + +def shortname(shortname_prefix, shortname, index): + return '%s_%s_%d' % (shortname_prefix, shortname, index) + + +def run_one_scenario(scenario_config): + jobset.message('START', 'Run scenario: %s' % scenario_config['name']) + server_jobs = {} + server_addresses = {} + suppress_server_logs = True + try: + backend_addrs = [] + fallback_ips = [] + grpclb_ips = [] + shortname_prefix = scenario_config['name'] + # Start backends + for i in xrange(len(scenario_config['backend_configs'])): + backend_config = scenario_config['backend_configs'][i] + backend_shortname = shortname(shortname_prefix, 'backend_server', i) + backend_spec = backend_server_jobspec( + backend_config['transport_sec'], backend_shortname) + backend_job = dockerjob.DockerJob(backend_spec) + server_jobs[backend_shortname] = backend_job + backend_addrs.append('%s:%d' % (backend_job.ip_address(), + _BACKEND_SERVER_PORT)) + # Start fallbacks + for i in xrange(len(scenario_config['fallback_configs'])): + fallback_config = scenario_config['fallback_configs'][i] + fallback_shortname = shortname(shortname_prefix, 'fallback_server', + i) + fallback_spec = fallback_server_jobspec( + fallback_config['transport_sec'], fallback_shortname) + fallback_job = dockerjob.DockerJob(fallback_spec) + server_jobs[fallback_shortname] = fallback_job + fallback_ips.append(fallback_job.ip_address()) + # Start balancers + for i in xrange(len(scenario_config['balancer_configs'])): + balancer_config = scenario_config['balancer_configs'][i] + grpclb_shortname = shortname(shortname_prefix, 'grpclb_server', i) + grpclb_spec = grpclb_jobspec(balancer_config['transport_sec'], + balancer_config['short_stream'], + backend_addrs, grpclb_shortname) + grpclb_job = dockerjob.DockerJob(grpclb_spec) + server_jobs[grpclb_shortname] = grpclb_job + grpclb_ips.append(grpclb_job.ip_address()) + # Start DNS server + dns_server_shortname = shortname(shortname_prefix, 'dns_server', 0) + dns_server_spec = dns_server_in_docker_jobspec( + grpclb_ips, fallback_ips, dns_server_shortname, + scenario_config['cause_no_error_no_data_for_balancer_a_record']) + dns_server_job = dockerjob.DockerJob(dns_server_spec) + server_jobs[dns_server_shortname] = dns_server_job + # Get the IP address of the docker container running the DNS server. + # The DNS server is running on port 53 of that IP address. Note we will + # point the DNS resolvers of grpc clients under test to our controlled + # DNS server by effectively modifying the /etc/resolve.conf "nameserver" + # lists of their docker containers. + dns_server_ip = dns_server_job.ip_address() + wait_until_dns_server_is_up(dns_server_ip) + # Run clients + jobs = [] + for lang_name in languages: + # Skip languages that are known to not currently + # work for this test. + if not args.no_skips and lang_name in scenario_config.get( + 'skip_langs', []): + jobset.message('IDLE', + 'Skipping scenario: %s for language: %s\n' % + (scenario_config['name'], lang_name)) + continue + lang = _LANGUAGES[lang_name] + test_job = lb_client_interop_jobspec( + lang, + dns_server_ip, + docker_image=docker_images.get(lang.safename), + transport_security=scenario_config['transport_sec']) + jobs.append(test_job) + jobset.message('IDLE', 'Jobs to run: \n%s\n' % '\n'.join( + str(job) for job in jobs)) + num_failures, resultset = jobset.run( + jobs, newline_on_success=True, maxjobs=args.jobs) + report_utils.render_junit_xml_report(resultset, 'sponge_log.xml') + if num_failures: + suppress_server_logs = False + jobset.message( + 'FAILED', + 'Scenario: %s. Some tests failed' % scenario_config['name'], + do_newline=True) + else: + jobset.message( + 'SUCCESS', + 'Scenario: %s. All tests passed' % scenario_config['name'], + do_newline=True) + return num_failures + finally: + # Check if servers are still running. + for server, job in server_jobs.items(): + if not job.is_running(): + print('Server "%s" has exited prematurely.' % server) + suppress_failure = suppress_server_logs and not args.verbose + dockerjob.finish_jobs( + [j for j in six.itervalues(server_jobs)], + suppress_failure=suppress_failure) + + +num_failures = 0 +with open(args.scenarios_file, 'r') as scenarios_input: + all_scenarios = json.loads(scenarios_input.read()) + for scenario in all_scenarios: + if args.scenario_name: + if args.scenario_name != scenario['name']: + jobset.message('IDLE', + 'Skipping scenario: %s' % scenario['name']) + continue + num_failures += run_one_scenario(scenario) +if num_failures == 0: + sys.exit(0) +else: + sys.exit(1) diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 9bbc2e3e0e..ff6682c3cf 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -63,17 +63,17 @@ _SKIP_ADVANCED = [ 'unimplemented_service' ] +_SKIP_SPECIAL_STATUS_MESSAGE = ['special_status_message'] + _TEST_TIMEOUT = 3 * 60 # disable this test on core-based languages, # see https://github.com/grpc/grpc/issues/9779 _SKIP_DATA_FRAME_PADDING = ['data_frame_padding'] -# report suffix is important for reports to get picked up by internal CI -_INTERNAL_CL_XML_REPORT = 'sponge_log.xml' - -# report suffix is important for reports to get picked up by internal CI -_XML_REPORT = 'report.xml' +# report suffix "sponge_log.xml" is important for reports to get picked up by internal CI +_DOCKER_BUILD_XML_REPORT = 'interop_docker_build/sponge_log.xml' +_TESTS_XML_REPORT = 'interop_test/sponge_log.xml' class CXXLanguage: @@ -100,7 +100,7 @@ class CXXLanguage: return {} def unimplemented_test_cases(self): - return _SKIP_DATA_FRAME_PADDING + return _SKIP_DATA_FRAME_PADDING + _SKIP_SPECIAL_STATUS_MESSAGE def unimplemented_test_cases_server(self): return [] @@ -129,7 +129,7 @@ class CSharpLanguage: return {} def unimplemented_test_cases(self): - return _SKIP_SERVER_COMPRESSION + _SKIP_DATA_FRAME_PADDING + return _SKIP_SERVER_COMPRESSION + _SKIP_DATA_FRAME_PADDING + _SKIP_SPECIAL_STATUS_MESSAGE def unimplemented_test_cases_server(self): return _SKIP_COMPRESSION @@ -158,7 +158,7 @@ class CSharpCoreCLRLanguage: return {} def unimplemented_test_cases(self): - return _SKIP_SERVER_COMPRESSION + _SKIP_DATA_FRAME_PADDING + return _SKIP_SERVER_COMPRESSION + _SKIP_DATA_FRAME_PADDING + _SKIP_SPECIAL_STATUS_MESSAGE def unimplemented_test_cases_server(self): return _SKIP_COMPRESSION @@ -188,10 +188,10 @@ class DartLanguage: return {} def unimplemented_test_cases(self): - return _SKIP_COMPRESSION + return _SKIP_COMPRESSION + _SKIP_SPECIAL_STATUS_MESSAGE def unimplemented_test_cases_server(self): - return _SKIP_COMPRESSION + return _SKIP_COMPRESSION + _SKIP_SPECIAL_STATUS_MESSAGE def __str__(self): return 'dart' @@ -248,7 +248,7 @@ class JavaOkHttpClient: return {} def unimplemented_test_cases(self): - return _SKIP_DATA_FRAME_PADDING + return _SKIP_DATA_FRAME_PADDING + _SKIP_SPECIAL_STATUS_MESSAGE def __str__(self): return 'javaokhttp' @@ -309,7 +309,7 @@ class Http2Server: return {} def unimplemented_test_cases(self): - return _TEST_CASES + _SKIP_DATA_FRAME_PADDING + return _TEST_CASES + _SKIP_DATA_FRAME_PADDING + _SKIP_SPECIAL_STATUS_MESSAGE def unimplemented_test_cases_server(self): return _TEST_CASES @@ -339,7 +339,7 @@ class Http2Client: return {} def unimplemented_test_cases(self): - return _TEST_CASES + return _TEST_CASES + _SKIP_SPECIAL_STATUS_MESSAGE def unimplemented_test_cases_server(self): return _TEST_CASES @@ -431,7 +431,7 @@ class PHPLanguage: return {} def unimplemented_test_cases(self): - return _SKIP_COMPRESSION + _SKIP_DATA_FRAME_PADDING + return _SKIP_COMPRESSION + _SKIP_DATA_FRAME_PADDING + _SKIP_SPECIAL_STATUS_MESSAGE def unimplemented_test_cases_server(self): return [] @@ -456,7 +456,7 @@ class PHP7Language: return {} def unimplemented_test_cases(self): - return _SKIP_COMPRESSION + _SKIP_DATA_FRAME_PADDING + return _SKIP_COMPRESSION + _SKIP_DATA_FRAME_PADDING + _SKIP_SPECIAL_STATUS_MESSAGE def unimplemented_test_cases_server(self): return [] @@ -491,7 +491,7 @@ class ObjcLanguage: # cmdline argument. Here we return all but one test cases as unimplemented, # and depend upon ObjC test's behavior that it runs all cases even when # we tell it to run just one. - return _TEST_CASES[1:] + _SKIP_COMPRESSION + _SKIP_DATA_FRAME_PADDING + return _TEST_CASES[1:] + _SKIP_COMPRESSION + _SKIP_DATA_FRAME_PADDING + _SKIP_SPECIAL_STATUS_MESSAGE def unimplemented_test_cases_server(self): return _SKIP_COMPRESSION @@ -526,7 +526,7 @@ class RubyLanguage: return {} def unimplemented_test_cases(self): - return _SKIP_SERVER_COMPRESSION + _SKIP_DATA_FRAME_PADDING + return _SKIP_SERVER_COMPRESSION + _SKIP_DATA_FRAME_PADDING + _SKIP_SPECIAL_STATUS_MESSAGE def unimplemented_test_cases_server(self): return _SKIP_COMPRESSION @@ -610,7 +610,7 @@ _TEST_CASES = [ 'custom_metadata', 'status_code_and_message', 'unimplemented_method', 'client_compressed_unary', 'server_compressed_unary', 'client_compressed_streaming', 'server_compressed_streaming', - 'unimplemented_service' + 'unimplemented_service', 'special_status_message' ] _AUTH_TEST_CASES = [ @@ -688,6 +688,10 @@ def write_cmdlog_maybe(cmdlog, filename): if cmdlog: with open(filename, 'w') as logfile: logfile.write('#!/bin/bash\n') + logfile.write('# DO NOT MODIFY\n') + logfile.write( + '# This file is generated by run_interop_tests.py/create_testcases.sh\n' + ) logfile.writelines("%s\n" % line for line in cmdlog) print('Command log written to file %s' % filename) @@ -777,12 +781,14 @@ def cloud_to_prod_jobspec(language, ] if transport_security == 'tls': transport_security_options = ['--use_tls=true'] - elif transport_security == 'google_default_credentials' and language == 'c++': + elif transport_security == 'google_default_credentials' and str( + language) in ['c++', 'go', 'java', 'javaokhttp']: transport_security_options = [ '--custom_credentials_type=google_default_credentials' ] else: - print('Invalid transport security option.') + print('Invalid transport security option %s in cloud_to_prod_jobspec.' % + transport_security) sys.exit(1) cmdargs = cmdargs + transport_security_options environ = dict(language.cloud_to_prod_env(), **language.global_env()) @@ -817,8 +823,9 @@ def cloud_to_prod_jobspec(language, cmdline=cmdline, cwd=cwd, environ=environ, - shortname='%s:%s:%s:%s' % (suite_name, language, server_host_nickname, - test_case), + shortname='%s:%s:%s:%s:%s' % + (suite_name, language, server_host_nickname, test_case, + transport_security), timeout_seconds=_TEST_TIMEOUT, flake_retries=4 if args.allow_flakes else 0, timeout_retries=2 if args.allow_flakes else 0, @@ -848,7 +855,8 @@ def cloud_to_cloud_jobspec(language, elif transport_security == 'insecure': interop_only_options += ['--use_tls=false'] else: - print('Invalid transport security option.') + print('Invalid transport security option %s in cloud_to_cloud_jobspec.' + % transport_security) sys.exit(1) client_test_case = test_case @@ -903,8 +911,8 @@ def cloud_to_cloud_jobspec(language, cmdline=cmdline, cwd=cwd, environ=environ, - shortname='cloud_to_cloud:%s:%s_server:%s' % (language, server_name, - test_case), + shortname='cloud_to_cloud:%s:%s_server:%s:%s' % + (language, server_name, test_case, transport_security), timeout_seconds=_TEST_TIMEOUT, flake_retries=4 if args.allow_flakes else 0, timeout_retries=2 if args.allow_flakes else 0, @@ -929,7 +937,8 @@ def server_jobspec(language, elif transport_security == 'insecure': server_cmd += ['--use_tls=false'] else: - print('Invalid transport security option.') + print('Invalid transport security option %s in server_jobspec.' % + transport_security) sys.exit(1) cmdline = bash_cmdline(language.server_cmd(server_cmd)) environ = language.global_env() @@ -1152,8 +1161,9 @@ argp.add_argument( default=False, action='store_const', const=True, - help=('Put reports into subdirectories to improve ' - 'presentation of results by Internal CI.')) + help=( + '(Deprecated, has no effect) Put reports into subdirectories to improve ' + 'presentation of results by Internal CI.')) argp.add_argument( '--bq_result_table', default='', @@ -1242,8 +1252,12 @@ if args.use_docker: if args.verbose: print('Jobs to run: \n%s\n' % '\n'.join(str(j) for j in build_jobs)) - num_failures, _ = jobset.run( + num_failures, build_resultset = jobset.run( build_jobs, newline_on_success=True, maxjobs=args.jobs) + + report_utils.render_junit_xml_report(build_resultset, + _DOCKER_BUILD_XML_REPORT) + if num_failures == 0: jobset.message( 'SUCCESS', @@ -1306,7 +1320,7 @@ try: for language in languages: for test_case in _TEST_CASES: if not test_case in language.unimplemented_test_cases(): - if not test_case in _SKIP_ADVANCED + _SKIP_COMPRESSION: + if not test_case in _SKIP_ADVANCED + _SKIP_COMPRESSION + _SKIP_SPECIAL_STATUS_MESSAGE: tls_test_job = cloud_to_prod_jobspec( language, test_case, @@ -1318,7 +1332,9 @@ try: service_account_key_file, transport_security='tls') jobs.append(tls_test_job) - if language == 'c++': + if str(language) in [ + 'c++', 'go', 'java', 'javaokhttp' + ]: google_default_creds_test_job = cloud_to_prod_jobspec( language, test_case, @@ -1370,7 +1386,9 @@ try: service_account_key_file, transport_security='tls') jobs.append(tls_test_job) - if language == 'c++': + if str(language) in [ + 'go' + ]: # Add more languages to the list to turn on tests. google_default_creds_test_job = cloud_to_prod_jobspec( language, test_case, @@ -1378,6 +1396,7 @@ try: prod_servers[server_host_nickname], docker_image=docker_images.get( str(language)), + auth=True, manual_cmd_log=client_manual_cmd_log, service_account_key_file=args. service_account_key_file, @@ -1503,10 +1522,7 @@ try: write_cmdlog_maybe(server_manual_cmd_log, 'interop_server_cmds.sh') write_cmdlog_maybe(client_manual_cmd_log, 'interop_client_cmds.sh') - xml_report_name = _XML_REPORT - if args.internal_ci: - xml_report_name = _INTERNAL_CL_XML_REPORT - report_utils.render_junit_xml_report(resultset, xml_report_name) + report_utils.render_junit_xml_report(resultset, _TESTS_XML_REPORT) for name, job in resultset.items(): if "http2" in name: diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index 04e706fa5b..c6e67eaf56 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -41,6 +41,11 @@ os.chdir(_ROOT) _REMOTE_HOST_USERNAME = 'jenkins' +_SCENARIO_TIMEOUT = 3 * 60 +_WORKER_TIMEOUT = 3 * 60 +_NETPERF_TIMEOUT = 60 +_QUIT_WORKER_TIMEOUT = 2 * 60 + class QpsWorkerJob: """Encapsulates a qps worker server job.""" @@ -85,15 +90,14 @@ def create_qpsworker_job(language, cmdline = perf_cmd + ['-o', '%s-perf.data' % perf_file_base_name ] + cmdline - worker_timeout = 3 * 60 + worker_timeout = _WORKER_TIMEOUT if remote_host: user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, remote_host) ssh_cmd = ['ssh'] cmdline = ['timeout', '%s' % (worker_timeout + 30)] + cmdline ssh_cmd.extend([ str(user_at_host), - 'cd ~/performance_workspace/grpc/ && python tools/run_tests/start_port_server.py && %s' - % ' '.join(cmdline) + 'cd ~/performance_workspace/grpc/ && %s' % ' '.join(cmdline) ]) cmdline = ssh_cmd @@ -132,7 +136,7 @@ def create_scenario_jobspec(scenario_json, return jobset.JobSpec( cmdline=[cmd], shortname='qps_json_driver.%s' % scenario_json['name'], - timeout_seconds=12 * 60, + timeout_seconds=_SCENARIO_TIMEOUT, shell=True, verbose_success=True) @@ -140,7 +144,7 @@ def create_scenario_jobspec(scenario_json, def create_quit_jobspec(workers, remote_host=None): """Runs quit using QPS driver.""" # setting QPS_WORKERS env variable here makes sure it works with SSH too. - cmd = 'QPS_WORKERS="%s" bins/opt/qps_json_driver --quit' % ','.join( + cmd = 'QPS_WORKERS="%s" cmake/build/qps_json_driver --quit' % ','.join( w.host_and_port for w in workers) if remote_host: user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, remote_host) @@ -150,7 +154,7 @@ def create_quit_jobspec(workers, remote_host=None): return jobset.JobSpec( cmdline=[cmd], shortname='qps_json_driver.quit', - timeout_seconds=3 * 60, + timeout_seconds=_QUIT_WORKER_TIMEOUT, shell=True, verbose_success=True) @@ -182,7 +186,7 @@ def create_netperf_jobspec(server_host='localhost', return jobset.JobSpec( cmdline=[cmd], shortname='netperf', - timeout_seconds=60, + timeout_seconds=_NETPERF_TIMEOUT, shell=True, verbose_success=True) @@ -270,6 +274,12 @@ def build_on_remote_hosts(hosts, 'CONFIG': 'opt'}, timeout_seconds=build_timeout)) if build_local: + # start port server locally + build_jobs.append( + jobset.JobSpec( + cmdline=['python', 'tools/run_tests/start_port_server.py'], + shortname='local_start_port_server', + timeout_seconds=2 * 60)) # Build locally as well build_jobs.append( jobset.JobSpec( diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index ecb5e1d899..a1f2aaab2f 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -61,7 +61,7 @@ _FORCE_ENVIRON_FOR_WRAPPERS = { } _POLLING_STRATEGIES = { - 'linux': ['epollex', 'epollsig', 'epoll1', 'poll', 'poll-cv'], + 'linux': ['epollex', 'epoll1', 'poll', 'poll-cv'], 'mac': ['poll'], } @@ -759,12 +759,16 @@ class PythonLanguage(object): self.python_manager_name(), _docker_arch_suffix(self.args.arch)) def python_manager_name(self): - if self.args.compiler in ['python3.5', 'python3.6']: - return 'pyenv' + if self.args.compiler in [ + 'python2.7', 'python3.5', 'python3.6', 'python3.7' + ]: + return 'stretch_' + self.args.compiler[len('python'):] elif self.args.compiler == 'python_alpine': return 'alpine' - else: + elif self.args.compiler == 'python3.4': return 'jessie' + else: + return 'stretch_3.7' def _get_pythons(self, args): if args.arch == 'x86': @@ -825,6 +829,12 @@ class PythonLanguage(object): minor='6', bits=bits, config_vars=config_vars) + python37_config = _python_config_generator( + name='py37', + major='3', + minor='7', + bits=bits, + config_vars=config_vars) pypy27_config = _pypy_config_generator( name='pypy', major='2', config_vars=config_vars) pypy32_config = _pypy_config_generator( @@ -836,7 +846,7 @@ class PythonLanguage(object): else: return ( python27_config, - python34_config, + python37_config, ) elif args.compiler == 'python2.7': return (python27_config,) @@ -846,6 +856,8 @@ class PythonLanguage(object): return (python35_config,) elif args.compiler == 'python3.6': return (python36_config,) + elif args.compiler == 'python3.7': + return (python37_config,) elif args.compiler == 'pypy': return (pypy27_config,) elif args.compiler == 'pypy3': @@ -858,6 +870,7 @@ class PythonLanguage(object): python34_config, python35_config, python36_config, + python37_config, ) else: raise Exception('Compiler %s not supported.' % args.compiler) @@ -922,20 +935,13 @@ class CSharpLanguage(object): self.config = config self.args = args if self.platform == 'windows': - _check_compiler(self.args.compiler, ['coreclr', 'default']) + _check_compiler(self.args.compiler, ['default', 'coreclr']) _check_arch(self.args.arch, ['default']) self._cmake_arch_option = 'x64' - self._make_options = [] else: _check_compiler(self.args.compiler, ['default', 'coreclr']) self._docker_distro = 'jessie' - if self.platform == 'mac': - # TODO(jtattermusch): EMBED_ZLIB=true currently breaks the mac build - self._make_options = ['EMBED_OPENSSL=true'] - else: - self._make_options = ['EMBED_OPENSSL=true', 'EMBED_ZLIB=true'] - def test_specs(self): with open('src/csharp/tests.json') as f: tests_by_assembly = json.load(f) @@ -1010,7 +1016,7 @@ class CSharpLanguage(object): return ['grpc_csharp_ext'] def make_options(self): - return self._make_options + return [] def build_steps(self): if self.platform == 'windows': @@ -1028,7 +1034,9 @@ class CSharpLanguage(object): if self.platform == 'windows': return 'cmake/build/%s/Makefile' % self._cmake_arch_option else: - return 'Makefile' + # no need to set x86 specific flags as run_tests.py + # currently forbids x86 C# builds on both Linux and MacOS. + return 'cmake/build/Makefile' def dockerfile_dir(self): return 'tools/dockerfile/test/csharp_%s_%s' % ( @@ -1365,9 +1373,9 @@ argp.add_argument( choices=[ 'default', 'gcc4.4', 'gcc4.6', 'gcc4.8', 'gcc4.9', 'gcc5.3', 'gcc7.2', 'gcc_musl', 'clang3.4', 'clang3.5', 'clang3.6', 'clang3.7', 'clang7.0', - 'python2.7', 'python3.4', 'python3.5', 'python3.6', 'pypy', 'pypy3', - 'python_alpine', 'all_the_cpythons', 'electron1.3', 'electron1.6', - 'coreclr', 'cmake', 'cmake_vs2015', 'cmake_vs2017' + 'python2.7', 'python3.4', 'python3.5', 'python3.6', 'python3.7', 'pypy', + 'pypy3', 'python_alpine', 'all_the_cpythons', 'electron1.3', + 'electron1.6', 'coreclr', 'cmake', 'cmake_vs2015', 'cmake_vs2017' ], default='default', help= @@ -1430,7 +1438,7 @@ argp.add_argument( default=None, type=str, help='Only use the specified comma-delimited list of polling engines. ' - 'Example: --force_use_pollers epollsig,poll ' + 'Example: --force_use_pollers epoll1,poll ' ' (This flag has no effect if --force_default_poller flag is also used)') argp.add_argument( '--max_time', default=-1, type=int, help='Maximum test runtime in seconds') @@ -1506,12 +1514,12 @@ if args.travis: _FORCE_ENVIRON_FOR_WRAPPERS = {'GRPC_TRACE': 'api'} if 'all' in args.language: - lang_list = _LANGUAGES.keys() + lang_list = list(_LANGUAGES.keys()) else: lang_list = args.language # We don't support code coverage on some languages if 'gcov' in args.config: - for bad in ['grpc-node', 'objc', 'sanity']: + for bad in ['csharp', 'grpc-node', 'objc', 'sanity']: if bad in lang_list: lang_list.remove(bad) @@ -1713,9 +1721,9 @@ def _has_epollexclusive(): try: subprocess.check_call(binary) return True - except subprocess.CalledProcessError, e: + except subprocess.CalledProcessError as e: return False - except OSError, e: + except OSError as e: # For languages other than C and Windows the binary won't exist return False diff --git a/tools/run_tests/run_tests_matrix.py b/tools/run_tests/run_tests_matrix.py index 1bbec94ee1..d93add00cd 100755 --- a/tools/run_tests/run_tests_matrix.py +++ b/tools/run_tests/run_tests_matrix.py @@ -43,9 +43,6 @@ _OBJC_RUNTESTS_TIMEOUT = 90 * 60 # Number of jobs assigned to each run_tests.py instance _DEFAULT_INNER_JOBS = 2 -# report suffix is important for reports to get picked up by internal CI -_REPORT_SUFFIX = 'sponge_log.xml' - def _safe_report_name(name): """Reports with '+' in target name won't show correctly in ResultStore""" @@ -54,7 +51,15 @@ def _safe_report_name(name): def _report_filename(name): """Generates report file name with directory structure that leads to better presentation by internal CI""" - return '%s/%s' % (_safe_report_name(name), _REPORT_SUFFIX) + # 'sponge_log.xml' suffix must be there for results to get recognized by kokoro. + return '%s/%s' % (_safe_report_name(name), 'sponge_log.xml') + + +def _report_logfilename(name): + """Generates log file name that corresponds to name generated by _report_filename""" + # 'sponge_log.log' suffix must be there for log to get recognized as "target log" + # for the corresponding 'sponge_log.xml' report. + return '%s/%s' % (_safe_report_name(name), 'sponge_log.log') def _docker_jobspec(name, @@ -75,7 +80,8 @@ def _docker_jobspec(name, ] + runtests_args, environ=runtests_envs, shortname='run_tests_%s' % name, - timeout_seconds=timeout_seconds) + timeout_seconds=timeout_seconds, + logfilename=_report_logfilename(name)) return test_job @@ -102,7 +108,8 @@ def _workspace_jobspec(name, ] + runtests_args, environ=env, shortname='run_tests_%s' % name, - timeout_seconds=timeout_seconds) + timeout_seconds=timeout_seconds, + logfilename=_report_logfilename(name)) return test_job @@ -567,11 +574,8 @@ if __name__ == "__main__": print('Will run these tests:') for job in jobs: - if args.dry_run: - print(' %s: "%s"' % (job.shortname, ' '.join(job.cmdline))) - else: - print(' %s' % job.shortname) - print + print(' %s: "%s"' % (job.shortname, ' '.join(job.cmdline))) + print('') if args.dry_run: print('--dry_run was used, exiting') diff --git a/tools/run_tests/sanity/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh index 2c7c140716..8ea53dfec5 100755 --- a/tools/run_tests/sanity/check_submodules.sh +++ b/tools/run_tests/sanity/check_submodules.sh @@ -30,7 +30,7 @@ cat << EOF | awk '{ print $1 }' | sort > "$want_submodules" 5b7683f49e1e9223cf9927b24f6fd3d6bd82e3f8 third_party/benchmark (v1.2.0) 73594cde8c9a52a102c4341c244c833aa61b9c06 third_party/bloaty (remotes/origin/wide-14-g73594cd) b29b21a81b32ec273f118f589f46d56ad3332420 third_party/boringssl (remotes/origin/chromium-stable) - 8149b351bf797bd80e063787886b7618f508e451 third_party/boringssl-with-bazel (version_for_cocoapods_10.0-434-g8149b351) + afc30d43eef92979b05776ec0963c9cede5fb80f third_party/boringssl-with-bazel (fips-20180716-116-gafc30d43e) 3be1924221e1326df520f8498d704a5c4c8d0cce third_party/cares/cares (cares-1_13_0) 30dbc81fb5ffdc98ea9b14b1918bfe4e8779b26e third_party/gflags (v2.2.0-5-g30dbc81) ec44c6c1675c25b9827aacd08c02433cccde7780 third_party/googletest (release-1.8.0) diff --git a/tools/run_tests/sanity/core_banned_functions.py b/tools/run_tests/sanity/core_banned_functions.py index 8afd826453..2a5dcda5be 100755 --- a/tools/run_tests/sanity/core_banned_functions.py +++ b/tools/run_tests/sanity/core_banned_functions.py @@ -24,7 +24,8 @@ os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../../..')) # map of banned function signature to whitelist BANNED_EXCEPT = { 'grpc_resource_quota_ref(': ['src/core/lib/iomgr/resource_quota.cc'], - 'grpc_resource_quota_unref(': ['src/core/lib/iomgr/resource_quota.cc'], + 'grpc_resource_quota_unref(': + ['src/core/lib/iomgr/resource_quota.cc', 'src/core/lib/surface/server.cc'], 'grpc_slice_buffer_destroy(': ['src/core/lib/slice/slice_buffer.cc'], 'grpc_slice_buffer_reset_and_unref(': ['src/core/lib/slice/slice_buffer.cc'], |