diff options
Diffstat (limited to 'tools')
| -rwxr-xr-x | tools/dockerfile/push_testing_images.sh | 64 | ||||
| -rw-r--r-- | tools/dockerfile/test/sanity/Dockerfile | 8 | ||||
| -rw-r--r-- | tools/doxygen/Doxyfile.core.internal | 8 | ||||
| -rwxr-xr-x | tools/run_tests/dockerize/build_and_run_docker.sh | 11 | ||||
| -rwxr-xr-x | tools/run_tests/jobset.py | 12 | ||||
| -rw-r--r-- | tools/run_tests/package_targets.py | 41 | ||||
| -rwxr-xr-x | tools/run_tests/run_tests.py | 6 | ||||
| -rw-r--r-- | tools/run_tests/sources_and_headers.json | 54 |
8 files changed, 157 insertions, 47 deletions
diff --git a/tools/dockerfile/push_testing_images.sh b/tools/dockerfile/push_testing_images.sh new file mode 100755 index 0000000000..f1ee8d59dd --- /dev/null +++ b/tools/dockerfile/push_testing_images.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Builds selected testing docker images and pushes them to dockerhub. +# Useful for testing environments where it's impractical (or impossible) +# to rely on docker images being cached locally after they've been built +# for the first time (which might be costly especially for some images). +# NOTE: gRPC docker images intended to be used by end users are NOT +# pushed using this script (they're built automatically by dockerhub). +# This script is only for "internal" images we use when testing gRPC. + +set -ex + +cd $(dirname $0)/../.. +git_root=$(pwd) +cd - + +DOCKERHUB_ORGANIZATION=grpctesting + +for DOCKERFILE_DIR in tools/dockerfile/test/fuzzer +do + # Generate image name based on Dockerfile checksum. That works well as long + # as can count on dockerfiles being written in a way that changing the logical + # contents of the docker image always changes the SHA (e.g. using "ADD file" + # cmd in the dockerfile in not ok as contents of the added file will not be + # reflected in the SHA). + DOCKER_IMAGE_NAME=$(basename $DOCKERFILE_DIR)_$(sha1sum $DOCKERFILE_DIR/Dockerfile | cut -f1 -d\ ) + + # skip the image if it already exists in the repo + curl --silent -f -lSL https://registry.hub.docker.com/v2/repositories/${DOCKERHUB_ORGANIZATION}/${DOCKER_IMAGE_NAME}/tags/latest > /dev/null \ + && continue + + docker build -t ${DOCKERHUB_ORGANIZATION}/${DOCKER_IMAGE_NAME} ${DOCKERFILE_DIR} + + # "docker login" needs to be run in advance + docker push ${DOCKERHUB_ORGANIZATION}/${DOCKER_IMAGE_NAME} +done diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile index f4b4831a64..6b19ac845b 100644 --- a/tools/dockerfile/test/sanity/Dockerfile +++ b/tools/dockerfile/test/sanity/Dockerfile @@ -93,9 +93,11 @@ RUN pip install simplejson mako #====================================== # More sanity test dependencies (bazel) RUN apt-get install -y openjdk-8-jdk -# TOOD(jtattermusch): pin the bazel version -RUN git clone https://github.com/bazelbuild/bazel.git /bazel -RUN cd /bazel && ./compile.sh +# Check out Bazel version 0.4.1 since this version allows running +# ./compile.sh without a local protoc dependency +# TODO(mattkwong): install dependencies to support latest Bazel version if newer +# version is needed +RUN git clone https://github.com/bazelbuild/bazel.git /bazel && cd /bazel && git checkout tags/0.4.1 && ./compile.sh RUN ln -s /bazel/output/bazel /bin/ #=================== diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index a629bdc542..6572bd4ddf 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -919,9 +919,9 @@ src/core/lib/security/credentials/oauth2/oauth2_credentials.h \ src/core/lib/security/credentials/plugin/plugin_credentials.h \ src/core/lib/security/credentials/ssl/ssl_credentials.h \ src/core/lib/security/transport/auth_filters.h \ -src/core/lib/security/transport/handshake.h \ src/core/lib/security/transport/secure_endpoint.h \ src/core/lib/security/transport/security_connector.h \ +src/core/lib/security/transport/security_handshaker.h \ src/core/lib/security/transport/tsi_error.h \ src/core/lib/security/util/b64.h \ src/core/lib/security/util/json_util.h \ @@ -930,6 +930,7 @@ src/core/lib/tsi/ssl_transport_security.h \ src/core/lib/tsi/ssl_types.h \ src/core/lib/tsi/transport_security.h \ src/core/lib/tsi/transport_security_interface.h \ +src/core/ext/transport/chttp2/server/chttp2_server.h \ src/core/ext/client_channel/client_channel.h \ src/core/ext/client_channel/client_channel_factory.h \ src/core/ext/client_channel/connector.h \ @@ -945,6 +946,7 @@ src/core/ext/client_channel/resolver_registry.h \ src/core/ext/client_channel/subchannel.h \ src/core/ext/client_channel/subchannel_index.h \ src/core/ext/client_channel/uri_parser.h \ +src/core/ext/transport/chttp2/client/chttp2_connector.h \ src/core/ext/lb_policy/grpclb/grpclb.h \ src/core/ext/lb_policy/grpclb/load_balancer_api.h \ src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h \ @@ -1117,9 +1119,9 @@ src/core/lib/security/credentials/oauth2/oauth2_credentials.c \ src/core/lib/security/credentials/plugin/plugin_credentials.c \ src/core/lib/security/credentials/ssl/ssl_credentials.c \ src/core/lib/security/transport/client_auth_filter.c \ -src/core/lib/security/transport/handshake.c \ src/core/lib/security/transport/secure_endpoint.c \ src/core/lib/security/transport/security_connector.c \ +src/core/lib/security/transport/security_handshaker.c \ src/core/lib/security/transport/server_auth_filter.c \ src/core/lib/security/transport/tsi_error.c \ src/core/lib/security/util/b64.c \ @@ -1128,6 +1130,7 @@ src/core/lib/surface/init_secure.c \ src/core/lib/tsi/fake_transport_security.c \ src/core/lib/tsi/ssl_transport_security.c \ src/core/lib/tsi/transport_security.c \ +src/core/ext/transport/chttp2/server/chttp2_server.c \ src/core/ext/transport/chttp2/client/secure/secure_channel_create.c \ src/core/ext/client_channel/channel_connectivity.c \ src/core/ext/client_channel/client_channel.c \ @@ -1147,6 +1150,7 @@ src/core/ext/client_channel/resolver_registry.c \ src/core/ext/client_channel/subchannel.c \ src/core/ext/client_channel/subchannel_index.c \ src/core/ext/client_channel/uri_parser.c \ +src/core/ext/transport/chttp2/client/chttp2_connector.c \ src/core/ext/transport/chttp2/server/insecure/server_chttp2.c \ src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.c \ src/core/ext/transport/chttp2/client/insecure/channel_create.c \ diff --git a/tools/run_tests/dockerize/build_and_run_docker.sh b/tools/run_tests/dockerize/build_and_run_docker.sh index 1ef34b2f96..f52f16ebd6 100755 --- a/tools/run_tests/dockerize/build_and_run_docker.sh +++ b/tools/run_tests/dockerize/build_and_run_docker.sh @@ -41,13 +41,20 @@ cd - # DOCKERFILE_DIR - Directory in which Dockerfile file is located. # DOCKER_RUN_SCRIPT - Script to run under docker (relative to grpc repo root) # OUTPUT_DIR - Directory that will be copied from inside docker after finishing. +# DOCKERHUB_ORGANIZATION - If set, pull a prebuilt image from given dockerhub org. # $@ - Extra args to pass to docker run # Use image name based on Dockerfile location checksum DOCKER_IMAGE_NAME=$(basename $DOCKERFILE_DIR)_$(sha1sum $DOCKERFILE_DIR/Dockerfile | cut -f1 -d\ ) -# Make sure docker image has been built. Should be instantaneous if so. -docker build -t $DOCKER_IMAGE_NAME $DOCKERFILE_DIR +if [ "$DOCKERHUB_ORGANIZATION" != "" ] +then + DOCKER_IMAGE_NAME=$DOCKERHUB_ORGANIZATION/$DOCKER_IMAGE_NAME + docker pull $DOCKER_IMAGE_NAME +else + # Make sure docker image has been built. Should be instantaneous if so. + docker build -t $DOCKER_IMAGE_NAME $DOCKERFILE_DIR +fi # Choose random name for docker container CONTAINER_NAME="build_and_run_docker_$(uuidgen)" diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py index b84eb3b5d7..2acc7971f6 100755 --- a/tools/run_tests/jobset.py +++ b/tools/run_tests/jobset.py @@ -49,12 +49,22 @@ measure_cpu_costs = False _DEFAULT_MAX_JOBS = 16 * multiprocessing.cpu_count() _MAX_RESULT_SIZE = 8192 + +# NOTE: If you change this, please make sure to test reviewing the +# github PR with http://reviewable.io, which is known to add UTF-8 +# characters to the PR description, which leak into the environment here +# and cause failures. +def strip_non_ascii_chars(s): + return ''.join(c for c in s if ord(c) < 128) + + def sanitized_environment(env): sanitized = {} for key, value in env.items(): - sanitized[str(key).encode()] = str(value).encode() + sanitized[strip_non_ascii_chars(key)] = strip_non_ascii_chars(value) return sanitized + def platform_string(): if platform.system() == 'Windows': return 'windows' diff --git a/tools/run_tests/package_targets.py b/tools/run_tests/package_targets.py index 2802957ff1..673affeac0 100644 --- a/tools/run_tests/package_targets.py +++ b/tools/run_tests/package_targets.py @@ -71,50 +71,28 @@ def create_jobspec(name, cmdline, environ=None, cwd=None, shell=False, class CSharpPackage: """Builds C# nuget packages.""" - def __init__(self, linux=False, use_dotnet_cli=True): + def __init__(self, linux=False): self.linux = linux - self.use_dotnet_cli = use_dotnet_cli - self.labels = ['package', 'csharp'] - - if use_dotnet_cli: - if linux: - self.name = 'csharp_package_dotnetcli_linux' - self.labels += ['linux'] - else: - self.name = 'csharp_package_dotnetcli_windows' - self.labels += ['windows'] + if linux: + self.name = 'csharp_package_dotnetcli_linux' + self.labels += ['linux'] else: - # official packages built with dotnet cli rather than nuget pack - self.name = 'csharp_package_obsolete' - self.labels += ['obsolete'] - + self.name = 'csharp_package_dotnetcli_windows' + self.labels += ['windows'] def pre_build_jobspecs(self): - # The older, obsolete build uses nuget only instead of dotnet cli - if 'obsolete' in self.labels: - return [create_jobspec('prebuild_%s' % self.name, - ['tools\\run_tests\\pre_build_csharp.bat'], - shell=True, - flake_retries=5, - timeout_retries=2)] - else: - return [] + return [] def build_jobspec(self): - if self.use_dotnet_cli and self.linux: + if self.linux: return create_docker_jobspec( self.name, 'tools/dockerfile/test/csharp_coreclr_x64', 'src/csharp/build_packages_dotnetcli.sh') - elif self.use_dotnet_cli: - return create_jobspec(self.name, - ['build_packages_dotnetcli.bat'], - cwd='src\\csharp', - shell=True) else: return create_jobspec(self.name, - ['build_packages.bat'], + ['build_packages_dotnetcli.bat'], cwd='src\\csharp', shell=True) @@ -194,7 +172,6 @@ def targets(): """Gets list of supported targets""" return [CSharpPackage(), CSharpPackage(linux=True), - CSharpPackage(use_dotnet_cli=False), NodePackage(), RubyPackage(), PythonPackage(), diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index e503b82f0e..1544ff369d 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -231,6 +231,9 @@ class CLanguage(object): 'GRPC_POLL_STRATEGY': polling_strategy, 'GRPC_VERBOSITY': 'DEBUG'} shortname_ext = '' if polling_strategy=='all' else ' GRPC_POLL_STRATEGY=%s' % polling_strategy + timeout_scaling = 1 + if polling_strategy == 'poll-cv': + timeout_scaling *= 5 if self.config.build_config in target['exclude_configs']: continue if self.args.iomgr_platform in target.get('exclude_iomgrs', []): @@ -269,6 +272,7 @@ class CLanguage(object): out.append(self.config.job_spec(cmdline, shortname='%s --gtest_filter=%s %s' % (binary, test, shortname_ext), cpu_cost=cpu_cost, + timeout_seconds=_DEFAULT_TIMEOUT_SECONDS * timeout_scaling, environ=env)) else: cmdline = [binary] + target['args'] @@ -279,7 +283,7 @@ class CLanguage(object): shortname_ext, cpu_cost=cpu_cost, flaky=target.get('flaky', False), - timeout_seconds=target.get('timeout_seconds', _DEFAULT_TIMEOUT_SECONDS), + timeout_seconds=target.get('timeout_seconds', _DEFAULT_TIMEOUT_SECONDS) * timeout_scaling, environ=env)) elif self.args.regex == '.*' or self.platform == 'windows': print('\nWARNING: binary not found, skipping', binary) diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index abdcfe501b..2e6877ccac 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -7219,9 +7219,9 @@ "src/core/lib/security/credentials/plugin/plugin_credentials.h", "src/core/lib/security/credentials/ssl/ssl_credentials.h", "src/core/lib/security/transport/auth_filters.h", - "src/core/lib/security/transport/handshake.h", "src/core/lib/security/transport/secure_endpoint.h", "src/core/lib/security/transport/security_connector.h", + "src/core/lib/security/transport/security_handshaker.h", "src/core/lib/security/transport/tsi_error.h", "src/core/lib/security/util/b64.h", "src/core/lib/security/util/json_util.h" @@ -7260,12 +7260,12 @@ "src/core/lib/security/credentials/ssl/ssl_credentials.h", "src/core/lib/security/transport/auth_filters.h", "src/core/lib/security/transport/client_auth_filter.c", - "src/core/lib/security/transport/handshake.c", - "src/core/lib/security/transport/handshake.h", "src/core/lib/security/transport/secure_endpoint.c", "src/core/lib/security/transport/secure_endpoint.h", "src/core/lib/security/transport/security_connector.c", "src/core/lib/security/transport/security_connector.h", + "src/core/lib/security/transport/security_handshaker.c", + "src/core/lib/security/transport/security_handshaker.h", "src/core/lib/security/transport/server_auth_filter.c", "src/core/lib/security/transport/tsi_error.c", "src/core/lib/security/transport/tsi_error.h", @@ -7432,9 +7432,29 @@ "deps": [ "gpr", "grpc_base", - "grpc_client_channel", "grpc_transport_chttp2" ], + "headers": [ + "src/core/ext/transport/chttp2/client/chttp2_connector.h" + ], + "is_filegroup": true, + "language": "c", + "name": "grpc_transport_chttp2_client_connector", + "src": [ + "src/core/ext/transport/chttp2/client/chttp2_connector.c", + "src/core/ext/transport/chttp2/client/chttp2_connector.h" + ], + "third_party": false, + "type": "filegroup" + }, + { + "deps": [ + "gpr", + "grpc_base", + "grpc_client_channel", + "grpc_transport_chttp2", + "grpc_transport_chttp2_client_connector" + ], "headers": [], "is_filegroup": true, "language": "c", @@ -7452,7 +7472,8 @@ "grpc_base", "grpc_client_channel", "grpc_secure", - "grpc_transport_chttp2" + "grpc_transport_chttp2", + "grpc_transport_chttp2_client_connector" ], "headers": [], "is_filegroup": true, @@ -7470,6 +7491,26 @@ "grpc_base", "grpc_transport_chttp2" ], + "headers": [ + "src/core/ext/transport/chttp2/server/chttp2_server.h" + ], + "is_filegroup": true, + "language": "c", + "name": "grpc_transport_chttp2_server", + "src": [ + "src/core/ext/transport/chttp2/server/chttp2_server.c", + "src/core/ext/transport/chttp2/server/chttp2_server.h" + ], + "third_party": false, + "type": "filegroup" + }, + { + "deps": [ + "gpr", + "grpc_base", + "grpc_transport_chttp2", + "grpc_transport_chttp2_server" + ], "headers": [], "is_filegroup": true, "language": "c", @@ -7486,7 +7527,8 @@ "gpr", "grpc_base", "grpc_secure", - "grpc_transport_chttp2" + "grpc_transport_chttp2", + "grpc_transport_chttp2_server" ], "headers": [], "is_filegroup": true, |