diff options
Diffstat (limited to 'tools')
-rw-r--r-- | tools/distrib/python/grpcio_tools/README.rst | 2 | ||||
-rw-r--r-- | tools/distrib/python/grpcio_tools/grpc/tools/main.cc | 1 | ||||
-rw-r--r-- | tools/dockerfile/test/csharp_coreclr_x64/Dockerfile | 34 | ||||
-rw-r--r-- | tools/doxygen/Doxyfile.core.internal | 5 | ||||
-rwxr-xr-x | tools/run_tests/build_csharp_coreclr.sh | 4 | ||||
-rwxr-xr-x | tools/run_tests/build_package_csharp_coreclr.sh | 59 | ||||
-rwxr-xr-x | tools/run_tests/build_package_ruby.sh | 1 | ||||
-rwxr-xr-x | tools/run_tests/dockerjob.py | 4 | ||||
-rwxr-xr-x | tools/run_tests/jobset.py | 66 | ||||
-rw-r--r-- | tools/run_tests/package_targets.py | 26 | ||||
-rwxr-xr-x | tools/run_tests/run_interop_tests.py | 6 | ||||
-rwxr-xr-x | tools/run_tests/run_performance_tests.py | 4 | ||||
-rwxr-xr-x | tools/run_tests/run_tests.py | 75 | ||||
-rw-r--r-- | tools/run_tests/sources_and_headers.json | 68 | ||||
-rw-r--r-- | tools/run_tests/tests.json | 649 |
15 files changed, 810 insertions, 194 deletions
diff --git a/tools/distrib/python/grpcio_tools/README.rst b/tools/distrib/python/grpcio_tools/README.rst index e9f137493b..8946a1d5b3 100644 --- a/tools/distrib/python/grpcio_tools/README.rst +++ b/tools/distrib/python/grpcio_tools/README.rst @@ -53,7 +53,7 @@ GCC-like stuff, but you may end up having a bad time. :: $ export REPO_ROOT=grpc # REPO_ROOT can be any directory of your choice - $ git clone https://github.com/grpc/grpc.git $REPO_ROOT + $ git clone -b $(curl -L http://grpc.io/release) https://github.com/grpc/grpc $REPO_ROOT $ cd $REPO_ROOT $ git submodule update --init diff --git a/tools/distrib/python/grpcio_tools/grpc/tools/main.cc b/tools/distrib/python/grpcio_tools/grpc/tools/main.cc index 81675b4e6f..8391839513 100644 --- a/tools/distrib/python/grpcio_tools/grpc/tools/main.cc +++ b/tools/distrib/python/grpcio_tools/grpc/tools/main.cc @@ -45,7 +45,6 @@ int protoc_main(int argc, char* argv[]) { // gRPC Python grpc_python_generator::GeneratorConfiguration grpc_py_config; - grpc_py_config.beta_package_root = "grpc.beta"; grpc_python_generator::PythonGrpcGenerator grpc_py_generator(grpc_py_config); cli.RegisterGenerator("--grpc_python_out", &grpc_py_generator, "Generate Python source file."); diff --git a/tools/dockerfile/test/csharp_coreclr_x64/Dockerfile b/tools/dockerfile/test/csharp_coreclr_x64/Dockerfile index 9dfc040d73..fd7215716d 100644 --- a/tools/dockerfile/test/csharp_coreclr_x64/Dockerfile +++ b/tools/dockerfile/test/csharp_coreclr_x64/Dockerfile @@ -27,7 +27,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -FROM microsoft/dotnet:1.0.0-preview1 +FROM debian:jessie # Install Git and basic packages. RUN apt-get update && apt-get install -y \ @@ -63,6 +63,38 @@ RUN apt-get update && apt-get install -y \ # Build profiling RUN apt-get update && apt-get install -y time && apt-get clean +#================ +# C# dependencies + +# Update to a newer version of mono +RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF +RUN echo "deb http://download.mono-project.com/repo/debian wheezy main" | tee /etc/apt/sources.list.d/mono-xamarin.list +RUN echo "deb http://download.mono-project.com/repo/debian wheezy-apache24-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list +RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libjpeg62-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list +RUN echo "deb http://download.mono-project.com/repo/debian wheezy-libtiff-compat main" | tee -a /etc/apt/sources.list.d/mono-xamarin.list + +# Install dependencies +RUN apt-get update && apt-get -y dist-upgrade && apt-get install -y \ + mono-devel \ + ca-certificates-mono \ + nuget \ + && apt-get clean + + +# Install dotnet SDK based on https://www.microsoft.com/net/core#debian +RUN apt-get update && apt-get install -y curl libunwind8 gettext +RUN curl -sSL -o dotnet.tar.gz https://go.microsoft.com/fwlink/?LinkID=809130 +RUN mkdir -p /opt/dotnet && tar zxf dotnet.tar.gz -C /opt/dotnet +RUN ln -s /opt/dotnet/dotnet /usr/local/bin + +# Trigger the population of the local package cache +ENV NUGET_XMLDOC_MODE skip +RUN mkdir warmup \ + && cd warmup \ + && dotnet new \ + && cd .. \ + && rm -rf warmup + # Prepare ccache RUN ln -s /usr/bin/ccache /usr/local/bin/gcc RUN ln -s /usr/bin/ccache /usr/local/bin/g++ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 1c75c941c2..b846237689 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -808,6 +808,7 @@ src/core/lib/iomgr/closure.h \ src/core/lib/iomgr/endpoint.h \ src/core/lib/iomgr/endpoint_pair.h \ src/core/lib/iomgr/error.h \ +src/core/lib/iomgr/ev_epoll_linux.h \ src/core/lib/iomgr/ev_poll_and_epoll_posix.h \ src/core/lib/iomgr/ev_poll_posix.h \ src/core/lib/iomgr/ev_posix.h \ @@ -818,6 +819,7 @@ src/core/lib/iomgr/iomgr.h \ src/core/lib/iomgr/iomgr_internal.h \ src/core/lib/iomgr/iomgr_posix.h \ src/core/lib/iomgr/load_file.h \ +src/core/lib/iomgr/network_status_tracker.h \ src/core/lib/iomgr/polling_entity.h \ src/core/lib/iomgr/pollset.h \ src/core/lib/iomgr/pollset_set.h \ @@ -859,7 +861,6 @@ src/core/lib/surface/event_string.h \ src/core/lib/surface/init.h \ src/core/lib/surface/lame_client.h \ src/core/lib/surface/server.h \ -src/core/lib/surface/surface_trace.h \ src/core/lib/transport/byte_stream.h \ src/core/lib/transport/connectivity_state.h \ src/core/lib/transport/metadata.h \ @@ -963,6 +964,7 @@ src/core/lib/iomgr/endpoint.c \ src/core/lib/iomgr/endpoint_pair_posix.c \ src/core/lib/iomgr/endpoint_pair_windows.c \ src/core/lib/iomgr/error.c \ +src/core/lib/iomgr/ev_epoll_linux.c \ src/core/lib/iomgr/ev_poll_and_epoll_posix.c \ src/core/lib/iomgr/ev_poll_posix.c \ src/core/lib/iomgr/ev_posix.c \ @@ -973,6 +975,7 @@ src/core/lib/iomgr/iomgr.c \ src/core/lib/iomgr/iomgr_posix.c \ src/core/lib/iomgr/iomgr_windows.c \ src/core/lib/iomgr/load_file.c \ +src/core/lib/iomgr/network_status_tracker.c \ src/core/lib/iomgr/polling_entity.c \ src/core/lib/iomgr/pollset_set_windows.c \ src/core/lib/iomgr/pollset_windows.c \ diff --git a/tools/run_tests/build_csharp_coreclr.sh b/tools/run_tests/build_csharp_coreclr.sh index 68c19cb6c9..733b1a2083 100755 --- a/tools/run_tests/build_csharp_coreclr.sh +++ b/tools/run_tests/build_csharp_coreclr.sh @@ -36,7 +36,3 @@ cd $(dirname $0)/../../src/csharp dotnet restore . dotnet build -f netstandard1.5 --configuration $MSBUILD_CONFIG '**/project.json' - -# Grpc.IntegrationTesting doesn't get built by the previous command for some reason. -# TODO(jtattermusch): get rid of the hack -dotnet build -f netstandard1.5 --configuration $MSBUILD_CONFIG Grpc.IntegrationTesting/project.json diff --git a/tools/run_tests/build_package_csharp_coreclr.sh b/tools/run_tests/build_package_csharp_coreclr.sh new file mode 100755 index 0000000000..e1c363da50 --- /dev/null +++ b/tools/run_tests/build_package_csharp_coreclr.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -ex + +cd $(dirname $0)/../.. + +mkdir -p artifacts/ + +cd src/csharp + +# IMPORTANT: NuGet packages generated by dotnet CLI are considered experimental. +# The official nugets are generated by src/csharp/build_packages.bat + +mkdir -p nativelibs/windows_x86 nativelibs/windows_x64 \ + nativelibs/linux_x86 nativelibs/linux_x64 \ + nativelibs/macosx_x86 nativelibs/macosx_x64 + +cp $EXTERNAL_GIT_ROOT/architecture=x86,language=csharp,platform=windows/artifacts/* nativelibs/windows_x86 || true +cp $EXTERNAL_GIT_ROOT/architecture=x64,language=csharp,platform=windows/artifacts/* nativelibs/windows_x64 || true +cp $EXTERNAL_GIT_ROOT/architecture=x86,language=csharp,platform=linux/artifacts/* nativelibs/linux_x86 || true +cp $EXTERNAL_GIT_ROOT/architecture=x64,language=csharp,platform=linux/artifacts/* nativelibs/linux_x64 || true +cp $EXTERNAL_GIT_ROOT/architecture=x86,language=csharp,platform=macos/artifacts/* nativelibs/macosx_x86 || true +cp $EXTERNAL_GIT_ROOT/architecture=x64,language=csharp,platform=macos/artifacts/* nativelibs/macosx_x64 || true + +dotnet restore . + +dotnet pack --configuration Release Grpc.Core/project.json --output ../../artifacts +dotnet pack --configuration Release Grpc.Auth/project.json --output ../../artifacts +dotnet pack --configuration Release Grpc.HealthCheck/project.json --output ../../artifacts + +tar -czf ../../artifacts/csharp_nugets_experimental.tar.gz ../../artifacts/*.nupkg diff --git a/tools/run_tests/build_package_ruby.sh b/tools/run_tests/build_package_ruby.sh index 4116e29deb..0a755bddb0 100755 --- a/tools/run_tests/build_package_ruby.sh +++ b/tools/run_tests/build_package_ruby.sh @@ -59,6 +59,7 @@ for arch in {x86,x64}; do input_dir="$EXTERNAL_GIT_ROOT/architecture=$arch,language=protoc,platform=$plat/artifacts" output_dir="$base/src/ruby/tools/bin/${ruby_arch}-${plat}" mkdir -p $output_dir/google/protobuf + mkdir -p $output_dir/google/protobuf/compiler # needed for plugin.proto cp $input_dir/protoc* $output_dir/ cp $input_dir/grpc_ruby_plugin* $output_dir/ for proto in "${well_known_protos[@]}"; do diff --git a/tools/run_tests/dockerjob.py b/tools/run_tests/dockerjob.py index 326c4faed9..e4ca3b7faa 100755 --- a/tools/run_tests/dockerjob.py +++ b/tools/run_tests/dockerjob.py @@ -104,7 +104,7 @@ class DockerJob: def __init__(self, spec): self._spec = spec - self._job = jobset.Job(spec, bin_hash=None, newline_on_success=True, travis=True, add_env={}) + self._job = jobset.Job(spec, newline_on_success=True, travis=True, add_env={}) self._container_name = spec.container_name def mapped_port(self, port): @@ -118,4 +118,4 @@ class DockerJob: def is_running(self): """Polls a job and returns True if given job is still running.""" - return self._job.state(jobset.NoCache()) == jobset._RUNNING + return self._job.state() == jobset._RUNNING diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py index d3259e724d..4fe77487f9 100755 --- a/tools/run_tests/jobset.py +++ b/tools/run_tests/jobset.py @@ -29,7 +29,6 @@ """Run a group of subprocesses and then finish.""" -import hashlib import multiprocessing import os import platform @@ -149,7 +148,7 @@ def which(filename): class JobSpec(object): """Specifies what to run for a job.""" - def __init__(self, cmdline, shortname=None, environ=None, hash_targets=None, + def __init__(self, cmdline, shortname=None, environ=None, cwd=None, shell=False, timeout_seconds=5*60, flake_retries=0, timeout_retries=0, kill_handler=None, cpu_cost=1.0, verbose_success=False): @@ -157,19 +156,14 @@ class JobSpec(object): Arguments: cmdline: a list of arguments to pass as the command line environ: a dictionary of environment variables to set in the child process - hash_targets: which files to include in the hash representing the jobs version - (or empty, indicating the job should not be hashed) kill_handler: a handler that will be called whenever job.kill() is invoked cpu_cost: number of cores per second this job needs """ if environ is None: environ = {} - if hash_targets is None: - hash_targets = [] self.cmdline = cmdline self.environ = environ self.shortname = cmdline[0] if shortname is None else shortname - self.hash_targets = hash_targets or [] self.cwd = cwd self.shell = shell self.timeout_seconds = timeout_seconds @@ -180,7 +174,7 @@ class JobSpec(object): self.verbose_success = verbose_success def identity(self): - return '%r %r %r' % (self.cmdline, self.environ, self.hash_targets) + return '%r %r' % (self.cmdline, self.environ) def __hash__(self): return hash(self.identity()) @@ -205,9 +199,8 @@ class JobResult(object): class Job(object): """Manages one job.""" - def __init__(self, spec, bin_hash, newline_on_success, travis, add_env): + def __init__(self, spec, newline_on_success, travis, add_env): self._spec = spec - self._bin_hash = bin_hash self._newline_on_success = newline_on_success self._travis = travis self._add_env = add_env.copy() @@ -249,7 +242,7 @@ class Job(object): self._process = try_start() self._state = _RUNNING - def state(self, update_cache): + def state(self): """Poll current state of the job. Prints messages at completion.""" def stdout(self=self): self._tempfile.seek(0) @@ -293,8 +286,6 @@ class Job(object): stdout() if self._spec.verbose_success else None, do_newline=self._newline_on_success or self._travis) self.result.state = 'PASSED' - if self._bin_hash: - update_cache.finished(self._spec.identity(), self._bin_hash) elif (self._state == _RUNNING and self._spec.timeout_seconds is not None and time.time() - self._start > self._spec.timeout_seconds): @@ -329,7 +320,7 @@ class Jobset(object): """Manages one run of jobs.""" def __init__(self, check_cancelled, maxjobs, newline_on_success, travis, - stop_on_failure, add_env, cache): + stop_on_failure, add_env): self._running = set() self._check_cancelled = check_cancelled self._cancelled = False @@ -338,9 +329,7 @@ class Jobset(object): self._maxjobs = maxjobs self._newline_on_success = newline_on_success self._travis = travis - self._cache = cache self._stop_on_failure = stop_on_failure - self._hashes = {} self._add_env = add_env self.resultset = {} self._remaining = None @@ -367,29 +356,13 @@ class Jobset(object): if current_cpu_cost + spec.cpu_cost <= self._maxjobs: break self.reap() if self.cancelled(): return False - if spec.hash_targets: - if spec.identity() in self._hashes: - bin_hash = self._hashes[spec.identity()] - else: - bin_hash = hashlib.sha1() - for fn in spec.hash_targets: - with open(which(fn)) as f: - bin_hash.update(f.read()) - bin_hash = bin_hash.hexdigest() - self._hashes[spec.identity()] = bin_hash - should_run = self._cache.should_run(spec.identity(), bin_hash) - else: - bin_hash = None - should_run = True - if should_run: - job = Job(spec, - bin_hash, - self._newline_on_success, - self._travis, - self._add_env) - self._running.add(job) - if not self.resultset.has_key(job.GetSpec().shortname): - self.resultset[job.GetSpec().shortname] = [] + job = Job(spec, + self._newline_on_success, + self._travis, + self._add_env) + self._running.add(job) + if not self.resultset.has_key(job.GetSpec().shortname): + self.resultset[job.GetSpec().shortname] = [] return True def reap(self): @@ -397,7 +370,7 @@ class Jobset(object): while self._running: dead = set() for job in self._running: - st = job.state(self._cache) + st = job.state() if st == _RUNNING: continue if st == _FAILURE or st == _KILLED: self._failures += 1 @@ -450,15 +423,6 @@ def _never_cancelled(): return False -# cache class that caches nothing -class NoCache(object): - def should_run(self, cmdline, bin_hash): - return True - - def finished(self, cmdline, bin_hash): - pass - - def tag_remaining(xs): staging = [] for x in xs: @@ -477,12 +441,10 @@ def run(cmdlines, travis=False, infinite_runs=False, stop_on_failure=False, - cache=None, add_env={}): js = Jobset(check_cancelled, maxjobs if maxjobs is not None else _DEFAULT_MAX_JOBS, - newline_on_success, travis, stop_on_failure, add_env, - cache if cache is not None else NoCache()) + newline_on_success, travis, stop_on_failure, add_env) for cmdline, remaining in tag_remaining(cmdlines): if not js.start(cmdline): break diff --git a/tools/run_tests/package_targets.py b/tools/run_tests/package_targets.py index 820b539b59..39a11a243d 100644 --- a/tools/run_tests/package_targets.py +++ b/tools/run_tests/package_targets.py @@ -71,18 +71,29 @@ def create_jobspec(name, cmdline, environ=None, cwd=None, shell=False, class CSharpPackage: """Builds C# nuget packages.""" - def __init__(self): - self.name = 'csharp_package' - self.labels = ['package', 'csharp', 'windows'] + def __init__(self, use_coreclr=False): + self.use_coreclr = use_coreclr + self.name = 'csharp_package_coreclr' if use_coreclr else 'csharp_package' + self.labels = ['package', 'csharp'] + if use_coreclr: + self.labels += ['linux'] + else: + self.labels += ['windows'] def pre_build_jobspecs(self): return [] def build_jobspec(self): - return create_jobspec(self.name, - ['build_packages.bat'], - cwd='src\\csharp', - shell=True) + if self.use_coreclr: + return create_docker_jobspec( + self.name, + 'tools/dockerfile/test/csharp_coreclr_x64', + 'tools/run_tests/build_package_csharp_coreclr.sh') + else: + return create_jobspec(self.name, + ['build_packages.bat'], + cwd='src\\csharp', + shell=True) def __str__(self): return self.name @@ -159,6 +170,7 @@ class PHPPackage: def targets(): """Gets list of supported targets""" return [CSharpPackage(), + CSharpPackage(use_coreclr=True), NodePackage(), RubyPackage(), PythonPackage(), diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index d76dd4b7d2..adf9adaf13 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -470,7 +470,8 @@ def cloud_to_prod_jobspec(language, test_case, server_host_name, flake_retries=5 if args.allow_flakes else 0, timeout_retries=2 if args.allow_flakes else 0, kill_handler=_job_kill_handler) - test_job.container_name = container_name + if docker_image: + test_job.container_name = container_name return test_job @@ -506,7 +507,8 @@ def cloud_to_cloud_jobspec(language, test_case, server_name, server_host, flake_retries=5 if args.allow_flakes else 0, timeout_retries=2 if args.allow_flakes else 0, kill_handler=_job_kill_handler) - test_job.container_name = container_name + if docker_image: + test_job.container_name = container_name return test_job diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index f037d0d17d..14901caf07 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -61,11 +61,11 @@ class QpsWorkerJob: self._spec = spec self.language = language self.host_and_port = host_and_port - self._job = jobset.Job(spec, bin_hash=None, newline_on_success=True, travis=True, add_env={}) + self._job = jobset.Job(spec, newline_on_success=True, travis=True, add_env={}) def is_running(self): """Polls a job and returns True if given job is still running.""" - return self._job.state(jobset.NoCache()) == jobset._RUNNING + return self._job.state() == jobset._RUNNING def kill(self): return self._job.kill() diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 396699d8fc..c1254275e6 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -33,7 +33,6 @@ import argparse import ast import glob -import hashlib import itertools import json import multiprocessing @@ -63,7 +62,7 @@ _FORCE_ENVIRON_FOR_WRAPPERS = {} _POLLING_STRATEGIES = { - 'linux': ['poll', 'legacy'] + 'linux': ['epoll', 'poll', 'legacy'] } @@ -78,24 +77,18 @@ class Config(object): if environ is None: environ = {} self.build_config = config - self.allow_hashing = (config != 'gcov') self.environ = environ self.environ['CONFIG'] = config self.tool_prefix = tool_prefix self.timeout_multiplier = timeout_multiplier - def job_spec(self, cmdline, hash_targets, timeout_seconds=5*60, + def job_spec(self, cmdline, timeout_seconds=5*60, shortname=None, environ={}, cpu_cost=1.0, flaky=False): """Construct a jobset.JobSpec for a test under this config Args: cmdline: a list of strings specifying the command line the test would like to run - hash_targets: either None (don't do caching of test results), or - a list of strings specifying files to include in a - binary hash to check if a test has changed - -- if used, all artifacts needed to run the test must - be listed """ actual_environ = self.environ.copy() for k, v in environ.iteritems(): @@ -105,8 +98,6 @@ class Config(object): environ=actual_environ, cpu_cost=cpu_cost, timeout_seconds=(self.timeout_multiplier * timeout_seconds if timeout_seconds else None), - hash_targets=hash_targets - if self.allow_hashing else None, flake_retries=5 if flaky or args.allow_flakes else 0, timeout_retries=3 if args.allow_flakes else 0) @@ -399,7 +390,6 @@ class PythonLanguage(object): if self.config.build_config != 'gcov': return [self.config.job_spec( ['tools/run_tests/run_python.sh', tox_env], - None, environ=dict(environment.items() + [('GRPC_PYTHON_TESTRUNNER_FILTER', suite_name)]), shortname='%s.test.%s' % (tox_env, suite_name), @@ -408,7 +398,6 @@ class PythonLanguage(object): for tox_env in self._tox_envs] else: return [self.config.job_spec(['tools/run_tests/run_python.sh', tox_env], - None, environ=environment, shortname='%s.test.coverage' % tox_env, timeout_seconds=15*60) @@ -425,7 +414,7 @@ class PythonLanguage(object): return [] def build_steps(self): - return [['tools/run_tests/build_python.sh', tox_env] + return [['tools/run_tests/build_python.sh', tox_env] for tox_env in self._tox_envs] def post_tests_steps(self): @@ -460,7 +449,7 @@ class RubyLanguage(object): _check_compiler(self.args.compiler, ['default']) def test_specs(self): - return [self.config.job_spec(['tools/run_tests/run_ruby.sh'], None, + return [self.config.job_spec(['tools/run_tests/run_ruby.sh'], timeout_seconds=10*60, environ=_FORCE_ENVIRON_FOR_WRAPPERS)] @@ -533,7 +522,7 @@ class CSharpLanguage(object): if self.platform == 'linux': assembly_subdir += '/netstandard1.5/debian.8-x64' assembly_extension = '' - if self.platform == 'mac': + elif self.platform == 'mac': assembly_subdir += '/netstandard1.5/osx.10.11-x64' assembly_extension = '' else: @@ -673,7 +662,7 @@ class Sanity(object): def test_specs(self): import yaml with open('tools/run_tests/sanity/sanity_tests.yaml', 'r') as f: - return [self.config.job_spec(cmd['script'].split(), None, + return [self.config.job_spec(cmd['script'].split(), timeout_seconds=None, environ={'TEST': 'true'}, cpu_cost=cmd.get('cpu_cost', 1)) for cmd in yaml.load(f)] @@ -1061,46 +1050,6 @@ runs_per_test = args.runs_per_test forever = args.forever -class TestCache(object): - """Cache for running tests.""" - - def __init__(self, use_cache_results): - self._last_successful_run = {} - self._use_cache_results = use_cache_results - self._last_save = time.time() - - def should_run(self, cmdline, bin_hash): - if cmdline not in self._last_successful_run: - return True - if self._last_successful_run[cmdline] != bin_hash: - return True - if not self._use_cache_results: - return True - return False - - def finished(self, cmdline, bin_hash): - self._last_successful_run[cmdline] = bin_hash - if time.time() - self._last_save > 1: - self.save() - - def dump(self): - return [{'cmdline': k, 'hash': v} - for k, v in self._last_successful_run.iteritems()] - - def parse(self, exdump): - self._last_successful_run = dict((o['cmdline'], o['hash']) for o in exdump) - - def save(self): - with open('.run_tests_cache', 'w') as f: - f.write(json.dumps(self.dump())) - self._last_save = time.time() - - def maybe_load(self): - if os.path.exists('.run_tests_cache'): - with open('.run_tests_cache') as f: - self.parse(json.loads(f.read())) - - def _start_port_server(port_server_port): # check if a compatible port server is running # if incompatible (version mismatch) ==> start a new one @@ -1109,7 +1058,7 @@ def _start_port_server(port_server_port): try: version = int(urllib2.urlopen( 'http://localhost:%d/version_number' % port_server_port, - timeout=1).read()) + timeout=10).read()) print 'detected port server running version %d' % version running = True except Exception as e: @@ -1220,7 +1169,7 @@ class BuildAndRunError(object): # returns a list of things that failed (or an empty list on success) def _build_and_run( - check_cancelled, newline_on_success, cache, xml_report=None, build_only=False): + check_cancelled, newline_on_success, xml_report=None, build_only=False): """Do one pass of building & running tests.""" # build latest sequentially num_failures, resultset = jobset.run( @@ -1269,7 +1218,6 @@ def _build_and_run( all_runs, check_cancelled, newline_on_success=newline_on_success, travis=args.travis, infinite_runs=infinite_runs, maxjobs=args.jobs, stop_on_failure=args.stop_on_failure, - cache=cache if not xml_report else None, add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}) if resultset: for k, v in sorted(resultset.items()): @@ -1298,14 +1246,9 @@ def _build_and_run( if num_test_failures: out.append(BuildAndRunError.TEST) - if cache: cache.save() - return out -test_cache = TestCache(runs_per_test == 1) -test_cache.maybe_load() - if forever: success = True while True: @@ -1315,7 +1258,6 @@ if forever: previous_success = success errors = _build_and_run(check_cancelled=have_files_changed, newline_on_success=False, - cache=test_cache, build_only=args.build_only) == 0 if not previous_success and not errors: jobset.message('SUCCESS', @@ -1327,7 +1269,6 @@ if forever: else: errors = _build_and_run(check_cancelled=lambda: False, newline_on_success=args.newline_on_success, - cache=test_cache, xml_report=args.xml_report, build_only=args.build_only) if not errors: diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 00018834af..65de89c0ab 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -324,6 +324,22 @@ ], "headers": [], "language": "c", + "name": "ev_epoll_linux_test", + "src": [ + "test/core/iomgr/ev_epoll_linux_test.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "language": "c", "name": "fd_conservation_posix_test", "src": [ "test/core/iomgr/fd_conservation_posix_test.c" @@ -1704,22 +1720,6 @@ ], "headers": [], "language": "c", - "name": "timers_test", - "src": [ - "test/core/profiling/timers_test.c" - ], - "third_party": false, - "type": "target" - }, - { - "deps": [ - "gpr", - "gpr_test_util", - "grpc", - "grpc_test_util" - ], - "headers": [], - "language": "c", "name": "transport_connectivity_state_test", "src": [ "test/core/transport/connectivity_state_test.c" @@ -5387,6 +5387,7 @@ "test/core/end2end/tests/max_concurrent_streams.c", "test/core/end2end/tests/max_message_length.c", "test/core/end2end/tests/negative_deadline.c", + "test/core/end2end/tests/network_status_change.c", "test/core/end2end/tests/no_op.c", "test/core/end2end/tests/payload.c", "test/core/end2end/tests/ping.c", @@ -5446,6 +5447,7 @@ "test/core/end2end/tests/max_concurrent_streams.c", "test/core/end2end/tests/max_message_length.c", "test/core/end2end/tests/negative_deadline.c", + "test/core/end2end/tests/network_status_change.c", "test/core/end2end/tests/no_op.c", "test/core/end2end/tests/payload.c", "test/core/end2end/tests/ping.c", @@ -5683,19 +5685,6 @@ }, { "deps": [ - "grpc++_codegen_base" - ], - "headers": [], - "language": "c", - "name": "grpc++_codegen_base_src", - "src": [ - "src/cpp/codegen/codegen_init.cc" - ], - "third_party": false, - "type": "filegroup" - }, - { - "deps": [ "gpr", "grpc_codegen" ], @@ -5724,6 +5713,7 @@ "src/core/lib/iomgr/endpoint.h", "src/core/lib/iomgr/endpoint_pair.h", "src/core/lib/iomgr/error.h", + "src/core/lib/iomgr/ev_epoll_linux.h", "src/core/lib/iomgr/ev_poll_and_epoll_posix.h", "src/core/lib/iomgr/ev_poll_posix.h", "src/core/lib/iomgr/ev_posix.h", @@ -5734,6 +5724,7 @@ "src/core/lib/iomgr/iomgr_internal.h", "src/core/lib/iomgr/iomgr_posix.h", "src/core/lib/iomgr/load_file.h", + "src/core/lib/iomgr/network_status_tracker.h", "src/core/lib/iomgr/polling_entity.h", "src/core/lib/iomgr/pollset.h", "src/core/lib/iomgr/pollset_set.h", @@ -5775,7 +5766,6 @@ "src/core/lib/surface/init.h", "src/core/lib/surface/lame_client.h", "src/core/lib/surface/server.h", - "src/core/lib/surface/surface_trace.h", "src/core/lib/transport/byte_stream.h", "src/core/lib/transport/connectivity_state.h", "src/core/lib/transport/metadata.h", @@ -5829,6 +5819,8 @@ "src/core/lib/iomgr/endpoint_pair_windows.c", "src/core/lib/iomgr/error.c", "src/core/lib/iomgr/error.h", + "src/core/lib/iomgr/ev_epoll_linux.c", + "src/core/lib/iomgr/ev_epoll_linux.h", "src/core/lib/iomgr/ev_poll_and_epoll_posix.c", "src/core/lib/iomgr/ev_poll_and_epoll_posix.h", "src/core/lib/iomgr/ev_poll_posix.c", @@ -5849,6 +5841,8 @@ "src/core/lib/iomgr/iomgr_windows.c", "src/core/lib/iomgr/load_file.c", "src/core/lib/iomgr/load_file.h", + "src/core/lib/iomgr/network_status_tracker.c", + "src/core/lib/iomgr/network_status_tracker.h", "src/core/lib/iomgr/polling_entity.c", "src/core/lib/iomgr/polling_entity.h", "src/core/lib/iomgr/pollset.h", @@ -5938,7 +5932,6 @@ "src/core/lib/surface/metadata_array.c", "src/core/lib/surface/server.c", "src/core/lib/surface/server.h", - "src/core/lib/surface/surface_trace.h", "src/core/lib/surface/validate_metadata.c", "src/core/lib/surface/version.c", "src/core/lib/transport/byte_stream.c", @@ -6724,6 +6717,19 @@ }, { "deps": [ + "grpc++_codegen_base" + ], + "headers": [], + "language": "c++", + "name": "grpc++_codegen_base_src", + "src": [ + "src/cpp/codegen/codegen_init.cc" + ], + "third_party": false, + "type": "filegroup" + }, + { + "deps": [ "grpc++_codegen_base", "grpc++_config_proto" ], diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index c62058ede4..dedd55774b 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -380,6 +380,21 @@ { "args": [], "ci_platforms": [ + "linux" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "gtest": false, + "language": "c", + "name": "ev_epoll_linux_test", + "platforms": [ + "linux" + ] + }, + { + "args": [], + "ci_platforms": [ "linux", "mac", "posix" @@ -732,7 +747,7 @@ "posix", "windows" ], - "cpu_cost": 3, + "cpu_cost": 10, "exclude_configs": [], "flaky": false, "gtest": false, @@ -753,7 +768,7 @@ "posix", "windows" ], - "cpu_cost": 1, + "cpu_cost": 10, "exclude_configs": [], "flaky": false, "gtest": false, @@ -1811,27 +1826,6 @@ "flaky": false, "gtest": false, "language": "c", - "name": "timers_test", - "platforms": [ - "linux", - "mac", - "posix", - "windows" - ] - }, - { - "args": [], - "ci_platforms": [ - "linux", - "mac", - "posix", - "windows" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "flaky": false, - "gtest": false, - "language": "c", "name": "transport_connectivity_state_test", "platforms": [ "linux", @@ -4902,6 +4896,28 @@ }, { "args": [ + "network_status_change" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_census_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "no_op" ], "ci_platforms": [ @@ -5760,6 +5776,28 @@ }, { "args": [ + "network_status_change" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_compress_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "no_op" ], "ci_platforms": [ @@ -6594,6 +6632,27 @@ }, { "args": [ + "network_status_change" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_fakesec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "no_op" ], "ci_platforms": [ @@ -7329,6 +7388,26 @@ }, { "args": [ + "network_status_change" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_fd_test", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "no_op" ], "ci_platforms": [ @@ -8117,6 +8196,28 @@ }, { "args": [ + "network_status_change" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_full_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "no_op" ], "ci_platforms": [ @@ -8831,6 +8932,22 @@ }, { "args": [ + "network_status_change" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_full+pipe_test", + "platforms": [ + "linux" + ] + }, + { + "args": [ "no_op" ], "ci_platforms": [ @@ -9577,6 +9694,28 @@ }, { "args": [ + "network_status_change" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_full+trace_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "no_op" ], "ci_platforms": [ @@ -10435,6 +10574,28 @@ }, { "args": [ + "network_status_change" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_loadreporting_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "no_op" ], "ci_platforms": [ @@ -11269,6 +11430,27 @@ }, { "args": [ + "network_status_change" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_oauth2_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "no_op" ], "ci_platforms": [ @@ -12004,6 +12186,27 @@ }, { "args": [ + "network_status_change" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_proxy_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "no_op" ], "ci_platforms": [ @@ -12718,6 +12921,27 @@ }, { "args": [ + "network_status_change" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_sockpair_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "no_op" ], "ci_platforms": [ @@ -13411,6 +13635,27 @@ }, { "args": [ + "network_status_change" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_sockpair+trace_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "no_op" ], "ci_platforms": [ @@ -14125,6 +14370,27 @@ }, { "args": [ + "network_status_change" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_sockpair_1byte_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "no_op" ], "ci_platforms": [ @@ -14926,6 +15192,28 @@ }, { "args": [ + "network_status_change" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_ssl_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "no_op" ], "ci_platforms": [ @@ -15784,6 +16072,28 @@ }, { "args": [ + "network_status_change" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_ssl_cert_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "no_op" ], "ci_platforms": [ @@ -16534,6 +16844,27 @@ }, { "args": [ + "network_status_change" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_ssl_proxy_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "no_op" ], "ci_platforms": [ @@ -17267,6 +17598,26 @@ }, { "args": [ + "network_status_change" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_uds_test", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "no_op" ], "ci_platforms": [ @@ -18073,6 +18424,28 @@ }, { "args": [ + "network_status_change" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_census_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "no_op" ], "ci_platforms": [ @@ -18909,6 +19282,28 @@ }, { "args": [ + "network_status_change" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_compress_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "no_op" ], "ci_platforms": [ @@ -19639,6 +20034,26 @@ }, { "args": [ + "network_status_change" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_fd_nosec_test", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "no_op" ], "ci_platforms": [ @@ -20405,6 +20820,28 @@ }, { "args": [ + "network_status_change" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_full_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "no_op" ], "ci_platforms": [ @@ -21103,6 +21540,22 @@ }, { "args": [ + "network_status_change" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_full+pipe_nosec_test", + "platforms": [ + "linux" + ] + }, + { + "args": [ "no_op" ], "ci_platforms": [ @@ -21827,6 +22280,28 @@ }, { "args": [ + "network_status_change" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_full+trace_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "no_op" ], "ci_platforms": [ @@ -22663,6 +23138,28 @@ }, { "args": [ + "network_status_change" + ], + "ci_platforms": [ + "windows", + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_loadreporting_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "no_op" ], "ci_platforms": [ @@ -23392,6 +23889,27 @@ }, { "args": [ + "network_status_change" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_proxy_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "no_op" ], "ci_platforms": [ @@ -24085,6 +24603,27 @@ }, { "args": [ + "network_status_change" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_sockpair_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "no_op" ], "ci_platforms": [ @@ -24757,6 +25296,27 @@ }, { "args": [ + "network_status_change" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_sockpair+trace_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "no_op" ], "ci_platforms": [ @@ -25490,6 +26050,29 @@ }, { "args": [ + "network_status_change" + ], + "ci_platforms": [ + "windows", + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [ + "msan" + ], + "flaky": false, + "language": "c", + "name": "h2_sockpair_1byte_nosec_test", + "platforms": [ + "windows", + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "no_op" ], "ci_platforms": [ @@ -26229,6 +26812,26 @@ }, { "args": [ + "network_status_change" + ], + "ci_platforms": [ + "linux", + "mac", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "h2_uds_nosec_test", + "platforms": [ + "linux", + "mac", + "posix" + ] + }, + { + "args": [ "no_op" ], "ci_platforms": [ |