diff options
author | David Garcia Quintas <dgq@google.com> | 2016-05-02 10:56:54 -0700 |
---|---|---|
committer | David Garcia Quintas <dgq@google.com> | 2016-05-02 10:56:54 -0700 |
commit | 80721b20ca574df78e7e884a023425274e964a17 (patch) | |
tree | 3f18a170c1cdc49af7c48a787170a818eb2efa0f /tools/run_tests | |
parent | 1b2db6333d4783e18b8d72db7893a758c4fcd2b9 (diff) | |
parent | 1dd74dd3783daa3ac427add91b3b81deb7758200 (diff) |
Merge branch 'master' of github.com:grpc/grpc into codegen_remove_proto
Diffstat (limited to 'tools/run_tests')
-rwxr-xr-x | tools/run_tests/build_package_node.sh | 50 | ||||
-rwxr-xr-x | tools/run_tests/build_python.sh | 12 | ||||
-rwxr-xr-x | tools/run_tests/performance/kill_workers.sh | 54 | ||||
-rwxr-xr-x | tools/run_tests/performance/remote_host_prepare.sh | 7 | ||||
-rwxr-xr-x | tools/run_tests/performance/run_worker_python.sh | 35 | ||||
-rw-r--r-- | tools/run_tests/performance/scenario_config.py | 123 | ||||
-rwxr-xr-x | tools/run_tests/run_interop_tests.py | 3 | ||||
-rwxr-xr-x | tools/run_tests/run_performance_tests.py | 28 | ||||
-rwxr-xr-x | tools/run_tests/run_python.sh | 6 | ||||
-rwxr-xr-x | tools/run_tests/run_tests.py | 34 | ||||
-rw-r--r-- | tools/run_tests/sources_and_headers.json | 20 | ||||
-rw-r--r-- | tools/run_tests/stress_test/STRESS_CLIENT_SPEC.md | 4 | ||||
-rw-r--r-- | tools/run_tests/stress_test/configs/asan.json | 3 | ||||
-rw-r--r-- | tools/run_tests/stress_test/configs/opt-tsan-asan.json | 3 | ||||
-rw-r--r-- | tools/run_tests/stress_test/configs/opt.json | 3 | ||||
-rw-r--r-- | tools/run_tests/stress_test/configs/tsan.json | 3 | ||||
-rw-r--r-- | tools/run_tests/tests.json | 485 |
17 files changed, 818 insertions, 55 deletions
diff --git a/tools/run_tests/build_package_node.sh b/tools/run_tests/build_package_node.sh index 540c826311..6bc9466b63 100755 --- a/tools/run_tests/build_package_node.sh +++ b/tools/run_tests/build_package_node.sh @@ -35,10 +35,54 @@ set -ex cd $(dirname $0)/../.. -mkdir -p artifacts/ -cp -r $EXTERNAL_GIT_ROOT/architecture={x86,x64},language=node,platform={windows,linux,macos}/artifacts/* artifacts/ || true +base=$(pwd) +artifacts=$base/artifacts + +mkdir -p $artifacts +cp -r $EXTERNAL_GIT_ROOT/architecture={x86,x64},language=node,platform={windows,linux,macos}/artifacts/* $artifacts/ || true + +npm update +npm pack + +cp grpc-*.tgz $artifacts/grpc.tgz + +mkdir -p bin + +cd src/node/tools npm update npm pack +cp grpc-tools-*.tgz $artifacts/ +tools_version=$(npm list | grep -oP '(?<=grpc-tools@)\S+') + +output_dir=$artifacts/grpc-precompiled-binaries/node/grpc-tools/$tools_version +mkdir -p $output_dir -cp grpc-*.tgz artifacts/grpc.tgz +for arch in {x86,x64}; do + case arch in + x86) + node_arch=ia32 + ;; + *) + node_arch=$arch + ;; + esac + for plat in {windows,linux,macos}; do + case plat in + windows) + node_plat=win32 + ;; + macos) + node_plat=darwin + ;; + *) + node_plat=$plat + ;; + esac + rm bin/* + input_dir="$EXTERNAL_GIT_ROOT/architecture=$arch,language=protoc,platform=$plat/artifacts" + cp $input_dir/protoc* bin/ + cp $input_dir/grpc_node_plugin* bin/ + tar -czf $output_dir/$node_plat-$node_arch.tar.gz bin/ + done +done diff --git a/tools/run_tests/build_python.sh b/tools/run_tests/build_python.sh index 30d121007f..594c20b14c 100755 --- a/tools/run_tests/build_python.sh +++ b/tools/run_tests/build_python.sh @@ -33,6 +33,8 @@ set -ex # change to grpc repo root cd $(dirname $0)/../.. +TOX_PYTHON_ENV="$1" + ROOT=`pwd` export LD_LIBRARY_PATH=$ROOT/libs/$CONFIG export DYLD_LIBRARY_PATH=$ROOT/libs/$CONFIG @@ -47,9 +49,9 @@ then export GRPC_PYTHON_ENABLE_CYTHON_TRACING=1 fi -tox --notest +tox -e ${TOX_PYTHON_ENV} --notest -$ROOT/.tox/py27/bin/python $ROOT/setup.py build -$ROOT/.tox/py27/bin/python $ROOT/setup.py build_py -$ROOT/.tox/py27/bin/python $ROOT/setup.py build_ext --inplace -$ROOT/.tox/py27/bin/python $ROOT/setup.py gather --test +$ROOT/.tox/${TOX_PYTHON_ENV}/bin/python $ROOT/setup.py build +$ROOT/.tox/${TOX_PYTHON_ENV}/bin/python $ROOT/setup.py build_py +$ROOT/.tox/${TOX_PYTHON_ENV}/bin/python $ROOT/setup.py build_ext --inplace +$ROOT/.tox/${TOX_PYTHON_ENV}/bin/python $ROOT/setup.py gather --test diff --git a/tools/run_tests/performance/kill_workers.sh b/tools/run_tests/performance/kill_workers.sh new file mode 100755 index 0000000000..7a8763424d --- /dev/null +++ b/tools/run_tests/performance/kill_workers.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -ex + +cd $(dirname $0)/../../.. + +# Make sure there are no pre-existing QPS workers around before starting +# the performance test suite + +# C++ +killall -9 qps_worker || true + +# C# +ps -C mono -o pid=,cmd= | grep QpsWorker | awk '{print $1}' | xargs kill -9 + +# Ruby +ps -C ruby -o pid=,cmd= | grep 'qps/worker.rb' | awk '{print $1}' | xargs kill -9 + +# Node +ps -C node -o pid=,cmd= | grep 'performance/worker.js' | awk '{print $1}' | xargs kill -9 + +# Python +ps -C python -o pid=,cmd= | grep 'qps_worker.py' | awk '{print $1}' | xargs kill -9 + +# Java +jps | grep LoadWorker | awk '{print $1}' | xargs kill -9 diff --git a/tools/run_tests/performance/remote_host_prepare.sh b/tools/run_tests/performance/remote_host_prepare.sh index 17cfa1a599..d7f539a74e 100755 --- a/tools/run_tests/performance/remote_host_prepare.sh +++ b/tools/run_tests/performance/remote_host_prepare.sh @@ -41,10 +41,9 @@ ssh "${USER_AT_HOST}" "rm -rf ~/performance_workspace && mkdir -p ~/performance_ # could also kill jenkins. ssh "${USER_AT_HOST}" "killall -9 qps_worker mono node ruby || true" -# Kill all java LoadWorker processes. We can't just killall java -# as one of the processes might be jenkins. -ssh "${USER_AT_HOST}" 'kill -9 $(jps | grep LoadWorker | cut -f1 -d" ") || true' - # push the current sources to the slave and unpack it. scp ../grpc.tar "${USER_AT_HOST}:~/performance_workspace" ssh "${USER_AT_HOST}" "tar -xf ~/performance_workspace/grpc.tar -C ~/performance_workspace" + +# For consistency with local run, invoke the kill_workers script remotely. +ssh "${USER_AT_HOST}" "~/performance_workspace/grpc/tools/run_tests/performance/kill_workers.sh" diff --git a/tools/run_tests/performance/run_worker_python.sh b/tools/run_tests/performance/run_worker_python.sh new file mode 100755 index 0000000000..0da8deda58 --- /dev/null +++ b/tools/run_tests/performance/run_worker_python.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -ex + +cd $(dirname $0)/../../.. + +PYTHONPATH=src/python/grpcio:src/python/gens .tox/py27/bin/python src/python/grpcio/tests/qps/qps_worker.py $@ diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 55657f8d8a..ddbe237569 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -456,6 +456,128 @@ class NodeLanguage: def __str__(self): return 'node' +class PythonLanguage: + + def __init__(self): + self.safename = 'python' + + def worker_cmdline(self): + return ['tools/run_tests/performance/run_worker_python.sh'] + + def worker_port_offset(self): + return 500 + + def scenarios(self): + yield { + 'name': 'python_to_cpp_protobuf_streaming_ping_pong', + 'num_servers': 1, + 'num_clients': 1, + 'client_config': { + 'client_type': 'ASYNC_CLIENT', + 'security_params': SECURE_SECARGS, + 'outstanding_rpcs_per_channel': 1, + 'client_channels': 1, + 'async_client_threads': 1, + 'rpc_type': 'STREAMING', + 'load_params': { + 'closed_loop': {} + }, + 'payload_config': EMPTY_PROTO_PAYLOAD, + 'histogram_params': HISTOGRAM_PARAMS, + }, + 'server_config': { + 'server_type': 'SYNC_SERVER', + 'security_params': SECURE_SECARGS, + 'core_limit': 0, + 'async_server_threads': 1, + }, + 'warmup_seconds': WARMUP_SECONDS, + 'benchmark_seconds': BENCHMARK_SECONDS, + 'SERVER_LANGUAGE': 'c++' + } + yield { + 'name': 'python_protobuf_sync_unary_ping_pong', + 'num_servers': 1, + 'num_clients': 1, + 'client_config': { + 'client_type': 'SYNC_CLIENT', + 'security_params': SECURE_SECARGS, + 'outstanding_rpcs_per_channel': 1, + 'client_channels': 1, + 'async_client_threads': 1, + 'rpc_type': 'UNARY', + 'load_params': { + 'closed_loop': {} + }, + 'payload_config': EMPTY_PROTO_PAYLOAD, + 'histogram_params': HISTOGRAM_PARAMS, + }, + 'server_config': { + 'server_type': 'SYNC_SERVER', + 'security_params': SECURE_SECARGS, + 'core_limit': 0, + 'async_server_threads': 1, + }, + 'warmup_seconds': WARMUP_SECONDS, + 'benchmark_seconds': BENCHMARK_SECONDS, + } + yield { + 'name': 'python_protobuf_async_unary_ping_pong', + 'num_servers': 1, + 'num_clients': 1, + 'client_config': { + 'client_type': 'ASYNC_CLIENT', + 'security_params': SECURE_SECARGS, + 'outstanding_rpcs_per_channel': 1, + 'client_channels': 1, + 'async_client_threads': 1, + 'rpc_type': 'UNARY', + 'load_params': { + 'closed_loop': {} + }, + 'payload_config': EMPTY_PROTO_PAYLOAD, + 'histogram_params': HISTOGRAM_PARAMS, + }, + 'server_config': { + 'server_type': 'SYNC_SERVER', + 'security_params': SECURE_SECARGS, + 'core_limit': 0, + 'async_server_threads': 1, + }, + 'warmup_seconds': WARMUP_SECONDS, + 'benchmark_seconds': BENCHMARK_SECONDS, + } + yield { + 'name': 'python_to_cpp_single_channel_throughput', + 'num_servers': 1, + 'num_clients': 1, + 'client_config': { + 'client_type': 'ASYNC_CLIENT', + 'security_params': SECURE_SECARGS, + 'outstanding_rpcs_per_channel': 1, + 'client_channels': 1, + 'async_client_threads': 1, + 'rpc_type': 'STREAMING', + 'load_params': { + 'closed_loop': {} + }, + 'payload_config': BIG_GENERIC_PAYLOAD, + 'histogram_params': HISTOGRAM_PARAMS, + }, + 'server_config': { + 'server_type': 'ASYNC_GENERIC_SERVER', + 'security_params': SECURE_SECARGS, + 'core_limit': SINGLE_MACHINE_CORES/2, + 'async_server_threads': 1, + 'payload_config': BIG_GENERIC_PAYLOAD, + }, + 'warmup_seconds': WARMUP_SECONDS, + 'benchmark_seconds': BENCHMARK_SECONDS, + 'SERVER_LANGUAGE': 'c++' + } + + def __str__(self): + return 'python' class RubyLanguage: @@ -562,4 +684,5 @@ LANGUAGES = { 'node' : NodeLanguage(), 'ruby' : RubyLanguage(), 'java' : JavaLanguage(), + 'python' : PythonLanguage(), } diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 18d4c1072b..758be9304d 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -111,8 +111,7 @@ class CSharpLanguage: return {} def unimplemented_test_cases(self): - # TODO: status_code_and_message doesn't work against node_server - return _SKIP_COMPRESSION + ['status_code_and_message'] + return _SKIP_COMPRESSION def unimplemented_test_cases_server(self): return _SKIP_COMPRESSION diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index ada341abf5..5519666e84 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -157,8 +157,9 @@ def archive_repo(languages): sys.exit(1) -def prepare_remote_hosts(hosts): - """Prepares remote hosts.""" +def prepare_remote_hosts(hosts, prepare_local=False): + """Prepares remote hosts (and maybe prepare localhost as well).""" + prepare_timeout = 5*60 prepare_jobs = [] for host in hosts: user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, host) @@ -167,13 +168,20 @@ def prepare_remote_hosts(hosts): cmdline=['tools/run_tests/performance/remote_host_prepare.sh'], shortname='remote_host_prepare.%s' % host, environ = {'USER_AT_HOST': user_at_host}, - timeout_seconds=5*60)) - jobset.message('START', 'Preparing remote hosts.', do_newline=True) + timeout_seconds=prepare_timeout)) + if prepare_local: + # Prepare localhost as well + prepare_jobs.append( + jobset.JobSpec( + cmdline=['tools/run_tests/performance/kill_workers.sh'], + shortname='local_prepare', + timeout_seconds=prepare_timeout)) + jobset.message('START', 'Preparing hosts.', do_newline=True) num_failures, _ = jobset.run( prepare_jobs, newline_on_success=True, maxjobs=10) if num_failures == 0: jobset.message('SUCCESS', - 'Remote hosts ready to start build.', + 'Prepare step completed successfully.', do_newline=True) else: jobset.message('FAILED', 'Failed to prepare remote hosts.', @@ -238,6 +246,9 @@ def start_qpsworkers(languages, worker_hosts): def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*', bq_result_table=None): """Create jobspecs for scenarios to run.""" + all_workers = [worker + for workers in workers_by_lang.values() + for worker in workers] scenarios = [] for language in languages: for scenario_json in language.scenarios(): @@ -263,9 +274,6 @@ def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*', scenarios.append(scenario) # the very last scenario requests shutting down the workers. - all_workers = [worker - for workers in workers_by_lang.values() - for worker in workers] scenarios.append(create_quit_jobspec(all_workers, remote_host=remote_host)) return scenarios @@ -322,7 +330,9 @@ if args.remote_driver_host: if remote_hosts: archive_repo(languages=[str(l) for l in languages]) - prepare_remote_hosts(remote_hosts) + prepare_remote_hosts(remote_hosts, prepare_local=True) +else: + prepare_remote_hosts([], prepare_local=True) build_local = False if not args.remote_driver_host: diff --git a/tools/run_tests/run_python.sh b/tools/run_tests/run_python.sh index a93ef2576d..7a3ce6b821 100755 --- a/tools/run_tests/run_python.sh +++ b/tools/run_tests/run_python.sh @@ -33,6 +33,8 @@ set -ex # change to grpc repo root cd $(dirname $0)/../.. +TOX_PYTHON_ENV="$1" + ROOT=`pwd` export LD_LIBRARY_PATH=$ROOT/libs/$CONFIG export DYLD_LIBRARY_PATH=$ROOT/libs/$CONFIG @@ -45,9 +47,9 @@ export GRPC_PYTHON_USE_PRECOMPILED_BINARIES=0 if [ "$CONFIG" = "gcov" ] then export GRPC_PYTHON_ENABLE_CYTHON_TRACING=1 - tox + tox -e ${TOX_PYTHON_ENV} else - $ROOT/.tox/py27/bin/python $ROOT/setup.py test_lite + $ROOT/.tox/${TOX_PYTHON_ENV}/bin/python $ROOT/setup.py test_lite fi mkdir -p $ROOT/reports diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 4b9898539d..dea481ef90 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -356,25 +356,20 @@ class PhpLanguage(object): class PythonLanguage(object): - def __init__(self): - self._build_python_versions = ['2.7'] - self._has_python_versions = [] - def configure(self, config, args): self.config = config self.args = args - _check_compiler(self.args.compiler, ['default']) + self._tox_env = self._get_tox_env(self.args.compiler) def test_specs(self): # load list of known test suites with open('src/python/grpcio/tests/tests.json') as tests_json_file: tests_json = json.load(tests_json_file) environment = dict(_FORCE_ENVIRON_FOR_WRAPPERS) - environment['PYVER'] = '2.7' environment['PYTHONPATH'] = os.path.abspath('src/python/gens') if self.config.build_config != 'gcov': return [self.config.job_spec( - ['tools/run_tests/run_python.sh'], + ['tools/run_tests/run_python.sh', self._tox_env], None, environ=dict(environment.items() + [('GRPC_PYTHON_TESTRUNNER_FILTER', suite_name)]), @@ -399,18 +394,7 @@ class PythonLanguage(object): return [] def build_steps(self): - commands = [] - for python_version in self._build_python_versions: - try: - with open(os.devnull, 'w') as output: - subprocess.check_call(['which', 'python' + python_version], - stdout=output, stderr=output) - commands.append(['tools/run_tests/build_python.sh', python_version]) - self._has_python_versions.append(python_version) - except: - jobset.message('WARNING', 'Missing Python ' + python_version, - do_newline=True) - return commands + return [['tools/run_tests/build_python.sh', self._tox_env]] def post_tests_steps(self): return [] @@ -421,6 +405,15 @@ class PythonLanguage(object): def dockerfile_dir(self): return 'tools/dockerfile/test/python_jessie_%s' % _docker_arch_suffix(self.args.arch) + def _get_tox_env(self, compiler): + """Returns name of tox environment based on selected compiler.""" + if compiler == 'python2.7' or compiler == 'default': + return 'py27' + elif compiler == 'python3.4': + return 'py34' + else: + raise Exception('Compiler %s not supported.' % compiler) + def __str__(self): return 'python' @@ -808,7 +801,8 @@ argp.add_argument('--compiler', choices=['default', 'gcc4.4', 'gcc4.9', 'gcc5.3', 'clang3.4', 'clang3.6', - 'vs2010', 'vs2013', 'vs2015'], + 'vs2010', 'vs2013', 'vs2015', + 'python2.7', 'python3.4'], default='default', help='Selects compiler to use. Allowed values depend on the platform and language.') argp.add_argument('--build_only', diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index cfe204840d..3b3a49a5b2 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -2128,6 +2128,24 @@ { "deps": [ "gpr", + "grpc", + "grpc++" + ], + "headers": [ + "src/proto/grpc/testing/compiler_test.grpc.pb.h", + "src/proto/grpc/testing/compiler_test.pb.h" + ], + "language": "c++", + "name": "golden_file_test", + "src": [ + "test/cpp/codegen/golden_file_test.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", "gpr_test_util", "grpc", "grpc++", @@ -5497,6 +5515,7 @@ "src/core/lib/support/string.c", "src/core/lib/support/string.h", "src/core/lib/support/string_posix.c", + "src/core/lib/support/string_util_win32.c", "src/core/lib/support/string_win32.c", "src/core/lib/support/string_win32.h", "src/core/lib/support/subprocess_posix.c", @@ -5515,6 +5534,7 @@ "src/core/lib/support/time_win32.c", "src/core/lib/support/tls_pthread.c", "src/core/lib/support/tmpfile.h", + "src/core/lib/support/tmpfile_msys.c", "src/core/lib/support/tmpfile_posix.c", "src/core/lib/support/tmpfile_win32.c", "src/core/lib/support/wrap_memcpy.c" diff --git a/tools/run_tests/stress_test/STRESS_CLIENT_SPEC.md b/tools/run_tests/stress_test/STRESS_CLIENT_SPEC.md index 62ca8aff2c..9f079beebc 100644 --- a/tools/run_tests/stress_test/STRESS_CLIENT_SPEC.md +++ b/tools/run_tests/stress_test/STRESS_CLIENT_SPEC.md @@ -6,8 +6,8 @@ This document specifies the features a stress test client should implement in or --------------
**1.** A stress test client should be able to repeatedly execute one or more of the existing 'interop test cases'. It may just be a wrapper around the existing interop test client. The exact command line arguments the client should support are listed in _Table 1_ below.
-**2.** The stress test client must implement a metrics server defined by _[metrics.proto](https://github.com/grpc/grpc/blob/master/src/proto/grpc/testing/metrics.proto)_ and must expose _qps_ as a long-valued Gauge. The client can track the overall _qps_ in one Gauge or in multiple Gauges (for example: One per Channel or Stub).
- The framework periodically queries the _qps_ by calling the `GetAllGauges()` method (the framework assumes that all the returned Gauges are _qps_ Gauges) and uses this to determine if the stress test client is running or crashed or stalled.
+**2.** The stress test client must implement a metrics server defined by _[metrics.proto](https://github.com/grpc/grpc/blob/master/src/proto/grpc/testing/metrics.proto)_ and must expose _qps_ as a `Long`-valued Gauge. The client can track the overall _qps_ in one Gauge or in multiple Gauges (for example: One per Channel or Stub).
+ The framework periodically queries the _qps_ by calling the `GetAllGauges()` method (the framework assumes that all the returned Gauges are _qps_ Gauges and adds them up to determine the final qps) and uses this to determine if the stress test client is running or crashed or stalled.
> *Note:* In this context, the term _**qps**_ means _interop test cases per second_ (not _messages per second_ or _rpc calls per second_)
diff --git a/tools/run_tests/stress_test/configs/asan.json b/tools/run_tests/stress_test/configs/asan.json index c558855314..cb9f55763b 100644 --- a/tools/run_tests/stress_test/configs/asan.json +++ b/tools/run_tests/stress_test/configs/asan.json @@ -16,8 +16,7 @@ "num_channels_per_server":5, "num_stubs_per_channel":10, "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", - "metrics_port": 8081, - "metrics_collection_interval_secs":120 + "metrics_port": 8081 }, "metricsPort": 8081, "metricsArgs": { diff --git a/tools/run_tests/stress_test/configs/opt-tsan-asan.json b/tools/run_tests/stress_test/configs/opt-tsan-asan.json index 4f172ef30b..936d15169e 100644 --- a/tools/run_tests/stress_test/configs/opt-tsan-asan.json +++ b/tools/run_tests/stress_test/configs/opt-tsan-asan.json @@ -26,8 +26,7 @@ "num_channels_per_server":5, "num_stubs_per_channel":10, "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", - "metrics_port": 8081, - "metrics_collection_interval_secs": 60 + "metrics_port": 8081 }, "metricsPort": 8081, "metricsArgs": { diff --git a/tools/run_tests/stress_test/configs/opt.json b/tools/run_tests/stress_test/configs/opt.json index 75505186f2..f45b824048 100644 --- a/tools/run_tests/stress_test/configs/opt.json +++ b/tools/run_tests/stress_test/configs/opt.json @@ -16,8 +16,7 @@ "num_channels_per_server":5, "num_stubs_per_channel":10, "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", - "metrics_port": 8081, - "metrics_collection_interval_secs": 60 + "metrics_port": 8081 }, "metricsPort": 8081, "metricsArgs": { diff --git a/tools/run_tests/stress_test/configs/tsan.json b/tools/run_tests/stress_test/configs/tsan.json index a7ec08313d..6ef3bdf7ea 100644 --- a/tools/run_tests/stress_test/configs/tsan.json +++ b/tools/run_tests/stress_test/configs/tsan.json @@ -16,8 +16,7 @@ "num_channels_per_server":5, "num_stubs_per_channel":10, "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", - "metrics_port": 8081, - "metrics_collection_interval_secs":120 + "metrics_port": 8081 }, "metricsPort": 8081, "metricsArgs": { diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index 5b9a8fade2..0fd77854d2 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -2280,6 +2280,27 @@ "flaky": false, "gtest": true, "language": "c++", + "name": "golden_file_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ] + }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "gtest": true, + "language": "c++", "name": "grpclb_api_test", "platforms": [ "linux", @@ -23466,6 +23487,22 @@ }, { "args": [ + "test/core/end2end/fuzzers/api_fuzzer_corpus/04a5f10d2ebc712cf13c05b5ed0fafb31b42737c" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "api_fuzzer_one_entry", + "platforms": [ + "linux" + ] + }, + { + "args": [ "test/core/end2end/fuzzers/api_fuzzer_corpus/04e01f399f194434b2b724877df64828e8f52c14" ], "ci_platforms": [ @@ -25530,6 +25567,22 @@ }, { "args": [ + "test/core/end2end/fuzzers/api_fuzzer_corpus/38a55e83e685617cdf72e95f1303857b627ae346" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "api_fuzzer_one_entry", + "platforms": [ + "linux" + ] + }, + { + "args": [ "test/core/end2end/fuzzers/api_fuzzer_corpus/38c609f72f5a2cf977788afef9c34652f754add0" ], "ci_platforms": [ @@ -25722,6 +25775,22 @@ }, { "args": [ + "test/core/end2end/fuzzers/api_fuzzer_corpus/40b4b92460c4e76a39af9042fb3d86d491a98e16" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "api_fuzzer_one_entry", + "platforms": [ + "linux" + ] + }, + { + "args": [ "test/core/end2end/fuzzers/api_fuzzer_corpus/42324d3d9e013cd43d4feeed1b48fbe1ea18a732" ], "ci_platforms": [ @@ -26890,6 +26959,22 @@ }, { "args": [ + "test/core/end2end/fuzzers/api_fuzzer_corpus/6914f5f380c83ff9e3e90fc60d5048e47e5e77d9" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "api_fuzzer_one_entry", + "platforms": [ + "linux" + ] + }, + { + "args": [ "test/core/end2end/fuzzers/api_fuzzer_corpus/696ea30e2e1490f2f31b153641b2c29152ded5c2" ], "ci_platforms": [ @@ -27402,6 +27487,22 @@ }, { "args": [ + "test/core/end2end/fuzzers/api_fuzzer_corpus/7abe8c414aa1418157c2d7ae5e70a84ffb61c027" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "api_fuzzer_one_entry", + "platforms": [ + "linux" + ] + }, + { + "args": [ "test/core/end2end/fuzzers/api_fuzzer_corpus/7be89fb64b3d931387e8a5b1ef51bf9cda18006a" ], "ci_platforms": [ @@ -28026,6 +28127,22 @@ }, { "args": [ + "test/core/end2end/fuzzers/api_fuzzer_corpus/8c5bbcc6935d43c94a0c4ce4a5da01c04fd223d8" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "api_fuzzer_one_entry", + "platforms": [ + "linux" + ] + }, + { + "args": [ "test/core/end2end/fuzzers/api_fuzzer_corpus/8d7bb385d6b13b0e689a1e81e29113746218ba99" ], "ci_platforms": [ @@ -29354,6 +29471,22 @@ }, { "args": [ + "test/core/end2end/fuzzers/api_fuzzer_corpus/b5bcc7f39420e997ec6f8e3c70ef49b8f1afb361" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "api_fuzzer_one_entry", + "platforms": [ + "linux" + ] + }, + { + "args": [ "test/core/end2end/fuzzers/api_fuzzer_corpus/b5daec8e0821e8626c9b93ece56ccfef0511346b" ], "ci_platforms": [ @@ -32650,6 +32783,22 @@ }, { "args": [ + "test/core/end2end/fuzzers/api_fuzzer_corpus/f755b44ff2221c971ca2bfaffc69e002ba982730" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "api_fuzzer_one_entry", + "platforms": [ + "linux" + ] + }, + { + "args": [ "test/core/end2end/fuzzers/api_fuzzer_corpus/f788d2b893fe39fe24582acffa6a70f1ca4e3037" ], "ci_platforms": [ @@ -33962,6 +34111,22 @@ }, { "args": [ + "test/core/end2end/fuzzers/client_fuzzer_corpus/1e64080289ea4168304417f3fbd86b01d7d6f431" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "client_fuzzer_one_entry", + "platforms": [ + "linux" + ] + }, + { + "args": [ "test/core/end2end/fuzzers/client_fuzzer_corpus/1e84d42fcf18bbf81ef6e8a16a0c57abbf8d292a" ], "ci_platforms": [ @@ -34122,6 +34287,22 @@ }, { "args": [ + "test/core/end2end/fuzzers/client_fuzzer_corpus/20ee437b7f456ebb19d98d94d9feb1d5e9174c65" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "client_fuzzer_one_entry", + "platforms": [ + "linux" + ] + }, + { + "args": [ "test/core/end2end/fuzzers/client_fuzzer_corpus/2166c7093c424a2136c4cb8b10d0b124047320d4" ], "ci_platforms": [ @@ -34506,6 +34687,22 @@ }, { "args": [ + "test/core/end2end/fuzzers/client_fuzzer_corpus/2c1ecf05c5dde692ed16502294e9570ac3b02600" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "client_fuzzer_one_entry", + "platforms": [ + "linux" + ] + }, + { + "args": [ "test/core/end2end/fuzzers/client_fuzzer_corpus/2c342f8715556398d49bcf3343b5a249d968e19e" ], "ci_platforms": [ @@ -37658,6 +37855,22 @@ }, { "args": [ + "test/core/end2end/fuzzers/client_fuzzer_corpus/aa878edb0100e876e00e310ae221b220fdb5e028" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "client_fuzzer_one_entry", + "platforms": [ + "linux" + ] + }, + { + "args": [ "test/core/end2end/fuzzers/client_fuzzer_corpus/aaada46c7f3bff58c2dd6f4a8394135ed5f253ee" ], "ci_platforms": [ @@ -40106,6 +40319,22 @@ }, { "args": [ + "test/core/end2end/fuzzers/client_fuzzer_corpus/slow-unit-53cf4d25741d5f6e7ad9147b286ff0b40cb500a9" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "client_fuzzer_one_entry", + "platforms": [ + "linux" + ] + }, + { + "args": [ "test/core/transport/chttp2/hpack_parser_corpus/0141fcddc9807ee093313b2256f1306fbbdc6cda" ], "ci_platforms": [ @@ -57514,6 +57743,22 @@ }, { "args": [ + "test/core/end2end/fuzzers/server_fuzzer_corpus/03a304b82629155af693978c2b53439e553f6450" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "server_fuzzer_one_entry", + "platforms": [ + "linux" + ] + }, + { + "args": [ "test/core/end2end/fuzzers/server_fuzzer_corpus/03b9be1fa172dff5d1543be079b9c64fa2c9a278" ], "ci_platforms": [ @@ -57530,6 +57775,22 @@ }, { "args": [ + "test/core/end2end/fuzzers/server_fuzzer_corpus/052c8f28e5884bb48f0d504461272cd3a5893215" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "server_fuzzer_one_entry", + "platforms": [ + "linux" + ] + }, + { + "args": [ "test/core/end2end/fuzzers/server_fuzzer_corpus/05c3a0390d0f52d241728926fa901599a47e4606" ], "ci_platforms": [ @@ -58522,6 +58783,22 @@ }, { "args": [ + "test/core/end2end/fuzzers/server_fuzzer_corpus/2c4c7e2ed6d977ec119b040b328ad09808909a70" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "server_fuzzer_one_entry", + "platforms": [ + "linux" + ] + }, + { + "args": [ "test/core/end2end/fuzzers/server_fuzzer_corpus/2c6660ba.bin" ], "ci_platforms": [ @@ -59290,6 +59567,22 @@ }, { "args": [ + "test/core/end2end/fuzzers/server_fuzzer_corpus/4d982c41efad2242f8c06630c23c68146153b47b" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "server_fuzzer_one_entry", + "platforms": [ + "linux" + ] + }, + { + "args": [ "test/core/end2end/fuzzers/server_fuzzer_corpus/4db3d4075ed27f2a2311f85dd1d6df028cc5d083" ], "ci_platforms": [ @@ -60378,6 +60671,22 @@ }, { "args": [ + "test/core/end2end/fuzzers/server_fuzzer_corpus/830e3f794c53f7b284eb5c635b2943db9ee9aaee" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "server_fuzzer_one_entry", + "platforms": [ + "linux" + ] + }, + { + "args": [ "test/core/end2end/fuzzers/server_fuzzer_corpus/8338ebee.bin" ], "ci_platforms": [ @@ -62810,6 +63119,22 @@ }, { "args": [ + "test/core/end2end/fuzzers/server_fuzzer_corpus/slow-unit-0aa52e00ddd54f8e129430852c2da95650c354b0" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "server_fuzzer_one_entry", + "platforms": [ + "linux" + ] + }, + { + "args": [ "test/core/end2end/fuzzers/server_fuzzer_corpus/slow-unit-13501419f349b7855d2e94060bd08b28923d1f37" ], "ci_platforms": [ @@ -62906,6 +63231,22 @@ }, { "args": [ + "test/core/end2end/fuzzers/server_fuzzer_corpus/slow-unit-3cec540a680b108dda1e0a8e0bfb2d44e5a4a4e8" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "server_fuzzer_one_entry", + "platforms": [ + "linux" + ] + }, + { + "args": [ "test/core/end2end/fuzzers/server_fuzzer_corpus/slow-unit-4c6da955e4c101b81a62b2f8e934d94a62ae534b" ], "ci_platforms": [ @@ -63066,6 +63407,22 @@ }, { "args": [ + "test/core/end2end/fuzzers/server_fuzzer_corpus/slow-unit-84f22ffca68c6e1590a44aa9f6dd0cef1f680c77" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "server_fuzzer_one_entry", + "platforms": [ + "linux" + ] + }, + { + "args": [ "test/core/end2end/fuzzers/server_fuzzer_corpus/slow-unit-93cd6b3f9786ee107a0e2d135b40d13f96e652ed" ], "ci_platforms": [ @@ -63146,6 +63503,70 @@ }, { "args": [ + "test/core/end2end/fuzzers/server_fuzzer_corpus/slow-unit-adaac86cf1aa1e98e95240c5f92c3708456c3624" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "server_fuzzer_one_entry", + "platforms": [ + "linux" + ] + }, + { + "args": [ + "test/core/end2end/fuzzers/server_fuzzer_corpus/slow-unit-b281f018cc919301131cf3ed28449cfbd24b6bbf" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "server_fuzzer_one_entry", + "platforms": [ + "linux" + ] + }, + { + "args": [ + "test/core/end2end/fuzzers/server_fuzzer_corpus/slow-unit-ba0016a62a8576a57f000b90c364847ef6b12dcc" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "server_fuzzer_one_entry", + "platforms": [ + "linux" + ] + }, + { + "args": [ + "test/core/end2end/fuzzers/server_fuzzer_corpus/slow-unit-ba17346b8e46e6a05aaa7342a959a7c5ab0f1471" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "server_fuzzer_one_entry", + "platforms": [ + "linux" + ] + }, + { + "args": [ "test/core/end2end/fuzzers/server_fuzzer_corpus/slow-unit-ba2b1fde90cc70d9abae22c4c4cb051aae8aa148" ], "ci_platforms": [ @@ -63226,6 +63647,22 @@ }, { "args": [ + "test/core/end2end/fuzzers/server_fuzzer_corpus/slow-unit-ccafab6afdc6474610023b47bd7b3e1b9ea4647b" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "server_fuzzer_one_entry", + "platforms": [ + "linux" + ] + }, + { + "args": [ "test/core/end2end/fuzzers/server_fuzzer_corpus/slow-unit-d3c3cba3897fafec97665411ea1f94a89bb4de7b" ], "ci_platforms": [ @@ -63258,6 +63695,22 @@ }, { "args": [ + "test/core/end2end/fuzzers/server_fuzzer_corpus/slow-unit-dc57e96cd02ba32fa4a99c97b6490e9879d30be5" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "server_fuzzer_one_entry", + "platforms": [ + "linux" + ] + }, + { + "args": [ "test/core/end2end/fuzzers/server_fuzzer_corpus/slow-unit-ddfe613d8791b2d377e14fbdffb18b84a89d49b6" ], "ci_platforms": [ @@ -63290,6 +63743,38 @@ }, { "args": [ + "test/core/end2end/fuzzers/server_fuzzer_corpus/slow-unit-f6c1042f96e15183dcc13b9658d971cc29426d53" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "server_fuzzer_one_entry", + "platforms": [ + "linux" + ] + }, + { + "args": [ + "test/core/end2end/fuzzers/server_fuzzer_corpus/slow-unit-f9a2773d6502fd4b1ffa73df3c550b0da63af833" + ], + "ci_platforms": [ + "linux" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "flaky": false, + "language": "c", + "name": "server_fuzzer_one_entry", + "platforms": [ + "linux" + ] + }, + { + "args": [ "test/core/client_config/uri_corpus/02d156dc5e6f2c11c90c2e06fcee04adf036a342" ], "ci_platforms": [ |