diff options
author | 2016-04-15 14:40:08 -0700 | |
---|---|---|
committer | 2016-04-15 14:40:08 -0700 | |
commit | ff71c38be9bf8d8527a6a44655f100b86564f886 (patch) | |
tree | 8cd55f9a769a37a2e8cd78ae95f7a819b823a724 /tools/run_tests | |
parent | 2196c7e253c2932473c9909bdb58b4931b5c677f (diff) | |
parent | ec138dd3cb2f592f2333a12f70282fd861a4e8f2 (diff) |
Merge branch 'master' of github.com:grpc/grpc into ip_parse_refactor
Diffstat (limited to 'tools/run_tests')
-rwxr-xr-x | tools/run_tests/performance/bq_upload_result.py | 103 | ||||
-rwxr-xr-x | tools/run_tests/performance/run_qps_driver.sh | 40 | ||||
-rw-r--r-- | tools/run_tests/performance/scenario_config.py | 127 | ||||
-rw-r--r-- | tools/run_tests/performance/scenario_result_schema.json | 202 | ||||
-rwxr-xr-x | tools/run_tests/run_performance_tests.py | 60 | ||||
-rw-r--r-- | tools/run_tests/sources_and_headers.json | 3 | ||||
-rw-r--r-- | tools/run_tests/stress_test/configs/asan.json | 6 | ||||
-rw-r--r-- | tools/run_tests/stress_test/configs/opt.json | 2 | ||||
-rw-r--r-- | tools/run_tests/stress_test/configs/tsan.json | 6 | ||||
-rwxr-xr-x | tools/run_tests/stress_test/run_on_gke.py | 18 | ||||
-rw-r--r-- | tools/run_tests/tests.json | 48 |
11 files changed, 555 insertions, 60 deletions
diff --git a/tools/run_tests/performance/bq_upload_result.py b/tools/run_tests/performance/bq_upload_result.py new file mode 100755 index 0000000000..0f53ba5d02 --- /dev/null +++ b/tools/run_tests/performance/bq_upload_result.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python2.7 +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Uploads performance benchmark result file to bigquery. + +import argparse +import json +import os +import sys +import uuid + + +gcp_utils_dir = os.path.abspath(os.path.join( + os.path.dirname(__file__), '../../gcp/utils')) +sys.path.append(gcp_utils_dir) +import big_query_utils + + +_PROJECT_ID='grpc-testing' + + +def _upload_scenario_result_to_bigquery(dataset_id, table_id, result_file): + bq = big_query_utils.create_big_query() + _create_results_table(bq, dataset_id, table_id) + + with open(result_file, 'r') as f: + scenario_result = json.loads(f.read()) + + if not _insert_result(bq, dataset_id, table_id, scenario_result): + print 'Error uploading result to bigquery.' + sys.exit(1) + + +def _insert_result(bq, dataset_id, table_id, scenario_result): + _flatten_result_inplace(scenario_result) + row = big_query_utils.make_row(str(uuid.uuid4()), scenario_result) + return big_query_utils.insert_rows(bq, + _PROJECT_ID, + dataset_id, + table_id, + [row]) + + +def _create_results_table(bq, dataset_id, table_id): + with open(os.path.dirname(__file__) + '/scenario_result_schema.json', 'r') as f: + table_schema = json.loads(f.read()) + desc = 'Results of performance benchmarks.' + return big_query_utils.create_table2(bq, _PROJECT_ID, dataset_id, + table_id, table_schema, desc) + + +def _flatten_result_inplace(scenario_result): + """Bigquery is not really great for handling deeply nested data + and repeated fields. To maintain values of some fields while keeping + the schema relatively simple, we artificially leave some of the fields + as JSON strings. + """ + scenario_result['scenario']['clientConfig'] = json.dumps(scenario_result['scenario']['clientConfig']) + scenario_result['scenario']['serverConfig'] = json.dumps(scenario_result['scenario']['serverConfig']) + scenario_result['latencies'] = json.dumps(scenario_result['latencies']) + for stats in scenario_result['clientStats']: + stats['latencies'] = json.dumps(stats['latencies']) + scenario_result['serverCores'] = json.dumps(scenario_result['serverCores']) + + +argp = argparse.ArgumentParser(description='Upload result to big query.') +argp.add_argument('--bq_result_table', required=True, default=None, type=str, + help='Bigquery "dataset.table" to upload results to.') +argp.add_argument('--file_to_upload', default='scenario_result.json', type=str, + help='Report file to upload.') + +args = argp.parse_args() + +dataset_id, table_id = args.bq_result_table.split('.', 2) +_upload_scenario_result_to_bigquery(dataset_id, table_id, args.file_to_upload) +print 'Successfully uploaded %s to BigQuery.\n' % args.file_to_upload diff --git a/tools/run_tests/performance/run_qps_driver.sh b/tools/run_tests/performance/run_qps_driver.sh new file mode 100755 index 0000000000..c8c6890df9 --- /dev/null +++ b/tools/run_tests/performance/run_qps_driver.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -ex + +cd $(dirname $0)/../../.. + +bins/opt/qps_json_driver "$@" + +if [ "$BQ_RESULT_TABLE" != "" ] +then + tools/run_tests/performance/bq_upload_result.py --bq_result_table="$BQ_RESULT_TABLE" +fi diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 7a82d257e4..bd10b6f032 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -33,6 +33,11 @@ SINGLE_MACHINE_CORES=8 WARMUP_SECONDS=5 BENCHMARK_SECONDS=30 +HISTOGRAM_PARAMS = { + 'resolution': 0.01, + 'max_possible': 60e9, +} + EMPTY_GENERIC_PAYLOAD = { 'bytebuf_params': { 'req_size': 0, @@ -83,7 +88,7 @@ class CXXLanguage: secargs = None yield { - 'name': 'generic_async_streaming_ping_pong_%s' + 'name': 'cpp_generic_async_streaming_ping_pong_%s' % secstr, 'num_servers': 1, 'num_clients': 1, @@ -98,6 +103,7 @@ class CXXLanguage: 'closed_loop': {} }, 'payload_config': EMPTY_GENERIC_PAYLOAD, + 'histogram_params': HISTOGRAM_PARAMS, }, 'server_config': { 'server_type': 'ASYNC_GENERIC_SERVER', @@ -110,7 +116,7 @@ class CXXLanguage: 'benchmark_seconds': BENCHMARK_SECONDS } yield { - 'name': 'generic_async_streaming_qps_unconstrained_%s' + 'name': 'cpp_generic_async_streaming_qps_unconstrained_%s' % secstr, 'num_servers': 1, 'num_clients': 0, @@ -125,6 +131,7 @@ class CXXLanguage: 'closed_loop': {} }, 'payload_config': EMPTY_GENERIC_PAYLOAD, + 'histogram_params': HISTOGRAM_PARAMS, }, 'server_config': { 'server_type': 'ASYNC_GENERIC_SERVER', @@ -137,7 +144,7 @@ class CXXLanguage: 'benchmark_seconds': BENCHMARK_SECONDS } yield { - 'name': 'generic_async_streaming_qps_one_server_core_%s' + 'name': 'cpp_generic_async_streaming_qps_one_server_core_%s' % secstr, 'num_servers': 1, 'num_clients': 0, @@ -152,6 +159,7 @@ class CXXLanguage: 'closed_loop': {} }, 'payload_config': EMPTY_GENERIC_PAYLOAD, + 'histogram_params': HISTOGRAM_PARAMS, }, 'server_config': { 'server_type': 'ASYNC_GENERIC_SERVER', @@ -164,7 +172,7 @@ class CXXLanguage: 'benchmark_seconds': BENCHMARK_SECONDS } yield { - 'name': 'protobuf_async_qps_unconstrained_%s' + 'name': 'cpp_protobuf_async_streaming_qps_unconstrained_%s' % secstr, 'num_servers': 1, 'num_clients': 0, @@ -178,20 +186,20 @@ class CXXLanguage: 'load_params': { 'closed_loop': {} }, - 'payload_config': EMPTY_GENERIC_PAYLOAD, + 'payload_config': EMPTY_PROTO_PAYLOAD, + 'histogram_params': HISTOGRAM_PARAMS, }, 'server_config': { - 'server_type': 'ASYNC_GENERIC_SERVER', + 'server_type': 'ASYNC_SERVER', 'security_params': secargs, 'core_limit': SINGLE_MACHINE_CORES/2, 'async_server_threads': 1, - 'payload_config': EMPTY_GENERIC_PAYLOAD, }, 'warmup_seconds': WARMUP_SECONDS, 'benchmark_seconds': BENCHMARK_SECONDS } yield { - 'name': 'single_channel_throughput_%s' + 'name': 'cpp_single_channel_throughput_%s' % secstr, 'num_servers': 1, 'num_clients': 1, @@ -206,6 +214,7 @@ class CXXLanguage: 'closed_loop': {} }, 'payload_config': BIG_GENERIC_PAYLOAD, + 'histogram_params': HISTOGRAM_PARAMS, }, 'server_config': { 'server_type': 'ASYNC_GENERIC_SERVER', @@ -218,7 +227,7 @@ class CXXLanguage: 'benchmark_seconds': BENCHMARK_SECONDS } yield { - 'name': 'protobuf_async_ping_pong_%s' + 'name': 'cpp_protobuf_async_ping_pong_%s' % secstr, 'num_servers': 1, 'num_clients': 1, @@ -233,13 +242,13 @@ class CXXLanguage: 'closed_loop': {} }, 'payload_config': EMPTY_PROTO_PAYLOAD, + 'histogram_params': HISTOGRAM_PARAMS, }, 'server_config': { - 'server_type': 'ASYNC_GENERIC_SERVER', + 'server_type': 'ASYNC_SERVER', 'security_params': secargs, 'core_limit': SINGLE_MACHINE_CORES/2, 'async_server_threads': 1, - 'payload_config': EMPTY_PROTO_PAYLOAD, }, 'warmup_seconds': WARMUP_SECONDS, 'benchmark_seconds': BENCHMARK_SECONDS @@ -262,8 +271,9 @@ class CSharpLanguage: def scenarios(self): # TODO(jtattermusch): add more scenarios + secargs = None yield { - 'name': 'csharp_async_generic_streaming_ping_pong', + 'name': 'csharp_generic_async_streaming_ping_pong', 'num_servers': 1, 'num_clients': 1, 'client_config': { @@ -277,17 +287,97 @@ class CSharpLanguage: 'closed_loop': {} }, 'payload_config': EMPTY_GENERIC_PAYLOAD, + 'histogram_params': HISTOGRAM_PARAMS, }, 'server_config': { 'server_type': 'ASYNC_GENERIC_SERVER', 'security_params': secargs, - 'core_limit': SINGLE_MACHINE_CORES/2, + 'core_limit': 0, 'async_server_threads': 1, 'payload_config': EMPTY_GENERIC_PAYLOAD, }, 'warmup_seconds': WARMUP_SECONDS, 'benchmark_seconds': BENCHMARK_SECONDS } + yield { + 'name': 'csharp_protobuf_async_unary_ping_pong', + 'num_servers': 1, + 'num_clients': 1, + 'client_config': { + 'client_type': 'ASYNC_CLIENT', + 'security_params': secargs, + 'outstanding_rpcs_per_channel': 1, + 'client_channels': 1, + 'async_client_threads': 1, + 'rpc_type': 'UNARY', + 'load_params': { + 'closed_loop': {} + }, + 'payload_config': EMPTY_PROTO_PAYLOAD, + 'histogram_params': HISTOGRAM_PARAMS, + }, + 'server_config': { + 'server_type': 'ASYNC_SERVER', + 'security_params': secargs, + 'core_limit': 0, + 'async_server_threads': 1, + }, + 'warmup_seconds': WARMUP_SECONDS, + 'benchmark_seconds': BENCHMARK_SECONDS + } + yield { + 'name': 'csharp_protobuf_sync_to_async_unary_ping_pong', + 'num_servers': 1, + 'num_clients': 1, + 'client_config': { + 'client_type': 'SYNC_CLIENT', + 'security_params': secargs, + 'outstanding_rpcs_per_channel': 1, + 'client_channels': 1, + 'async_client_threads': 1, + 'rpc_type': 'UNARY', + 'load_params': { + 'closed_loop': {} + }, + 'payload_config': EMPTY_PROTO_PAYLOAD, + 'histogram_params': HISTOGRAM_PARAMS, + }, + 'server_config': { + 'server_type': 'ASYNC_SERVER', + 'security_params': secargs, + 'core_limit': 0, + 'async_server_threads': 1, + }, + 'warmup_seconds': WARMUP_SECONDS, + 'benchmark_seconds': BENCHMARK_SECONDS + } + yield { + 'name': 'csharp_to_cpp_protobuf_sync_unary_ping_pong', + 'num_servers': 1, + 'num_clients': 1, + 'client_config': { + 'client_type': 'SYNC_CLIENT', + 'security_params': secargs, + 'outstanding_rpcs_per_channel': 1, + 'client_channels': 1, + 'async_client_threads': 1, + 'rpc_type': 'UNARY', + 'load_params': { + 'closed_loop': {} + }, + 'payload_config': EMPTY_PROTO_PAYLOAD, + 'histogram_params': HISTOGRAM_PARAMS, + }, + 'server_config': { + 'server_type': 'SYNC_SERVER', + 'security_params': secargs, + 'core_limit': 0, + 'async_server_threads': 1, + }, + 'warmup_seconds': WARMUP_SECONDS, + 'benchmark_seconds': BENCHMARK_SECONDS, + 'SERVER_LANGUAGE': 'c++' # recognized by run_performance_tests.py + } def __str__(self): return 'csharp' @@ -307,8 +397,9 @@ class NodeLanguage: def scenarios(self): # TODO(jtattermusch): add more scenarios + secargs = None yield { - 'name': 'node_sync_unary_ping_pong_protobuf', + 'name': 'node_protobuf_unary_ping_pong', 'num_servers': 1, 'num_clients': 1, 'client_config': { @@ -317,18 +408,18 @@ class NodeLanguage: 'outstanding_rpcs_per_channel': 1, 'client_channels': 1, 'async_client_threads': 1, - 'rpc_type': 'STREAMING', + 'rpc_type': 'UNARY', 'load_params': { 'closed_loop': {} }, 'payload_config': EMPTY_PROTO_PAYLOAD, + 'histogram_params': HISTOGRAM_PARAMS, }, 'server_config': { - 'server_type': 'ASYNC_GENERIC_SERVER', + 'server_type': 'ASYNC_SERVER', 'security_params': secargs, - 'core_limit': SINGLE_MACHINE_CORES/2, + 'core_limit': 0, 'async_server_threads': 1, - 'payload_config': EMPTY_PROTO_PAYLOAD, }, 'warmup_seconds': WARMUP_SECONDS, 'benchmark_seconds': BENCHMARK_SECONDS diff --git a/tools/run_tests/performance/scenario_result_schema.json b/tools/run_tests/performance/scenario_result_schema.json new file mode 100644 index 0000000000..10d24a2517 --- /dev/null +++ b/tools/run_tests/performance/scenario_result_schema.json @@ -0,0 +1,202 @@ +[ + { + "name": "metadata", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "buildNumber", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "buildUrl", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "jobName", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "gitCommit", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "gitActualCommit", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "created", + "type": "TIMESTAMP", + "mode": "NULLABLE" + } + ] + }, + { + "name": "scenario", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "name", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "clientConfig", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "numClients", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "serverConfig", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "numServers", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "warmupSeconds", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "benchmarkSeconds", + "type": "INTEGER", + "mode": "NULLABLE" + } + ] + }, + { + "name": "latencies", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "clientStats", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "latencies", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "timeElapsed", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "timeUser", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "timeSystem", + "type": "FLOAT", + "mode": "NULLABLE" + } + ] + }, + { + "name": "serverStats", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "timeElapsed", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "timeUser", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "timeSystem", + "type": "FLOAT", + "mode": "NULLABLE" + } + ] + }, + { + "name": "serverCores", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "summary", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "qps", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "qps_per_server_core", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "server_system_time", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "server_user_time", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "client_system_time", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "client_user_time", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "latency_50", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "latency_90", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "latency_95", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "latency_99", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "latency_999", + "type": "FLOAT", + "mode": "NULLABLE" + } + ] + } +] diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index 51ed35f760..beedd819ad 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -37,10 +37,12 @@ import json import multiprocessing import os import pipes +import re import subprocess import sys import tempfile import time +import traceback import uuid import performance.scenario_config as scenario_config @@ -82,6 +84,8 @@ def create_qpsworker_job(language, shortname=None, else: host_and_port='localhost:%s' % port + # TODO(jtattermusch): with some care, we can calculate the right timeout + # of a worker from the sum of warmup + benchmark times for all the scenarios jobspec = jobset.JobSpec( cmdline=cmdline, shortname=shortname, @@ -89,14 +93,19 @@ def create_qpsworker_job(language, shortname=None, return QpsWorkerJob(jobspec, language, host_and_port) -def create_scenario_jobspec(scenario_json, workers, remote_host=None): +def create_scenario_jobspec(scenario_json, workers, remote_host=None, + bq_result_table=None): """Runs one scenario using QPS driver.""" # setting QPS_WORKERS env variable here makes sure it works with SSH too. - cmd = 'QPS_WORKERS="%s" bins/opt/qps_json_driver ' % ','.join(workers) - cmd += '--scenarios_json=%s' % pipes.quote(json.dumps({'scenarios': [scenario_json]})) + cmd = 'QPS_WORKERS="%s" ' % ','.join(workers) + if bq_result_table: + cmd += 'BQ_RESULT_TABLE="%s" ' % bq_result_table + cmd += 'tools/run_tests/performance/run_qps_driver.sh ' + cmd += '--scenarios_json=%s ' % pipes.quote(json.dumps({'scenarios': [scenario_json]})) + cmd += '--scenario_result_file=scenario_result.json' if remote_host: user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, remote_host) - cmd = 'ssh %s "cd ~/performance_workspace/grpc/ && %s"' % (user_at_host, cmd) + cmd = 'ssh %s "cd ~/performance_workspace/grpc/ && "%s' % (user_at_host, pipes.quote(cmd)) return jobset.JobSpec( cmdline=[cmd], @@ -112,7 +121,7 @@ def create_quit_jobspec(workers, remote_host=None): cmd = 'QPS_WORKERS="%s" bins/opt/qps_driver --quit' % ','.join(workers) if remote_host: user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, remote_host) - cmd = 'ssh %s "cd ~/performance_workspace/grpc/ && %s"' % (user_at_host, cmd) + cmd = 'ssh %s "cd ~/performance_workspace/grpc/ && "%s' % (user_at_host, pipes.quote(cmd)) return jobset.JobSpec( cmdline=[cmd], @@ -221,15 +230,32 @@ def start_qpsworkers(languages, worker_hosts): for worker_idx, worker in enumerate(workers)] -def create_scenarios(languages, workers_by_lang, remote_host=None): +def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*', + bq_result_table=None): """Create jobspecs for scenarios to run.""" scenarios = [] for language in languages: for scenario_json in language.scenarios(): - scenario = create_scenario_jobspec(scenario_json, - workers_by_lang[str(language)], - remote_host=remote_host) - scenarios.append(scenario) + if re.search(args.regex, scenario_json['name']): + workers = workers_by_lang[str(language)] + # 'SERVER_LANGUAGE' is an indicator for this script to pick + # a server in different language. It doesn't belong to the Scenario + # schema, so we also need to remove it. + custom_server_lang = scenario_json.pop('SERVER_LANGUAGE', None) + if custom_server_lang: + if not workers_by_lang.get(custom_server_lang, []): + print 'Warning: Skipping scenario %s as' % scenario_json['name'] + print('SERVER_LANGUAGE is set to %s yet the language has ' + 'not been selected with -l' % custom_server_lang) + continue + for idx in range(0, scenario_json['num_servers']): + # replace first X workers by workers of a different language + workers[idx] = workers_by_lang[custom_server_lang][idx] + scenario = create_scenario_jobspec(scenario_json, + workers, + remote_host=remote_host, + bq_result_table=bq_result_table) + scenarios.append(scenario) # the very last scenario requests shutting down the workers. all_workers = [worker @@ -268,6 +294,10 @@ argp.add_argument('--remote_worker_host', nargs='+', default=[], help='Worker hosts where to start QPS workers.') +argp.add_argument('-r', '--regex', default='.*', type=str, + help='Regex to select scenarios to run.') +argp.add_argument('--bq_result_table', default=None, type=str, + help='Bigquery "dataset.table" to upload results to.') args = argp.parse_args() @@ -276,6 +306,7 @@ languages = set(scenario_config.LANGUAGES[l] scenario_config.LANGUAGES.iterkeys() if x == 'all' else [x] for x in args.language)) + # Put together set of remote hosts where to run and build remote_hosts = set() if args.remote_worker_host: @@ -295,6 +326,9 @@ build_on_remote_hosts(remote_hosts, languages=[str(l) for l in languages], build qpsworker_jobs = start_qpsworkers(languages, args.remote_worker_host) +# TODO(jtattermusch): see https://github.com/grpc/grpc/issues/6174 +time.sleep(5) + # get list of worker addresses for each language. worker_addresses = dict([(str(language), []) for language in languages]) for job in qpsworker_jobs: @@ -303,7 +337,9 @@ for job in qpsworker_jobs: try: scenarios = create_scenarios(languages, workers_by_lang=worker_addresses, - remote_host=args.remote_driver_host) + remote_host=args.remote_driver_host, + regex=args.regex, + bq_result_table=args.bq_result_table) if not scenarios: raise Exception('No scenarios to run') @@ -318,5 +354,7 @@ try: jobset.message('FAILED', 'Some of the scenarios failed.', do_newline=True) sys.exit(1) +except: + traceback.print_exc() finally: finish_qps_workers(qpsworker_jobs) diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index d277e4c3b1..7978f12d53 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -6229,6 +6229,7 @@ "test/core/end2end/fixtures/proxy.h", "test/core/iomgr/endpoint_tests.h", "test/core/util/grpc_profiler.h", + "test/core/util/memory_counters.h", "test/core/util/mock_endpoint.h", "test/core/util/parse_hexstring.h", "test/core/util/port.h", @@ -6246,6 +6247,8 @@ "test/core/iomgr/endpoint_tests.h", "test/core/util/grpc_profiler.c", "test/core/util/grpc_profiler.h", + "test/core/util/memory_counters.c", + "test/core/util/memory_counters.h", "test/core/util/mock_endpoint.c", "test/core/util/mock_endpoint.h", "test/core/util/parse_hexstring.c", diff --git a/tools/run_tests/stress_test/configs/asan.json b/tools/run_tests/stress_test/configs/asan.json index 768088d93d..c558855314 100644 --- a/tools/run_tests/stress_test/configs/asan.json +++ b/tools/run_tests/stress_test/configs/asan.json @@ -11,13 +11,13 @@ "baseTemplates": { "default": { "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", - "pollIntervalSecs": 60, + "pollIntervalSecs": 120, "clientArgs": { "num_channels_per_server":5, "num_stubs_per_channel":10, "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", "metrics_port": 8081, - "metrics_collection_interval_secs":60 + "metrics_collection_interval_secs":120 }, "metricsPort": 8081, "metricsArgs": { @@ -66,7 +66,7 @@ "stress-client-asan": { "clientTemplate": "cxx_client_asan", "dockerImage": "grpc_stress_cxx_asan", - "numInstances": 20, + "numInstances": 5, "serverPodSpec": "stress-server-asan" } } diff --git a/tools/run_tests/stress_test/configs/opt.json b/tools/run_tests/stress_test/configs/opt.json index ffd4a704c3..75505186f2 100644 --- a/tools/run_tests/stress_test/configs/opt.json +++ b/tools/run_tests/stress_test/configs/opt.json @@ -66,7 +66,7 @@ "stress-client-opt": { "clientTemplate": "cxx_client_opt", "dockerImage": "grpc_stress_cxx_opt", - "numInstances": 10, + "numInstances": 15, "serverPodSpec": "stress-server-opt" } } diff --git a/tools/run_tests/stress_test/configs/tsan.json b/tools/run_tests/stress_test/configs/tsan.json index f8d3f878e1..a7ec08313d 100644 --- a/tools/run_tests/stress_test/configs/tsan.json +++ b/tools/run_tests/stress_test/configs/tsan.json @@ -11,13 +11,13 @@ "baseTemplates": { "default": { "wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py", - "pollIntervalSecs": 60, + "pollIntervalSecs": 120, "clientArgs": { "num_channels_per_server":5, "num_stubs_per_channel":10, "test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1", "metrics_port": 8081, - "metrics_collection_interval_secs":60 + "metrics_collection_interval_secs":120 }, "metricsPort": 8081, "metricsArgs": { @@ -66,7 +66,7 @@ "stress-client-tsan": { "clientTemplate": "cxx_client_tsan", "dockerImage": "grpc_stress_cxx_tsan", - "numInstances": 20, + "numInstances": 5, "serverPodSpec": "stress-server-tsan" } } diff --git a/tools/run_tests/stress_test/run_on_gke.py b/tools/run_tests/stress_test/run_on_gke.py index db3ba28346..916c890cbd 100755 --- a/tools/run_tests/stress_test/run_on_gke.py +++ b/tools/run_tests/stress_test/run_on_gke.py @@ -604,6 +604,17 @@ def run_tests(config): return is_success +def tear_down(config): + gke = Gke(config.global_settings.gcp_project_id, '', '', + config.global_settings.summary_table_id, + config.global_settings.qps_table_id, + config.global_settings.kubernetes_proxy_port) + for name, server_pod_spec in config.server_pod_specs_dict.iteritems(): + gke.delete_servers(server_pod_spec) + for name, client_pod_spec in config.client_pod_specs_dict.iteritems(): + gke.delete_clients(client_pod_spec) + + argp = argparse.ArgumentParser( description='Launch stress tests in GKE', formatter_class=argparse.ArgumentDefaultsHelpFormatter) @@ -614,6 +625,7 @@ argp.add_argument('--config_file', required=True, type=str, help='The test config file') +argp.add_argument('--tear_down', action='store_true', default=False) if __name__ == '__main__': args = argp.parse_args() @@ -636,5 +648,11 @@ if __name__ == '__main__': os.path.dirname(sys.argv[0]), '../../..')) os.chdir(grpc_root) + # Note that tear_down is only in cases where we want to manually tear down a + # test that for some reason run_tests() could not cleanup + if args.tear_down: + tear_down(config) + sys.exit(1) + if not run_tests(config): sys.exit(1) diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index f8c658672b..ad03f16420 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -22035,7 +22035,7 @@ { "args": [ "--scenario_json", - "'{\"name\": \"generic_async_streaming_ping_pong_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 1}'" + "'{\"name\": \"cpp_generic_async_streaming_ping_pong_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'" ], "boringssl": true, "ci_platforms": [ @@ -22056,12 +22056,12 @@ "posix", "windows" ], - "shortname": "json_run_localhost:generic_async_streaming_ping_pong_secure" + "shortname": "json_run_localhost:cpp_generic_async_streaming_ping_pong_secure" }, { "args": [ "--scenario_json", - "'{\"name\": \"generic_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 0}'" + "'{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'" ], "boringssl": true, "ci_platforms": [ @@ -22082,12 +22082,12 @@ "posix", "windows" ], - "shortname": "json_run_localhost:generic_async_streaming_qps_unconstrained_secure" + "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_unconstrained_secure" }, { "args": [ "--scenario_json", - "'{\"name\": \"generic_async_streaming_qps_one_server_core_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 0}'" + "'{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'" ], "boringssl": true, "ci_platforms": [ @@ -22108,12 +22108,12 @@ "posix", "windows" ], - "shortname": "json_run_localhost:generic_async_streaming_qps_one_server_core_secure" + "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_one_server_core_secure" }, { "args": [ "--scenario_json", - "'{\"name\": \"protobuf_async_qps_unconstrained_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 0}'" + "'{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'" ], "boringssl": true, "ci_platforms": [ @@ -22134,12 +22134,12 @@ "posix", "windows" ], - "shortname": "json_run_localhost:protobuf_async_qps_unconstrained_secure" + "shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_secure" }, { "args": [ "--scenario_json", - "'{\"name\": \"single_channel_throughput_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 1}'" + "'{\"name\": \"cpp_single_channel_throughput_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'" ], "boringssl": true, "ci_platforms": [ @@ -22160,12 +22160,12 @@ "posix", "windows" ], - "shortname": "json_run_localhost:single_channel_throughput_secure" + "shortname": "json_run_localhost:cpp_single_channel_throughput_secure" }, { "args": [ "--scenario_json", - "'{\"name\": \"protobuf_async_ping_pong_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 1}'" + "'{\"name\": \"cpp_protobuf_async_ping_pong_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'" ], "boringssl": true, "ci_platforms": [ @@ -22186,12 +22186,12 @@ "posix", "windows" ], - "shortname": "json_run_localhost:protobuf_async_ping_pong_secure" + "shortname": "json_run_localhost:cpp_protobuf_async_ping_pong_secure" }, { "args": [ "--scenario_json", - "'{\"name\": \"generic_async_streaming_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 1}'" + "'{\"name\": \"cpp_generic_async_streaming_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'" ], "boringssl": true, "ci_platforms": [ @@ -22212,12 +22212,12 @@ "posix", "windows" ], - "shortname": "json_run_localhost:generic_async_streaming_ping_pong_insecure" + "shortname": "json_run_localhost:cpp_generic_async_streaming_ping_pong_insecure" }, { "args": [ "--scenario_json", - "'{\"name\": \"generic_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 0}'" + "'{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'" ], "boringssl": true, "ci_platforms": [ @@ -22238,12 +22238,12 @@ "posix", "windows" ], - "shortname": "json_run_localhost:generic_async_streaming_qps_unconstrained_insecure" + "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_unconstrained_insecure" }, { "args": [ "--scenario_json", - "'{\"name\": \"generic_async_streaming_qps_one_server_core_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 0}'" + "'{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'" ], "boringssl": true, "ci_platforms": [ @@ -22264,12 +22264,12 @@ "posix", "windows" ], - "shortname": "json_run_localhost:generic_async_streaming_qps_one_server_core_insecure" + "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_one_server_core_insecure" }, { "args": [ "--scenario_json", - "'{\"name\": \"protobuf_async_qps_unconstrained_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 0}'" + "'{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'" ], "boringssl": true, "ci_platforms": [ @@ -22290,12 +22290,12 @@ "posix", "windows" ], - "shortname": "json_run_localhost:protobuf_async_qps_unconstrained_insecure" + "shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_insecure" }, { "args": [ "--scenario_json", - "'{\"name\": \"single_channel_throughput_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 1}'" + "'{\"name\": \"cpp_single_channel_throughput_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 65536, \"req_size\": 65536}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'" ], "boringssl": true, "ci_platforms": [ @@ -22316,12 +22316,12 @@ "posix", "windows" ], - "shortname": "json_run_localhost:single_channel_throughput_insecure" + "shortname": "json_run_localhost:cpp_single_channel_throughput_insecure" }, { "args": [ "--scenario_json", - "'{\"name\": \"protobuf_async_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}}, \"num_clients\": 1}'" + "'{\"name\": \"cpp_protobuf_async_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 4, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'" ], "boringssl": true, "ci_platforms": [ @@ -22342,7 +22342,7 @@ "posix", "windows" ], - "shortname": "json_run_localhost:protobuf_async_ping_pong_insecure" + "shortname": "json_run_localhost:cpp_protobuf_async_ping_pong_insecure" }, { "args": [ |