diff options
author | 2016-10-03 15:23:02 -0700 | |
---|---|---|
committer | 2016-10-03 15:23:02 -0700 | |
commit | 5f806d321afee630f98796429856c57584ca66ea (patch) | |
tree | fbb7fbe98b2c1995be3550e4085e28b9fd2b3b6f /tools/run_tests | |
parent | 33382d0f537c5c793b46742089ebeb42d764ac45 (diff) | |
parent | 9070ab6610bd8cd0dbe6922a049a3b094c908bd1 (diff) |
Merge branch 'master' into rpc_mgr
Diffstat (limited to 'tools/run_tests')
-rw-r--r-- | tools/run_tests/performance/scenario_config.py | 529 | ||||
-rwxr-xr-x | tools/run_tests/run_performance_tests.py | 45 | ||||
-rwxr-xr-x | tools/run_tests/sanity/check_submodules.sh | 2 | ||||
-rw-r--r-- | tools/run_tests/sources_and_headers.json | 35 | ||||
-rw-r--r-- | tools/run_tests/tests.json | 21 |
5 files changed, 282 insertions, 350 deletions
diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 8166fbd419..725ca299bd 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -31,49 +31,52 @@ import math -WARMUP_SECONDS = 5 -JAVA_WARMUP_SECONDS = 15 # Java needs more warmup time for JIT to kick in. -BENCHMARK_SECONDS = 30 +WARMUP_SECONDS=5 +JAVA_WARMUP_SECONDS=15 # Java needs more warmup time for JIT to kick in. +BENCHMARK_SECONDS=30 -SMOKETEST = 'smoketest' -SCALABLE = 'scalable' -SWEEP = 'sweep' -DEFAULT_CATEGORIES = [SCALABLE, SMOKETEST] +SMOKETEST='smoketest' +SCALABLE='scalable' +SWEEP='sweep' +DEFAULT_CATEGORIES=[SCALABLE, SMOKETEST] SECURE_SECARGS = {'use_test_ca': True, 'server_host_override': 'foo.test.google.fr'} HISTOGRAM_PARAMS = { - 'resolution': 0.01, - 'max_possible': 60e9, + 'resolution': 0.01, + 'max_possible': 60e9, } EMPTY_GENERIC_PAYLOAD = { - 'bytebuf_params': { - 'req_size': 0, - 'resp_size': 0, - } + 'bytebuf_params': { + 'req_size': 0, + 'resp_size': 0, + } } EMPTY_PROTO_PAYLOAD = { - 'simple_params': { - 'req_size': 0, - 'resp_size': 0, - } + 'simple_params': { + 'req_size': 0, + 'resp_size': 0, + } } BIG_GENERIC_PAYLOAD = { - 'bytebuf_params': { - 'req_size': 65536, - 'resp_size': 65536, - } + 'bytebuf_params': { + 'req_size': 65536, + 'resp_size': 65536, + } } # target number of RPCs outstanding on across all client channels in # non-ping-pong tests (since we can only specify per-channel numbers, the # actual target will be slightly higher) -OUTSTANDING_REQUESTS = {'async': 6400, 'sync': 1000} +OUTSTANDING_REQUESTS={ + 'async': 6400, + 'sync': 1000 +} # wide is the number of client channels in multi-channel tests (1 otherwise) -WIDE = 64 +WIDE=64 def _get_secargs(is_secure): @@ -99,10 +102,8 @@ def geometric_progression(start, stop, step): n *= step -def _ping_pong_scenario(name, - rpc_type, - client_type, - server_type, +def _ping_pong_scenario(name, rpc_type, + client_type, server_type, secure=True, use_generic_payload=False, unconstrained_client=None, @@ -116,29 +117,29 @@ def _ping_pong_scenario(name, outstanding=None): """Creates a basic ping pong scenario.""" scenario = { - 'name': name, - 'num_servers': 1, - 'num_clients': 1, - 'client_config': { - 'client_type': client_type, - 'security_params': _get_secargs(secure), - 'outstanding_rpcs_per_channel': 1, - 'client_channels': 1, - 'async_client_threads': 1, - 'rpc_type': rpc_type, - 'load_params': { - 'closed_loop': {} - }, - 'histogram_params': HISTOGRAM_PARAMS, + 'name': name, + 'num_servers': 1, + 'num_clients': 1, + 'client_config': { + 'client_type': client_type, + 'security_params': _get_secargs(secure), + 'outstanding_rpcs_per_channel': 1, + 'client_channels': 1, + 'async_client_threads': 1, + 'rpc_type': rpc_type, + 'load_params': { + 'closed_loop': {} }, - 'server_config': { - 'server_type': server_type, - 'security_params': _get_secargs(secure), - 'core_limit': server_core_limit, - 'async_server_threads': async_server_threads, - }, - 'warmup_seconds': warmup_seconds, - 'benchmark_seconds': BENCHMARK_SECONDS + 'histogram_params': HISTOGRAM_PARAMS, + }, + 'server_config': { + 'server_type': server_type, + 'security_params': _get_secargs(secure), + 'core_limit': server_core_limit, + 'async_server_threads': async_server_threads, + }, + 'warmup_seconds': warmup_seconds, + 'benchmark_seconds': BENCHMARK_SECONDS } if use_generic_payload: if server_type != 'ASYNC_GENERIC_SERVER': @@ -150,8 +151,7 @@ def _ping_pong_scenario(name, scenario['client_config']['payload_config'] = EMPTY_PROTO_PAYLOAD if unconstrained_client: - outstanding_calls = outstanding if outstanding is not None else OUTSTANDING_REQUESTS[ - unconstrained_client] + outstanding_calls = outstanding if outstanding is not None else OUTSTANDING_REQUESTS[unconstrained_client] wide = channels if channels is not None else WIDE deep = int(math.ceil(1.0 * outstanding_calls / wide)) @@ -190,16 +190,14 @@ class CXXLanguage: # TODO(ctiller): add 70% load latency test for secure in [True, False]: secstr = 'secure' if secure else 'insecure' - smoketest_categories = [SMOKETEST] if secure else [] + smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] yield _ping_pong_scenario( 'cpp_generic_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING', client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', - use_generic_payload=True, - server_core_limit=1, - async_server_threads=1, + use_generic_payload=True, server_core_limit=1, async_server_threads=1, secure=secure, categories=smoketest_categories) @@ -208,71 +206,49 @@ class CXXLanguage: rpc_type='STREAMING', client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', - unconstrained_client='async', - use_generic_payload=True, + unconstrained_client='async', use_generic_payload=True, secure=secure, - categories=smoketest_categories + [SCALABLE]) + categories=smoketest_categories+[SCALABLE]) yield _ping_pong_scenario( 'cpp_generic_async_streaming_qps_one_server_core_%s' % secstr, rpc_type='STREAMING', client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', - unconstrained_client='async', - use_generic_payload=True, - server_core_limit=1, - async_server_threads=1, + unconstrained_client='async', use_generic_payload=True, + server_core_limit=1, async_server_threads=1, secure=secure) - yield _ping_pong_scenario( - 'cpp_protobuf_async_client_sync_server_unary_qps_unconstrained_%s' % - (secstr), - rpc_type='UNARY', - client_type='ASYNC_CLIENT', - server_type='SYNC_SERVER', - unconstrained_client='async', - secure=secure, - categories=smoketest_categories + [SCALABLE]) - for rpc_type in ['unary', 'streaming']: for synchronicity in ['sync', 'async']: yield _ping_pong_scenario( - 'cpp_protobuf_%s_%s_ping_pong_%s' % - (synchronicity, rpc_type, secstr), + 'cpp_protobuf_%s_%s_ping_pong_%s' % (synchronicity, rpc_type, secstr), rpc_type=rpc_type.upper(), client_type='%s_CLIENT' % synchronicity.upper(), server_type='%s_SERVER' % synchronicity.upper(), - server_core_limit=1, - async_server_threads=1, + server_core_limit=1, async_server_threads=1, secure=secure) yield _ping_pong_scenario( - 'cpp_protobuf_%s_%s_qps_unconstrained_%s' % - (synchronicity, rpc_type, secstr), + 'cpp_protobuf_%s_%s_qps_unconstrained_%s' % (synchronicity, rpc_type, secstr), rpc_type=rpc_type.upper(), client_type='%s_CLIENT' % synchronicity.upper(), server_type='%s_SERVER' % synchronicity.upper(), unconstrained_client=synchronicity, secure=secure, - categories=smoketest_categories + [SCALABLE]) + categories=smoketest_categories+[SCALABLE]) for channels in geometric_progression(1, 20000, math.sqrt(10)): for outstanding in geometric_progression(1, 200000, math.sqrt(10)): - if synchronicity == 'sync' and outstanding > 1200: - continue - if outstanding < channels: - continue - yield _ping_pong_scenario( - 'cpp_protobuf_%s_%s_qps_unconstrained_%s_%d_channels_%d_outstanding' - % (synchronicity, rpc_type, secstr, channels, outstanding), - rpc_type=rpc_type.upper(), - client_type='%s_CLIENT' % synchronicity.upper(), - server_type='%s_SERVER' % synchronicity.upper(), - unconstrained_client=synchronicity, - secure=secure, - categories=[SWEEP], - channels=channels, - outstanding=outstanding) + if synchronicity == 'sync' and outstanding > 1200: continue + if outstanding < channels: continue + yield _ping_pong_scenario( + 'cpp_protobuf_%s_%s_qps_unconstrained_%s_%d_channels_%d_outstanding' % (synchronicity, rpc_type, secstr, channels, outstanding), + rpc_type=rpc_type.upper(), + client_type='%s_CLIENT' % synchronicity.upper(), + server_type='%s_SERVER' % synchronicity.upper(), + unconstrained_client=synchronicity, secure=secure, + categories=[SWEEP], channels=channels, outstanding=outstanding) def __str__(self): return 'c++' @@ -291,94 +267,66 @@ class CSharpLanguage: def scenarios(self): yield _ping_pong_scenario( - 'csharp_generic_async_streaming_ping_pong', - rpc_type='STREAMING', - client_type='ASYNC_CLIENT', - server_type='ASYNC_GENERIC_SERVER', + 'csharp_generic_async_streaming_ping_pong', rpc_type='STREAMING', + client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', use_generic_payload=True, - categories=[SMOKETEST]) + categories=[SMOKETEST, SCALABLE]) yield _ping_pong_scenario( - 'csharp_protobuf_async_streaming_ping_pong', - rpc_type='STREAMING', - client_type='ASYNC_CLIENT', - server_type='ASYNC_SERVER') + 'csharp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER') yield _ping_pong_scenario( - 'csharp_protobuf_async_unary_ping_pong', - rpc_type='UNARY', - client_type='ASYNC_CLIENT', - server_type='ASYNC_SERVER', - categories=[SMOKETEST]) + 'csharp_protobuf_async_unary_ping_pong', rpc_type='UNARY', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + categories=[SMOKETEST, SCALABLE]) yield _ping_pong_scenario( - 'csharp_protobuf_sync_to_async_unary_ping_pong', - rpc_type='UNARY', - client_type='SYNC_CLIENT', - server_type='ASYNC_SERVER') + 'csharp_protobuf_sync_to_async_unary_ping_pong', rpc_type='UNARY', + client_type='SYNC_CLIENT', server_type='ASYNC_SERVER') yield _ping_pong_scenario( - 'csharp_protobuf_async_unary_qps_unconstrained', - rpc_type='UNARY', - client_type='ASYNC_CLIENT', - server_type='ASYNC_SERVER', + 'csharp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', unconstrained_client='async', - categories=[SMOKETEST, SCALABLE]) + categories=[SMOKETEST,SCALABLE]) yield _ping_pong_scenario( - 'csharp_protobuf_async_streaming_qps_unconstrained', - rpc_type='STREAMING', - client_type='ASYNC_CLIENT', - server_type='ASYNC_SERVER', + 'csharp_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', unconstrained_client='async', categories=[SCALABLE]) yield _ping_pong_scenario( - 'csharp_to_cpp_protobuf_sync_unary_ping_pong', - rpc_type='UNARY', - client_type='SYNC_CLIENT', - server_type='SYNC_SERVER', - server_language='c++', - server_core_limit=1, - async_server_threads=1, - categories=[SMOKETEST]) + 'csharp_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + server_language='c++', server_core_limit=1, async_server_threads=1, + categories=[SMOKETEST, SCALABLE]) yield _ping_pong_scenario( - 'csharp_to_cpp_protobuf_async_streaming_ping_pong', - rpc_type='STREAMING', - client_type='ASYNC_CLIENT', - server_type='ASYNC_SERVER', - server_language='c++', - server_core_limit=1, - async_server_threads=1) + 'csharp_to_cpp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + server_language='c++', server_core_limit=1, async_server_threads=1) yield _ping_pong_scenario( - 'csharp_to_cpp_protobuf_async_unary_qps_unconstrained', - rpc_type='UNARY', - client_type='ASYNC_CLIENT', - server_type='ASYNC_SERVER', - unconstrained_client='async', - server_language='c++', + 'csharp_to_cpp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + unconstrained_client='async', server_language='c++', categories=[SCALABLE]) yield _ping_pong_scenario( - 'csharp_to_cpp_protobuf_sync_to_async_unary_qps_unconstrained', - rpc_type='UNARY', - client_type='SYNC_CLIENT', - server_type='ASYNC_SERVER', - unconstrained_client='sync', - server_language='c++', + 'csharp_to_cpp_protobuf_sync_to_async_unary_qps_unconstrained', rpc_type='UNARY', + client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', + unconstrained_client='sync', server_language='c++', categories=[SCALABLE]) yield _ping_pong_scenario( - 'cpp_to_csharp_protobuf_async_unary_qps_unconstrained', - rpc_type='UNARY', - client_type='ASYNC_CLIENT', - server_type='ASYNC_SERVER', - unconstrained_client='async', - client_language='c++', + 'cpp_to_csharp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + unconstrained_client='async', client_language='c++', categories=[SCALABLE]) + def __str__(self): return 'csharp' @@ -408,19 +356,15 @@ class NodeLanguage: # client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER') yield _ping_pong_scenario( - 'node_protobuf_unary_ping_pong', - rpc_type='UNARY', - client_type='ASYNC_CLIENT', - server_type='ASYNC_SERVER', - categories=[SMOKETEST]) + 'node_protobuf_unary_ping_pong', rpc_type='UNARY', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', + categories=[SCALABLE, SMOKETEST]) yield _ping_pong_scenario( - 'node_protobuf_async_unary_qps_unconstrained', - rpc_type='UNARY', - client_type='ASYNC_CLIENT', - server_type='ASYNC_SERVER', + 'node_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', unconstrained_client='async', - categories=[SMOKETEST]) + categories=[SCALABLE, SMOKETEST]) # TODO(jtattermusch): make this scenario work #yield _ping_pong_scenario( @@ -443,7 +387,6 @@ class NodeLanguage: def __str__(self): return 'node' - class PythonLanguage: def __init__(self): @@ -457,69 +400,48 @@ class PythonLanguage: def scenarios(self): yield _ping_pong_scenario( - 'python_generic_sync_streaming_ping_pong', - rpc_type='STREAMING', - client_type='SYNC_CLIENT', - server_type='ASYNC_GENERIC_SERVER', + 'python_generic_sync_streaming_ping_pong', rpc_type='STREAMING', + client_type='SYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', use_generic_payload=True, - categories=[SMOKETEST]) + categories=[SMOKETEST, SCALABLE]) yield _ping_pong_scenario( - 'python_protobuf_sync_streaming_ping_pong', - rpc_type='STREAMING', - client_type='SYNC_CLIENT', - server_type='ASYNC_SERVER') + 'python_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', + client_type='SYNC_CLIENT', server_type='ASYNC_SERVER') yield _ping_pong_scenario( - 'python_protobuf_async_unary_ping_pong', - rpc_type='UNARY', - client_type='ASYNC_CLIENT', - server_type='ASYNC_SERVER') + 'python_protobuf_async_unary_ping_pong', rpc_type='UNARY', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER') yield _ping_pong_scenario( - 'python_protobuf_sync_unary_ping_pong', - rpc_type='UNARY', - client_type='SYNC_CLIENT', - server_type='ASYNC_SERVER', - categories=[SMOKETEST]) + 'python_protobuf_sync_unary_ping_pong', rpc_type='UNARY', + client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', + categories=[SMOKETEST, SCALABLE]) yield _ping_pong_scenario( - 'python_protobuf_sync_unary_qps_unconstrained', - rpc_type='UNARY', - client_type='SYNC_CLIENT', - server_type='ASYNC_SERVER', + 'python_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY', + client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', unconstrained_client='sync') yield _ping_pong_scenario( - 'python_protobuf_sync_streaming_qps_unconstrained', - rpc_type='STREAMING', - client_type='SYNC_CLIENT', - server_type='ASYNC_SERVER', + 'python_protobuf_sync_streaming_qps_unconstrained', rpc_type='STREAMING', + client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', unconstrained_client='sync') yield _ping_pong_scenario( - 'python_to_cpp_protobuf_sync_unary_ping_pong', - rpc_type='UNARY', - client_type='SYNC_CLIENT', - server_type='ASYNC_SERVER', - server_language='c++', - server_core_limit=1, - async_server_threads=1, - categories=[SMOKETEST]) + 'python_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', + client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', + server_language='c++', server_core_limit=1, async_server_threads=1, + categories=[SMOKETEST, SCALABLE]) yield _ping_pong_scenario( - 'python_to_cpp_protobuf_sync_streaming_ping_pong', - rpc_type='STREAMING', - client_type='SYNC_CLIENT', - server_type='ASYNC_SERVER', - server_language='c++', - server_core_limit=1, - async_server_threads=1) + 'python_to_cpp_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', + client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', + server_language='c++', server_core_limit=1, async_server_threads=1) def __str__(self): return 'python' - class RubyLanguage: def __init__(self): @@ -534,50 +456,34 @@ class RubyLanguage: def scenarios(self): yield _ping_pong_scenario( - 'ruby_protobuf_sync_streaming_ping_pong', - rpc_type='STREAMING', - client_type='SYNC_CLIENT', - server_type='SYNC_SERVER', - categories=[SMOKETEST]) + 'ruby_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + categories=[SMOKETEST, SCALABLE]) yield _ping_pong_scenario( - 'ruby_protobuf_unary_ping_pong', - rpc_type='UNARY', - client_type='SYNC_CLIENT', - server_type='SYNC_SERVER', - categories=[SMOKETEST]) + 'ruby_protobuf_unary_ping_pong', rpc_type='UNARY', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + categories=[SMOKETEST, SCALABLE]) yield _ping_pong_scenario( - 'ruby_protobuf_sync_unary_qps_unconstrained', - rpc_type='UNARY', - client_type='SYNC_CLIENT', - server_type='SYNC_SERVER', + 'ruby_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', unconstrained_client='sync') yield _ping_pong_scenario( - 'ruby_protobuf_sync_streaming_qps_unconstrained', - rpc_type='STREAMING', - client_type='SYNC_CLIENT', - server_type='SYNC_SERVER', + 'ruby_protobuf_sync_streaming_qps_unconstrained', rpc_type='STREAMING', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', unconstrained_client='sync') yield _ping_pong_scenario( - 'ruby_to_cpp_protobuf_sync_unary_ping_pong', - rpc_type='UNARY', - client_type='SYNC_CLIENT', - server_type='SYNC_SERVER', - server_language='c++', - server_core_limit=1, - async_server_threads=1) + 'ruby_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + server_language='c++', server_core_limit=1, async_server_threads=1) yield _ping_pong_scenario( - 'ruby_to_cpp_protobuf_sync_streaming_ping_pong', - rpc_type='STREAMING', - client_type='SYNC_CLIENT', - server_type='SYNC_SERVER', - server_language='c++', - server_core_limit=1, - async_server_threads=1) + 'ruby_to_cpp_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', + server_language='c++', server_core_limit=1, async_server_threads=1) def __str__(self): return 'ruby' @@ -598,88 +504,61 @@ class JavaLanguage: def scenarios(self): for secure in [True, False]: secstr = 'secure' if secure else 'insecure' - smoketest_categories = [SMOKETEST] if secure else [] + smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] yield _ping_pong_scenario( - 'java_generic_async_streaming_ping_pong_%s' % secstr, - rpc_type='STREAMING', - client_type='ASYNC_CLIENT', - server_type='ASYNC_GENERIC_SERVER', - use_generic_payload=True, - async_server_threads=1, - secure=secure, - warmup_seconds=JAVA_WARMUP_SECONDS, + 'java_generic_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING', + client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', + use_generic_payload=True, async_server_threads=1, + secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS, categories=smoketest_categories) yield _ping_pong_scenario( - 'java_protobuf_async_streaming_ping_pong_%s' % secstr, - rpc_type='STREAMING', - client_type='ASYNC_CLIENT', - server_type='ASYNC_SERVER', + 'java_protobuf_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', async_server_threads=1, - secure=secure, - warmup_seconds=JAVA_WARMUP_SECONDS) + secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS) yield _ping_pong_scenario( - 'java_protobuf_async_unary_ping_pong_%s' % secstr, - rpc_type='UNARY', - client_type='ASYNC_CLIENT', - server_type='ASYNC_SERVER', + 'java_protobuf_async_unary_ping_pong_%s' % secstr, rpc_type='UNARY', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', async_server_threads=1, - secure=secure, - warmup_seconds=JAVA_WARMUP_SECONDS, + secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS, categories=smoketest_categories) yield _ping_pong_scenario( - 'java_protobuf_unary_ping_pong_%s' % secstr, - rpc_type='UNARY', - client_type='SYNC_CLIENT', - server_type='SYNC_SERVER', + 'java_protobuf_unary_ping_pong_%s' % secstr, rpc_type='UNARY', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', async_server_threads=1, - secure=secure, - warmup_seconds=JAVA_WARMUP_SECONDS) + secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS) yield _ping_pong_scenario( - 'java_protobuf_async_unary_qps_unconstrained_%s' % secstr, - rpc_type='UNARY', - client_type='ASYNC_CLIENT', - server_type='ASYNC_SERVER', + 'java_protobuf_async_unary_qps_unconstrained_%s' % secstr, rpc_type='UNARY', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', unconstrained_client='async', - secure=secure, - warmup_seconds=JAVA_WARMUP_SECONDS, - categories=smoketest_categories + [SCALABLE]) + secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS, + categories=smoketest_categories+[SCALABLE]) yield _ping_pong_scenario( - 'java_protobuf_async_streaming_qps_unconstrained_%s' % secstr, - rpc_type='STREAMING', - client_type='ASYNC_CLIENT', - server_type='ASYNC_SERVER', + 'java_protobuf_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING', + client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', unconstrained_client='async', - secure=secure, - warmup_seconds=JAVA_WARMUP_SECONDS, + secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS, categories=[SCALABLE]) yield _ping_pong_scenario( - 'java_generic_async_streaming_qps_unconstrained_%s' % secstr, - rpc_type='STREAMING', - client_type='ASYNC_CLIENT', - server_type='ASYNC_GENERIC_SERVER', - unconstrained_client='async', - use_generic_payload=True, - secure=secure, - warmup_seconds=JAVA_WARMUP_SECONDS, + 'java_generic_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING', + client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', + unconstrained_client='async', use_generic_payload=True, + secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS, categories=[SCALABLE]) yield _ping_pong_scenario( - 'java_generic_async_streaming_qps_one_server_core_%s' % secstr, - rpc_type='STREAMING', - client_type='ASYNC_CLIENT', - server_type='ASYNC_GENERIC_SERVER', - unconstrained_client='async', - use_generic_payload=True, + 'java_generic_async_streaming_qps_one_server_core_%s' % secstr, rpc_type='STREAMING', + client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', + unconstrained_client='async', use_generic_payload=True, async_server_threads=1, - secure=secure, - warmup_seconds=JAVA_WARMUP_SECONDS) + secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS) # TODO(jtattermusch): add scenarios java vs C++ @@ -702,53 +581,42 @@ class GoLanguage: def scenarios(self): for secure in [True, False]: secstr = 'secure' if secure else 'insecure' - smoketest_categories = [SMOKETEST] if secure else [] + smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] # ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server, # but that's mostly because of lack of better name of the enum value. yield _ping_pong_scenario( - 'go_generic_sync_streaming_ping_pong_%s' % secstr, - rpc_type='STREAMING', - client_type='SYNC_CLIENT', - server_type='ASYNC_GENERIC_SERVER', - use_generic_payload=True, - async_server_threads=1, + 'go_generic_sync_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING', + client_type='SYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', + use_generic_payload=True, async_server_threads=1, secure=secure, categories=smoketest_categories) yield _ping_pong_scenario( - 'go_protobuf_sync_streaming_ping_pong_%s' % secstr, - rpc_type='STREAMING', - client_type='SYNC_CLIENT', - server_type='SYNC_SERVER', + 'go_protobuf_sync_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', async_server_threads=1, secure=secure) yield _ping_pong_scenario( - 'go_protobuf_sync_unary_ping_pong_%s' % secstr, - rpc_type='UNARY', - client_type='SYNC_CLIENT', - server_type='SYNC_SERVER', + 'go_protobuf_sync_unary_ping_pong_%s' % secstr, rpc_type='UNARY', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', async_server_threads=1, secure=secure, categories=smoketest_categories) # unconstrained_client='async' is intended (client uses goroutines) yield _ping_pong_scenario( - 'go_protobuf_sync_unary_qps_unconstrained_%s' % secstr, - rpc_type='UNARY', - client_type='SYNC_CLIENT', - server_type='SYNC_SERVER', + 'go_protobuf_sync_unary_qps_unconstrained_%s' % secstr, rpc_type='UNARY', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', unconstrained_client='async', secure=secure, - categories=smoketest_categories + [SCALABLE]) + categories=smoketest_categories+[SCALABLE]) # unconstrained_client='async' is intended (client uses goroutines) yield _ping_pong_scenario( - 'go_protobuf_sync_streaming_qps_unconstrained_%s' % secstr, - rpc_type='STREAMING', - client_type='SYNC_CLIENT', - server_type='SYNC_SERVER', + 'go_protobuf_sync_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING', + client_type='SYNC_CLIENT', server_type='SYNC_SERVER', unconstrained_client='async', secure=secure, categories=[SCALABLE]) @@ -757,12 +625,9 @@ class GoLanguage: # ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server, # but that's mostly because of lack of better name of the enum value. yield _ping_pong_scenario( - 'go_generic_sync_streaming_qps_unconstrained_%s' % secstr, - rpc_type='STREAMING', - client_type='SYNC_CLIENT', - server_type='ASYNC_GENERIC_SERVER', - unconstrained_client='async', - use_generic_payload=True, + 'go_generic_sync_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING', + client_type='SYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', + unconstrained_client='async', use_generic_payload=True, secure=secure, categories=[SCALABLE]) @@ -773,11 +638,11 @@ class GoLanguage: LANGUAGES = { - 'c++': CXXLanguage(), - 'csharp': CSharpLanguage(), - 'node': NodeLanguage(), - 'ruby': RubyLanguage(), - 'java': JavaLanguage(), - 'python': PythonLanguage(), - 'go': GoLanguage(), + 'c++' : CXXLanguage(), + 'csharp' : CSharpLanguage(), + 'node' : NodeLanguage(), + 'ruby' : RubyLanguage(), + 'java' : JavaLanguage(), + 'python' : PythonLanguage(), + 'go' : GoLanguage(), } diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index 3f028f177b..92149115fe 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -280,7 +280,7 @@ def create_qpsworkers(languages, worker_hosts): for worker_idx, worker in enumerate(workers)] -Scenario = collections.namedtuple('Scenario', 'jobspec workers') +Scenario = collections.namedtuple('Scenario', 'jobspec workers name') def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*', @@ -307,7 +307,7 @@ def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*', create_netperf_jobspec(server_host=netperf_server, client_host=netperf_client, bq_result_table=bq_result_table), - _NO_WORKERS)) + _NO_WORKERS, 'netperf')) for language in languages: for scenario_json in language.scenarios(): @@ -347,7 +347,8 @@ def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*', [w.host_and_port for w in workers], remote_host=remote_host, bq_result_table=bq_result_table), - workers) + workers, + scenario_json['name']) scenarios.append(scenario) return scenarios @@ -382,6 +383,11 @@ argp.add_argument('--remote_worker_host', nargs='+', default=[], help='Worker hosts where to start QPS workers.') +argp.add_argument('--dry_run', + default=False, + action='store_const', + const=True, + help='Just list scenarios to be run, but don\'t run them.') argp.add_argument('-r', '--regex', default='.*', type=str, help='Regex to select scenarios to run.') argp.add_argument('--bq_result_table', default=None, type=str, @@ -412,16 +418,18 @@ if args.remote_worker_host: if args.remote_driver_host: remote_hosts.add(args.remote_driver_host) -if remote_hosts: - archive_repo(languages=[str(l) for l in languages]) - prepare_remote_hosts(remote_hosts, prepare_local=True) -else: - prepare_remote_hosts([], prepare_local=True) +if not args.dry_run: + if remote_hosts: + archive_repo(languages=[str(l) for l in languages]) + prepare_remote_hosts(remote_hosts, prepare_local=True) + else: + prepare_remote_hosts([], prepare_local=True) build_local = False if not args.remote_driver_host: build_local = True -build_on_remote_hosts(remote_hosts, languages=[str(l) for l in languages], build_local=build_local) +if not args.dry_run: + build_on_remote_hosts(remote_hosts, languages=[str(l) for l in languages], build_local=build_local) qpsworker_jobs = create_qpsworkers(languages, args.remote_worker_host) @@ -443,11 +451,14 @@ if not scenarios: raise Exception('No scenarios to run') for scenario in scenarios: - try: - for worker in scenario.workers: - worker.start() - jobset.run([scenario.jobspec, - create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host)], - newline_on_success=True, maxjobs=1) - finally: - finish_qps_workers(scenario.workers) + if args.dry_run: + print(scenario.name) + else: + try: + for worker in scenario.workers: + worker.start() + jobset.run([scenario.jobspec, + create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host)], + newline_on_success=True, maxjobs=1) + finally: + finish_qps_workers(scenario.workers) diff --git a/tools/run_tests/sanity/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh index a33f5b0cc3..6410646cd1 100755 --- a/tools/run_tests/sanity/check_submodules.sh +++ b/tools/run_tests/sanity/check_submodules.sh @@ -44,7 +44,7 @@ cat << EOF | awk '{ print $1 }' | sort > $want_submodules c880e42ba1c8032d4cdde2aba0541d8a9d9fa2e9 third_party/boringssl (version_for_cocoapods_2.0-100-gc880e42) 05b155ff59114735ec8cd089f669c4c3d8f59029 third_party/gflags (v2.1.0-45-g05b155f) c99458533a9b4c743ed51537e25989ea55944908 third_party/googletest (release-1.7.0) - 1a586735085e817b1f52e53feec92ce418049f69 third_party/protobuf (v3.0.2) + a428e42072765993ff674fda72863c9f1aa2d268 third_party/protobuf (v3.1.0) 50893291621658f355bc5b4d450a8d06a563053d third_party/zlib (v1.2.8) bcad91771b7f0bff28a1cac1981d7ef2b9bcef3c third_party/thrift EOF diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 936703223e..a0c9480a4e 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -346,6 +346,22 @@ "headers": [], "is_filegroup": false, "language": "c", + "name": "connection_refused_test", + "src": [ + "test/core/end2end/connection_refused_test.c" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "language": "c", "name": "dns_resolver_connectivity_test", "src": [ "test/core/client_config/resolvers/dns_resolver_connectivity_test.c" @@ -2489,6 +2505,20 @@ "headers": [], "is_filegroup": false, "language": "c++", + "name": "grpc_php_plugin", + "src": [ + "src/compiler/php_plugin.cc" + ], + "third_party": false, + "type": "target" + }, + { + "deps": [ + "grpc_plugin_support" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", "name": "grpc_python_plugin", "src": [ "src/compiler/python_plugin.cc" @@ -5047,6 +5077,8 @@ "src/compiler/node_generator_helpers.h", "src/compiler/objective_c_generator.h", "src/compiler/objective_c_generator_helpers.h", + "src/compiler/php_generator.h", + "src/compiler/php_generator_helpers.h", "src/compiler/python_generator.h", "src/compiler/ruby_generator.h", "src/compiler/ruby_generator_helpers-inl.h", @@ -5071,6 +5103,9 @@ "src/compiler/objective_c_generator.cc", "src/compiler/objective_c_generator.h", "src/compiler/objective_c_generator_helpers.h", + "src/compiler/php_generator.cc", + "src/compiler/php_generator.h", + "src/compiler/php_generator_helpers.h", "src/compiler/python_generator.cc", "src/compiler/python_generator.h", "src/compiler/ruby_generator.cc", diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index bc95f4274f..a9f44e9ff7 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -392,6 +392,27 @@ "flaky": false, "gtest": false, "language": "c", + "name": "connection_refused_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ] + }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 0.1, + "exclude_configs": [], + "flaky": false, + "gtest": false, + "language": "c", "name": "dns_resolver_connectivity_test", "platforms": [ "linux", |