diff options
Diffstat (limited to 'tools/run_tests/performance/scenario_config.py')
| -rw-r--r-- | tools/run_tests/performance/scenario_config.py | 154 |
1 files changed, 83 insertions, 71 deletions
diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index 4dfd01fc66..725ca299bd 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -29,12 +29,16 @@ # performance scenario configuration for various languages +import math + WARMUP_SECONDS=5 JAVA_WARMUP_SECONDS=15 # Java needs more warmup time for JIT to kick in. BENCHMARK_SECONDS=30 SMOKETEST='smoketest' SCALABLE='scalable' +SWEEP='sweep' +DEFAULT_CATEGORIES=[SCALABLE, SMOKETEST] SECURE_SECARGS = {'use_test_ca': True, 'server_host_override': 'foo.test.google.fr'} @@ -63,17 +67,17 @@ BIG_GENERIC_PAYLOAD = { } } -# deep is the number of RPCs outstanding on a channel in non-ping-pong tests -# (the value used is 1 otherwise) -DEEP=100 +# target number of RPCs outstanding on across all client channels in +# non-ping-pong tests (since we can only specify per-channel numbers, the +# actual target will be slightly higher) +OUTSTANDING_REQUESTS={ + 'async': 6400, + 'sync': 1000 +} # wide is the number of client channels in multi-channel tests (1 otherwise) WIDE=64 -# For most synchronous clients, DEEP*WIDE threads will be created. -SYNC_DEEP=10 -SYNC_WIDE=8 - def _get_secargs(is_secure): if is_secure: @@ -91,6 +95,13 @@ def remove_nonproto_fields(scenario): return scenario +def geometric_progression(start, stop, step): + n = start + while n < stop: + yield int(round(n)) + n *= step + + def _ping_pong_scenario(name, rpc_type, client_type, server_type, secure=True, @@ -101,7 +112,9 @@ def _ping_pong_scenario(name, rpc_type, server_core_limit=0, async_server_threads=0, warmup_seconds=WARMUP_SECONDS, - categories=[]): + categories=DEFAULT_CATEGORIES, + channels=None, + outstanding=None): """Creates a basic ping pong scenario.""" scenario = { 'name': name, @@ -138,14 +151,9 @@ def _ping_pong_scenario(name, rpc_type, scenario['client_config']['payload_config'] = EMPTY_PROTO_PAYLOAD if unconstrained_client: - if unconstrained_client == 'async': - deep = DEEP - wide = WIDE - elif unconstrained_client == 'sync': - deep = SYNC_DEEP - wide = SYNC_WIDE - else: - raise Exception('Illegal value of unconstrained_client option.') + outstanding_calls = outstanding if outstanding is not None else OUTSTANDING_REQUESTS[unconstrained_client] + wide = channels if channels is not None else WIDE + deep = int(math.ceil(1.0 * outstanding_calls / wide)) scenario['num_clients'] = 0 # use as many client as available. scenario['client_config']['outstanding_rpcs_per_channel'] = deep @@ -182,62 +190,66 @@ class CXXLanguage: # TODO(ctiller): add 70% load latency test for secure in [True, False]: secstr = 'secure' if secure else 'insecure' - smoketest_categories = [SMOKETEST] if secure else [] + smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] yield _ping_pong_scenario( - 'cpp_generic_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING', - client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', + 'cpp_generic_async_streaming_ping_pong_%s' % secstr, + rpc_type='STREAMING', + client_type='ASYNC_CLIENT', + server_type='ASYNC_GENERIC_SERVER', use_generic_payload=True, server_core_limit=1, async_server_threads=1, secure=secure, categories=smoketest_categories) yield _ping_pong_scenario( - 'cpp_protobuf_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING', - client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - server_core_limit=1, async_server_threads=1, - secure=secure) - - yield _ping_pong_scenario( - 'cpp_protobuf_async_unary_ping_pong_%s' % secstr, rpc_type='UNARY', - client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - server_core_limit=1, async_server_threads=1, - secure=secure, - categories=smoketest_categories) - - yield _ping_pong_scenario( - 'cpp_protobuf_sync_unary_ping_pong_%s' % secstr, rpc_type='UNARY', - client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - server_core_limit=1, async_server_threads=1, - secure=secure) - - yield _ping_pong_scenario( - 'cpp_protobuf_async_unary_qps_unconstrained_%s' % secstr, rpc_type='UNARY', - client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - unconstrained_client='async', - secure=secure, - categories=smoketest_categories+[SCALABLE]) - - yield _ping_pong_scenario( - 'cpp_protobuf_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING', - client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - unconstrained_client='async', - secure=secure, - categories=[SCALABLE]) - - yield _ping_pong_scenario( - 'cpp_generic_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING', - client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', + 'cpp_generic_async_streaming_qps_unconstrained_%s' % secstr, + rpc_type='STREAMING', + client_type='ASYNC_CLIENT', + server_type='ASYNC_GENERIC_SERVER', unconstrained_client='async', use_generic_payload=True, secure=secure, categories=smoketest_categories+[SCALABLE]) yield _ping_pong_scenario( - 'cpp_generic_async_streaming_qps_one_server_core_%s' % secstr, rpc_type='STREAMING', - client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', + 'cpp_generic_async_streaming_qps_one_server_core_%s' % secstr, + rpc_type='STREAMING', + client_type='ASYNC_CLIENT', + server_type='ASYNC_GENERIC_SERVER', unconstrained_client='async', use_generic_payload=True, server_core_limit=1, async_server_threads=1, secure=secure) + for rpc_type in ['unary', 'streaming']: + for synchronicity in ['sync', 'async']: + yield _ping_pong_scenario( + 'cpp_protobuf_%s_%s_ping_pong_%s' % (synchronicity, rpc_type, secstr), + rpc_type=rpc_type.upper(), + client_type='%s_CLIENT' % synchronicity.upper(), + server_type='%s_SERVER' % synchronicity.upper(), + server_core_limit=1, async_server_threads=1, + secure=secure) + + yield _ping_pong_scenario( + 'cpp_protobuf_%s_%s_qps_unconstrained_%s' % (synchronicity, rpc_type, secstr), + rpc_type=rpc_type.upper(), + client_type='%s_CLIENT' % synchronicity.upper(), + server_type='%s_SERVER' % synchronicity.upper(), + unconstrained_client=synchronicity, + secure=secure, + categories=smoketest_categories+[SCALABLE]) + + for channels in geometric_progression(1, 20000, math.sqrt(10)): + for outstanding in geometric_progression(1, 200000, math.sqrt(10)): + if synchronicity == 'sync' and outstanding > 1200: continue + if outstanding < channels: continue + yield _ping_pong_scenario( + 'cpp_protobuf_%s_%s_qps_unconstrained_%s_%d_channels_%d_outstanding' % (synchronicity, rpc_type, secstr, channels, outstanding), + rpc_type=rpc_type.upper(), + client_type='%s_CLIENT' % synchronicity.upper(), + server_type='%s_SERVER' % synchronicity.upper(), + unconstrained_client=synchronicity, secure=secure, + categories=[SWEEP], channels=channels, outstanding=outstanding) + def __str__(self): return 'c++' @@ -258,7 +270,7 @@ class CSharpLanguage: 'csharp_generic_async_streaming_ping_pong', rpc_type='STREAMING', client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', use_generic_payload=True, - categories=[SMOKETEST]) + categories=[SMOKETEST, SCALABLE]) yield _ping_pong_scenario( 'csharp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING', @@ -267,7 +279,7 @@ class CSharpLanguage: yield _ping_pong_scenario( 'csharp_protobuf_async_unary_ping_pong', rpc_type='UNARY', client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - categories=[SMOKETEST]) + categories=[SMOKETEST, SCALABLE]) yield _ping_pong_scenario( 'csharp_protobuf_sync_to_async_unary_ping_pong', rpc_type='UNARY', @@ -289,7 +301,7 @@ class CSharpLanguage: 'csharp_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', client_type='SYNC_CLIENT', server_type='SYNC_SERVER', server_language='c++', server_core_limit=1, async_server_threads=1, - categories=[SMOKETEST]) + categories=[SMOKETEST, SCALABLE]) yield _ping_pong_scenario( 'csharp_to_cpp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING', @@ -346,13 +358,13 @@ class NodeLanguage: yield _ping_pong_scenario( 'node_protobuf_unary_ping_pong', rpc_type='UNARY', client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - categories=[SMOKETEST]) + categories=[SCALABLE, SMOKETEST]) yield _ping_pong_scenario( 'node_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY', client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', unconstrained_client='async', - categories=[SMOKETEST]) + categories=[SCALABLE, SMOKETEST]) # TODO(jtattermusch): make this scenario work #yield _ping_pong_scenario( @@ -391,7 +403,7 @@ class PythonLanguage: 'python_generic_sync_streaming_ping_pong', rpc_type='STREAMING', client_type='SYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', use_generic_payload=True, - categories=[SMOKETEST]) + categories=[SMOKETEST, SCALABLE]) yield _ping_pong_scenario( 'python_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', @@ -404,7 +416,7 @@ class PythonLanguage: yield _ping_pong_scenario( 'python_protobuf_sync_unary_ping_pong', rpc_type='UNARY', client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', - categories=[SMOKETEST]) + categories=[SMOKETEST, SCALABLE]) yield _ping_pong_scenario( 'python_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY', @@ -420,7 +432,7 @@ class PythonLanguage: 'python_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', server_language='c++', server_core_limit=1, async_server_threads=1, - categories=[SMOKETEST]) + categories=[SMOKETEST, SCALABLE]) yield _ping_pong_scenario( 'python_to_cpp_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', @@ -446,12 +458,12 @@ class RubyLanguage: yield _ping_pong_scenario( 'ruby_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - categories=[SMOKETEST]) + categories=[SMOKETEST, SCALABLE]) yield _ping_pong_scenario( 'ruby_protobuf_unary_ping_pong', rpc_type='UNARY', client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - categories=[SMOKETEST]) + categories=[SMOKETEST, SCALABLE]) yield _ping_pong_scenario( 'ruby_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY', @@ -492,7 +504,7 @@ class JavaLanguage: def scenarios(self): for secure in [True, False]: secstr = 'secure' if secure else 'insecure' - smoketest_categories = [SMOKETEST] if secure else [] + smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] yield _ping_pong_scenario( 'java_generic_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING', @@ -548,7 +560,7 @@ class JavaLanguage: async_server_threads=1, secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS) - # TODO(jtattermusch): add scenarios java vs C++ + # TODO(jtattermusch): add scenarios java vs C++ def __str__(self): return 'java' @@ -569,10 +581,10 @@ class GoLanguage: def scenarios(self): for secure in [True, False]: secstr = 'secure' if secure else 'insecure' - smoketest_categories = [SMOKETEST] if secure else [] + smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] # ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server, - # but that's mostly because of lack of better name of the enum value. + # but that's mostly because of lack of better name of the enum value. yield _ping_pong_scenario( 'go_generic_sync_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING', client_type='SYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', @@ -619,7 +631,7 @@ class GoLanguage: secure=secure, categories=[SCALABLE]) - # TODO(jtattermusch): add scenarios go vs C++ + # TODO(jtattermusch): add scenarios go vs C++ def __str__(self): return 'go' |