diff options
Diffstat (limited to 'tools/run_tests/performance/scenario_config.py')
| -rw-r--r-- | tools/run_tests/performance/scenario_config.py | 1852 |
1 files changed, 1030 insertions, 822 deletions
diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index cafac3dfad..e92de5898c 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -16,66 +16,64 @@ import math -WARMUP_SECONDS=5 -JAVA_WARMUP_SECONDS=15 # Java needs more warmup time for JIT to kick in. -BENCHMARK_SECONDS=30 - -SMOKETEST='smoketest' -SCALABLE='scalable' -INPROC='inproc' -SWEEP='sweep' -DEFAULT_CATEGORIES=[SCALABLE, SMOKETEST] - -SECURE_SECARGS = {'use_test_ca': True, - 'server_host_override': 'foo.test.google.fr'} +WARMUP_SECONDS = 5 +JAVA_WARMUP_SECONDS = 15 # Java needs more warmup time for JIT to kick in. +BENCHMARK_SECONDS = 30 + +SMOKETEST = 'smoketest' +SCALABLE = 'scalable' +INPROC = 'inproc' +SWEEP = 'sweep' +DEFAULT_CATEGORIES = [SCALABLE, SMOKETEST] + +SECURE_SECARGS = { + 'use_test_ca': True, + 'server_host_override': 'foo.test.google.fr' +} HISTOGRAM_PARAMS = { - 'resolution': 0.01, - 'max_possible': 60e9, + 'resolution': 0.01, + 'max_possible': 60e9, } # target number of RPCs outstanding on across all client channels in # non-ping-pong tests (since we can only specify per-channel numbers, the # actual target will be slightly higher) -OUTSTANDING_REQUESTS={ - 'async': 6400, - 'async-limited': 800, - 'sync': 1000 -} +OUTSTANDING_REQUESTS = {'async': 6400, 'async-limited': 800, 'sync': 1000} # wide is the number of client channels in multi-channel tests (1 otherwise) -WIDE=64 +WIDE = 64 def _get_secargs(is_secure): - if is_secure: - return SECURE_SECARGS - else: - return None + if is_secure: + return SECURE_SECARGS + else: + return None def remove_nonproto_fields(scenario): - """Remove special-purpose that contains some extra info about the scenario + """Remove special-purpose that contains some extra info about the scenario but don't belong to the ScenarioConfig protobuf message""" - scenario.pop('CATEGORIES', None) - scenario.pop('CLIENT_LANGUAGE', None) - scenario.pop('SERVER_LANGUAGE', None) - scenario.pop('EXCLUDED_POLL_ENGINES', None) - return scenario + scenario.pop('CATEGORIES', None) + scenario.pop('CLIENT_LANGUAGE', None) + scenario.pop('SERVER_LANGUAGE', None) + scenario.pop('EXCLUDED_POLL_ENGINES', None) + return scenario def geometric_progression(start, stop, step): - n = start - while n < stop: - yield int(round(n)) - n *= step + n = start + while n < stop: + yield int(round(n)) + n *= step def _payload_type(use_generic_payload, req_size, resp_size): r = {} sizes = { - 'req_size': req_size, - 'resp_size': resp_size, + 'req_size': req_size, + 'resp_size': resp_size, } if use_generic_payload: r['bytebuf_params'] = sizes @@ -83,6 +81,7 @@ def _payload_type(use_generic_payload, req_size, resp_size): r['simple_params'] = sizes return r + def _load_params(offered_load): r = {} if offered_load is None: @@ -93,21 +92,25 @@ def _load_params(offered_load): r['poisson'] = load return r + def _add_channel_arg(config, key, value): - if 'channel_args' in config: - channel_args = config['channel_args'] - else: - channel_args = [] - config['channel_args'] = channel_args - arg = {'name': key} - if isinstance(value, int): - arg['int_value'] = value - else: - arg['str_value'] = value - channel_args.append(arg) - -def _ping_pong_scenario(name, rpc_type, - client_type, server_type, + if 'channel_args' in config: + channel_args = config['channel_args'] + else: + channel_args = [] + config['channel_args'] = channel_args + arg = {'name': key} + if isinstance(value, int): + arg['int_value'] = value + else: + arg['str_value'] = value + channel_args.append(arg) + + +def _ping_pong_scenario(name, + rpc_type, + client_type, + server_type, secure=True, use_generic_payload=False, req_size=0, @@ -128,824 +131,1029 @@ def _ping_pong_scenario(name, rpc_type, excluded_poll_engines=[], minimal_stack=False, offered_load=None): - """Creates a basic ping pong scenario.""" - scenario = { - 'name': name, - 'num_servers': 1, - 'num_clients': 1, - 'client_config': { - 'client_type': client_type, - 'security_params': _get_secargs(secure), - 'outstanding_rpcs_per_channel': 1, - 'client_channels': 1, - 'async_client_threads': 1, - 'threads_per_cq': client_threads_per_cq, - 'rpc_type': rpc_type, - 'histogram_params': HISTOGRAM_PARAMS, - 'channel_args': [], - }, - 'server_config': { - 'server_type': server_type, - 'security_params': _get_secargs(secure), - 'async_server_threads': async_server_threads, - 'threads_per_cq': server_threads_per_cq, - 'channel_args': [], - }, - 'warmup_seconds': warmup_seconds, - 'benchmark_seconds': BENCHMARK_SECONDS - } - if resource_quota_size: - scenario['server_config']['resource_quota_size'] = resource_quota_size - if use_generic_payload: - if server_type != 'ASYNC_GENERIC_SERVER': - raise Exception('Use ASYNC_GENERIC_SERVER for generic payload.') - scenario['server_config']['payload_config'] = _payload_type(use_generic_payload, req_size, resp_size) - - scenario['client_config']['payload_config'] = _payload_type(use_generic_payload, req_size, resp_size) - - # Optimization target of 'throughput' does not work well with epoll1 polling - # engine. Use the default value of 'blend' - optimization_target = 'throughput' - - if unconstrained_client: - outstanding_calls = outstanding if outstanding is not None else OUTSTANDING_REQUESTS[unconstrained_client] - # clamp buffer usage to something reasonable (16 gig for now) - MAX_MEMORY_USE = 16 * 1024 * 1024 * 1024 - if outstanding_calls * max(req_size, resp_size) > MAX_MEMORY_USE: - outstanding_calls = max(1, MAX_MEMORY_USE / max(req_size, resp_size)) - wide = channels if channels is not None else WIDE - deep = int(math.ceil(1.0 * outstanding_calls / wide)) - - scenario['num_clients'] = num_clients if num_clients is not None else 0 # use as many clients as available. - scenario['client_config']['outstanding_rpcs_per_channel'] = deep - scenario['client_config']['client_channels'] = wide - scenario['client_config']['async_client_threads'] = 0 - if offered_load is not None: + """Creates a basic ping pong scenario.""" + scenario = { + 'name': name, + 'num_servers': 1, + 'num_clients': 1, + 'client_config': { + 'client_type': client_type, + 'security_params': _get_secargs(secure), + 'outstanding_rpcs_per_channel': 1, + 'client_channels': 1, + 'async_client_threads': 1, + 'threads_per_cq': client_threads_per_cq, + 'rpc_type': rpc_type, + 'histogram_params': HISTOGRAM_PARAMS, + 'channel_args': [], + }, + 'server_config': { + 'server_type': server_type, + 'security_params': _get_secargs(secure), + 'async_server_threads': async_server_threads, + 'threads_per_cq': server_threads_per_cq, + 'channel_args': [], + }, + 'warmup_seconds': warmup_seconds, + 'benchmark_seconds': BENCHMARK_SECONDS + } + if resource_quota_size: + scenario['server_config']['resource_quota_size'] = resource_quota_size + if use_generic_payload: + if server_type != 'ASYNC_GENERIC_SERVER': + raise Exception('Use ASYNC_GENERIC_SERVER for generic payload.') + scenario['server_config']['payload_config'] = _payload_type( + use_generic_payload, req_size, resp_size) + + scenario['client_config']['payload_config'] = _payload_type( + use_generic_payload, req_size, resp_size) + + # Optimization target of 'throughput' does not work well with epoll1 polling + # engine. Use the default value of 'blend' + optimization_target = 'throughput' + + if unconstrained_client: + outstanding_calls = outstanding if outstanding is not None else OUTSTANDING_REQUESTS[ + unconstrained_client] + # clamp buffer usage to something reasonable (16 gig for now) + MAX_MEMORY_USE = 16 * 1024 * 1024 * 1024 + if outstanding_calls * max(req_size, resp_size) > MAX_MEMORY_USE: + outstanding_calls = max(1, MAX_MEMORY_USE / max(req_size, + resp_size)) + wide = channels if channels is not None else WIDE + deep = int(math.ceil(1.0 * outstanding_calls / wide)) + + scenario[ + 'num_clients'] = num_clients if num_clients is not None else 0 # use as many clients as available. + scenario['client_config']['outstanding_rpcs_per_channel'] = deep + scenario['client_config']['client_channels'] = wide + scenario['client_config']['async_client_threads'] = 0 + if offered_load is not None: + optimization_target = 'latency' + else: + scenario['client_config']['outstanding_rpcs_per_channel'] = 1 + scenario['client_config']['client_channels'] = 1 + scenario['client_config']['async_client_threads'] = 1 optimization_target = 'latency' - else: - scenario['client_config']['outstanding_rpcs_per_channel'] = 1 - scenario['client_config']['client_channels'] = 1 - scenario['client_config']['async_client_threads'] = 1 - optimization_target = 'latency' - - scenario['client_config']['load_params'] = _load_params(offered_load) - - optimization_channel_arg = { - 'name': 'grpc.optimization_target', - 'str_value': optimization_target - } - scenario['client_config']['channel_args'].append(optimization_channel_arg) - scenario['server_config']['channel_args'].append(optimization_channel_arg) - - if minimal_stack: - _add_channel_arg(scenario['client_config'], 'grpc.minimal_stack', 1) - _add_channel_arg(scenario['server_config'], 'grpc.minimal_stack', 1) - - if messages_per_stream: - scenario['client_config']['messages_per_stream'] = messages_per_stream - if client_language: - # the CLIENT_LANGUAGE field is recognized by run_performance_tests.py - scenario['CLIENT_LANGUAGE'] = client_language - if server_language: - # the SERVER_LANGUAGE field is recognized by run_performance_tests.py - scenario['SERVER_LANGUAGE'] = server_language - if categories: - scenario['CATEGORIES'] = categories - if len(excluded_poll_engines): - # The polling engines for which this scenario is excluded - scenario['EXCLUDED_POLL_ENGINES'] = excluded_poll_engines - return scenario + + scenario['client_config']['load_params'] = _load_params(offered_load) + + optimization_channel_arg = { + 'name': 'grpc.optimization_target', + 'str_value': optimization_target + } + scenario['client_config']['channel_args'].append(optimization_channel_arg) + scenario['server_config']['channel_args'].append(optimization_channel_arg) + + if minimal_stack: + _add_channel_arg(scenario['client_config'], 'grpc.minimal_stack', 1) + _add_channel_arg(scenario['server_config'], 'grpc.minimal_stack', 1) + + if messages_per_stream: + scenario['client_config']['messages_per_stream'] = messages_per_stream + if client_language: + # the CLIENT_LANGUAGE field is recognized by run_performance_tests.py + scenario['CLIENT_LANGUAGE'] = client_language + if server_language: + # the SERVER_LANGUAGE field is recognized by run_performance_tests.py + scenario['SERVER_LANGUAGE'] = server_language + if categories: + scenario['CATEGORIES'] = categories + if len(excluded_poll_engines): + # The polling engines for which this scenario is excluded + scenario['EXCLUDED_POLL_ENGINES'] = excluded_poll_engines + return scenario class CXXLanguage: - def __init__(self): - self.safename = 'cxx' - - def worker_cmdline(self): - return ['bins/opt/qps_worker'] - - def worker_port_offset(self): - return 0 - - def scenarios(self): - # TODO(ctiller): add 70% load latency test - yield _ping_pong_scenario( - 'cpp_protobuf_async_unary_1channel_100rpcs_1MB', rpc_type='UNARY', - client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - req_size=1024*1024, resp_size=1024*1024, - unconstrained_client='async', outstanding=100, channels=1, - num_clients=1, - secure=False, - categories=[SMOKETEST] + [INPROC] + [SCALABLE]) - - yield _ping_pong_scenario( - 'cpp_protobuf_async_streaming_from_client_1channel_1MB', rpc_type='STREAMING_FROM_CLIENT', - client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - req_size=1024*1024, resp_size=1024*1024, - unconstrained_client='async', outstanding=1, channels=1, - num_clients=1, - secure=False, - categories=[SMOKETEST] + [INPROC] + [SCALABLE]) - - yield _ping_pong_scenario( - 'cpp_protobuf_async_unary_75Kqps_600channel_60Krpcs_300Breq_50Bresp', - rpc_type='UNARY', client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - req_size=300, resp_size=50, - unconstrained_client='async', outstanding=30000, channels=300, - offered_load=37500, secure=False, - async_server_threads=16, server_threads_per_cq=1, - categories=[SMOKETEST] + [SCALABLE]) - - for secure in [True, False]: - secstr = 'secure' if secure else 'insecure' - smoketest_categories = ([SMOKETEST] if secure else [INPROC]) + [SCALABLE] - - yield _ping_pong_scenario( - 'cpp_generic_async_streaming_ping_pong_%s' % secstr, - rpc_type='STREAMING', - client_type='ASYNC_CLIENT', - server_type='ASYNC_GENERIC_SERVER', - use_generic_payload=True, async_server_threads=1, - secure=secure, - categories=smoketest_categories) - - yield _ping_pong_scenario( - 'cpp_generic_async_streaming_qps_unconstrained_%s' % secstr, - rpc_type='STREAMING', - client_type='ASYNC_CLIENT', - server_type='ASYNC_GENERIC_SERVER', - unconstrained_client='async', use_generic_payload=True, - secure=secure, - minimal_stack=not secure, - categories=smoketest_categories+[SCALABLE]) - - for mps in geometric_progression(1, 20, 10): - yield _ping_pong_scenario( - 'cpp_generic_async_streaming_qps_unconstrained_%smps_%s' % (mps, secstr), - rpc_type='STREAMING', + def __init__(self): + self.safename = 'cxx' + + def worker_cmdline(self): + return ['bins/opt/qps_worker'] + + def worker_port_offset(self): + return 0 + + def scenarios(self): + # TODO(ctiller): add 70% load latency test + yield _ping_pong_scenario( + 'cpp_protobuf_async_unary_1channel_100rpcs_1MB', + rpc_type='UNARY', client_type='ASYNC_CLIENT', - server_type='ASYNC_GENERIC_SERVER', - unconstrained_client='async', use_generic_payload=True, - secure=secure, messages_per_stream=mps, - minimal_stack=not secure, - categories=smoketest_categories+[SCALABLE]) + server_type='ASYNC_SERVER', + req_size=1024 * 1024, + resp_size=1024 * 1024, + unconstrained_client='async', + outstanding=100, + channels=1, + num_clients=1, + secure=False, + categories=[SMOKETEST] + [INPROC] + [SCALABLE]) - for mps in geometric_progression(1, 200, math.sqrt(10)): yield _ping_pong_scenario( - 'cpp_generic_async_streaming_qps_unconstrained_%smps_%s' % (mps, secstr), - rpc_type='STREAMING', + 'cpp_protobuf_async_streaming_from_client_1channel_1MB', + rpc_type='STREAMING_FROM_CLIENT', client_type='ASYNC_CLIENT', - server_type='ASYNC_GENERIC_SERVER', - unconstrained_client='async', use_generic_payload=True, - secure=secure, messages_per_stream=mps, - minimal_stack=not secure, - categories=[SWEEP]) - - yield _ping_pong_scenario( - 'cpp_generic_async_streaming_qps_1channel_1MBmsg_%s' % secstr, - rpc_type='STREAMING', - req_size=1024*1024, - resp_size=1024*1024, - client_type='ASYNC_CLIENT', - server_type='ASYNC_GENERIC_SERVER', - unconstrained_client='async', use_generic_payload=True, - secure=secure, - minimal_stack=not secure, - categories=smoketest_categories+[SCALABLE], - channels=1, outstanding=100) - - yield _ping_pong_scenario( - 'cpp_generic_async_streaming_qps_unconstrained_64KBmsg_%s' % secstr, - rpc_type='STREAMING', - req_size=64*1024, - resp_size=64*1024, - client_type='ASYNC_CLIENT', - server_type='ASYNC_GENERIC_SERVER', - unconstrained_client='async', use_generic_payload=True, - secure=secure, - minimal_stack=not secure, - categories=smoketest_categories+[SCALABLE]) - - # TODO(https://github.com/grpc/grpc/issues/11500) Re-enable this test - #yield _ping_pong_scenario( - # 'cpp_generic_async_streaming_qps_unconstrained_1cq_%s' % secstr, - # rpc_type='STREAMING', - # client_type='ASYNC_CLIENT', - # server_type='ASYNC_GENERIC_SERVER', - # unconstrained_client='async-limited', use_generic_payload=True, - # secure=secure, - # client_threads_per_cq=1000000, server_threads_per_cq=1000000, - # categories=smoketest_categories+[SCALABLE]) - - yield _ping_pong_scenario( - 'cpp_generic_async_streaming_qps_unconstrained_2waysharedcq_%s' % secstr, - rpc_type='STREAMING', - client_type='ASYNC_CLIENT', - server_type='ASYNC_GENERIC_SERVER', - unconstrained_client='async', use_generic_payload=True, - secure=secure, - client_threads_per_cq=2, server_threads_per_cq=2, - categories=smoketest_categories+[SCALABLE]) - - #yield _ping_pong_scenario( - # 'cpp_protobuf_async_streaming_qps_unconstrained_1cq_%s' % secstr, - # rpc_type='STREAMING', - # client_type='ASYNC_CLIENT', - # server_type='ASYNC_SERVER', - # unconstrained_client='async-limited', - # secure=secure, - # client_threads_per_cq=1000000, server_threads_per_cq=1000000, - # categories=smoketest_categories+[SCALABLE]) - - yield _ping_pong_scenario( - 'cpp_protobuf_async_streaming_qps_unconstrained_2waysharedcq_%s' % secstr, - rpc_type='STREAMING', - client_type='ASYNC_CLIENT', - server_type='ASYNC_SERVER', - unconstrained_client='async', - secure=secure, - client_threads_per_cq=2, server_threads_per_cq=2, - categories=smoketest_categories+[SCALABLE]) - - #yield _ping_pong_scenario( - # 'cpp_protobuf_async_unary_qps_unconstrained_1cq_%s' % secstr, - # rpc_type='UNARY', - # client_type='ASYNC_CLIENT', - # server_type='ASYNC_SERVER', - # unconstrained_client='async-limited', - # secure=secure, - # client_threads_per_cq=1000000, server_threads_per_cq=1000000, - # categories=smoketest_categories+[SCALABLE]) - - yield _ping_pong_scenario( - 'cpp_protobuf_async_unary_qps_unconstrained_2waysharedcq_%s' % secstr, - rpc_type='UNARY', - client_type='ASYNC_CLIENT', - server_type='ASYNC_SERVER', - unconstrained_client='async', - secure=secure, - client_threads_per_cq=2, server_threads_per_cq=2, - categories=smoketest_categories+[SCALABLE]) - - yield _ping_pong_scenario( - 'cpp_generic_async_streaming_qps_one_server_core_%s' % secstr, - rpc_type='STREAMING', - client_type='ASYNC_CLIENT', - server_type='ASYNC_GENERIC_SERVER', - unconstrained_client='async-limited', use_generic_payload=True, - async_server_threads=1, - minimal_stack=not secure, - secure=secure) - - yield _ping_pong_scenario( - 'cpp_protobuf_async_client_sync_server_unary_qps_unconstrained_%s' % - (secstr), - rpc_type='UNARY', - client_type='ASYNC_CLIENT', - server_type='SYNC_SERVER', - unconstrained_client='async', - secure=secure, - minimal_stack=not secure, - categories=smoketest_categories + [SCALABLE], - excluded_poll_engines = ['poll-cv']) - - yield _ping_pong_scenario( - 'cpp_protobuf_async_client_unary_1channel_64wide_128Breq_8MBresp_%s' % - (secstr), - rpc_type='UNARY', - client_type='ASYNC_CLIENT', - server_type='ASYNC_SERVER', - channels=1, - outstanding=64, - req_size=128, - resp_size=8*1024*1024, - secure=secure, - minimal_stack=not secure, - categories=smoketest_categories + [SCALABLE]) - - yield _ping_pong_scenario( - 'cpp_protobuf_async_client_sync_server_streaming_qps_unconstrained_%s' % secstr, - rpc_type='STREAMING', - client_type='ASYNC_CLIENT', - server_type='SYNC_SERVER', - unconstrained_client='async', - secure=secure, - minimal_stack=not secure, - categories=smoketest_categories+[SCALABLE], - excluded_poll_engines = ['poll-cv']) - - yield _ping_pong_scenario( - 'cpp_protobuf_async_unary_ping_pong_%s_1MB' % secstr, rpc_type='UNARY', - client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - req_size=1024*1024, resp_size=1024*1024, - secure=secure, - minimal_stack=not secure, - categories=smoketest_categories + [SCALABLE]) - - for rpc_type in ['unary', 'streaming', 'streaming_from_client', 'streaming_from_server']: - for synchronicity in ['sync', 'async']: - yield _ping_pong_scenario( - 'cpp_protobuf_%s_%s_ping_pong_%s' % (synchronicity, rpc_type, secstr), - rpc_type=rpc_type.upper(), - client_type='%s_CLIENT' % synchronicity.upper(), - server_type='%s_SERVER' % synchronicity.upper(), - async_server_threads=1, - minimal_stack=not secure, - secure=secure) - - for size in geometric_progression(1, 1024*1024*1024+1, 8): - yield _ping_pong_scenario( - 'cpp_protobuf_%s_%s_qps_unconstrained_%s_%db' % (synchronicity, rpc_type, secstr, size), - rpc_type=rpc_type.upper(), - req_size=size, - resp_size=size, - client_type='%s_CLIENT' % synchronicity.upper(), - server_type='%s_SERVER' % synchronicity.upper(), - unconstrained_client=synchronicity, - secure=secure, - minimal_stack=not secure, - categories=[SWEEP]) - - yield _ping_pong_scenario( - 'cpp_protobuf_%s_%s_qps_unconstrained_%s' % (synchronicity, rpc_type, secstr), - rpc_type=rpc_type.upper(), - client_type='%s_CLIENT' % synchronicity.upper(), - server_type='%s_SERVER' % synchronicity.upper(), - unconstrained_client=synchronicity, - secure=secure, - minimal_stack=not secure, - server_threads_per_cq=3, - client_threads_per_cq=3, - categories=smoketest_categories+[SCALABLE]) - - # TODO(vjpai): Re-enable this test. It has a lot of timeouts - # and hasn't yet been conclusively identified as a test failure - # or race in the library - # yield _ping_pong_scenario( - # 'cpp_protobuf_%s_%s_qps_unconstrained_%s_500kib_resource_quota' % (synchronicity, rpc_type, secstr), - # rpc_type=rpc_type.upper(), - # client_type='%s_CLIENT' % synchronicity.upper(), - # server_type='%s_SERVER' % synchronicity.upper(), - # unconstrained_client=synchronicity, - # secure=secure, - # categories=smoketest_categories+[SCALABLE], - # resource_quota_size=500*1024) - - if rpc_type == 'streaming': + server_type='ASYNC_SERVER', + req_size=1024 * 1024, + resp_size=1024 * 1024, + unconstrained_client='async', + outstanding=1, + channels=1, + num_clients=1, + secure=False, + categories=[SMOKETEST] + [INPROC] + [SCALABLE]) + + yield _ping_pong_scenario( + 'cpp_protobuf_async_unary_75Kqps_600channel_60Krpcs_300Breq_50Bresp', + rpc_type='UNARY', + client_type='ASYNC_CLIENT', + server_type='ASYNC_SERVER', + req_size=300, + resp_size=50, + unconstrained_client='async', + outstanding=30000, + channels=300, + offered_load=37500, + secure=False, + async_server_threads=16, + server_threads_per_cq=1, + categories=[SMOKETEST] + [SCALABLE]) + + for secure in [True, False]: + secstr = 'secure' if secure else 'insecure' + smoketest_categories = ([SMOKETEST] + if secure else [INPROC]) + [SCALABLE] + + yield _ping_pong_scenario( + 'cpp_generic_async_streaming_ping_pong_%s' % secstr, + rpc_type='STREAMING', + client_type='ASYNC_CLIENT', + server_type='ASYNC_GENERIC_SERVER', + use_generic_payload=True, + async_server_threads=1, + secure=secure, + categories=smoketest_categories) + + yield _ping_pong_scenario( + 'cpp_generic_async_streaming_qps_unconstrained_%s' % secstr, + rpc_type='STREAMING', + client_type='ASYNC_CLIENT', + server_type='ASYNC_GENERIC_SERVER', + unconstrained_client='async', + use_generic_payload=True, + secure=secure, + minimal_stack=not secure, + categories=smoketest_categories + [SCALABLE]) + for mps in geometric_progression(1, 20, 10): - yield _ping_pong_scenario( - 'cpp_protobuf_%s_%s_qps_unconstrained_%smps_%s' % (synchronicity, rpc_type, mps, secstr), - rpc_type=rpc_type.upper(), - client_type='%s_CLIENT' % synchronicity.upper(), - server_type='%s_SERVER' % synchronicity.upper(), - unconstrained_client=synchronicity, - secure=secure, messages_per_stream=mps, - minimal_stack=not secure, - categories=smoketest_categories+[SCALABLE]) + yield _ping_pong_scenario( + 'cpp_generic_async_streaming_qps_unconstrained_%smps_%s' % + (mps, secstr), + rpc_type='STREAMING', + client_type='ASYNC_CLIENT', + server_type='ASYNC_GENERIC_SERVER', + unconstrained_client='async', + use_generic_payload=True, + secure=secure, + messages_per_stream=mps, + minimal_stack=not secure, + categories=smoketest_categories + [SCALABLE]) for mps in geometric_progression(1, 200, math.sqrt(10)): - yield _ping_pong_scenario( - 'cpp_protobuf_%s_%s_qps_unconstrained_%smps_%s' % (synchronicity, rpc_type, mps, secstr), - rpc_type=rpc_type.upper(), - client_type='%s_CLIENT' % synchronicity.upper(), - server_type='%s_SERVER' % synchronicity.upper(), - unconstrained_client=synchronicity, - secure=secure, messages_per_stream=mps, - minimal_stack=not secure, - categories=[SWEEP]) - - for channels in geometric_progression(1, 20000, math.sqrt(10)): - for outstanding in geometric_progression(1, 200000, math.sqrt(10)): - if synchronicity == 'sync' and outstanding > 1200: continue - if outstanding < channels: continue yield _ping_pong_scenario( - 'cpp_protobuf_%s_%s_qps_unconstrained_%s_%d_channels_%d_outstanding' % (synchronicity, rpc_type, secstr, channels, outstanding), - rpc_type=rpc_type.upper(), - client_type='%s_CLIENT' % synchronicity.upper(), - server_type='%s_SERVER' % synchronicity.upper(), - unconstrained_client=synchronicity, secure=secure, + 'cpp_generic_async_streaming_qps_unconstrained_%smps_%s' % + (mps, secstr), + rpc_type='STREAMING', + client_type='ASYNC_CLIENT', + server_type='ASYNC_GENERIC_SERVER', + unconstrained_client='async', + use_generic_payload=True, + secure=secure, + messages_per_stream=mps, minimal_stack=not secure, - categories=[SWEEP], channels=channels, outstanding=outstanding) - - def __str__(self): - return 'c++' + categories=[SWEEP]) + + yield _ping_pong_scenario( + 'cpp_generic_async_streaming_qps_1channel_1MBmsg_%s' % secstr, + rpc_type='STREAMING', + req_size=1024 * 1024, + resp_size=1024 * 1024, + client_type='ASYNC_CLIENT', + server_type='ASYNC_GENERIC_SERVER', + unconstrained_client='async', + use_generic_payload=True, + secure=secure, + minimal_stack=not secure, + categories=smoketest_categories + [SCALABLE], + channels=1, + outstanding=100) + + yield _ping_pong_scenario( + 'cpp_generic_async_streaming_qps_unconstrained_64KBmsg_%s' % + secstr, + rpc_type='STREAMING', + req_size=64 * 1024, + resp_size=64 * 1024, + client_type='ASYNC_CLIENT', + server_type='ASYNC_GENERIC_SERVER', + unconstrained_client='async', + use_generic_payload=True, + secure=secure, + minimal_stack=not secure, + categories=smoketest_categories + [SCALABLE]) + + # TODO(https://github.com/grpc/grpc/issues/11500) Re-enable this test + #yield _ping_pong_scenario( + # 'cpp_generic_async_streaming_qps_unconstrained_1cq_%s' % secstr, + # rpc_type='STREAMING', + # client_type='ASYNC_CLIENT', + # server_type='ASYNC_GENERIC_SERVER', + # unconstrained_client='async-limited', use_generic_payload=True, + # secure=secure, + # client_threads_per_cq=1000000, server_threads_per_cq=1000000, + # categories=smoketest_categories+[SCALABLE]) + + yield _ping_pong_scenario( + 'cpp_generic_async_streaming_qps_unconstrained_2waysharedcq_%s' + % secstr, + rpc_type='STREAMING', + client_type='ASYNC_CLIENT', + server_type='ASYNC_GENERIC_SERVER', + unconstrained_client='async', + use_generic_payload=True, + secure=secure, + client_threads_per_cq=2, + server_threads_per_cq=2, + categories=smoketest_categories + [SCALABLE]) + + #yield _ping_pong_scenario( + # 'cpp_protobuf_async_streaming_qps_unconstrained_1cq_%s' % secstr, + # rpc_type='STREAMING', + # client_type='ASYNC_CLIENT', + # server_type='ASYNC_SERVER', + # unconstrained_client='async-limited', + # secure=secure, + # client_threads_per_cq=1000000, server_threads_per_cq=1000000, + # categories=smoketest_categories+[SCALABLE]) + + yield _ping_pong_scenario( + 'cpp_protobuf_async_streaming_qps_unconstrained_2waysharedcq_%s' + % secstr, + rpc_type='STREAMING', + client_type='ASYNC_CLIENT', + server_type='ASYNC_SERVER', + unconstrained_client='async', + secure=secure, + client_threads_per_cq=2, + server_threads_per_cq=2, + categories=smoketest_categories + [SCALABLE]) + + #yield _ping_pong_scenario( + # 'cpp_protobuf_async_unary_qps_unconstrained_1cq_%s' % secstr, + # rpc_type='UNARY', + # client_type='ASYNC_CLIENT', + # server_type='ASYNC_SERVER', + # unconstrained_client='async-limited', + # secure=secure, + # client_threads_per_cq=1000000, server_threads_per_cq=1000000, + # categories=smoketest_categories+[SCALABLE]) + + yield _ping_pong_scenario( + 'cpp_protobuf_async_unary_qps_unconstrained_2waysharedcq_%s' % + secstr, + rpc_type='UNARY', + client_type='ASYNC_CLIENT', + server_type='ASYNC_SERVER', + unconstrained_client='async', + secure=secure, + client_threads_per_cq=2, + server_threads_per_cq=2, + categories=smoketest_categories + [SCALABLE]) + + yield _ping_pong_scenario( + 'cpp_generic_async_streaming_qps_one_server_core_%s' % secstr, + rpc_type='STREAMING', + client_type='ASYNC_CLIENT', + server_type='ASYNC_GENERIC_SERVER', + unconstrained_client='async-limited', + use_generic_payload=True, + async_server_threads=1, + minimal_stack=not secure, + secure=secure) + + yield _ping_pong_scenario( + 'cpp_protobuf_async_client_sync_server_unary_qps_unconstrained_%s' + % (secstr), + rpc_type='UNARY', + client_type='ASYNC_CLIENT', + server_type='SYNC_SERVER', + unconstrained_client='async', + secure=secure, + minimal_stack=not secure, + categories=smoketest_categories + [SCALABLE], + excluded_poll_engines=['poll-cv']) + + yield _ping_pong_scenario( + 'cpp_protobuf_async_client_unary_1channel_64wide_128Breq_8MBresp_%s' + % (secstr), + rpc_type='UNARY', + client_type='ASYNC_CLIENT', + server_type='ASYNC_SERVER', + channels=1, + outstanding=64, + req_size=128, + resp_size=8 * 1024 * 1024, + secure=secure, + minimal_stack=not secure, + categories=smoketest_categories + [SCALABLE]) + + yield _ping_pong_scenario( + 'cpp_protobuf_async_client_sync_server_streaming_qps_unconstrained_%s' + % secstr, + rpc_type='STREAMING', + client_type='ASYNC_CLIENT', + server_type='SYNC_SERVER', + unconstrained_client='async', + secure=secure, + minimal_stack=not secure, + categories=smoketest_categories + [SCALABLE], + excluded_poll_engines=['poll-cv']) + + yield _ping_pong_scenario( + 'cpp_protobuf_async_unary_ping_pong_%s_1MB' % secstr, + rpc_type='UNARY', + client_type='ASYNC_CLIENT', + server_type='ASYNC_SERVER', + req_size=1024 * 1024, + resp_size=1024 * 1024, + secure=secure, + minimal_stack=not secure, + categories=smoketest_categories + [SCALABLE]) + + for rpc_type in [ + 'unary', 'streaming', 'streaming_from_client', + 'streaming_from_server' + ]: + for synchronicity in ['sync', 'async']: + yield _ping_pong_scenario( + 'cpp_protobuf_%s_%s_ping_pong_%s' % + (synchronicity, rpc_type, secstr), + rpc_type=rpc_type.upper(), + client_type='%s_CLIENT' % synchronicity.upper(), + server_type='%s_SERVER' % synchronicity.upper(), + async_server_threads=1, + minimal_stack=not secure, + secure=secure) + + for size in geometric_progression(1, 1024 * 1024 * 1024 + 1, + 8): + yield _ping_pong_scenario( + 'cpp_protobuf_%s_%s_qps_unconstrained_%s_%db' % + (synchronicity, rpc_type, secstr, size), + rpc_type=rpc_type.upper(), + req_size=size, + resp_size=size, + client_type='%s_CLIENT' % synchronicity.upper(), + server_type='%s_SERVER' % synchronicity.upper(), + unconstrained_client=synchronicity, + secure=secure, + minimal_stack=not secure, + categories=[SWEEP]) + + yield _ping_pong_scenario( + 'cpp_protobuf_%s_%s_qps_unconstrained_%s' % + (synchronicity, rpc_type, secstr), + rpc_type=rpc_type.upper(), + client_type='%s_CLIENT' % synchronicity.upper(), + server_type='%s_SERVER' % synchronicity.upper(), + unconstrained_client=synchronicity, + secure=secure, + minimal_stack=not secure, + server_threads_per_cq=3, + client_threads_per_cq=3, + categories=smoketest_categories + [SCALABLE]) + + # TODO(vjpai): Re-enable this test. It has a lot of timeouts + # and hasn't yet been conclusively identified as a test failure + # or race in the library + # yield _ping_pong_scenario( + # 'cpp_protobuf_%s_%s_qps_unconstrained_%s_500kib_resource_quota' % (synchronicity, rpc_type, secstr), + # rpc_type=rpc_type.upper(), + # client_type='%s_CLIENT' % synchronicity.upper(), + # server_type='%s_SERVER' % synchronicity.upper(), + # unconstrained_client=synchronicity, + # secure=secure, + # categories=smoketest_categories+[SCALABLE], + # resource_quota_size=500*1024) + + if rpc_type == 'streaming': + for mps in geometric_progression(1, 20, 10): + yield _ping_pong_scenario( + 'cpp_protobuf_%s_%s_qps_unconstrained_%smps_%s' + % (synchronicity, rpc_type, mps, secstr), + rpc_type=rpc_type.upper(), + client_type='%s_CLIENT' % synchronicity.upper(), + server_type='%s_SERVER' % synchronicity.upper(), + unconstrained_client=synchronicity, + secure=secure, + messages_per_stream=mps, + minimal_stack=not secure, + categories=smoketest_categories + [SCALABLE]) + + for mps in geometric_progression(1, 200, math.sqrt(10)): + yield _ping_pong_scenario( + 'cpp_protobuf_%s_%s_qps_unconstrained_%smps_%s' + % (synchronicity, rpc_type, mps, secstr), + rpc_type=rpc_type.upper(), + client_type='%s_CLIENT' % synchronicity.upper(), + server_type='%s_SERVER' % synchronicity.upper(), + unconstrained_client=synchronicity, + secure=secure, + messages_per_stream=mps, + minimal_stack=not secure, + categories=[SWEEP]) + + for channels in geometric_progression(1, 20000, + math.sqrt(10)): + for outstanding in geometric_progression(1, 200000, + math.sqrt(10)): + if synchronicity == 'sync' and outstanding > 1200: + continue + if outstanding < channels: continue + yield _ping_pong_scenario( + 'cpp_protobuf_%s_%s_qps_unconstrained_%s_%d_channels_%d_outstanding' + % (synchronicity, rpc_type, secstr, channels, + outstanding), + rpc_type=rpc_type.upper(), + client_type='%s_CLIENT' % synchronicity.upper(), + server_type='%s_SERVER' % synchronicity.upper(), + unconstrained_client=synchronicity, + secure=secure, + minimal_stack=not secure, + categories=[SWEEP], + channels=channels, + outstanding=outstanding) + + def __str__(self): + return 'c++' class CSharpLanguage: - def __init__(self): - self.safename = str(self) - - def worker_cmdline(self): - return ['tools/run_tests/performance/run_worker_csharp.sh'] - - def worker_port_offset(self): - return 100 - - def scenarios(self): - yield _ping_pong_scenario( - 'csharp_generic_async_streaming_ping_pong', rpc_type='STREAMING', - client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', - use_generic_payload=True, - categories=[SMOKETEST, SCALABLE]) - - yield _ping_pong_scenario( - 'csharp_generic_async_streaming_ping_pong_insecure_1MB', rpc_type='STREAMING', - client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', - req_size=1024*1024, resp_size=1024*1024, - use_generic_payload=True, - secure=False, - categories=[SMOKETEST, SCALABLE]) - - yield _ping_pong_scenario( - 'csharp_generic_async_streaming_qps_unconstrained_insecure', rpc_type='STREAMING', - client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', - unconstrained_client='async', use_generic_payload=True, - secure=False, - categories=[SMOKETEST, SCALABLE]) - - yield _ping_pong_scenario( - 'csharp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING', - client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER') - - yield _ping_pong_scenario( - 'csharp_protobuf_async_unary_ping_pong', rpc_type='UNARY', - client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - categories=[SMOKETEST, SCALABLE]) - - yield _ping_pong_scenario( - 'csharp_protobuf_sync_to_async_unary_ping_pong', rpc_type='UNARY', - client_type='SYNC_CLIENT', server_type='ASYNC_SERVER') - - yield _ping_pong_scenario( - 'csharp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY', - client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - unconstrained_client='async', - categories=[SMOKETEST,SCALABLE]) - - yield _ping_pong_scenario( - 'csharp_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING', - client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - unconstrained_client='async', - categories=[SCALABLE]) - - yield _ping_pong_scenario( - 'csharp_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', - client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - server_language='c++', async_server_threads=1, - categories=[SMOKETEST, SCALABLE]) - - yield _ping_pong_scenario( - 'csharp_to_cpp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING', - client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - server_language='c++', async_server_threads=1) - - yield _ping_pong_scenario( - 'csharp_to_cpp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY', - client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - unconstrained_client='async', server_language='c++', - categories=[SCALABLE]) - - yield _ping_pong_scenario( - 'csharp_to_cpp_protobuf_sync_to_async_unary_qps_unconstrained', rpc_type='UNARY', - client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', - unconstrained_client='sync', server_language='c++', - categories=[SCALABLE]) - - yield _ping_pong_scenario( - 'cpp_to_csharp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY', - client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - unconstrained_client='async', client_language='c++', - categories=[SCALABLE]) - - yield _ping_pong_scenario( - 'csharp_protobuf_async_unary_ping_pong_1MB', rpc_type='UNARY', - client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - req_size=1024*1024, resp_size=1024*1024, - categories=[SMOKETEST, SCALABLE]) - - def __str__(self): - return 'csharp' + def __init__(self): + self.safename = str(self) + + def worker_cmdline(self): + return ['tools/run_tests/performance/run_worker_csharp.sh'] + + def worker_port_offset(self): + return 100 + + def scenarios(self): + yield _ping_pong_scenario( + 'csharp_generic_async_streaming_ping_pong', + rpc_type='STREAMING', + client_type='ASYNC_CLIENT', + server_type='ASYNC_GENERIC_SERVER', + use_generic_payload=True, + categories=[SMOKETEST, SCALABLE]) + + yield _ping_pong_scenario( + 'csharp_generic_async_streaming_ping_pong_insecure_1MB', + rpc_type='STREAMING', + client_type='ASYNC_CLIENT', + server_type='ASYNC_GENERIC_SERVER', + req_size=1024 * 1024, + resp_size=1024 * 1024, + use_generic_payload=True, + secure=False, + categories=[SMOKETEST, SCALABLE]) + + yield _ping_pong_scenario( + 'csharp_generic_async_streaming_qps_unconstrained_insecure', + rpc_type='STREAMING', + client_type='ASYNC_CLIENT', + server_type='ASYNC_GENERIC_SERVER', + unconstrained_client='async', + use_generic_payload=True, + secure=False, + categories=[SMOKETEST, SCALABLE]) + + yield _ping_pong_scenario( + 'csharp_protobuf_async_streaming_ping_pong', + rpc_type='STREAMING', + client_type='ASYNC_CLIENT', + server_type='ASYNC_SERVER') + + yield _ping_pong_scenario( + 'csharp_protobuf_async_unary_ping_pong', + rpc_type='UNARY', + client_type='ASYNC_CLIENT', + server_type='ASYNC_SERVER', + categories=[SMOKETEST, SCALABLE]) + + yield _ping_pong_scenario( + 'csharp_protobuf_sync_to_async_unary_ping_pong', + rpc_type='UNARY', + client_type='SYNC_CLIENT', + server_type='ASYNC_SERVER') + + yield _ping_pong_scenario( + 'csharp_protobuf_async_unary_qps_unconstrained', + rpc_type='UNARY', + client_type='ASYNC_CLIENT', + server_type='ASYNC_SERVER', + unconstrained_client='async', + categories=[SMOKETEST, SCALABLE]) + + yield _ping_pong_scenario( + 'csharp_protobuf_async_streaming_qps_unconstrained', + rpc_type='STREAMING', + client_type='ASYNC_CLIENT', + server_type='ASYNC_SERVER', + unconstrained_client='async', + categories=[SCALABLE]) + + yield _ping_pong_scenario( + 'csharp_to_cpp_protobuf_sync_unary_ping_pong', + rpc_type='UNARY', + client_type='SYNC_CLIENT', + server_type='SYNC_SERVER', + server_language='c++', + async_server_threads=1, + categories=[SMOKETEST, SCALABLE]) + + yield _ping_pong_scenario( + 'csharp_to_cpp_protobuf_async_streaming_ping_pong', + rpc_type='STREAMING', + client_type='ASYNC_CLIENT', + server_type='ASYNC_SERVER', + server_language='c++', + async_server_threads=1) + + yield _ping_pong_scenario( + 'csharp_to_cpp_protobuf_async_unary_qps_unconstrained', + rpc_type='UNARY', + client_type='ASYNC_CLIENT', + server_type='ASYNC_SERVER', + unconstrained_client='async', + server_language='c++', + categories=[SCALABLE]) + + yield _ping_pong_scenario( + 'csharp_to_cpp_protobuf_sync_to_async_unary_qps_unconstrained', + rpc_type='UNARY', + client_type='SYNC_CLIENT', + server_type='ASYNC_SERVER', + unconstrained_client='sync', + server_language='c++', + categories=[SCALABLE]) + + yield _ping_pong_scenario( + 'cpp_to_csharp_protobuf_async_unary_qps_unconstrained', + rpc_type='UNARY', + client_type='ASYNC_CLIENT', + server_type='ASYNC_SERVER', + unconstrained_client='async', + client_language='c++', + categories=[SCALABLE]) + + yield _ping_pong_scenario( + 'csharp_protobuf_async_unary_ping_pong_1MB', + rpc_type='UNARY', + client_type='ASYNC_CLIENT', + server_type='ASYNC_SERVER', + req_size=1024 * 1024, + resp_size=1024 * 1024, + categories=[SMOKETEST, SCALABLE]) + + def __str__(self): + return 'csharp' + class PythonLanguage: - def __init__(self): - self.safename = 'python' - - def worker_cmdline(self): - return ['tools/run_tests/performance/run_worker_python.sh'] - - def worker_port_offset(self): - return 500 - - def scenarios(self): - yield _ping_pong_scenario( - 'python_generic_sync_streaming_ping_pong', rpc_type='STREAMING', - client_type='SYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', - use_generic_payload=True, - categories=[SMOKETEST, SCALABLE]) - - yield _ping_pong_scenario( - 'python_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', - client_type='SYNC_CLIENT', server_type='ASYNC_SERVER') - - yield _ping_pong_scenario( - 'python_protobuf_async_unary_ping_pong', rpc_type='UNARY', - client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER') - - yield _ping_pong_scenario( - 'python_protobuf_sync_unary_ping_pong', rpc_type='UNARY', - client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', - categories=[SMOKETEST, SCALABLE]) - - yield _ping_pong_scenario( - 'python_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY', - client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', - unconstrained_client='sync') - - yield _ping_pong_scenario( - 'python_protobuf_sync_streaming_qps_unconstrained', rpc_type='STREAMING', - client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', - unconstrained_client='sync') - - yield _ping_pong_scenario( - 'python_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', - client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', - server_language='c++', async_server_threads=1, - categories=[SMOKETEST, SCALABLE]) - - yield _ping_pong_scenario( - 'python_to_cpp_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', - client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', - server_language='c++', async_server_threads=1) - - yield _ping_pong_scenario( - 'python_protobuf_sync_unary_ping_pong_1MB', rpc_type='UNARY', - client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', - req_size=1024*1024, resp_size=1024*1024, - categories=[SMOKETEST, SCALABLE]) - - def __str__(self): - return 'python' + def __init__(self): + self.safename = 'python' + + def worker_cmdline(self): + return ['tools/run_tests/performance/run_worker_python.sh'] + + def worker_port_offset(self): + return 500 + + def scenarios(self): + yield _ping_pong_scenario( + 'python_generic_sync_streaming_ping_pong', + rpc_type='STREAMING', + client_type='SYNC_CLIENT', + server_type='ASYNC_GENERIC_SERVER', + use_generic_payload=True, + categories=[SMOKETEST, SCALABLE]) + + yield _ping_pong_scenario( + 'python_protobuf_sync_streaming_ping_pong', + rpc_type='STREAMING', + client_type='SYNC_CLIENT', + server_type='ASYNC_SERVER') + + yield _ping_pong_scenario( + 'python_protobuf_async_unary_ping_pong', + rpc_type='UNARY', + client_type='ASYNC_CLIENT', + server_type='ASYNC_SERVER') + + yield _ping_pong_scenario( + 'python_protobuf_sync_unary_ping_pong', + rpc_type='UNARY', + client_type='SYNC_CLIENT', + server_type='ASYNC_SERVER', + categories=[SMOKETEST, SCALABLE]) + + yield _ping_pong_scenario( + 'python_protobuf_sync_unary_qps_unconstrained', + rpc_type='UNARY', + client_type='SYNC_CLIENT', + server_type='ASYNC_SERVER', + unconstrained_client='sync') + + yield _ping_pong_scenario( + 'python_protobuf_sync_streaming_qps_unconstrained', + rpc_type='STREAMING', + client_type='SYNC_CLIENT', + server_type='ASYNC_SERVER', + unconstrained_client='sync') + + yield _ping_pong_scenario( + 'python_to_cpp_protobuf_sync_unary_ping_pong', + rpc_type='UNARY', + client_type='SYNC_CLIENT', + server_type='ASYNC_SERVER', + server_language='c++', + async_server_threads=1, + categories=[SMOKETEST, SCALABLE]) + + yield _ping_pong_scenario( + 'python_to_cpp_protobuf_sync_streaming_ping_pong', + rpc_type='STREAMING', + client_type='SYNC_CLIENT', + server_type='ASYNC_SERVER', + server_language='c++', + async_server_threads=1) + + yield _ping_pong_scenario( + 'python_protobuf_sync_unary_ping_pong_1MB', + rpc_type='UNARY', + client_type='SYNC_CLIENT', + server_type='ASYNC_SERVER', + req_size=1024 * 1024, + resp_size=1024 * 1024, + categories=[SMOKETEST, SCALABLE]) + + def __str__(self): + return 'python' + class RubyLanguage: - def __init__(self): - pass - self.safename = str(self) + def __init__(self): + pass + self.safename = str(self) - def worker_cmdline(self): - return ['tools/run_tests/performance/run_worker_ruby.sh'] + def worker_cmdline(self): + return ['tools/run_tests/performance/run_worker_ruby.sh'] - def worker_port_offset(self): - return 300 + def worker_port_offset(self): + return 300 - def scenarios(self): - yield _ping_pong_scenario( - 'ruby_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', - client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - categories=[SMOKETEST, SCALABLE]) + def scenarios(self): + yield _ping_pong_scenario( + 'ruby_protobuf_sync_streaming_ping_pong', + rpc_type='STREAMING', + client_type='SYNC_CLIENT', + server_type='SYNC_SERVER', + categories=[SMOKETEST, SCALABLE]) - yield _ping_pong_scenario( - 'ruby_protobuf_unary_ping_pong', rpc_type='UNARY', - client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - categories=[SMOKETEST, SCALABLE]) + yield _ping_pong_scenario( + 'ruby_protobuf_unary_ping_pong', + rpc_type='UNARY', + client_type='SYNC_CLIENT', + server_type='SYNC_SERVER', + categories=[SMOKETEST, SCALABLE]) - yield _ping_pong_scenario( - 'ruby_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY', - client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - unconstrained_client='sync') + yield _ping_pong_scenario( + 'ruby_protobuf_sync_unary_qps_unconstrained', + rpc_type='UNARY', + client_type='SYNC_CLIENT', + server_type='SYNC_SERVER', + unconstrained_client='sync') - yield _ping_pong_scenario( - 'ruby_protobuf_sync_streaming_qps_unconstrained', rpc_type='STREAMING', - client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - unconstrained_client='sync') + yield _ping_pong_scenario( + 'ruby_protobuf_sync_streaming_qps_unconstrained', + rpc_type='STREAMING', + client_type='SYNC_CLIENT', + server_type='SYNC_SERVER', + unconstrained_client='sync') - yield _ping_pong_scenario( - 'ruby_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY', - client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - server_language='c++', async_server_threads=1) + yield _ping_pong_scenario( + 'ruby_to_cpp_protobuf_sync_unary_ping_pong', + rpc_type='UNARY', + client_type='SYNC_CLIENT', + server_type='SYNC_SERVER', + server_language='c++', + async_server_threads=1) - yield _ping_pong_scenario( - 'ruby_to_cpp_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING', - client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - server_language='c++', async_server_threads=1) + yield _ping_pong_scenario( + 'ruby_to_cpp_protobuf_sync_streaming_ping_pong', + rpc_type='STREAMING', + client_type='SYNC_CLIENT', + server_type='SYNC_SERVER', + server_language='c++', + async_server_threads=1) - yield _ping_pong_scenario( - 'ruby_protobuf_unary_ping_pong_1MB', rpc_type='UNARY', - client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - req_size=1024*1024, resp_size=1024*1024, - categories=[SMOKETEST, SCALABLE]) + yield _ping_pong_scenario( + 'ruby_protobuf_unary_ping_pong_1MB', + rpc_type='UNARY', + client_type='SYNC_CLIENT', + server_type='SYNC_SERVER', + req_size=1024 * 1024, + resp_size=1024 * 1024, + categories=[SMOKETEST, SCALABLE]) - def __str__(self): - return 'ruby' + def __str__(self): + return 'ruby' class Php7Language: - def __init__(self, php7_protobuf_c=False): - pass - self.php7_protobuf_c=php7_protobuf_c - self.safename = str(self) - - def worker_cmdline(self): - if self.php7_protobuf_c: - return ['tools/run_tests/performance/run_worker_php.sh', '--use_protobuf_c_extension'] - return ['tools/run_tests/performance/run_worker_php.sh'] - - def worker_port_offset(self): - if self.php7_protobuf_c: - return 900 - return 800 - - def scenarios(self): - php7_extension_mode='php7_protobuf_php_extension' - if self.php7_protobuf_c: - php7_extension_mode='php7_protobuf_c_extension' - - yield _ping_pong_scenario( - '%s_to_cpp_protobuf_sync_unary_ping_pong' % php7_extension_mode, - rpc_type='UNARY', client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - server_language='c++', async_server_threads=1) - - yield _ping_pong_scenario( - '%s_to_cpp_protobuf_sync_streaming_ping_pong' % php7_extension_mode, - rpc_type='STREAMING', client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - server_language='c++', async_server_threads=1) - - # TODO(ddyihai): Investigate why when async_server_threads=1/CPU usage 340%, the QPS performs - # better than async_server_threads=0/CPU usage 490%. - yield _ping_pong_scenario( - '%s_to_cpp_protobuf_sync_unary_qps_unconstrained' % php7_extension_mode, - rpc_type='UNARY', client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', - server_language='c++', outstanding=1, async_server_threads=1, unconstrained_client='sync') - - yield _ping_pong_scenario( - '%s_to_cpp_protobuf_sync_streaming_qps_unconstrained' % php7_extension_mode, - rpc_type='STREAMING', client_type='SYNC_CLIENT', server_type='ASYNC_SERVER', - server_language='c++', outstanding=1, async_server_threads=1, unconstrained_client='sync') - - def __str__(self): - if self.php7_protobuf_c: - return 'php7_protobuf_c' - return 'php7' + def __init__(self, php7_protobuf_c=False): + pass + self.php7_protobuf_c = php7_protobuf_c + self.safename = str(self) + + def worker_cmdline(self): + if self.php7_protobuf_c: + return [ + 'tools/run_tests/performance/run_worker_php.sh', + '--use_protobuf_c_extension' + ] + return ['tools/run_tests/performance/run_worker_php.sh'] + + def worker_port_offset(self): + if self.php7_protobuf_c: + return 900 + return 800 + + def scenarios(self): + php7_extension_mode = 'php7_protobuf_php_extension' + if self.php7_protobuf_c: + php7_extension_mode = 'php7_protobuf_c_extension' + + yield _ping_pong_scenario( + '%s_to_cpp_protobuf_sync_unary_ping_pong' % php7_extension_mode, + rpc_type='UNARY', + client_type='SYNC_CLIENT', + server_type='SYNC_SERVER', + server_language='c++', + async_server_threads=1) + + yield _ping_pong_scenario( + '%s_to_cpp_protobuf_sync_streaming_ping_pong' % php7_extension_mode, + rpc_type='STREAMING', + client_type='SYNC_CLIENT', + server_type='SYNC_SERVER', + server_language='c++', + async_server_threads=1) + + # TODO(ddyihai): Investigate why when async_server_threads=1/CPU usage 340%, the QPS performs + # better than async_server_threads=0/CPU usage 490%. + yield _ping_pong_scenario( + '%s_to_cpp_protobuf_sync_unary_qps_unconstrained' % + php7_extension_mode, + rpc_type='UNARY', + client_type='SYNC_CLIENT', + server_type='ASYNC_SERVER', + server_language='c++', + outstanding=1, + async_server_threads=1, + unconstrained_client='sync') + + yield _ping_pong_scenario( + '%s_to_cpp_protobuf_sync_streaming_qps_unconstrained' % + php7_extension_mode, + rpc_type='STREAMING', + client_type='SYNC_CLIENT', + server_type='ASYNC_SERVER', + server_language='c++', + outstanding=1, + async_server_threads=1, + unconstrained_client='sync') + + def __str__(self): + if self.php7_protobuf_c: + return 'php7_protobuf_c' + return 'php7' + class JavaLanguage: - def __init__(self): - pass - self.safename = str(self) - - def worker_cmdline(self): - return ['tools/run_tests/performance/run_worker_java.sh'] - - def worker_port_offset(self): - return 400 - - def scenarios(self): - for secure in [True, False]: - secstr = 'secure' if secure else 'insecure' - smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] - - yield _ping_pong_scenario( - 'java_generic_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING', - client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', - use_generic_payload=True, async_server_threads=1, - secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS, - categories=smoketest_categories) - - yield _ping_pong_scenario( - 'java_protobuf_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING', - client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - async_server_threads=1, - secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS) - - yield _ping_pong_scenario( - 'java_protobuf_async_unary_ping_pong_%s' % secstr, rpc_type='UNARY', - client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - async_server_threads=1, - secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS, - categories=smoketest_categories) - - yield _ping_pong_scenario( - 'java_protobuf_unary_ping_pong_%s' % secstr, rpc_type='UNARY', - client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - async_server_threads=1, - secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS) - - yield _ping_pong_scenario( - 'java_protobuf_async_unary_qps_unconstrained_%s' % secstr, rpc_type='UNARY', - client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - unconstrained_client='async', - secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS, - categories=smoketest_categories+[SCALABLE]) - - yield _ping_pong_scenario( - 'java_protobuf_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING', - client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER', - unconstrained_client='async', - secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS, - categories=[SCALABLE]) - - yield _ping_pong_scenario( - 'java_generic_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING', - client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', - unconstrained_client='async', use_generic_payload=True, - secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS, - categories=[SCALABLE]) - - yield _ping_pong_scenario( - 'java_generic_async_streaming_qps_one_server_core_%s' % secstr, rpc_type='STREAMING', - client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', - unconstrained_client='async-limited', use_generic_payload=True, - async_server_threads=1, - secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS) - - # TODO(jtattermusch): add scenarios java vs C++ - - def __str__(self): - return 'java' + def __init__(self): + pass + self.safename = str(self) + + def worker_cmdline(self): + return ['tools/run_tests/performance/run_worker_java.sh'] + + def worker_port_offset(self): + return 400 + + def scenarios(self): + for secure in [True, False]: + secstr = 'secure' if secure else 'insecure' + smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] + + yield _ping_pong_scenario( + 'java_generic_async_streaming_ping_pong_%s' % secstr, + rpc_type='STREAMING', + client_type='ASYNC_CLIENT', + server_type='ASYNC_GENERIC_SERVER', + use_generic_payload=True, + async_server_threads=1, + secure=secure, + warmup_seconds=JAVA_WARMUP_SECONDS, + categories=smoketest_categories) + + yield _ping_pong_scenario( + 'java_protobuf_async_streaming_ping_pong_%s' % secstr, + rpc_type='STREAMING', + client_type='ASYNC_CLIENT', + server_type='ASYNC_SERVER', + async_server_threads=1, + secure=secure, + warmup_seconds=JAVA_WARMUP_SECONDS) + + yield _ping_pong_scenario( + 'java_protobuf_async_unary_ping_pong_%s' % secstr, + rpc_type='UNARY', + client_type='ASYNC_CLIENT', + server_type='ASYNC_SERVER', + async_server_threads=1, + secure=secure, + warmup_seconds=JAVA_WARMUP_SECONDS, + categories=smoketest_categories) + + yield _ping_pong_scenario( + 'java_protobuf_unary_ping_pong_%s' % secstr, + rpc_type='UNARY', + client_type='SYNC_CLIENT', + server_type='SYNC_SERVER', + async_server_threads=1, + secure=secure, + warmup_seconds=JAVA_WARMUP_SECONDS) + + yield _ping_pong_scenario( + 'java_protobuf_async_unary_qps_unconstrained_%s' % secstr, + rpc_type='UNARY', + client_type='ASYNC_CLIENT', + server_type='ASYNC_SERVER', + unconstrained_client='async', + secure=secure, + warmup_seconds=JAVA_WARMUP_SECONDS, + categories=smoketest_categories + [SCALABLE]) + + yield _ping_pong_scenario( + 'java_protobuf_async_streaming_qps_unconstrained_%s' % secstr, + rpc_type='STREAMING', + client_type='ASYNC_CLIENT', + server_type='ASYNC_SERVER', + unconstrained_client='async', + secure=secure, + warmup_seconds=JAVA_WARMUP_SECONDS, + categories=[SCALABLE]) + + yield _ping_pong_scenario( + 'java_generic_async_streaming_qps_unconstrained_%s' % secstr, + rpc_type='STREAMING', + client_type='ASYNC_CLIENT', + server_type='ASYNC_GENERIC_SERVER', + unconstrained_client='async', + use_generic_payload=True, + secure=secure, + warmup_seconds=JAVA_WARMUP_SECONDS, + categories=[SCALABLE]) + + yield _ping_pong_scenario( + 'java_generic_async_streaming_qps_one_server_core_%s' % secstr, + rpc_type='STREAMING', + client_type='ASYNC_CLIENT', + server_type='ASYNC_GENERIC_SERVER', + unconstrained_client='async-limited', + use_generic_payload=True, + async_server_threads=1, + secure=secure, + warmup_seconds=JAVA_WARMUP_SECONDS) + + # TODO(jtattermusch): add scenarios java vs C++ + + def __str__(self): + return 'java' class GoLanguage: - def __init__(self): - pass - self.safename = str(self) - - def worker_cmdline(self): - return ['tools/run_tests/performance/run_worker_go.sh'] - - def worker_port_offset(self): - return 600 - - def scenarios(self): - for secure in [True, False]: - secstr = 'secure' if secure else 'insecure' - smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] - - # ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server, - # but that's mostly because of lack of better name of the enum value. - yield _ping_pong_scenario( - 'go_generic_sync_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING', - client_type='SYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', - use_generic_payload=True, async_server_threads=1, - secure=secure, - categories=smoketest_categories) - - yield _ping_pong_scenario( - 'go_protobuf_sync_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING', - client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - async_server_threads=1, - secure=secure) - - yield _ping_pong_scenario( - 'go_protobuf_sync_unary_ping_pong_%s' % secstr, rpc_type='UNARY', - client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - async_server_threads=1, - secure=secure, - categories=smoketest_categories) - - # unconstrained_client='async' is intended (client uses goroutines) - yield _ping_pong_scenario( - 'go_protobuf_sync_unary_qps_unconstrained_%s' % secstr, rpc_type='UNARY', - client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - unconstrained_client='async', - secure=secure, - categories=smoketest_categories+[SCALABLE]) - - # unconstrained_client='async' is intended (client uses goroutines) - yield _ping_pong_scenario( - 'go_protobuf_sync_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING', - client_type='SYNC_CLIENT', server_type='SYNC_SERVER', - unconstrained_client='async', - secure=secure, - categories=[SCALABLE]) - - # unconstrained_client='async' is intended (client uses goroutines) - # ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server, - # but that's mostly because of lack of better name of the enum value. - yield _ping_pong_scenario( - 'go_generic_sync_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING', - client_type='SYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER', - unconstrained_client='async', use_generic_payload=True, - secure=secure, - categories=[SCALABLE]) - - # TODO(jtattermusch): add scenarios go vs C++ - - def __str__(self): - return 'go' + def __init__(self): + pass + self.safename = str(self) + + def worker_cmdline(self): + return ['tools/run_tests/performance/run_worker_go.sh'] + + def worker_port_offset(self): + return 600 + + def scenarios(self): + for secure in [True, False]: + secstr = 'secure' if secure else 'insecure' + smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] + + # ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server, + # but that's mostly because of lack of better name of the enum value. + yield _ping_pong_scenario( + 'go_generic_sync_streaming_ping_pong_%s' % secstr, + rpc_type='STREAMING', + client_type='SYNC_CLIENT', + server_type='ASYNC_GENERIC_SERVER', + use_generic_payload=True, + async_server_threads=1, + secure=secure, + categories=smoketest_categories) + + yield _ping_pong_scenario( + 'go_protobuf_sync_streaming_ping_pong_%s' % secstr, + rpc_type='STREAMING', + client_type='SYNC_CLIENT', + server_type='SYNC_SERVER', + async_server_threads=1, + secure=secure) + + yield _ping_pong_scenario( + 'go_protobuf_sync_unary_ping_pong_%s' % secstr, + rpc_type='UNARY', + client_type='SYNC_CLIENT', + server_type='SYNC_SERVER', + async_server_threads=1, + secure=secure, + categories=smoketest_categories) + + # unconstrained_client='async' is intended (client uses goroutines) + yield _ping_pong_scenario( + 'go_protobuf_sync_unary_qps_unconstrained_%s' % secstr, + rpc_type='UNARY', + client_type='SYNC_CLIENT', + server_type='SYNC_SERVER', + unconstrained_client='async', + secure=secure, + categories=smoketest_categories + [SCALABLE]) + + # unconstrained_client='async' is intended (client uses goroutines) + yield _ping_pong_scenario( + 'go_protobuf_sync_streaming_qps_unconstrained_%s' % secstr, + rpc_type='STREAMING', + client_type='SYNC_CLIENT', + server_type='SYNC_SERVER', + unconstrained_client='async', + secure=secure, + categories=[SCALABLE]) + + # unconstrained_client='async' is intended (client uses goroutines) + # ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server, + # but that's mostly because of lack of better name of the enum value. + yield _ping_pong_scenario( + 'go_generic_sync_streaming_qps_unconstrained_%s' % secstr, + rpc_type='STREAMING', + client_type='SYNC_CLIENT', + server_type='ASYNC_GENERIC_SERVER', + unconstrained_client='async', + use_generic_payload=True, + secure=secure, + categories=[SCALABLE]) + + # TODO(jtattermusch): add scenarios go vs C++ + + def __str__(self): + return 'go' LANGUAGES = { - 'c++' : CXXLanguage(), - 'csharp' : CSharpLanguage(), - 'ruby' : RubyLanguage(), - 'php7' : Php7Language(), - 'php7_protobuf_c' : Php7Language(php7_protobuf_c=True), - 'java' : JavaLanguage(), - 'python' : PythonLanguage(), - 'go' : GoLanguage(), + 'c++': CXXLanguage(), + 'csharp': CSharpLanguage(), + 'ruby': RubyLanguage(), + 'php7': Php7Language(), + 'php7_protobuf_c': Php7Language(php7_protobuf_c=True), + 'java': JavaLanguage(), + 'python': PythonLanguage(), + 'go': GoLanguage(), } |