diff options
author | Nicolas "Pixel" Noble <pixel@nobis-crew.org> | 2016-04-28 00:01:16 +0200 |
---|---|---|
committer | Nicolas "Pixel" Noble <pixel@nobis-crew.org> | 2016-04-28 00:01:16 +0200 |
commit | acbd8675a283a51a409991ac1393294c9af726e7 (patch) | |
tree | 01fa0ca4b463fd84edb3d3fbe2751ad76229b9d9 /tools/run_tests/performance/scenario_config.py | |
parent | 23df75a97054822e1a7f89ab5a0601d6bd4b3aa6 (diff) | |
parent | cec42984a0374465de9b2626f16b0efa960a66d0 (diff) |
Merge branch 'master' of https://github.com/grpc/grpc into what-the-fuzz
Diffstat (limited to 'tools/run_tests/performance/scenario_config.py')
-rw-r--r-- | tools/run_tests/performance/scenario_config.py | 565 |
1 files changed, 565 insertions, 0 deletions
diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py new file mode 100644 index 0000000000..55657f8d8a --- /dev/null +++ b/tools/run_tests/performance/scenario_config.py @@ -0,0 +1,565 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# performance scenario configuration for various languages + +SINGLE_MACHINE_CORES=8 +WARMUP_SECONDS=5 +JAVA_WARMUP_SECONDS=15 # Java needs more warmup time for JIT to kick in. +BENCHMARK_SECONDS=30 + +SECURE_SECARGS = {'use_test_ca': True, + 'server_host_override': 'foo.test.google.fr'} + +HISTOGRAM_PARAMS = { + 'resolution': 0.01, + 'max_possible': 60e9, +} + +EMPTY_GENERIC_PAYLOAD = { + 'bytebuf_params': { + 'req_size': 0, + 'resp_size': 0, + } +} +EMPTY_PROTO_PAYLOAD = { + 'simple_params': { + 'req_size': 0, + 'resp_size': 0, + } +} +BIG_GENERIC_PAYLOAD = { + 'bytebuf_params': { + 'req_size': 65536, + 'resp_size': 65536, + } +} + +# deep is the number of RPCs outstanding on a channel in non-ping-pong tests +# (the value used is 1 otherwise) +DEEP=100 + +# wide is the number of client channels in multi-channel tests (1 otherwise) +WIDE=64 + + +class CXXLanguage: + + def __init__(self): + self.safename = 'cxx' + + def worker_cmdline(self): + return ['bins/opt/qps_worker'] + + def worker_port_offset(self): + return 0 + + def scenarios(self): + # TODO(ctiller): add 70% load latency test + for secure in [True, False]: + if secure: + secstr = 'secure' + secargs = SECURE_SECARGS + else: + secstr = 'insecure' + secargs = None + + yield { + 'name': 'cpp_generic_async_streaming_ping_pong_%s' + % secstr, + 'num_servers': 1, + 'num_clients': 1, + 'client_config': { + 'client_type': 'ASYNC_CLIENT', + 'security_params': secargs, + 'outstanding_rpcs_per_channel': 1, + 'client_channels': 1, + 'async_client_threads': 1, + 'rpc_type': 'STREAMING', + 'load_params': { + 'closed_loop': {} + }, + 'payload_config': EMPTY_GENERIC_PAYLOAD, + 'histogram_params': HISTOGRAM_PARAMS, + }, + 'server_config': { + 'server_type': 'ASYNC_GENERIC_SERVER', + 'security_params': secargs, + 'core_limit': 1, + 'async_server_threads': 1, + 'payload_config': EMPTY_GENERIC_PAYLOAD, + }, + 'warmup_seconds': WARMUP_SECONDS, + 'benchmark_seconds': BENCHMARK_SECONDS + } + yield { + 'name': 'cpp_generic_async_streaming_qps_unconstrained_%s' + % secstr, + 'num_servers': 1, + 'num_clients': 0, + 'client_config': { + 'client_type': 'ASYNC_CLIENT', + 'security_params': secargs, + 'outstanding_rpcs_per_channel': DEEP, + 'client_channels': WIDE, + 'async_client_threads': 0, + 'rpc_type': 'STREAMING', + 'load_params': { + 'closed_loop': {} + }, + 'payload_config': EMPTY_GENERIC_PAYLOAD, + 'histogram_params': HISTOGRAM_PARAMS, + }, + 'server_config': { + 'server_type': 'ASYNC_GENERIC_SERVER', + 'security_params': secargs, + 'core_limit': SINGLE_MACHINE_CORES/2, + 'async_server_threads': 0, + 'payload_config': EMPTY_GENERIC_PAYLOAD, + }, + 'warmup_seconds': WARMUP_SECONDS, + 'benchmark_seconds': BENCHMARK_SECONDS + } + yield { + 'name': 'cpp_generic_async_streaming_qps_one_server_core_%s' + % secstr, + 'num_servers': 1, + 'num_clients': 0, + 'client_config': { + 'client_type': 'ASYNC_CLIENT', + 'security_params': secargs, + 'outstanding_rpcs_per_channel': DEEP, + 'client_channels': WIDE, + 'async_client_threads': 0, + 'rpc_type': 'STREAMING', + 'load_params': { + 'closed_loop': {} + }, + 'payload_config': EMPTY_GENERIC_PAYLOAD, + 'histogram_params': HISTOGRAM_PARAMS, + }, + 'server_config': { + 'server_type': 'ASYNC_GENERIC_SERVER', + 'security_params': secargs, + 'core_limit': 1, + 'async_server_threads': 1, + 'payload_config': EMPTY_GENERIC_PAYLOAD, + }, + 'warmup_seconds': WARMUP_SECONDS, + 'benchmark_seconds': BENCHMARK_SECONDS + } + yield { + 'name': 'cpp_protobuf_async_streaming_qps_unconstrained_%s' + % secstr, + 'num_servers': 1, + 'num_clients': 0, + 'client_config': { + 'client_type': 'ASYNC_CLIENT', + 'security_params': secargs, + 'outstanding_rpcs_per_channel': DEEP, + 'client_channels': WIDE, + 'async_client_threads': 0, + 'rpc_type': 'STREAMING', + 'load_params': { + 'closed_loop': {} + }, + 'payload_config': EMPTY_PROTO_PAYLOAD, + 'histogram_params': HISTOGRAM_PARAMS, + }, + 'server_config': { + 'server_type': 'ASYNC_SERVER', + 'security_params': secargs, + 'core_limit': SINGLE_MACHINE_CORES/2, + 'async_server_threads': 0, + }, + 'warmup_seconds': WARMUP_SECONDS, + 'benchmark_seconds': BENCHMARK_SECONDS + } + yield { + 'name': 'cpp_protobuf_async_streaming_ping_pong_%s' + % secstr, + 'num_servers': 1, + 'num_clients': 1, + 'client_config': { + 'client_type': 'ASYNC_CLIENT', + 'security_params': secargs, + 'outstanding_rpcs_per_channel': 1, + 'client_channels': 1, + 'async_client_threads': 1, + 'rpc_type': 'STREAMING', + 'load_params': { + 'closed_loop': {} + }, + 'payload_config': EMPTY_PROTO_PAYLOAD, + 'histogram_params': HISTOGRAM_PARAMS, + }, + 'server_config': { + 'server_type': 'ASYNC_SERVER', + 'security_params': secargs, + 'core_limit': 1, + 'async_server_threads': 1, + }, + 'warmup_seconds': WARMUP_SECONDS, + 'benchmark_seconds': BENCHMARK_SECONDS + } + yield { + 'name': 'cpp_protobuf_sync_unary_ping_pong_%s' + % secstr, + 'num_servers': 1, + 'num_clients': 1, + 'client_config': { + 'client_type': 'SYNC_CLIENT', + 'security_params': secargs, + 'outstanding_rpcs_per_channel': 1, + 'client_channels': 1, + 'async_client_threads': 0, + 'rpc_type': 'UNARY', + 'load_params': { + 'closed_loop': {} + }, + 'payload_config': EMPTY_PROTO_PAYLOAD, + 'histogram_params': HISTOGRAM_PARAMS, + }, + 'server_config': { + 'server_type': 'SYNC_SERVER', + 'security_params': secargs, + 'core_limit': 1, + 'async_server_threads': 0, + }, + 'warmup_seconds': WARMUP_SECONDS, + 'benchmark_seconds': BENCHMARK_SECONDS + } + yield { + 'name': 'cpp_protobuf_async_unary_ping_pong_%s' + % secstr, + 'num_servers': 1, + 'num_clients': 1, + 'client_config': { + 'client_type': 'ASYNC_CLIENT', + 'security_params': secargs, + 'outstanding_rpcs_per_channel': 1, + 'client_channels': 1, + 'async_client_threads': 1, + 'rpc_type': 'UNARY', + 'load_params': { + 'closed_loop': {} + }, + 'payload_config': EMPTY_PROTO_PAYLOAD, + 'histogram_params': HISTOGRAM_PARAMS, + }, + 'server_config': { + 'server_type': 'ASYNC_SERVER', + 'security_params': secargs, + 'core_limit': 1, + 'async_server_threads': 1, + }, + 'warmup_seconds': WARMUP_SECONDS, + 'benchmark_seconds': BENCHMARK_SECONDS + } + + def __str__(self): + return 'c++' + + +class CSharpLanguage: + + def __init__(self): + self.safename = str(self) + + def worker_cmdline(self): + return ['tools/run_tests/performance/run_worker_csharp.sh'] + + def worker_port_offset(self): + return 100 + + def scenarios(self): + secargs = SECURE_SECARGS + yield { + 'name': 'csharp_generic_async_streaming_ping_pong', + 'num_servers': 1, + 'num_clients': 1, + 'client_config': { + 'client_type': 'ASYNC_CLIENT', + 'security_params': secargs, + 'outstanding_rpcs_per_channel': 1, + 'client_channels': 1, + 'async_client_threads': 1, + 'rpc_type': 'STREAMING', + 'load_params': { + 'closed_loop': {} + }, + 'payload_config': EMPTY_GENERIC_PAYLOAD, + 'histogram_params': HISTOGRAM_PARAMS, + }, + 'server_config': { + 'server_type': 'ASYNC_GENERIC_SERVER', + 'security_params': secargs, + 'core_limit': 0, + 'async_server_threads': 0, + 'payload_config': EMPTY_GENERIC_PAYLOAD, + }, + 'warmup_seconds': WARMUP_SECONDS, + 'benchmark_seconds': BENCHMARK_SECONDS + } + yield { + 'name': 'csharp_protobuf_async_unary_ping_pong', + 'num_servers': 1, + 'num_clients': 1, + 'client_config': { + 'client_type': 'ASYNC_CLIENT', + 'security_params': secargs, + 'outstanding_rpcs_per_channel': 1, + 'client_channels': 1, + 'async_client_threads': 1, + 'rpc_type': 'UNARY', + 'load_params': { + 'closed_loop': {} + }, + 'payload_config': EMPTY_PROTO_PAYLOAD, + 'histogram_params': HISTOGRAM_PARAMS, + }, + 'server_config': { + 'server_type': 'ASYNC_SERVER', + 'security_params': secargs, + 'core_limit': 0, + 'async_server_threads': 0, + }, + 'warmup_seconds': WARMUP_SECONDS, + 'benchmark_seconds': BENCHMARK_SECONDS + } + yield { + 'name': 'csharp_protobuf_sync_to_async_unary_ping_pong', + 'num_servers': 1, + 'num_clients': 1, + 'client_config': { + 'client_type': 'SYNC_CLIENT', + 'security_params': secargs, + 'outstanding_rpcs_per_channel': 1, + 'client_channels': 1, + 'async_client_threads': 1, + 'rpc_type': 'UNARY', + 'load_params': { + 'closed_loop': {} + }, + 'payload_config': EMPTY_PROTO_PAYLOAD, + 'histogram_params': HISTOGRAM_PARAMS, + }, + 'server_config': { + 'server_type': 'ASYNC_SERVER', + 'security_params': secargs, + 'core_limit': 0, + 'async_server_threads': 0, + }, + 'warmup_seconds': WARMUP_SECONDS, + 'benchmark_seconds': BENCHMARK_SECONDS + } + yield { + 'name': 'csharp_to_cpp_protobuf_sync_unary_ping_pong', + 'num_servers': 1, + 'num_clients': 1, + 'client_config': { + 'client_type': 'SYNC_CLIENT', + 'security_params': secargs, + 'outstanding_rpcs_per_channel': 1, + 'client_channels': 1, + 'async_client_threads': 1, + 'rpc_type': 'UNARY', + 'load_params': { + 'closed_loop': {} + }, + 'payload_config': EMPTY_PROTO_PAYLOAD, + 'histogram_params': HISTOGRAM_PARAMS, + }, + 'server_config': { + 'server_type': 'SYNC_SERVER', + 'security_params': secargs, + 'core_limit': 1, + 'async_server_threads': 1, + }, + 'warmup_seconds': WARMUP_SECONDS, + 'benchmark_seconds': BENCHMARK_SECONDS, + 'SERVER_LANGUAGE': 'c++' # recognized by run_performance_tests.py + } + + def __str__(self): + return 'csharp' + + +class NodeLanguage: + + def __init__(self): + pass + self.safename = str(self) + + def worker_cmdline(self): + return ['tools/run_tests/performance/run_worker_node.sh'] + + def worker_port_offset(self): + return 200 + + def scenarios(self): + # TODO(jtattermusch): add more scenarios + secargs = SECURE_SECARGS + yield { + 'name': 'node_protobuf_unary_ping_pong', + 'num_servers': 1, + 'num_clients': 1, + 'client_config': { + 'client_type': 'ASYNC_CLIENT', + 'security_params': secargs, + 'outstanding_rpcs_per_channel': 1, + 'client_channels': 1, + 'async_client_threads': 1, + 'rpc_type': 'UNARY', + 'load_params': { + 'closed_loop': {} + }, + 'payload_config': EMPTY_PROTO_PAYLOAD, + 'histogram_params': HISTOGRAM_PARAMS, + }, + 'server_config': { + 'server_type': 'ASYNC_SERVER', + 'security_params': secargs, + 'core_limit': 0, + 'async_server_threads': 1, + }, + 'warmup_seconds': WARMUP_SECONDS, + 'benchmark_seconds': BENCHMARK_SECONDS + } + + def __str__(self): + return 'node' + + +class RubyLanguage: + + def __init__(self): + pass + self.safename = str(self) + + def worker_cmdline(self): + return ['tools/run_tests/performance/run_worker_ruby.sh'] + + def worker_port_offset(self): + return 300 + + def scenarios(self): + # TODO(jtattermusch): add more scenarios + secargs = SECURE_SECARGS + yield { + 'name': 'ruby_protobuf_unary_ping_pong', + 'num_servers': 1, + 'num_clients': 1, + 'client_config': { + 'client_type': 'SYNC_CLIENT', + 'security_params': secargs, + 'outstanding_rpcs_per_channel': 1, + 'client_channels': 1, + 'async_client_threads': 1, + 'rpc_type': 'UNARY', + 'load_params': { + 'closed_loop': {} + }, + 'payload_config': EMPTY_PROTO_PAYLOAD, + 'histogram_params': HISTOGRAM_PARAMS, + }, + 'server_config': { + 'server_type': 'SYNC_SERVER', + 'security_params': secargs, + 'core_limit': 0, + 'async_server_threads': 1, + }, + 'warmup_seconds': WARMUP_SECONDS, + 'benchmark_seconds': BENCHMARK_SECONDS + } + + def __str__(self): + return 'ruby' + + +class JavaLanguage: + + def __init__(self): + pass + self.safename = str(self) + + def worker_cmdline(self): + return ['tools/run_tests/performance/run_worker_java.sh'] + + def worker_port_offset(self): + return 400 + + def scenarios(self): + # TODO(jtattermusch): add more scenarios + for secure in [True, False]: + if secure: + secstr = 'secure' + secargs = SECURE_SECARGS + else: + secstr = 'insecure' + secargs = None + + yield { + 'name': 'java_protobuf_unary_ping_pong_%s' % secstr, + 'num_servers': 1, + 'num_clients': 1, + 'client_config': { + 'client_type': 'SYNC_CLIENT', + 'security_params': secargs, + 'outstanding_rpcs_per_channel': 1, + 'client_channels': 1, + 'async_client_threads': 1, + 'rpc_type': 'UNARY', + 'load_params': { + 'closed_loop': {} + }, + 'payload_config': EMPTY_PROTO_PAYLOAD, + 'histogram_params': HISTOGRAM_PARAMS, + }, + 'server_config': { + 'server_type': 'SYNC_SERVER', + 'security_params': secargs, + 'core_limit': 0, + 'async_server_threads': 1, + }, + 'warmup_seconds': JAVA_WARMUP_SECONDS, + 'benchmark_seconds': BENCHMARK_SECONDS + } + + def __str__(self): + return 'java' + + +LANGUAGES = { + 'c++' : CXXLanguage(), + 'csharp' : CSharpLanguage(), + 'node' : NodeLanguage(), + 'ruby' : RubyLanguage(), + 'java' : JavaLanguage(), +} |