aboutsummaryrefslogtreecommitdiffhomepage
path: root/tools/run_tests/performance/scenario_config.py
diff options
context:
space:
mode:
Diffstat (limited to 'tools/run_tests/performance/scenario_config.py')
-rw-r--r--tools/run_tests/performance/scenario_config.py565
1 files changed, 565 insertions, 0 deletions
diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py
new file mode 100644
index 0000000000..55657f8d8a
--- /dev/null
+++ b/tools/run_tests/performance/scenario_config.py
@@ -0,0 +1,565 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# performance scenario configuration for various languages
+
+SINGLE_MACHINE_CORES=8
+WARMUP_SECONDS=5
+JAVA_WARMUP_SECONDS=15 # Java needs more warmup time for JIT to kick in.
+BENCHMARK_SECONDS=30
+
+SECURE_SECARGS = {'use_test_ca': True,
+ 'server_host_override': 'foo.test.google.fr'}
+
+HISTOGRAM_PARAMS = {
+ 'resolution': 0.01,
+ 'max_possible': 60e9,
+}
+
+EMPTY_GENERIC_PAYLOAD = {
+ 'bytebuf_params': {
+ 'req_size': 0,
+ 'resp_size': 0,
+ }
+}
+EMPTY_PROTO_PAYLOAD = {
+ 'simple_params': {
+ 'req_size': 0,
+ 'resp_size': 0,
+ }
+}
+BIG_GENERIC_PAYLOAD = {
+ 'bytebuf_params': {
+ 'req_size': 65536,
+ 'resp_size': 65536,
+ }
+}
+
+# deep is the number of RPCs outstanding on a channel in non-ping-pong tests
+# (the value used is 1 otherwise)
+DEEP=100
+
+# wide is the number of client channels in multi-channel tests (1 otherwise)
+WIDE=64
+
+
+class CXXLanguage:
+
+ def __init__(self):
+ self.safename = 'cxx'
+
+ def worker_cmdline(self):
+ return ['bins/opt/qps_worker']
+
+ def worker_port_offset(self):
+ return 0
+
+ def scenarios(self):
+ # TODO(ctiller): add 70% load latency test
+ for secure in [True, False]:
+ if secure:
+ secstr = 'secure'
+ secargs = SECURE_SECARGS
+ else:
+ secstr = 'insecure'
+ secargs = None
+
+ yield {
+ 'name': 'cpp_generic_async_streaming_ping_pong_%s'
+ % secstr,
+ 'num_servers': 1,
+ 'num_clients': 1,
+ 'client_config': {
+ 'client_type': 'ASYNC_CLIENT',
+ 'security_params': secargs,
+ 'outstanding_rpcs_per_channel': 1,
+ 'client_channels': 1,
+ 'async_client_threads': 1,
+ 'rpc_type': 'STREAMING',
+ 'load_params': {
+ 'closed_loop': {}
+ },
+ 'payload_config': EMPTY_GENERIC_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
+ },
+ 'server_config': {
+ 'server_type': 'ASYNC_GENERIC_SERVER',
+ 'security_params': secargs,
+ 'core_limit': 1,
+ 'async_server_threads': 1,
+ 'payload_config': EMPTY_GENERIC_PAYLOAD,
+ },
+ 'warmup_seconds': WARMUP_SECONDS,
+ 'benchmark_seconds': BENCHMARK_SECONDS
+ }
+ yield {
+ 'name': 'cpp_generic_async_streaming_qps_unconstrained_%s'
+ % secstr,
+ 'num_servers': 1,
+ 'num_clients': 0,
+ 'client_config': {
+ 'client_type': 'ASYNC_CLIENT',
+ 'security_params': secargs,
+ 'outstanding_rpcs_per_channel': DEEP,
+ 'client_channels': WIDE,
+ 'async_client_threads': 0,
+ 'rpc_type': 'STREAMING',
+ 'load_params': {
+ 'closed_loop': {}
+ },
+ 'payload_config': EMPTY_GENERIC_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
+ },
+ 'server_config': {
+ 'server_type': 'ASYNC_GENERIC_SERVER',
+ 'security_params': secargs,
+ 'core_limit': SINGLE_MACHINE_CORES/2,
+ 'async_server_threads': 0,
+ 'payload_config': EMPTY_GENERIC_PAYLOAD,
+ },
+ 'warmup_seconds': WARMUP_SECONDS,
+ 'benchmark_seconds': BENCHMARK_SECONDS
+ }
+ yield {
+ 'name': 'cpp_generic_async_streaming_qps_one_server_core_%s'
+ % secstr,
+ 'num_servers': 1,
+ 'num_clients': 0,
+ 'client_config': {
+ 'client_type': 'ASYNC_CLIENT',
+ 'security_params': secargs,
+ 'outstanding_rpcs_per_channel': DEEP,
+ 'client_channels': WIDE,
+ 'async_client_threads': 0,
+ 'rpc_type': 'STREAMING',
+ 'load_params': {
+ 'closed_loop': {}
+ },
+ 'payload_config': EMPTY_GENERIC_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
+ },
+ 'server_config': {
+ 'server_type': 'ASYNC_GENERIC_SERVER',
+ 'security_params': secargs,
+ 'core_limit': 1,
+ 'async_server_threads': 1,
+ 'payload_config': EMPTY_GENERIC_PAYLOAD,
+ },
+ 'warmup_seconds': WARMUP_SECONDS,
+ 'benchmark_seconds': BENCHMARK_SECONDS
+ }
+ yield {
+ 'name': 'cpp_protobuf_async_streaming_qps_unconstrained_%s'
+ % secstr,
+ 'num_servers': 1,
+ 'num_clients': 0,
+ 'client_config': {
+ 'client_type': 'ASYNC_CLIENT',
+ 'security_params': secargs,
+ 'outstanding_rpcs_per_channel': DEEP,
+ 'client_channels': WIDE,
+ 'async_client_threads': 0,
+ 'rpc_type': 'STREAMING',
+ 'load_params': {
+ 'closed_loop': {}
+ },
+ 'payload_config': EMPTY_PROTO_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
+ },
+ 'server_config': {
+ 'server_type': 'ASYNC_SERVER',
+ 'security_params': secargs,
+ 'core_limit': SINGLE_MACHINE_CORES/2,
+ 'async_server_threads': 0,
+ },
+ 'warmup_seconds': WARMUP_SECONDS,
+ 'benchmark_seconds': BENCHMARK_SECONDS
+ }
+ yield {
+ 'name': 'cpp_protobuf_async_streaming_ping_pong_%s'
+ % secstr,
+ 'num_servers': 1,
+ 'num_clients': 1,
+ 'client_config': {
+ 'client_type': 'ASYNC_CLIENT',
+ 'security_params': secargs,
+ 'outstanding_rpcs_per_channel': 1,
+ 'client_channels': 1,
+ 'async_client_threads': 1,
+ 'rpc_type': 'STREAMING',
+ 'load_params': {
+ 'closed_loop': {}
+ },
+ 'payload_config': EMPTY_PROTO_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
+ },
+ 'server_config': {
+ 'server_type': 'ASYNC_SERVER',
+ 'security_params': secargs,
+ 'core_limit': 1,
+ 'async_server_threads': 1,
+ },
+ 'warmup_seconds': WARMUP_SECONDS,
+ 'benchmark_seconds': BENCHMARK_SECONDS
+ }
+ yield {
+ 'name': 'cpp_protobuf_sync_unary_ping_pong_%s'
+ % secstr,
+ 'num_servers': 1,
+ 'num_clients': 1,
+ 'client_config': {
+ 'client_type': 'SYNC_CLIENT',
+ 'security_params': secargs,
+ 'outstanding_rpcs_per_channel': 1,
+ 'client_channels': 1,
+ 'async_client_threads': 0,
+ 'rpc_type': 'UNARY',
+ 'load_params': {
+ 'closed_loop': {}
+ },
+ 'payload_config': EMPTY_PROTO_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
+ },
+ 'server_config': {
+ 'server_type': 'SYNC_SERVER',
+ 'security_params': secargs,
+ 'core_limit': 1,
+ 'async_server_threads': 0,
+ },
+ 'warmup_seconds': WARMUP_SECONDS,
+ 'benchmark_seconds': BENCHMARK_SECONDS
+ }
+ yield {
+ 'name': 'cpp_protobuf_async_unary_ping_pong_%s'
+ % secstr,
+ 'num_servers': 1,
+ 'num_clients': 1,
+ 'client_config': {
+ 'client_type': 'ASYNC_CLIENT',
+ 'security_params': secargs,
+ 'outstanding_rpcs_per_channel': 1,
+ 'client_channels': 1,
+ 'async_client_threads': 1,
+ 'rpc_type': 'UNARY',
+ 'load_params': {
+ 'closed_loop': {}
+ },
+ 'payload_config': EMPTY_PROTO_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
+ },
+ 'server_config': {
+ 'server_type': 'ASYNC_SERVER',
+ 'security_params': secargs,
+ 'core_limit': 1,
+ 'async_server_threads': 1,
+ },
+ 'warmup_seconds': WARMUP_SECONDS,
+ 'benchmark_seconds': BENCHMARK_SECONDS
+ }
+
+ def __str__(self):
+ return 'c++'
+
+
+class CSharpLanguage:
+
+ def __init__(self):
+ self.safename = str(self)
+
+ def worker_cmdline(self):
+ return ['tools/run_tests/performance/run_worker_csharp.sh']
+
+ def worker_port_offset(self):
+ return 100
+
+ def scenarios(self):
+ secargs = SECURE_SECARGS
+ yield {
+ 'name': 'csharp_generic_async_streaming_ping_pong',
+ 'num_servers': 1,
+ 'num_clients': 1,
+ 'client_config': {
+ 'client_type': 'ASYNC_CLIENT',
+ 'security_params': secargs,
+ 'outstanding_rpcs_per_channel': 1,
+ 'client_channels': 1,
+ 'async_client_threads': 1,
+ 'rpc_type': 'STREAMING',
+ 'load_params': {
+ 'closed_loop': {}
+ },
+ 'payload_config': EMPTY_GENERIC_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
+ },
+ 'server_config': {
+ 'server_type': 'ASYNC_GENERIC_SERVER',
+ 'security_params': secargs,
+ 'core_limit': 0,
+ 'async_server_threads': 0,
+ 'payload_config': EMPTY_GENERIC_PAYLOAD,
+ },
+ 'warmup_seconds': WARMUP_SECONDS,
+ 'benchmark_seconds': BENCHMARK_SECONDS
+ }
+ yield {
+ 'name': 'csharp_protobuf_async_unary_ping_pong',
+ 'num_servers': 1,
+ 'num_clients': 1,
+ 'client_config': {
+ 'client_type': 'ASYNC_CLIENT',
+ 'security_params': secargs,
+ 'outstanding_rpcs_per_channel': 1,
+ 'client_channels': 1,
+ 'async_client_threads': 1,
+ 'rpc_type': 'UNARY',
+ 'load_params': {
+ 'closed_loop': {}
+ },
+ 'payload_config': EMPTY_PROTO_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
+ },
+ 'server_config': {
+ 'server_type': 'ASYNC_SERVER',
+ 'security_params': secargs,
+ 'core_limit': 0,
+ 'async_server_threads': 0,
+ },
+ 'warmup_seconds': WARMUP_SECONDS,
+ 'benchmark_seconds': BENCHMARK_SECONDS
+ }
+ yield {
+ 'name': 'csharp_protobuf_sync_to_async_unary_ping_pong',
+ 'num_servers': 1,
+ 'num_clients': 1,
+ 'client_config': {
+ 'client_type': 'SYNC_CLIENT',
+ 'security_params': secargs,
+ 'outstanding_rpcs_per_channel': 1,
+ 'client_channels': 1,
+ 'async_client_threads': 1,
+ 'rpc_type': 'UNARY',
+ 'load_params': {
+ 'closed_loop': {}
+ },
+ 'payload_config': EMPTY_PROTO_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
+ },
+ 'server_config': {
+ 'server_type': 'ASYNC_SERVER',
+ 'security_params': secargs,
+ 'core_limit': 0,
+ 'async_server_threads': 0,
+ },
+ 'warmup_seconds': WARMUP_SECONDS,
+ 'benchmark_seconds': BENCHMARK_SECONDS
+ }
+ yield {
+ 'name': 'csharp_to_cpp_protobuf_sync_unary_ping_pong',
+ 'num_servers': 1,
+ 'num_clients': 1,
+ 'client_config': {
+ 'client_type': 'SYNC_CLIENT',
+ 'security_params': secargs,
+ 'outstanding_rpcs_per_channel': 1,
+ 'client_channels': 1,
+ 'async_client_threads': 1,
+ 'rpc_type': 'UNARY',
+ 'load_params': {
+ 'closed_loop': {}
+ },
+ 'payload_config': EMPTY_PROTO_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
+ },
+ 'server_config': {
+ 'server_type': 'SYNC_SERVER',
+ 'security_params': secargs,
+ 'core_limit': 1,
+ 'async_server_threads': 1,
+ },
+ 'warmup_seconds': WARMUP_SECONDS,
+ 'benchmark_seconds': BENCHMARK_SECONDS,
+ 'SERVER_LANGUAGE': 'c++' # recognized by run_performance_tests.py
+ }
+
+ def __str__(self):
+ return 'csharp'
+
+
+class NodeLanguage:
+
+ def __init__(self):
+ pass
+ self.safename = str(self)
+
+ def worker_cmdline(self):
+ return ['tools/run_tests/performance/run_worker_node.sh']
+
+ def worker_port_offset(self):
+ return 200
+
+ def scenarios(self):
+ # TODO(jtattermusch): add more scenarios
+ secargs = SECURE_SECARGS
+ yield {
+ 'name': 'node_protobuf_unary_ping_pong',
+ 'num_servers': 1,
+ 'num_clients': 1,
+ 'client_config': {
+ 'client_type': 'ASYNC_CLIENT',
+ 'security_params': secargs,
+ 'outstanding_rpcs_per_channel': 1,
+ 'client_channels': 1,
+ 'async_client_threads': 1,
+ 'rpc_type': 'UNARY',
+ 'load_params': {
+ 'closed_loop': {}
+ },
+ 'payload_config': EMPTY_PROTO_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
+ },
+ 'server_config': {
+ 'server_type': 'ASYNC_SERVER',
+ 'security_params': secargs,
+ 'core_limit': 0,
+ 'async_server_threads': 1,
+ },
+ 'warmup_seconds': WARMUP_SECONDS,
+ 'benchmark_seconds': BENCHMARK_SECONDS
+ }
+
+ def __str__(self):
+ return 'node'
+
+
+class RubyLanguage:
+
+ def __init__(self):
+ pass
+ self.safename = str(self)
+
+ def worker_cmdline(self):
+ return ['tools/run_tests/performance/run_worker_ruby.sh']
+
+ def worker_port_offset(self):
+ return 300
+
+ def scenarios(self):
+ # TODO(jtattermusch): add more scenarios
+ secargs = SECURE_SECARGS
+ yield {
+ 'name': 'ruby_protobuf_unary_ping_pong',
+ 'num_servers': 1,
+ 'num_clients': 1,
+ 'client_config': {
+ 'client_type': 'SYNC_CLIENT',
+ 'security_params': secargs,
+ 'outstanding_rpcs_per_channel': 1,
+ 'client_channels': 1,
+ 'async_client_threads': 1,
+ 'rpc_type': 'UNARY',
+ 'load_params': {
+ 'closed_loop': {}
+ },
+ 'payload_config': EMPTY_PROTO_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
+ },
+ 'server_config': {
+ 'server_type': 'SYNC_SERVER',
+ 'security_params': secargs,
+ 'core_limit': 0,
+ 'async_server_threads': 1,
+ },
+ 'warmup_seconds': WARMUP_SECONDS,
+ 'benchmark_seconds': BENCHMARK_SECONDS
+ }
+
+ def __str__(self):
+ return 'ruby'
+
+
+class JavaLanguage:
+
+ def __init__(self):
+ pass
+ self.safename = str(self)
+
+ def worker_cmdline(self):
+ return ['tools/run_tests/performance/run_worker_java.sh']
+
+ def worker_port_offset(self):
+ return 400
+
+ def scenarios(self):
+ # TODO(jtattermusch): add more scenarios
+ for secure in [True, False]:
+ if secure:
+ secstr = 'secure'
+ secargs = SECURE_SECARGS
+ else:
+ secstr = 'insecure'
+ secargs = None
+
+ yield {
+ 'name': 'java_protobuf_unary_ping_pong_%s' % secstr,
+ 'num_servers': 1,
+ 'num_clients': 1,
+ 'client_config': {
+ 'client_type': 'SYNC_CLIENT',
+ 'security_params': secargs,
+ 'outstanding_rpcs_per_channel': 1,
+ 'client_channels': 1,
+ 'async_client_threads': 1,
+ 'rpc_type': 'UNARY',
+ 'load_params': {
+ 'closed_loop': {}
+ },
+ 'payload_config': EMPTY_PROTO_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
+ },
+ 'server_config': {
+ 'server_type': 'SYNC_SERVER',
+ 'security_params': secargs,
+ 'core_limit': 0,
+ 'async_server_threads': 1,
+ },
+ 'warmup_seconds': JAVA_WARMUP_SECONDS,
+ 'benchmark_seconds': BENCHMARK_SECONDS
+ }
+
+ def __str__(self):
+ return 'java'
+
+
+LANGUAGES = {
+ 'c++' : CXXLanguage(),
+ 'csharp' : CSharpLanguage(),
+ 'node' : NodeLanguage(),
+ 'ruby' : RubyLanguage(),
+ 'java' : JavaLanguage(),
+}
generated by cgit v1.2.3 (git 2.39.1) at 2024-06-28 00:58:01 +0000