aboutsummaryrefslogtreecommitdiffhomepage
path: root/tools/run_tests/performance/scenario_config.py
diff options
context:
space:
mode:
Diffstat (limited to 'tools/run_tests/performance/scenario_config.py')
-rw-r--r--tools/run_tests/performance/scenario_config.py294
1 files changed, 267 insertions, 27 deletions
diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py
index 7a82d257e4..cf3c8ae80a 100644
--- a/tools/run_tests/performance/scenario_config.py
+++ b/tools/run_tests/performance/scenario_config.py
@@ -31,8 +31,14 @@
SINGLE_MACHINE_CORES=8
WARMUP_SECONDS=5
+JAVA_WARMUP_SECONDS=15 # Java needs more warmup time for JIT to kick in.
BENCHMARK_SECONDS=30
+HISTOGRAM_PARAMS = {
+ 'resolution': 0.01,
+ 'max_possible': 60e9,
+}
+
EMPTY_GENERIC_PAYLOAD = {
'bytebuf_params': {
'req_size': 0,
@@ -83,7 +89,7 @@ class CXXLanguage:
secargs = None
yield {
- 'name': 'generic_async_streaming_ping_pong_%s'
+ 'name': 'cpp_generic_async_streaming_ping_pong_%s'
% secstr,
'num_servers': 1,
'num_clients': 1,
@@ -98,11 +104,12 @@ class CXXLanguage:
'closed_loop': {}
},
'payload_config': EMPTY_GENERIC_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
'server_type': 'ASYNC_GENERIC_SERVER',
'security_params': secargs,
- 'core_limit': SINGLE_MACHINE_CORES/2,
+ 'core_limit': 1,
'async_server_threads': 1,
'payload_config': EMPTY_GENERIC_PAYLOAD,
},
@@ -110,7 +117,7 @@ class CXXLanguage:
'benchmark_seconds': BENCHMARK_SECONDS
}
yield {
- 'name': 'generic_async_streaming_qps_unconstrained_%s'
+ 'name': 'cpp_generic_async_streaming_qps_unconstrained_%s'
% secstr,
'num_servers': 1,
'num_clients': 0,
@@ -119,25 +126,26 @@ class CXXLanguage:
'security_params': secargs,
'outstanding_rpcs_per_channel': DEEP,
'client_channels': WIDE,
- 'async_client_threads': 1,
+ 'async_client_threads': 0,
'rpc_type': 'STREAMING',
'load_params': {
'closed_loop': {}
},
'payload_config': EMPTY_GENERIC_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
'server_type': 'ASYNC_GENERIC_SERVER',
'security_params': secargs,
'core_limit': SINGLE_MACHINE_CORES/2,
- 'async_server_threads': 1,
+ 'async_server_threads': 0,
'payload_config': EMPTY_GENERIC_PAYLOAD,
},
'warmup_seconds': WARMUP_SECONDS,
'benchmark_seconds': BENCHMARK_SECONDS
}
yield {
- 'name': 'generic_async_streaming_qps_one_server_core_%s'
+ 'name': 'cpp_generic_async_streaming_qps_one_server_core_%s'
% secstr,
'num_servers': 1,
'num_clients': 0,
@@ -146,12 +154,13 @@ class CXXLanguage:
'security_params': secargs,
'outstanding_rpcs_per_channel': DEEP,
'client_channels': WIDE,
- 'async_client_threads': 1,
+ 'async_client_threads': 0,
'rpc_type': 'STREAMING',
'load_params': {
'closed_loop': {}
},
'payload_config': EMPTY_GENERIC_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
'server_type': 'ASYNC_GENERIC_SERVER',
@@ -164,7 +173,7 @@ class CXXLanguage:
'benchmark_seconds': BENCHMARK_SECONDS
}
yield {
- 'name': 'protobuf_async_qps_unconstrained_%s'
+ 'name': 'cpp_protobuf_async_streaming_qps_unconstrained_%s'
% secstr,
'num_servers': 1,
'num_clients': 0,
@@ -173,52 +182,53 @@ class CXXLanguage:
'security_params': secargs,
'outstanding_rpcs_per_channel': DEEP,
'client_channels': WIDE,
- 'async_client_threads': 1,
+ 'async_client_threads': 0,
'rpc_type': 'STREAMING',
'load_params': {
'closed_loop': {}
},
- 'payload_config': EMPTY_GENERIC_PAYLOAD,
+ 'payload_config': EMPTY_PROTO_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
- 'server_type': 'ASYNC_GENERIC_SERVER',
+ 'server_type': 'ASYNC_SERVER',
'security_params': secargs,
'core_limit': SINGLE_MACHINE_CORES/2,
- 'async_server_threads': 1,
- 'payload_config': EMPTY_GENERIC_PAYLOAD,
+ 'async_server_threads': 0,
},
'warmup_seconds': WARMUP_SECONDS,
'benchmark_seconds': BENCHMARK_SECONDS
}
yield {
- 'name': 'single_channel_throughput_%s'
+ 'name': 'cpp_single_channel_throughput_%s'
% secstr,
'num_servers': 1,
'num_clients': 1,
'client_config': {
'client_type': 'ASYNC_CLIENT',
'security_params': secargs,
- 'outstanding_rpcs_per_channel': 1,
+ 'outstanding_rpcs_per_channel': DEEP,
'client_channels': 1,
- 'async_client_threads': 1,
+ 'async_client_threads': 0,
'rpc_type': 'STREAMING',
'load_params': {
'closed_loop': {}
},
'payload_config': BIG_GENERIC_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
'server_type': 'ASYNC_GENERIC_SERVER',
'security_params': secargs,
'core_limit': SINGLE_MACHINE_CORES/2,
- 'async_server_threads': 1,
+ 'async_server_threads': 0,
'payload_config': BIG_GENERIC_PAYLOAD,
},
'warmup_seconds': WARMUP_SECONDS,
'benchmark_seconds': BENCHMARK_SECONDS
}
yield {
- 'name': 'protobuf_async_ping_pong_%s'
+ 'name': 'cpp_protobuf_async_streaming_ping_pong_%s'
% secstr,
'num_servers': 1,
'num_clients': 1,
@@ -233,13 +243,67 @@ class CXXLanguage:
'closed_loop': {}
},
'payload_config': EMPTY_PROTO_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
- 'server_type': 'ASYNC_GENERIC_SERVER',
+ 'server_type': 'ASYNC_SERVER',
'security_params': secargs,
- 'core_limit': SINGLE_MACHINE_CORES/2,
+ 'core_limit': 1,
'async_server_threads': 1,
+ },
+ 'warmup_seconds': WARMUP_SECONDS,
+ 'benchmark_seconds': BENCHMARK_SECONDS
+ }
+ yield {
+ 'name': 'cpp_protobuf_sync_unary_ping_pong_%s'
+ % secstr,
+ 'num_servers': 1,
+ 'num_clients': 1,
+ 'client_config': {
+ 'client_type': 'SYNC_CLIENT',
+ 'security_params': secargs,
+ 'outstanding_rpcs_per_channel': 1,
+ 'client_channels': 1,
+ 'async_client_threads': 0,
+ 'rpc_type': 'UNARY',
+ 'load_params': {
+ 'closed_loop': {}
+ },
'payload_config': EMPTY_PROTO_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
+ },
+ 'server_config': {
+ 'server_type': 'SYNC_SERVER',
+ 'security_params': secargs,
+ 'core_limit': 1,
+ 'async_server_threads': 0,
+ },
+ 'warmup_seconds': WARMUP_SECONDS,
+ 'benchmark_seconds': BENCHMARK_SECONDS
+ }
+ yield {
+ 'name': 'cpp_protobuf_async_unary_ping_pong_%s'
+ % secstr,
+ 'num_servers': 1,
+ 'num_clients': 1,
+ 'client_config': {
+ 'client_type': 'ASYNC_CLIENT',
+ 'security_params': secargs,
+ 'outstanding_rpcs_per_channel': 1,
+ 'client_channels': 1,
+ 'async_client_threads': 1,
+ 'rpc_type': 'UNARY',
+ 'load_params': {
+ 'closed_loop': {}
+ },
+ 'payload_config': EMPTY_PROTO_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
+ },
+ 'server_config': {
+ 'server_type': 'ASYNC_SERVER',
+ 'security_params': secargs,
+ 'core_limit': 1,
+ 'async_server_threads': 1,
},
'warmup_seconds': WARMUP_SECONDS,
'benchmark_seconds': BENCHMARK_SECONDS
@@ -262,8 +326,9 @@ class CSharpLanguage:
def scenarios(self):
# TODO(jtattermusch): add more scenarios
+ secargs = None
yield {
- 'name': 'csharp_async_generic_streaming_ping_pong',
+ 'name': 'csharp_generic_async_streaming_ping_pong',
'num_servers': 1,
'num_clients': 1,
'client_config': {
@@ -277,17 +342,97 @@ class CSharpLanguage:
'closed_loop': {}
},
'payload_config': EMPTY_GENERIC_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
'server_type': 'ASYNC_GENERIC_SERVER',
'security_params': secargs,
- 'core_limit': SINGLE_MACHINE_CORES/2,
+ 'core_limit': 0,
'async_server_threads': 1,
'payload_config': EMPTY_GENERIC_PAYLOAD,
},
'warmup_seconds': WARMUP_SECONDS,
'benchmark_seconds': BENCHMARK_SECONDS
}
+ yield {
+ 'name': 'csharp_protobuf_async_unary_ping_pong',
+ 'num_servers': 1,
+ 'num_clients': 1,
+ 'client_config': {
+ 'client_type': 'ASYNC_CLIENT',
+ 'security_params': secargs,
+ 'outstanding_rpcs_per_channel': 1,
+ 'client_channels': 1,
+ 'async_client_threads': 1,
+ 'rpc_type': 'UNARY',
+ 'load_params': {
+ 'closed_loop': {}
+ },
+ 'payload_config': EMPTY_PROTO_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
+ },
+ 'server_config': {
+ 'server_type': 'ASYNC_SERVER',
+ 'security_params': secargs,
+ 'core_limit': 0,
+ 'async_server_threads': 1,
+ },
+ 'warmup_seconds': WARMUP_SECONDS,
+ 'benchmark_seconds': BENCHMARK_SECONDS
+ }
+ yield {
+ 'name': 'csharp_protobuf_sync_to_async_unary_ping_pong',
+ 'num_servers': 1,
+ 'num_clients': 1,
+ 'client_config': {
+ 'client_type': 'SYNC_CLIENT',
+ 'security_params': secargs,
+ 'outstanding_rpcs_per_channel': 1,
+ 'client_channels': 1,
+ 'async_client_threads': 1,
+ 'rpc_type': 'UNARY',
+ 'load_params': {
+ 'closed_loop': {}
+ },
+ 'payload_config': EMPTY_PROTO_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
+ },
+ 'server_config': {
+ 'server_type': 'ASYNC_SERVER',
+ 'security_params': secargs,
+ 'core_limit': 0,
+ 'async_server_threads': 1,
+ },
+ 'warmup_seconds': WARMUP_SECONDS,
+ 'benchmark_seconds': BENCHMARK_SECONDS
+ }
+ yield {
+ 'name': 'csharp_to_cpp_protobuf_sync_unary_ping_pong',
+ 'num_servers': 1,
+ 'num_clients': 1,
+ 'client_config': {
+ 'client_type': 'SYNC_CLIENT',
+ 'security_params': secargs,
+ 'outstanding_rpcs_per_channel': 1,
+ 'client_channels': 1,
+ 'async_client_threads': 1,
+ 'rpc_type': 'UNARY',
+ 'load_params': {
+ 'closed_loop': {}
+ },
+ 'payload_config': EMPTY_PROTO_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
+ },
+ 'server_config': {
+ 'server_type': 'SYNC_SERVER',
+ 'security_params': secargs,
+ 'core_limit': 0,
+ 'async_server_threads': 1,
+ },
+ 'warmup_seconds': WARMUP_SECONDS,
+ 'benchmark_seconds': BENCHMARK_SECONDS,
+ 'SERVER_LANGUAGE': 'c++' # recognized by run_performance_tests.py
+ }
def __str__(self):
return 'csharp'
@@ -307,8 +452,9 @@ class NodeLanguage:
def scenarios(self):
# TODO(jtattermusch): add more scenarios
+ secargs = None
yield {
- 'name': 'node_sync_unary_ping_pong_protobuf',
+ 'name': 'node_protobuf_unary_ping_pong',
'num_servers': 1,
'num_clients': 1,
'client_config': {
@@ -317,18 +463,18 @@ class NodeLanguage:
'outstanding_rpcs_per_channel': 1,
'client_channels': 1,
'async_client_threads': 1,
- 'rpc_type': 'STREAMING',
+ 'rpc_type': 'UNARY',
'load_params': {
'closed_loop': {}
},
'payload_config': EMPTY_PROTO_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
},
'server_config': {
- 'server_type': 'ASYNC_GENERIC_SERVER',
+ 'server_type': 'ASYNC_SERVER',
'security_params': secargs,
- 'core_limit': SINGLE_MACHINE_CORES/2,
+ 'core_limit': 0,
'async_server_threads': 1,
- 'payload_config': EMPTY_PROTO_PAYLOAD,
},
'warmup_seconds': WARMUP_SECONDS,
'benchmark_seconds': BENCHMARK_SECONDS
@@ -338,8 +484,102 @@ class NodeLanguage:
return 'node'
+class RubyLanguage:
+
+ def __init__(self):
+ pass
+ self.safename = str(self)
+
+ def worker_cmdline(self):
+ return ['tools/run_tests/performance/run_worker_ruby.sh']
+
+ def worker_port_offset(self):
+ return 300
+
+ def scenarios(self):
+ # TODO(jtattermusch): add more scenarios
+ secargs = None
+ yield {
+ 'name': 'ruby_protobuf_unary_ping_pong',
+ 'num_servers': 1,
+ 'num_clients': 1,
+ 'client_config': {
+ 'client_type': 'SYNC_CLIENT',
+ 'security_params': secargs,
+ 'outstanding_rpcs_per_channel': 1,
+ 'client_channels': 1,
+ 'async_client_threads': 1,
+ 'rpc_type': 'UNARY',
+ 'load_params': {
+ 'closed_loop': {}
+ },
+ 'payload_config': EMPTY_PROTO_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
+ },
+ 'server_config': {
+ 'server_type': 'SYNC_SERVER',
+ 'security_params': secargs,
+ 'core_limit': 0,
+ 'async_server_threads': 1,
+ },
+ 'warmup_seconds': WARMUP_SECONDS,
+ 'benchmark_seconds': BENCHMARK_SECONDS
+ }
+
+ def __str__(self):
+ return 'ruby'
+
+
+class JavaLanguage:
+
+ def __init__(self):
+ pass
+ self.safename = str(self)
+
+ def worker_cmdline(self):
+ return ['tools/run_tests/performance/run_worker_java.sh']
+
+ def worker_port_offset(self):
+ return 400
+
+ def scenarios(self):
+ # TODO(jtattermusch): add more scenarios
+ secargs = None
+ yield {
+ 'name': 'java_protobuf_unary_ping_pong',
+ 'num_servers': 1,
+ 'num_clients': 1,
+ 'client_config': {
+ 'client_type': 'SYNC_CLIENT',
+ 'security_params': secargs,
+ 'outstanding_rpcs_per_channel': 1,
+ 'client_channels': 1,
+ 'async_client_threads': 1,
+ 'rpc_type': 'UNARY',
+ 'load_params': {
+ 'closed_loop': {}
+ },
+ 'payload_config': EMPTY_PROTO_PAYLOAD,
+ 'histogram_params': HISTOGRAM_PARAMS,
+ },
+ 'server_config': {
+ 'server_type': 'SYNC_SERVER',
+ 'security_params': secargs,
+ 'core_limit': 0,
+ 'async_server_threads': 1,
+ },
+ 'warmup_seconds': JAVA_WARMUP_SECONDS,
+ 'benchmark_seconds': BENCHMARK_SECONDS
+ }
+
+ def __str__(self):
+ return 'java'
+
+
LANGUAGES = {
'c++' : CXXLanguage(),
'csharp' : CSharpLanguage(),
'node' : NodeLanguage(),
+ 'ruby' : RubyLanguage(),
+ 'java' : JavaLanguage(),
}
generated by cgit v1.2.3 (git 2.39.1) at 2025-01-23 09:02:07 +0000