Merge pull request #6158 from jtattermusch/performance_testing_polishing

Improvement to benchmarking

1 files changed, 109 insertions, 18 deletions

diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py
index 7a82d257e4..bd10b6f032 100644
--- a/tools/run_tests/performance/scenario_config.py
+++ b/tools/run_tests/performance/scenario_config.py
@@ -33,6 +33,11 @@ SINGLE_MACHINE_CORES=8
 WARMUP_SECONDS=5
 BENCHMARK_SECONDS=30
 
+HISTOGRAM_PARAMS = {
+  'resolution': 0.01,
+  'max_possible': 60e9,
+}
+
 EMPTY_GENERIC_PAYLOAD = {
   'bytebuf_params': {
     'req_size': 0,
@@ -83,7 +88,7 @@ class CXXLanguage:
         secargs = None
 
       yield {
-          'name': 'generic_async_streaming_ping_pong_%s'
+          'name': 'cpp_generic_async_streaming_ping_pong_%s'
                   % secstr,
           'num_servers': 1,
           'num_clients': 1,
@@ -98,6 +103,7 @@ class CXXLanguage:
               'closed_loop': {}
             },
             'payload_config': EMPTY_GENERIC_PAYLOAD,
+            'histogram_params': HISTOGRAM_PARAMS,
           },
           'server_config': {
             'server_type': 'ASYNC_GENERIC_SERVER',
@@ -110,7 +116,7 @@ class CXXLanguage:
           'benchmark_seconds': BENCHMARK_SECONDS
       }
       yield {
-          'name': 'generic_async_streaming_qps_unconstrained_%s'
+          'name': 'cpp_generic_async_streaming_qps_unconstrained_%s'
                   % secstr,
           'num_servers': 1,
           'num_clients': 0,
@@ -125,6 +131,7 @@ class CXXLanguage:
               'closed_loop': {}
             },
             'payload_config': EMPTY_GENERIC_PAYLOAD,
+            'histogram_params': HISTOGRAM_PARAMS,
           },
           'server_config': {
             'server_type': 'ASYNC_GENERIC_SERVER',
@@ -137,7 +144,7 @@ class CXXLanguage:
           'benchmark_seconds': BENCHMARK_SECONDS
       }
       yield {
-          'name': 'generic_async_streaming_qps_one_server_core_%s'
+          'name': 'cpp_generic_async_streaming_qps_one_server_core_%s'
                   % secstr,
           'num_servers': 1,
           'num_clients': 0,
@@ -152,6 +159,7 @@ class CXXLanguage:
               'closed_loop': {}
             },
             'payload_config': EMPTY_GENERIC_PAYLOAD,
+            'histogram_params': HISTOGRAM_PARAMS,
           },
           'server_config': {
             'server_type': 'ASYNC_GENERIC_SERVER',
@@ -164,7 +172,7 @@ class CXXLanguage:
           'benchmark_seconds': BENCHMARK_SECONDS
       }
       yield {
-          'name': 'protobuf_async_qps_unconstrained_%s'
+          'name': 'cpp_protobuf_async_streaming_qps_unconstrained_%s'
                   % secstr,
           'num_servers': 1,
           'num_clients': 0,
@@ -178,20 +186,20 @@ class CXXLanguage:
             'load_params': {
               'closed_loop': {}
             },
-            'payload_config': EMPTY_GENERIC_PAYLOAD,
+            'payload_config': EMPTY_PROTO_PAYLOAD,
+            'histogram_params': HISTOGRAM_PARAMS,
           },
           'server_config': {
-            'server_type': 'ASYNC_GENERIC_SERVER',
+            'server_type': 'ASYNC_SERVER',
             'security_params': secargs,
             'core_limit': SINGLE_MACHINE_CORES/2,
             'async_server_threads': 1,
-            'payload_config': EMPTY_GENERIC_PAYLOAD,
           },
           'warmup_seconds': WARMUP_SECONDS,
           'benchmark_seconds': BENCHMARK_SECONDS
       }
       yield {
-          'name': 'single_channel_throughput_%s'
+          'name': 'cpp_single_channel_throughput_%s'
                   % secstr,
           'num_servers': 1,
           'num_clients': 1,
@@ -206,6 +214,7 @@ class CXXLanguage:
               'closed_loop': {}
             },
             'payload_config': BIG_GENERIC_PAYLOAD,
+            'histogram_params': HISTOGRAM_PARAMS,
           },
           'server_config': {
             'server_type': 'ASYNC_GENERIC_SERVER',
@@ -218,7 +227,7 @@ class CXXLanguage:
           'benchmark_seconds': BENCHMARK_SECONDS
       }
       yield {
-          'name': 'protobuf_async_ping_pong_%s'
+          'name': 'cpp_protobuf_async_ping_pong_%s'
                   % secstr,
           'num_servers': 1,
           'num_clients': 1,
@@ -233,13 +242,13 @@ class CXXLanguage:
               'closed_loop': {}
             },
             'payload_config': EMPTY_PROTO_PAYLOAD,
+            'histogram_params': HISTOGRAM_PARAMS,
           },
           'server_config': {
-            'server_type': 'ASYNC_GENERIC_SERVER',
+            'server_type': 'ASYNC_SERVER',
             'security_params': secargs,
             'core_limit': SINGLE_MACHINE_CORES/2,
             'async_server_threads': 1,
-            'payload_config': EMPTY_PROTO_PAYLOAD,
           },
           'warmup_seconds': WARMUP_SECONDS,
           'benchmark_seconds': BENCHMARK_SECONDS
@@ -262,8 +271,9 @@ class CSharpLanguage:
 
   def scenarios(self):
     # TODO(jtattermusch): add more scenarios
+    secargs = None
     yield {
-        'name': 'csharp_async_generic_streaming_ping_pong',
+        'name': 'csharp_generic_async_streaming_ping_pong',
         'num_servers': 1,
         'num_clients': 1,
         'client_config': {
@@ -277,17 +287,97 @@ class CSharpLanguage:
             'closed_loop': {}
           },
           'payload_config': EMPTY_GENERIC_PAYLOAD,
+          'histogram_params': HISTOGRAM_PARAMS,
         },
         'server_config': {
           'server_type': 'ASYNC_GENERIC_SERVER',
           'security_params': secargs,
-          'core_limit': SINGLE_MACHINE_CORES/2,
+          'core_limit': 0,
           'async_server_threads': 1,
           'payload_config': EMPTY_GENERIC_PAYLOAD,
         },
         'warmup_seconds': WARMUP_SECONDS,
         'benchmark_seconds': BENCHMARK_SECONDS
     }
+    yield {
+        'name': 'csharp_protobuf_async_unary_ping_pong',
+        'num_servers': 1,
+        'num_clients': 1,
+        'client_config': {
+          'client_type': 'ASYNC_CLIENT',
+          'security_params': secargs,
+          'outstanding_rpcs_per_channel': 1,
+          'client_channels': 1,
+          'async_client_threads': 1,
+          'rpc_type': 'UNARY',
+          'load_params': {
+            'closed_loop': {}
+          },
+          'payload_config': EMPTY_PROTO_PAYLOAD,
+          'histogram_params': HISTOGRAM_PARAMS,
+        },
+        'server_config': {
+          'server_type': 'ASYNC_SERVER',
+          'security_params': secargs,
+          'core_limit': 0,
+          'async_server_threads': 1,
+        },
+        'warmup_seconds': WARMUP_SECONDS,
+        'benchmark_seconds': BENCHMARK_SECONDS
+    }
+    yield {
+        'name': 'csharp_protobuf_sync_to_async_unary_ping_pong',
+        'num_servers': 1,
+        'num_clients': 1,
+        'client_config': {
+          'client_type': 'SYNC_CLIENT',
+          'security_params': secargs,
+          'outstanding_rpcs_per_channel': 1,
+          'client_channels': 1,
+          'async_client_threads': 1,
+          'rpc_type': 'UNARY',
+          'load_params': {
+            'closed_loop': {}
+          },
+          'payload_config': EMPTY_PROTO_PAYLOAD,
+          'histogram_params': HISTOGRAM_PARAMS,
+        },
+        'server_config': {
+          'server_type': 'ASYNC_SERVER',
+          'security_params': secargs,
+          'core_limit': 0,
+          'async_server_threads': 1,
+        },
+        'warmup_seconds': WARMUP_SECONDS,
+        'benchmark_seconds': BENCHMARK_SECONDS
+    }
+    yield {
+        'name': 'csharp_to_cpp_protobuf_sync_unary_ping_pong',
+        'num_servers': 1,
+        'num_clients': 1,
+        'client_config': {
+          'client_type': 'SYNC_CLIENT',
+          'security_params': secargs,
+          'outstanding_rpcs_per_channel': 1,
+          'client_channels': 1,
+          'async_client_threads': 1,
+          'rpc_type': 'UNARY',
+          'load_params': {
+            'closed_loop': {}
+          },
+          'payload_config': EMPTY_PROTO_PAYLOAD,
+          'histogram_params': HISTOGRAM_PARAMS,
+        },
+        'server_config': {
+          'server_type': 'SYNC_SERVER',
+          'security_params': secargs,
+          'core_limit': 0,
+          'async_server_threads': 1,
+        },
+        'warmup_seconds': WARMUP_SECONDS,
+        'benchmark_seconds': BENCHMARK_SECONDS,
+        'SERVER_LANGUAGE': 'c++'  # recognized by run_performance_tests.py
+    }
 
   def __str__(self):
     return 'csharp'
@@ -307,8 +397,9 @@ class NodeLanguage:
 
   def scenarios(self):
     # TODO(jtattermusch): add more scenarios
+    secargs = None
     yield {
-        'name': 'node_sync_unary_ping_pong_protobuf',
+        'name': 'node_protobuf_unary_ping_pong',
         'num_servers': 1,
         'num_clients': 1,
         'client_config': {
@@ -317,18 +408,18 @@ class NodeLanguage:
           'outstanding_rpcs_per_channel': 1,
           'client_channels': 1,
           'async_client_threads': 1,
-          'rpc_type': 'STREAMING',
+          'rpc_type': 'UNARY',
           'load_params': {
             'closed_loop': {}
           },
           'payload_config': EMPTY_PROTO_PAYLOAD,
+          'histogram_params': HISTOGRAM_PARAMS,
         },
         'server_config': {
-          'server_type': 'ASYNC_GENERIC_SERVER',
+          'server_type': 'ASYNC_SERVER',
           'security_params': secargs,
-          'core_limit': SINGLE_MACHINE_CORES/2,
+          'core_limit': 0,
           'async_server_threads': 1,
-          'payload_config': EMPTY_PROTO_PAYLOAD,
         },
         'warmup_seconds': WARMUP_SECONDS,
         'benchmark_seconds': BENCHMARK_SECONDS