Merge pull request #6462 from jtattermusch/benchmarking_unify_scenarios

Improve benchmarking scenarios and enable "smoketest only" suite.

3 files changed, 458 insertions, 550 deletions

diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py
index ddbe237569..d393709623 100644
--- a/tools/run_tests/performance/scenario_config.py
+++ b/tools/run_tests/performance/scenario_config.py
@@ -34,6 +34,8 @@ WARMUP_SECONDS=5
 JAVA_WARMUP_SECONDS=15  # Java needs more warmup time for JIT to kick in.
 BENCHMARK_SECONDS=30
 
+SMOKETEST='smoketest'
+
 SECURE_SECARGS = {'use_test_ca': True,
                   'server_host_override': 'foo.test.google.fr'}
 
@@ -69,6 +71,86 @@ DEEP=100
 WIDE=64
 
 
+def _get_secargs(is_secure):
+  if is_secure:
+    return SECURE_SECARGS
+  else:
+    return None
+
+
+def remove_nonproto_fields(scenario):
+  """Remove special-purpose that contains some extra info about the scenario
+  but don't belong to the ScenarioConfig protobuf message"""
+  scenario.pop('CATEGORIES', None)
+  scenario.pop('SERVER_LANGUAGE', None)
+  return scenario
+
+
+def _ping_pong_scenario(name, rpc_type,
+                        client_type, server_type,
+                        secure=True,
+                        use_generic_payload=False,
+                        use_unconstrained_client=False,
+                        server_language=None,
+                        server_core_limit=0,
+                        async_server_threads=0,
+                        warmup_seconds=WARMUP_SECONDS,
+                        categories=[]):
+  """Creates a basic ping pong scenario."""
+  scenario = {
+    'name': name,
+    'num_servers': 1,
+    'num_clients': 1,
+    'client_config': {
+      'client_type': client_type,
+      'security_params': _get_secargs(secure),
+      'outstanding_rpcs_per_channel': 1,
+      'client_channels': 1,
+      'async_client_threads': 1,
+      'rpc_type': rpc_type,
+      'load_params': {
+        'closed_loop': {}
+      },
+      'histogram_params': HISTOGRAM_PARAMS,
+    },
+    'server_config': {
+      'server_type': server_type,
+      'security_params': _get_secargs(secure),
+      'core_limit': server_core_limit,
+      'async_server_threads': async_server_threads,
+    },
+    'warmup_seconds': warmup_seconds,
+    'benchmark_seconds': BENCHMARK_SECONDS
+  }
+  if use_generic_payload:
+    if server_type != 'ASYNC_GENERIC_SERVER':
+      raise Exception('Use ASYNC_GENERIC_SERVER for generic payload.')
+    scenario['client_config']['payload_config'] = EMPTY_GENERIC_PAYLOAD
+    scenario['server_config']['payload_config'] = EMPTY_GENERIC_PAYLOAD
+  else:
+    # For proto payload, only the client should get the config.
+    scenario['client_config']['payload_config'] = EMPTY_PROTO_PAYLOAD
+
+  if use_unconstrained_client:
+    scenario['num_clients'] = 0  # use as many client as available.
+    # TODO(jtattermusch): for SYNC_CLIENT, this will create 100*64 threads
+    # and that's probably too much (at least for wrapped languages).
+    scenario['client_config']['outstanding_rpcs_per_channel'] = DEEP
+    scenario['client_config']['client_channels'] = WIDE
+    scenario['client_config']['async_client_threads'] = 0
+  else:
+    scenario['client_config']['outstanding_rpcs_per_channel'] = 1
+    scenario['client_config']['client_channels'] = 1
+    scenario['client_config']['async_client_threads'] = 1
+
+  if server_language:
+    # the SERVER_LANGUAGE field is recognized by run_performance_tests.py
+    scenario['SERVER_LANGUAGE'] = server_language
+  if categories:
+    scenario['CATEGORIES'] = categories
+  return scenario
+
+
 class CXXLanguage:
 
   def __init__(self):
@@ -83,205 +165,64 @@ class CXXLanguage:
   def scenarios(self):
     # TODO(ctiller): add 70% load latency test
     for secure in [True, False]:
-      if secure:
-        secstr = 'secure'
-        secargs = SECURE_SECARGS
-      else:
-        secstr = 'insecure'
-        secargs = None
-
-      yield {
-          'name': 'cpp_generic_async_streaming_ping_pong_%s'
-                  % secstr,
-          'num_servers': 1,
-          'num_clients': 1,
-          'client_config': {
-            'client_type': 'ASYNC_CLIENT',
-            'security_params': secargs,
-            'outstanding_rpcs_per_channel': 1,
-            'client_channels': 1,
-            'async_client_threads': 1,
-            'rpc_type': 'STREAMING',
-            'load_params': {
-              'closed_loop': {}
-            },
-            'payload_config': EMPTY_GENERIC_PAYLOAD,
-            'histogram_params': HISTOGRAM_PARAMS,
-          },
-          'server_config': {
-            'server_type': 'ASYNC_GENERIC_SERVER',
-            'security_params': secargs,
-            'core_limit': 1,
-            'async_server_threads': 1,
-            'payload_config': EMPTY_GENERIC_PAYLOAD,
-          },
-          'warmup_seconds': WARMUP_SECONDS,
-          'benchmark_seconds': BENCHMARK_SECONDS
-      }
-      yield {
-          'name': 'cpp_generic_async_streaming_qps_unconstrained_%s'
-                  % secstr,
-          'num_servers': 1,
-          'num_clients': 0,
-          'client_config': {
-            'client_type': 'ASYNC_CLIENT',
-            'security_params': secargs,
-            'outstanding_rpcs_per_channel': DEEP,
-            'client_channels': WIDE,
-            'async_client_threads': 0,
-            'rpc_type': 'STREAMING',
-            'load_params': {
-              'closed_loop': {}
-            },
-            'payload_config': EMPTY_GENERIC_PAYLOAD,
-            'histogram_params': HISTOGRAM_PARAMS,
-          },
-          'server_config': {
-            'server_type': 'ASYNC_GENERIC_SERVER',
-            'security_params': secargs,
-            'core_limit': SINGLE_MACHINE_CORES/2,
-            'async_server_threads': 0,
-            'payload_config': EMPTY_GENERIC_PAYLOAD,
-          },
-          'warmup_seconds': WARMUP_SECONDS,
-          'benchmark_seconds': BENCHMARK_SECONDS
-      }
-      yield {
-          'name': 'cpp_generic_async_streaming_qps_one_server_core_%s'
-                  % secstr,
-          'num_servers': 1,
-          'num_clients': 0,
-          'client_config': {
-            'client_type': 'ASYNC_CLIENT',
-            'security_params': secargs,
-            'outstanding_rpcs_per_channel': DEEP,
-            'client_channels': WIDE,
-            'async_client_threads': 0,
-            'rpc_type': 'STREAMING',
-            'load_params': {
-              'closed_loop': {}
-            },
-            'payload_config': EMPTY_GENERIC_PAYLOAD,
-            'histogram_params': HISTOGRAM_PARAMS,
-          },
-          'server_config': {
-            'server_type': 'ASYNC_GENERIC_SERVER',
-            'security_params': secargs,
-            'core_limit': 1,
-            'async_server_threads': 1,
-            'payload_config': EMPTY_GENERIC_PAYLOAD,
-          },
-          'warmup_seconds': WARMUP_SECONDS,
-          'benchmark_seconds': BENCHMARK_SECONDS
-      }
-      yield {
-          'name': 'cpp_protobuf_async_streaming_qps_unconstrained_%s'
-                  % secstr,
-          'num_servers': 1,
-          'num_clients': 0,
-          'client_config': {
-            'client_type': 'ASYNC_CLIENT',
-            'security_params': secargs,
-            'outstanding_rpcs_per_channel': DEEP,
-            'client_channels': WIDE,
-            'async_client_threads': 0,
-            'rpc_type': 'STREAMING',
-            'load_params': {
-              'closed_loop': {}
-            },
-            'payload_config': EMPTY_PROTO_PAYLOAD,
-            'histogram_params': HISTOGRAM_PARAMS,
-          },
-          'server_config': {
-            'server_type': 'ASYNC_SERVER',
-            'security_params': secargs,
-            'core_limit': SINGLE_MACHINE_CORES/2,
-            'async_server_threads': 0,
-          },
-          'warmup_seconds': WARMUP_SECONDS,
-          'benchmark_seconds': BENCHMARK_SECONDS
-      }
-      yield {
-          'name': 'cpp_protobuf_async_streaming_ping_pong_%s'
-                  % secstr,
-          'num_servers': 1,
-          'num_clients': 1,
-          'client_config': {
-            'client_type': 'ASYNC_CLIENT',
-            'security_params': secargs,
-            'outstanding_rpcs_per_channel': 1,
-            'client_channels': 1,
-            'async_client_threads': 1,
-            'rpc_type': 'STREAMING',
-            'load_params': {
-              'closed_loop': {}
-            },
-            'payload_config': EMPTY_PROTO_PAYLOAD,
-            'histogram_params': HISTOGRAM_PARAMS,
-          },
-          'server_config': {
-            'server_type': 'ASYNC_SERVER',
-            'security_params': secargs,
-            'core_limit': 1,
-            'async_server_threads': 1,
-          },
-          'warmup_seconds': WARMUP_SECONDS,
-          'benchmark_seconds': BENCHMARK_SECONDS
-      }
-      yield {
-          'name': 'cpp_protobuf_sync_unary_ping_pong_%s'
-                  % secstr,
-          'num_servers': 1,
-          'num_clients': 1,
-          'client_config': {
-            'client_type': 'SYNC_CLIENT',
-            'security_params': secargs,
-            'outstanding_rpcs_per_channel': 1,
-            'client_channels': 1,
-            'async_client_threads': 0,
-            'rpc_type': 'UNARY',
-            'load_params': {
-              'closed_loop': {}
-            },
-            'payload_config': EMPTY_PROTO_PAYLOAD,
-            'histogram_params': HISTOGRAM_PARAMS,
-          },
-          'server_config': {
-            'server_type': 'SYNC_SERVER',
-            'security_params': secargs,
-            'core_limit': 1,
-            'async_server_threads': 0,
-          },
-          'warmup_seconds': WARMUP_SECONDS,
-          'benchmark_seconds': BENCHMARK_SECONDS
-      }
-      yield {
-          'name': 'cpp_protobuf_async_unary_ping_pong_%s'
-                  % secstr,
-          'num_servers': 1,
-          'num_clients': 1,
-          'client_config': {
-            'client_type': 'ASYNC_CLIENT',
-            'security_params': secargs,
-            'outstanding_rpcs_per_channel': 1,
-            'client_channels': 1,
-            'async_client_threads': 1,
-            'rpc_type': 'UNARY',
-            'load_params': {
-              'closed_loop': {}
-            },
-            'payload_config': EMPTY_PROTO_PAYLOAD,
-            'histogram_params': HISTOGRAM_PARAMS,
-          },
-          'server_config': {
-            'server_type': 'ASYNC_SERVER',
-            'security_params': secargs,
-            'core_limit': 1,
-            'async_server_threads': 1,
-          },
-          'warmup_seconds': WARMUP_SECONDS,
-          'benchmark_seconds': BENCHMARK_SECONDS
-      }
+      secstr = 'secure' if secure else 'insecure'
+      smoketest_categories = [SMOKETEST] if secure else None
+
+      yield _ping_pong_scenario(
+          'cpp_generic_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING',
+          client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
+          use_generic_payload=True, server_core_limit=1, async_server_threads=1,
+          secure=secure,
+          categories=smoketest_categories)
+
+      yield _ping_pong_scenario(
+          'cpp_protobuf_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING',
+          client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
+          server_core_limit=1, async_server_threads=1,
+          secure=secure)
+
+      yield _ping_pong_scenario(
+          'cpp_protobuf_async_unary_ping_pong_%s' % secstr, rpc_type='UNARY',
+          client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
+          server_core_limit=1, async_server_threads=1,
+          secure=secure,
+          categories=smoketest_categories)
+
+      yield _ping_pong_scenario(
+          'cpp_protobuf_sync_unary_ping_pong_%s' % secstr, rpc_type='UNARY',
+          client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
+          server_core_limit=1, async_server_threads=1,
+          secure=secure)
+
+      yield _ping_pong_scenario(
+          'cpp_protobuf_async_unary_qps_unconstrained_%s' % secstr, rpc_type='UNARY',
+          client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
+          server_core_limit=SINGLE_MACHINE_CORES/2,
+          use_unconstrained_client=True,
+          secure=secure,
+          categories=smoketest_categories)
+
+      yield _ping_pong_scenario(
+          'cpp_protobuf_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING',
+          client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
+          server_core_limit=SINGLE_MACHINE_CORES/2,
+          use_unconstrained_client=True,
+          secure=secure)
+
+      yield _ping_pong_scenario(
+          'cpp_generic_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING',
+          client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
+          use_unconstrained_client=True, use_generic_payload=True,
+          server_core_limit=SINGLE_MACHINE_CORES/2,
+          secure=secure,
+          categories=smoketest_categories)
+
+      yield _ping_pong_scenario(
+          'cpp_generic_async_streaming_qps_one_server_core_%s' % secstr, rpc_type='STREAMING',
+          client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
+          use_unconstrained_client=True, use_generic_payload=True,
+          server_core_limit=1, async_server_threads=1,
+          secure=secure)
 
   def __str__(self):
     return 'c++'
@@ -299,113 +240,48 @@ class CSharpLanguage:
     return 100
 
   def scenarios(self):
-    secargs = SECURE_SECARGS
-    yield {
-        'name': 'csharp_generic_async_streaming_ping_pong',
-        'num_servers': 1,
-        'num_clients': 1,
-        'client_config': {
-          'client_type': 'ASYNC_CLIENT',
-          'security_params': secargs,
-          'outstanding_rpcs_per_channel': 1,
-          'client_channels': 1,
-          'async_client_threads': 1,
-          'rpc_type': 'STREAMING',
-          'load_params': {
-            'closed_loop': {}
-          },
-          'payload_config': EMPTY_GENERIC_PAYLOAD,
-          'histogram_params': HISTOGRAM_PARAMS,
-        },
-        'server_config': {
-          'server_type': 'ASYNC_GENERIC_SERVER',
-          'security_params': secargs,
-          'core_limit': 0,
-          'async_server_threads': 0,
-          'payload_config': EMPTY_GENERIC_PAYLOAD,
-        },
-        'warmup_seconds': WARMUP_SECONDS,
-        'benchmark_seconds': BENCHMARK_SECONDS
-    }
-    yield {
-        'name': 'csharp_protobuf_async_unary_ping_pong',
-        'num_servers': 1,
-        'num_clients': 1,
-        'client_config': {
-          'client_type': 'ASYNC_CLIENT',
-          'security_params': secargs,
-          'outstanding_rpcs_per_channel': 1,
-          'client_channels': 1,
-          'async_client_threads': 1,
-          'rpc_type': 'UNARY',
-          'load_params': {
-            'closed_loop': {}
-          },
-          'payload_config': EMPTY_PROTO_PAYLOAD,
-          'histogram_params': HISTOGRAM_PARAMS,
-        },
-        'server_config': {
-          'server_type': 'ASYNC_SERVER',
-          'security_params': secargs,
-          'core_limit': 0,
-          'async_server_threads': 0,
-        },
-        'warmup_seconds': WARMUP_SECONDS,
-        'benchmark_seconds': BENCHMARK_SECONDS
-    }
-    yield {
-        'name': 'csharp_protobuf_sync_to_async_unary_ping_pong',
-        'num_servers': 1,
-        'num_clients': 1,
-        'client_config': {
-          'client_type': 'SYNC_CLIENT',
-          'security_params': secargs,
-          'outstanding_rpcs_per_channel': 1,
-          'client_channels': 1,
-          'async_client_threads': 1,
-          'rpc_type': 'UNARY',
-          'load_params': {
-            'closed_loop': {}
-          },
-          'payload_config': EMPTY_PROTO_PAYLOAD,
-          'histogram_params': HISTOGRAM_PARAMS,
-        },
-        'server_config': {
-          'server_type': 'ASYNC_SERVER',
-          'security_params': secargs,
-          'core_limit': 0,
-          'async_server_threads': 0,
-        },
-        'warmup_seconds': WARMUP_SECONDS,
-        'benchmark_seconds': BENCHMARK_SECONDS
-    }
-    yield {
-        'name': 'csharp_to_cpp_protobuf_sync_unary_ping_pong',
-        'num_servers': 1,
-        'num_clients': 1,
-        'client_config': {
-          'client_type': 'SYNC_CLIENT',
-          'security_params': secargs,
-          'outstanding_rpcs_per_channel': 1,
-          'client_channels': 1,
-          'async_client_threads': 1,
-          'rpc_type': 'UNARY',
-          'load_params': {
-            'closed_loop': {}
-          },
-          'payload_config': EMPTY_PROTO_PAYLOAD,
-          'histogram_params': HISTOGRAM_PARAMS,
-        },
-        'server_config': {
-          'server_type': 'SYNC_SERVER',
-          'security_params': secargs,
-          'core_limit': 1,
-          'async_server_threads': 1,
-        },
-        'warmup_seconds': WARMUP_SECONDS,
-        'benchmark_seconds': BENCHMARK_SECONDS,
-        'SERVER_LANGUAGE': 'c++'  # recognized by run_performance_tests.py
-    }
+    yield _ping_pong_scenario(
+        'csharp_generic_async_streaming_ping_pong', rpc_type='STREAMING',
+        client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
+        use_generic_payload=True,
+        categories=[SMOKETEST])
+
+    yield _ping_pong_scenario(
+        'csharp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING',
+        client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER')
+
+    yield _ping_pong_scenario(
+        'csharp_protobuf_async_unary_ping_pong', rpc_type='UNARY',
+        client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
+        categories=[SMOKETEST])
+
+    yield _ping_pong_scenario(
+        'csharp_protobuf_sync_to_async_unary_ping_pong', rpc_type='UNARY',
+        client_type='SYNC_CLIENT', server_type='ASYNC_SERVER')
+
+    # TODO(jtattermusch): scenario works locally but fails on jenkins
+    #yield _ping_pong_scenario(
+    #    'csharp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY',
+    #    client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
+    #    use_unconstrained_client=True,
+    #    categories=[SMOKETEST])
+
+    # TODO(jtattermusch): scenario works locally but fails on jenkins
+    #yield _ping_pong_scenario(
+    #    'csharp_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING',
+    #    client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
+    #    use_unconstrained_client=True)
+
+    yield _ping_pong_scenario(
+        'csharp_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY',
+        client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
+        server_language='c++', server_core_limit=1, async_server_threads=1,
+        categories=[SMOKETEST])
+
+    yield _ping_pong_scenario(
+        'csharp_to_cpp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING',
+        client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
+        server_language='c++', server_core_limit=1, async_server_threads=1)
 
   def __str__(self):
     return 'csharp'
@@ -424,34 +300,45 @@ class NodeLanguage:
     return 200
 
   def scenarios(self):
-    # TODO(jtattermusch): add more scenarios
-    secargs = SECURE_SECARGS
-    yield {
-        'name': 'node_protobuf_unary_ping_pong',
-        'num_servers': 1,
-        'num_clients': 1,
-        'client_config': {
-          'client_type': 'ASYNC_CLIENT',
-          'security_params': secargs,
-          'outstanding_rpcs_per_channel': 1,
-          'client_channels': 1,
-          'async_client_threads': 1,
-          'rpc_type': 'UNARY',
-          'load_params': {
-            'closed_loop': {}
-          },
-          'payload_config': EMPTY_PROTO_PAYLOAD,
-          'histogram_params': HISTOGRAM_PARAMS,
-        },
-        'server_config': {
-          'server_type': 'ASYNC_SERVER',
-          'security_params': secargs,
-          'core_limit': 0,
-          'async_server_threads': 1,
-        },
-        'warmup_seconds': WARMUP_SECONDS,
-        'benchmark_seconds': BENCHMARK_SECONDS
-    }
+    # TODO(jtattermusch): make this scenario work
+    #yield _ping_pong_scenario(
+    #    'node_generic_async_streaming_ping_pong', rpc_type='STREAMING',
+    #    client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
+    #    use_generic_payload=True)
+
+    # TODO(jtattermusch): make this scenario work
+    #yield _ping_pong_scenario(
+    #    'node_protobuf_async_streaming_ping_pong', rpc_type='STREAMING',
+    #    client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER')
+
+    yield _ping_pong_scenario(
+        'node_protobuf_unary_ping_pong', rpc_type='UNARY',
+        client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
+        categories=[SMOKETEST])
+
+    yield _ping_pong_scenario(
+        'node_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY',
+        client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
+        use_unconstrained_client=True,
+        categories=[SMOKETEST])
+
+    # TODO(jtattermusch): make this scenario work
+    #yield _ping_pong_scenario(
+    #    'node_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING',
+    #    client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
+    #    use_unconstrained_client=True)
+
+    # TODO(jtattermusch): make this scenario work
+    #yield _ping_pong_scenario(
+    #    'node_to_cpp_protobuf_async_unary_ping_pong', rpc_type='UNARY',
+    #    client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
+    #    server_language='c++', server_core_limit=1, async_server_threads=1)
+
+    # TODO(jtattermusch): make this scenario work
+    #yield _ping_pong_scenario(
+    #    'node_to_cpp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING',
+    #    client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
+    #    server_language='c++', server_core_limit=1, async_server_threads=1)
 
   def __str__(self):
     return 'node'
@@ -468,114 +355,52 @@ class PythonLanguage:
     return 500
 
   def scenarios(self):
-    yield {
-        'name': 'python_to_cpp_protobuf_streaming_ping_pong',
-        'num_servers': 1,
-        'num_clients': 1,
-        'client_config': {
-          'client_type': 'ASYNC_CLIENT',
-          'security_params': SECURE_SECARGS,
-          'outstanding_rpcs_per_channel': 1,
-          'client_channels': 1,
-          'async_client_threads': 1,
-          'rpc_type': 'STREAMING',
-          'load_params': {
-            'closed_loop': {}
-          },
-          'payload_config': EMPTY_PROTO_PAYLOAD,
-          'histogram_params': HISTOGRAM_PARAMS,
-        },
-        'server_config': {
-          'server_type': 'SYNC_SERVER',
-          'security_params': SECURE_SECARGS,
-          'core_limit': 0,
-          'async_server_threads': 1,
-        },
-        'warmup_seconds': WARMUP_SECONDS,
-        'benchmark_seconds': BENCHMARK_SECONDS,
-        'SERVER_LANGUAGE': 'c++' 
-    }
-    yield {
-        'name': 'python_protobuf_sync_unary_ping_pong',
-        'num_servers': 1,
-        'num_clients': 1,
-        'client_config': {
-          'client_type': 'SYNC_CLIENT',
-          'security_params': SECURE_SECARGS,
-          'outstanding_rpcs_per_channel': 1,
-          'client_channels': 1,
-          'async_client_threads': 1,
-          'rpc_type': 'UNARY',
-          'load_params': {
-            'closed_loop': {}
-          },
-          'payload_config': EMPTY_PROTO_PAYLOAD,
-          'histogram_params': HISTOGRAM_PARAMS,
-        },
-        'server_config': {
-          'server_type': 'SYNC_SERVER',
-          'security_params': SECURE_SECARGS,
-          'core_limit': 0,
-          'async_server_threads': 1,
-        },
-        'warmup_seconds': WARMUP_SECONDS,
-        'benchmark_seconds': BENCHMARK_SECONDS,
-    }
-    yield {
-        'name': 'python_protobuf_async_unary_ping_pong',
-        'num_servers': 1,
-        'num_clients': 1,
-        'client_config': {
-          'client_type': 'ASYNC_CLIENT',
-          'security_params': SECURE_SECARGS,
-          'outstanding_rpcs_per_channel': 1,
-          'client_channels': 1,
-          'async_client_threads': 1,
-            'rpc_type': 'UNARY',
-            'load_params': {
-              'closed_loop': {}
-            },
-            'payload_config': EMPTY_PROTO_PAYLOAD,
-            'histogram_params': HISTOGRAM_PARAMS,
-          },
-          'server_config': {
-            'server_type': 'SYNC_SERVER',
-            'security_params': SECURE_SECARGS,
-            'core_limit': 0,
-            'async_server_threads': 1,
-          },
-          'warmup_seconds': WARMUP_SECONDS,
-          'benchmark_seconds': BENCHMARK_SECONDS,
-    }
-    yield {
-        'name': 'python_to_cpp_single_channel_throughput',
-        'num_servers': 1,
-        'num_clients': 1,
-        'client_config': {
-          'client_type': 'ASYNC_CLIENT',
-          'security_params': SECURE_SECARGS,
-          'outstanding_rpcs_per_channel': 1,
-          'client_channels': 1,
-          'async_client_threads': 1,
-          'rpc_type': 'STREAMING',
-          'load_params': {
-            'closed_loop': {}
-          },
-          'payload_config': BIG_GENERIC_PAYLOAD,
-          'histogram_params': HISTOGRAM_PARAMS,
-        },
-        'server_config': {
-          'server_type': 'ASYNC_GENERIC_SERVER',
-          'security_params': SECURE_SECARGS,
-          'core_limit': SINGLE_MACHINE_CORES/2,
-          'async_server_threads': 1,
-          'payload_config': BIG_GENERIC_PAYLOAD,
-        },
-        'warmup_seconds': WARMUP_SECONDS,
-        'benchmark_seconds': BENCHMARK_SECONDS,
-        'SERVER_LANGUAGE': 'c++'
-    }
-      
+    # TODO(jtattermusch): this scenario reports QPS 0.0
+    yield _ping_pong_scenario(
+        'python_generic_async_streaming_ping_pong', rpc_type='STREAMING',
+        client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
+        use_generic_payload=True,
+        categories=[SMOKETEST])
+
+    # TODO(jtattermusch): make this scenario work
+    #yield _ping_pong_scenario(
+    #    'python_protobuf_async_streaming_ping_pong', rpc_type='STREAMING',
+    #    client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER')
+
+    # TODO(jtattermusch): make this scenario work
+    #yield _ping_pong_scenario(
+    #    'python_protobuf_async_unary_ping_pong', rpc_type='UNARY',
+    #    client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER')
+
+    yield _ping_pong_scenario(
+        'python_protobuf_sync_unary_ping_pong', rpc_type='UNARY',
+        client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
+        categories=[SMOKETEST])
+
+    # TODO(jtattermusch): make this scenario work
+    #yield _ping_pong_scenario(
+    #    'python_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY',
+    #    client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
+    #    use_unconstrained_client=True)
+
+    # TODO(jtattermusch): make this scenario work
+    #yield _ping_pong_scenario(
+    #    'python_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING',
+    #    client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
+    #    use_unconstrained_client=True)
+
+    yield _ping_pong_scenario(
+        'python_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY',
+        client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
+        server_language='c++', server_core_limit=1, async_server_threads=1,
+        categories=[SMOKETEST])
+
+    # TODO(jtattermusch): make this scenario work
+    #yield _ping_pong_scenario(
+    #    'python_to_cpp_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING',
+    #    client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
+    #    server_language='c++', server_core_limit=1, async_server_threads=1)
+
   def __str__(self):
     return 'python'
 
@@ -592,34 +417,37 @@ class RubyLanguage:
     return 300
 
   def scenarios(self):
-    # TODO(jtattermusch): add more scenarios
-    secargs = SECURE_SECARGS
-    yield {
-        'name': 'ruby_protobuf_unary_ping_pong',
-        'num_servers': 1,
-        'num_clients': 1,
-        'client_config': {
-          'client_type': 'SYNC_CLIENT',
-          'security_params': secargs,
-          'outstanding_rpcs_per_channel': 1,
-          'client_channels': 1,
-          'async_client_threads': 1,
-          'rpc_type': 'UNARY',
-          'load_params': {
-            'closed_loop': {}
-          },
-          'payload_config': EMPTY_PROTO_PAYLOAD,
-          'histogram_params': HISTOGRAM_PARAMS,
-        },
-        'server_config': {
-          'server_type': 'SYNC_SERVER',
-          'security_params': secargs,
-          'core_limit': 0,
-          'async_server_threads': 1,
-        },
-        'warmup_seconds': WARMUP_SECONDS,
-        'benchmark_seconds': BENCHMARK_SECONDS
-    }
+    yield _ping_pong_scenario(
+        'ruby_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING',
+        client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
+        categories=[SMOKETEST])
+
+    yield _ping_pong_scenario(
+        'ruby_protobuf_unary_ping_pong', rpc_type='UNARY',
+        client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
+        categories=[SMOKETEST])
+
+    # TODO: scenario reports QPS of 0.0
+    #yield _ping_pong_scenario(
+    #    'ruby_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY',
+    #    client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
+    #    use_unconstrained_client=True)
+
+    # TODO: scenario reports QPS of 0.0
+    #yield _ping_pong_scenario(
+    #    'ruby_protobuf_sync_streaming_qps_unconstrained', rpc_type='STREAMING',
+    #    client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
+    #    use_unconstrained_client=True)
+
+    yield _ping_pong_scenario(
+        'ruby_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY',
+        client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
+        server_language='c++', server_core_limit=1, async_server_threads=1)
+
+    yield _ping_pong_scenario(
+        'ruby_to_cpp_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING',
+        client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
+        server_language='c++', server_core_limit=1, async_server_threads=1)
 
   def __str__(self):
     return 'ruby'
@@ -638,41 +466,63 @@ class JavaLanguage:
     return 400
 
   def scenarios(self):
-    # TODO(jtattermusch): add more scenarios
     for secure in [True, False]:
-      if secure:
-        secstr = 'secure'
-        secargs = SECURE_SECARGS
-      else:
-        secstr = 'insecure'
-        secargs = None
-
-      yield {
-          'name': 'java_protobuf_unary_ping_pong_%s' % secstr,
-          'num_servers': 1,
-          'num_clients': 1,
-          'client_config': {
-            'client_type': 'SYNC_CLIENT',
-            'security_params': secargs,
-            'outstanding_rpcs_per_channel': 1,
-            'client_channels': 1,
-            'async_client_threads': 1,
-            'rpc_type': 'UNARY',
-            'load_params': {
-              'closed_loop': {}
-            },
-            'payload_config': EMPTY_PROTO_PAYLOAD,
-            'histogram_params': HISTOGRAM_PARAMS,
-          },
-          'server_config': {
-            'server_type': 'SYNC_SERVER',
-            'security_params': secargs,
-            'core_limit': 0,
-            'async_server_threads': 1,
-          },
-          'warmup_seconds': JAVA_WARMUP_SECONDS,
-          'benchmark_seconds': BENCHMARK_SECONDS
-      }
+      secstr = 'secure' if secure else 'insecure'
+      smoketest_categories = [SMOKETEST] if secure else None
+
+      yield _ping_pong_scenario(
+          'java_generic_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING',
+          client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
+          use_generic_payload=True, async_server_threads=1,
+          secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS,
+          categories=smoketest_categories)
+
+      yield _ping_pong_scenario(
+          'java_protobuf_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING',
+          client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
+          async_server_threads=1,
+          secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS)
+
+      yield _ping_pong_scenario(
+          'java_protobuf_async_unary_ping_pong_%s' % secstr, rpc_type='UNARY',
+          client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
+          async_server_threads=1,
+          secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS,
+          categories=smoketest_categories)
+
+      yield _ping_pong_scenario(
+          'java_protobuf_unary_ping_pong_%s' % secstr, rpc_type='UNARY',
+          client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
+          async_server_threads=1,
+          secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS)
+
+      yield _ping_pong_scenario(
+          'java_protobuf_async_unary_qps_unconstrained_%s' % secstr, rpc_type='UNARY',
+          client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
+          use_unconstrained_client=True,
+          secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS,
+          categories=smoketest_categories)
+
+      yield _ping_pong_scenario(
+          'java_protobuf_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING',
+          client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
+          use_unconstrained_client=True,
+          secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS)
+
+      yield _ping_pong_scenario(
+          'java_generic_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING',
+          client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
+          use_unconstrained_client=True, use_generic_payload=True,
+          secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS)
+
+      yield _ping_pong_scenario(
+          'java_generic_async_streaming_qps_one_server_core_%s' % secstr, rpc_type='STREAMING',
+          client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
+          use_unconstrained_client=True, use_generic_payload=True,
+          async_server_threads=1,
+          secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS)
+
+      # TODO(jtattermusch): add scenarios java vs C++ 
 
   def __str__(self):
     return 'java'
diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py
index 5519666e84..b1f5889e54 100755
--- a/tools/run_tests/run_performance_tests.py
+++ b/tools/run_tests/run_performance_tests.py
@@ -89,7 +89,7 @@ def create_qpsworker_job(language, shortname=None,
   jobspec = jobset.JobSpec(
       cmdline=cmdline,
       shortname=shortname,
-      timeout_seconds=30*60)
+      timeout_seconds=2*60*60)
   return QpsWorkerJob(jobspec, language, host_and_port)
 
 
@@ -244,7 +244,7 @@ def start_qpsworkers(languages, worker_hosts):
 
 
 def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*',
-                     bq_result_table=None):
+                     category='all', bq_result_table=None):
   """Create jobspecs for scenarios to run."""
   all_workers = [worker
                  for workers in workers_by_lang.values()
@@ -253,25 +253,26 @@ def create_scenarios(languages, workers_by_lang, remote_host=None, regex='.*',
   for language in languages:
     for scenario_json in language.scenarios():
       if re.search(args.regex, scenario_json['name']):
-        workers = workers_by_lang[str(language)]
-        # 'SERVER_LANGUAGE' is an indicator for this script to pick
-        # a server in different language. It doesn't belong to the Scenario
-        # schema, so we also need to remove it.
-        custom_server_lang = scenario_json.pop('SERVER_LANGUAGE', None)
-        if custom_server_lang:
-          if not workers_by_lang.get(custom_server_lang, []):
-            print 'Warning: Skipping scenario %s as' % scenario_json['name']
-            print('SERVER_LANGUAGE is set to %s yet the language has '
-                  'not been selected with -l' % custom_server_lang)
-            continue
-          for idx in range(0, scenario_json['num_servers']):
-            # replace first X workers by workers of a different language
-            workers[idx] = workers_by_lang[custom_server_lang][idx]
-        scenario = create_scenario_jobspec(scenario_json,
-                                           workers,
-                                           remote_host=remote_host,
-                                           bq_result_table=bq_result_table)
-        scenarios.append(scenario)
+        if category in scenario_json.get('CATEGORIES', []) or category == 'all':
+          workers = workers_by_lang[str(language)]
+          # 'SERVER_LANGUAGE' is an indicator for this script to pick
+          # a server in different language.
+          custom_server_lang = scenario_json.get('SERVER_LANGUAGE', None)
+          scenario_json = scenario_config.remove_nonproto_fields(scenario_json)
+          if custom_server_lang:
+            if not workers_by_lang.get(custom_server_lang, []):
+              print 'Warning: Skipping scenario %s as' % scenario_json['name']
+              print('SERVER_LANGUAGE is set to %s yet the language has '
+                    'not been selected with -l' % custom_server_lang)
+              continue
+            for idx in range(0, scenario_json['num_servers']):
+              # replace first X workers by workers of a different language
+              workers[idx] = workers_by_lang[custom_server_lang][idx]
+          scenario = create_scenario_jobspec(scenario_json,
+                                             workers,
+                                             remote_host=remote_host,
+                                             bq_result_table=bq_result_table)
+          scenarios.append(scenario)
 
   # the very last scenario requests shutting down the workers.
   scenarios.append(create_quit_jobspec(all_workers, remote_host=remote_host))
@@ -298,7 +299,7 @@ argp = argparse.ArgumentParser(description='Run performance tests.')
 argp.add_argument('-l', '--language',
                   choices=['all'] + sorted(scenario_config.LANGUAGES.keys()),
                   nargs='+',
-                  default=['all'],
+                  required=True,
                   help='Languages to benchmark.')
 argp.add_argument('--remote_driver_host',
                   default=None,
@@ -311,6 +312,10 @@ argp.add_argument('-r', '--regex', default='.*', type=str,
                   help='Regex to select scenarios to run.')
 argp.add_argument('--bq_result_table', default=None, type=str,
                   help='Bigquery "dataset.table" to upload results to.')
+argp.add_argument('--category',
+                  choices=['smoketest','all'],
+                  default='smoketest',
+                  help='Select a category of tests to run. Smoketest runs by default.')
 
 args = argp.parse_args()
 
@@ -354,6 +359,7 @@ try:
                                workers_by_lang=worker_addresses,
                                remote_host=args.remote_driver_host,
                                regex=args.regex,
+                               category=args.category,
                                bq_result_table=args.bq_result_table)
   if not scenarios:
     raise Exception('No scenarios to run')
diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json
index cf1154426f..79a9e73117 100644
--- a/tools/run_tests/tests.json
+++ b/tools/run_tests/tests.json
@@ -22942,7 +22942,7 @@
   {
     "args": [
       "--scenario_json", 
-      "'{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
+      "'{\"name\": \"cpp_protobuf_async_streaming_ping_pong_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
     ], 
     "boringssl": true, 
     "ci_platforms": [
@@ -22963,12 +22963,12 @@
       "posix", 
       "windows"
     ], 
-    "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_unconstrained_secure"
+    "shortname": "json_run_localhost:cpp_protobuf_async_streaming_ping_pong_secure"
   }, 
   {
     "args": [
       "--scenario_json", 
-      "'{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
+      "'{\"name\": \"cpp_protobuf_async_unary_ping_pong_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
     ], 
     "boringssl": true, 
     "ci_platforms": [
@@ -22989,12 +22989,12 @@
       "posix", 
       "windows"
     ], 
-    "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_one_server_core_secure"
+    "shortname": "json_run_localhost:cpp_protobuf_async_unary_ping_pong_secure"
   }, 
   {
     "args": [
       "--scenario_json", 
-      "'{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
+      "'{\"name\": \"cpp_protobuf_sync_unary_ping_pong_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
     ], 
     "boringssl": true, 
     "ci_platforms": [
@@ -23015,12 +23015,12 @@
       "posix", 
       "windows"
     ], 
-    "shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_secure"
+    "shortname": "json_run_localhost:cpp_protobuf_sync_unary_ping_pong_secure"
   }, 
   {
     "args": [
       "--scenario_json", 
-      "'{\"name\": \"cpp_protobuf_async_streaming_ping_pong_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
+      "'{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
     ], 
     "boringssl": true, 
     "ci_platforms": [
@@ -23041,12 +23041,12 @@
       "posix", 
       "windows"
     ], 
-    "shortname": "json_run_localhost:cpp_protobuf_async_streaming_ping_pong_secure"
+    "shortname": "json_run_localhost:cpp_protobuf_async_unary_qps_unconstrained_secure"
   }, 
   {
     "args": [
       "--scenario_json", 
-      "'{\"name\": \"cpp_protobuf_sync_unary_ping_pong_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
+      "'{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
     ], 
     "boringssl": true, 
     "ci_platforms": [
@@ -23067,12 +23067,12 @@
       "posix", 
       "windows"
     ], 
-    "shortname": "json_run_localhost:cpp_protobuf_sync_unary_ping_pong_secure"
+    "shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_secure"
   }, 
   {
     "args": [
       "--scenario_json", 
-      "'{\"name\": \"cpp_protobuf_async_unary_ping_pong_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
+      "'{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 4, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
     ], 
     "boringssl": true, 
     "ci_platforms": [
@@ -23093,7 +23093,33 @@
       "posix", 
       "windows"
     ], 
-    "shortname": "json_run_localhost:cpp_protobuf_async_unary_ping_pong_secure"
+    "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_unconstrained_secure"
+  }, 
+  {
+    "args": [
+      "--scenario_json", 
+      "'{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
+    ], 
+    "boringssl": true, 
+    "ci_platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "cpu_cost": 1000.0, 
+    "defaults": "boringssl", 
+    "exclude_configs": [], 
+    "flaky": false, 
+    "language": "c++", 
+    "name": "json_run_localhost", 
+    "platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_one_server_core_secure"
   }, 
   {
     "args": [
@@ -23124,7 +23150,7 @@
   {
     "args": [
       "--scenario_json", 
-      "'{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 4, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
+      "'{\"name\": \"cpp_protobuf_async_streaming_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
     ], 
     "boringssl": true, 
     "ci_platforms": [
@@ -23145,12 +23171,12 @@
       "posix", 
       "windows"
     ], 
-    "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_unconstrained_insecure"
+    "shortname": "json_run_localhost:cpp_protobuf_async_streaming_ping_pong_insecure"
   }, 
   {
     "args": [
       "--scenario_json", 
-      "'{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
+      "'{\"name\": \"cpp_protobuf_async_unary_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
     ], 
     "boringssl": true, 
     "ci_platforms": [
@@ -23171,12 +23197,12 @@
       "posix", 
       "windows"
     ], 
-    "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_one_server_core_insecure"
+    "shortname": "json_run_localhost:cpp_protobuf_async_unary_ping_pong_insecure"
   }, 
   {
     "args": [
       "--scenario_json", 
-      "'{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 4, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
+      "'{\"name\": \"cpp_protobuf_sync_unary_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
     ], 
     "boringssl": true, 
     "ci_platforms": [
@@ -23197,12 +23223,12 @@
       "posix", 
       "windows"
     ], 
-    "shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_insecure"
+    "shortname": "json_run_localhost:cpp_protobuf_sync_unary_ping_pong_insecure"
   }, 
   {
     "args": [
       "--scenario_json", 
-      "'{\"name\": \"cpp_protobuf_async_streaming_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
+      "'{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 4, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
     ], 
     "boringssl": true, 
     "ci_platforms": [
@@ -23223,12 +23249,12 @@
       "posix", 
       "windows"
     ], 
-    "shortname": "json_run_localhost:cpp_protobuf_async_streaming_ping_pong_insecure"
+    "shortname": "json_run_localhost:cpp_protobuf_async_unary_qps_unconstrained_insecure"
   }, 
   {
     "args": [
       "--scenario_json", 
-      "'{\"name\": \"cpp_protobuf_sync_unary_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 1, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
+      "'{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 4, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
     ], 
     "boringssl": true, 
     "ci_platforms": [
@@ -23249,12 +23275,12 @@
       "posix", 
       "windows"
     ], 
-    "shortname": "json_run_localhost:cpp_protobuf_sync_unary_ping_pong_insecure"
+    "shortname": "json_run_localhost:cpp_protobuf_async_streaming_qps_unconstrained_insecure"
   }, 
   {
     "args": [
       "--scenario_json", 
-      "'{\"name\": \"cpp_protobuf_async_unary_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
+      "'{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 4, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
     ], 
     "boringssl": true, 
     "ci_platforms": [
@@ -23275,7 +23301,33 @@
       "posix", 
       "windows"
     ], 
-    "shortname": "json_run_localhost:cpp_protobuf_async_unary_ping_pong_insecure"
+    "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_unconstrained_insecure"
+  }, 
+  {
+    "args": [
+      "--scenario_json", 
+      "'{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
+    ], 
+    "boringssl": true, 
+    "ci_platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "cpu_cost": 1000.0, 
+    "defaults": "boringssl", 
+    "exclude_configs": [], 
+    "flaky": false, 
+    "language": "c++", 
+    "name": "json_run_localhost", 
+    "platforms": [
+      "linux", 
+      "mac", 
+      "posix", 
+      "windows"
+    ], 
+    "shortname": "json_run_localhost:cpp_generic_async_streaming_qps_one_server_core_insecure"
   }, 
   {
     "args": [