yapf tools/run_tests/performance

5 files changed, 1657 insertions, 1150 deletions

diff --git a/tools/run_tests/performance/bq_upload_result.py b/tools/run_tests/performance/bq_upload_result.py
index 31819d6159..6702587557 100755
--- a/tools/run_tests/performance/bq_upload_result.py
+++ b/tools/run_tests/performance/bq_upload_result.py
@@ -26,146 +26,161 @@ import time
 import uuid
 import massage_qps_stats
 
-
-gcp_utils_dir = os.path.abspath(os.path.join(
-    os.path.dirname(__file__), '../../gcp/utils'))
+gcp_utils_dir = os.path.abspath(
+    os.path.join(os.path.dirname(__file__), '../../gcp/utils'))
 sys.path.append(gcp_utils_dir)
 import big_query_utils
 
-
-_PROJECT_ID='grpc-testing'
+_PROJECT_ID = 'grpc-testing'
 
 
 def _upload_netperf_latency_csv_to_bigquery(dataset_id, table_id, result_file):
-  with open(result_file, 'r') as f:
-    (col1, col2, col3) = f.read().split(',')
-    latency50 = float(col1.strip()) * 1000
-    latency90 = float(col2.strip()) * 1000
-    latency99 = float(col3.strip()) * 1000
-
-    scenario_result = {
-        'scenario': {
-          'name': 'netperf_tcp_rr'
-        },
-        'summary': {
-          'latency50': latency50,
-          'latency90': latency90,
-          'latency99': latency99
+    with open(result_file, 'r') as f:
+        (col1, col2, col3) = f.read().split(',')
+        latency50 = float(col1.strip()) * 1000
+        latency90 = float(col2.strip()) * 1000
+        latency99 = float(col3.strip()) * 1000
+
+        scenario_result = {
+            'scenario': {
+                'name': 'netperf_tcp_rr'
+            },
+            'summary': {
+                'latency50': latency50,
+                'latency90': latency90,
+                'latency99': latency99
+            }
         }
-    }
 
-  bq = big_query_utils.create_big_query()
-  _create_results_table(bq, dataset_id, table_id)
+    bq = big_query_utils.create_big_query()
+    _create_results_table(bq, dataset_id, table_id)
 
-  if not _insert_result(bq, dataset_id, table_id, scenario_result, flatten=False):
-    print('Error uploading result to bigquery.')
-    sys.exit(1)
+    if not _insert_result(
+            bq, dataset_id, table_id, scenario_result, flatten=False):
+        print('Error uploading result to bigquery.')
+        sys.exit(1)
 
 
 def _upload_scenario_result_to_bigquery(dataset_id, table_id, result_file):
-  with open(result_file, 'r') as f:
-    scenario_result = json.loads(f.read())
+    with open(result_file, 'r') as f:
+        scenario_result = json.loads(f.read())
 
-  bq = big_query_utils.create_big_query()
-  _create_results_table(bq, dataset_id, table_id)
+    bq = big_query_utils.create_big_query()
+    _create_results_table(bq, dataset_id, table_id)
 
-  if not _insert_result(bq, dataset_id, table_id, scenario_result):
-    print('Error uploading result to bigquery.')
-    sys.exit(1)
+    if not _insert_result(bq, dataset_id, table_id, scenario_result):
+        print('Error uploading result to bigquery.')
+        sys.exit(1)
 
 
 def _insert_result(bq, dataset_id, table_id, scenario_result, flatten=True):
-  if flatten:
-    _flatten_result_inplace(scenario_result)
-  _populate_metadata_inplace(scenario_result)
-  row = big_query_utils.make_row(str(uuid.uuid4()), scenario_result)
-  return big_query_utils.insert_rows(bq,
-                                     _PROJECT_ID,
-                                     dataset_id,
-                                     table_id,
-                                     [row])
+    if flatten:
+        _flatten_result_inplace(scenario_result)
+    _populate_metadata_inplace(scenario_result)
+    row = big_query_utils.make_row(str(uuid.uuid4()), scenario_result)
+    return big_query_utils.insert_rows(bq, _PROJECT_ID, dataset_id, table_id,
+                                       [row])
 
 
 def _create_results_table(bq, dataset_id, table_id):
-  with open(os.path.dirname(__file__) + '/scenario_result_schema.json', 'r') as f:
-    table_schema = json.loads(f.read())
-  desc = 'Results of performance benchmarks.'
-  return big_query_utils.create_table2(bq, _PROJECT_ID, dataset_id,
-                               table_id, table_schema, desc)
+    with open(os.path.dirname(__file__) + '/scenario_result_schema.json',
+              'r') as f:
+        table_schema = json.loads(f.read())
+    desc = 'Results of performance benchmarks.'
+    return big_query_utils.create_table2(bq, _PROJECT_ID, dataset_id, table_id,
+                                         table_schema, desc)
 
 
 def _flatten_result_inplace(scenario_result):
-  """Bigquery is not really great for handling deeply nested data
+    """Bigquery is not really great for handling deeply nested data
   and repeated fields. To maintain values of some fields while keeping
   the schema relatively simple, we artificially leave some of the fields
   as JSON strings.
   """
-  scenario_result['scenario']['clientConfig'] = json.dumps(scenario_result['scenario']['clientConfig'])
-  scenario_result['scenario']['serverConfig'] = json.dumps(scenario_result['scenario']['serverConfig'])
-  scenario_result['latencies'] = json.dumps(scenario_result['latencies'])
-  scenario_result['serverCpuStats'] = []
-  for stats in scenario_result['serverStats']:
-    scenario_result['serverCpuStats'].append(dict())
-    scenario_result['serverCpuStats'][-1]['totalCpuTime'] = stats.pop('totalCpuTime', None)
-    scenario_result['serverCpuStats'][-1]['idleCpuTime'] = stats.pop('idleCpuTime', None)
-  for stats in scenario_result['clientStats']:
-    stats['latencies'] = json.dumps(stats['latencies'])
-    stats.pop('requestResults', None)
-  scenario_result['serverCores'] = json.dumps(scenario_result['serverCores'])
-  scenario_result['clientSuccess'] = json.dumps(scenario_result['clientSuccess'])
-  scenario_result['serverSuccess'] = json.dumps(scenario_result['serverSuccess'])
-  scenario_result['requestResults'] = json.dumps(scenario_result.get('requestResults', []))
-  scenario_result['serverCpuUsage'] = scenario_result['summary'].pop('serverCpuUsage', None)
-  scenario_result['summary'].pop('successfulRequestsPerSecond', None)
-  scenario_result['summary'].pop('failedRequestsPerSecond', None)
-  massage_qps_stats.massage_qps_stats(scenario_result)
+    scenario_result['scenario']['clientConfig'] = json.dumps(
+        scenario_result['scenario']['clientConfig'])
+    scenario_result['scenario']['serverConfig'] = json.dumps(
+        scenario_result['scenario']['serverConfig'])
+    scenario_result['latencies'] = json.dumps(scenario_result['latencies'])
+    scenario_result['serverCpuStats'] = []
+    for stats in scenario_result['serverStats']:
+        scenario_result['serverCpuStats'].append(dict())
+        scenario_result['serverCpuStats'][-1]['totalCpuTime'] = stats.pop(
+            'totalCpuTime', None)
+        scenario_result['serverCpuStats'][-1]['idleCpuTime'] = stats.pop(
+            'idleCpuTime', None)
+    for stats in scenario_result['clientStats']:
+        stats['latencies'] = json.dumps(stats['latencies'])
+        stats.pop('requestResults', None)
+    scenario_result['serverCores'] = json.dumps(scenario_result['serverCores'])
+    scenario_result['clientSuccess'] = json.dumps(
+        scenario_result['clientSuccess'])
+    scenario_result['serverSuccess'] = json.dumps(
+        scenario_result['serverSuccess'])
+    scenario_result['requestResults'] = json.dumps(
+        scenario_result.get('requestResults', []))
+    scenario_result['serverCpuUsage'] = scenario_result['summary'].pop(
+        'serverCpuUsage', None)
+    scenario_result['summary'].pop('successfulRequestsPerSecond', None)
+    scenario_result['summary'].pop('failedRequestsPerSecond', None)
+    massage_qps_stats.massage_qps_stats(scenario_result)
 
 
 def _populate_metadata_inplace(scenario_result):
-  """Populates metadata based on environment variables set by Jenkins."""
-  # NOTE: Grabbing the Jenkins environment variables will only work if the
-  # driver is running locally on the same machine where Jenkins has started
-  # the job. For our setup, this is currently the case, so just assume that.
-  build_number = os.getenv('BUILD_NUMBER')
-  build_url = os.getenv('BUILD_URL')
-  job_name = os.getenv('JOB_NAME')
-  git_commit = os.getenv('GIT_COMMIT')
-  # actual commit is the actual head of PR that is getting tested
-  git_actual_commit = os.getenv('ghprbActualCommit')
-
-  utc_timestamp = str(calendar.timegm(time.gmtime()))
-  metadata = {'created': utc_timestamp}
-
-  if build_number:
-    metadata['buildNumber'] = build_number
-  if build_url:
-    metadata['buildUrl'] = build_url
-  if job_name:
-    metadata['jobName'] = job_name
-  if git_commit:
-    metadata['gitCommit'] = git_commit
-  if git_actual_commit:
-    metadata['gitActualCommit'] = git_actual_commit
-
-  scenario_result['metadata'] = metadata
+    """Populates metadata based on environment variables set by Jenkins."""
+    # NOTE: Grabbing the Jenkins environment variables will only work if the
+    # driver is running locally on the same machine where Jenkins has started
+    # the job. For our setup, this is currently the case, so just assume that.
+    build_number = os.getenv('BUILD_NUMBER')
+    build_url = os.getenv('BUILD_URL')
+    job_name = os.getenv('JOB_NAME')
+    git_commit = os.getenv('GIT_COMMIT')
+    # actual commit is the actual head of PR that is getting tested
+    git_actual_commit = os.getenv('ghprbActualCommit')
+
+    utc_timestamp = str(calendar.timegm(time.gmtime()))
+    metadata = {'created': utc_timestamp}
+
+    if build_number:
+        metadata['buildNumber'] = build_number
+    if build_url:
+        metadata['buildUrl'] = build_url
+    if job_name:
+        metadata['jobName'] = job_name
+    if git_commit:
+        metadata['gitCommit'] = git_commit
+    if git_actual_commit:
+        metadata['gitActualCommit'] = git_actual_commit
+
+    scenario_result['metadata'] = metadata
 
 
 argp = argparse.ArgumentParser(description='Upload result to big query.')
-argp.add_argument('--bq_result_table', required=True, default=None, type=str,
-                  help='Bigquery "dataset.table" to upload results to.')
-argp.add_argument('--file_to_upload', default='scenario_result.json', type=str,
-                  help='Report file to upload.')
-argp.add_argument('--file_format',
-                  choices=['scenario_result','netperf_latency_csv'],
-                  default='scenario_result',
-                  help='Format of the file to upload.')
+argp.add_argument(
+    '--bq_result_table',
+    required=True,
+    default=None,
+    type=str,
+    help='Bigquery "dataset.table" to upload results to.')
+argp.add_argument(
+    '--file_to_upload',
+    default='scenario_result.json',
+    type=str,
+    help='Report file to upload.')
+argp.add_argument(
+    '--file_format',
+    choices=['scenario_result', 'netperf_latency_csv'],
+    default='scenario_result',
+    help='Format of the file to upload.')
 
 args = argp.parse_args()
 
 dataset_id, table_id = args.bq_result_table.split('.', 2)
 
 if args.file_format == 'netperf_latency_csv':
-  _upload_netperf_latency_csv_to_bigquery(dataset_id, table_id, args.file_to_upload)
+    _upload_netperf_latency_csv_to_bigquery(dataset_id, table_id,
+                                            args.file_to_upload)
 else:
-  _upload_scenario_result_to_bigquery(dataset_id, table_id, args.file_to_upload)
+    _upload_scenario_result_to_bigquery(dataset_id, table_id,
+                                        args.file_to_upload)
 print('Successfully uploaded %s to BigQuery.\n' % args.file_to_upload)
diff --git a/tools/run_tests/performance/massage_qps_stats.py b/tools/run_tests/performance/massage_qps_stats.py
index 48c57581a5..37f6e7aae0 100644
--- a/tools/run_tests/performance/massage_qps_stats.py
+++ b/tools/run_tests/performance/massage_qps_stats.py
@@ -15,182 +15,456 @@
 # Autogenerated by tools/codegen/core/gen_stats_data.py
 
 import massage_qps_stats_helpers
+
+
 def massage_qps_stats(scenario_result):
-  for stats in scenario_result["serverStats"] + scenario_result["clientStats"]:
-    if "coreStats" not in stats: return
-    core_stats = stats["coreStats"]
-    del stats["coreStats"]
-    stats["core_client_calls_created"] = massage_qps_stats_helpers.counter(core_stats, "client_calls_created")
-    stats["core_server_calls_created"] = massage_qps_stats_helpers.counter(core_stats, "server_calls_created")
-    stats["core_cqs_created"] = massage_qps_stats_helpers.counter(core_stats, "cqs_created")
-    stats["core_client_channels_created"] = massage_qps_stats_helpers.counter(core_stats, "client_channels_created")
-    stats["core_client_subchannels_created"] = massage_qps_stats_helpers.counter(core_stats, "client_subchannels_created")
-    stats["core_server_channels_created"] = massage_qps_stats_helpers.counter(core_stats, "server_channels_created")
-    stats["core_syscall_poll"] = massage_qps_stats_helpers.counter(core_stats, "syscall_poll")
-    stats["core_syscall_wait"] = massage_qps_stats_helpers.counter(core_stats, "syscall_wait")
-    stats["core_pollset_kick"] = massage_qps_stats_helpers.counter(core_stats, "pollset_kick")
-    stats["core_pollset_kicked_without_poller"] = massage_qps_stats_helpers.counter(core_stats, "pollset_kicked_without_poller")
-    stats["core_pollset_kicked_again"] = massage_qps_stats_helpers.counter(core_stats, "pollset_kicked_again")
-    stats["core_pollset_kick_wakeup_fd"] = massage_qps_stats_helpers.counter(core_stats, "pollset_kick_wakeup_fd")
-    stats["core_pollset_kick_wakeup_cv"] = massage_qps_stats_helpers.counter(core_stats, "pollset_kick_wakeup_cv")
-    stats["core_pollset_kick_own_thread"] = massage_qps_stats_helpers.counter(core_stats, "pollset_kick_own_thread")
-    stats["core_histogram_slow_lookups"] = massage_qps_stats_helpers.counter(core_stats, "histogram_slow_lookups")
-    stats["core_syscall_write"] = massage_qps_stats_helpers.counter(core_stats, "syscall_write")
-    stats["core_syscall_read"] = massage_qps_stats_helpers.counter(core_stats, "syscall_read")
-    stats["core_tcp_backup_pollers_created"] = massage_qps_stats_helpers.counter(core_stats, "tcp_backup_pollers_created")
-    stats["core_tcp_backup_poller_polls"] = massage_qps_stats_helpers.counter(core_stats, "tcp_backup_poller_polls")
-    stats["core_http2_op_batches"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_batches")
-    stats["core_http2_op_cancel"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_cancel")
-    stats["core_http2_op_send_initial_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_send_initial_metadata")
-    stats["core_http2_op_send_message"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_send_message")
-    stats["core_http2_op_send_trailing_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_send_trailing_metadata")
-    stats["core_http2_op_recv_initial_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_recv_initial_metadata")
-    stats["core_http2_op_recv_message"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_recv_message")
-    stats["core_http2_op_recv_trailing_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_op_recv_trailing_metadata")
-    stats["core_http2_settings_writes"] = massage_qps_stats_helpers.counter(core_stats, "http2_settings_writes")
-    stats["core_http2_pings_sent"] = massage_qps_stats_helpers.counter(core_stats, "http2_pings_sent")
-    stats["core_http2_writes_begun"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_begun")
-    stats["core_http2_writes_offloaded"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_offloaded")
-    stats["core_http2_writes_continued"] = massage_qps_stats_helpers.counter(core_stats, "http2_writes_continued")
-    stats["core_http2_partial_writes"] = massage_qps_stats_helpers.counter(core_stats, "http2_partial_writes")
-    stats["core_http2_initiate_write_due_to_initial_write"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_initial_write")
-    stats["core_http2_initiate_write_due_to_start_new_stream"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_start_new_stream")
-    stats["core_http2_initiate_write_due_to_send_message"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_send_message")
-    stats["core_http2_initiate_write_due_to_send_initial_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_send_initial_metadata")
-    stats["core_http2_initiate_write_due_to_send_trailing_metadata"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_send_trailing_metadata")
-    stats["core_http2_initiate_write_due_to_retry_send_ping"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_retry_send_ping")
-    stats["core_http2_initiate_write_due_to_continue_pings"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_continue_pings")
-    stats["core_http2_initiate_write_due_to_goaway_sent"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_goaway_sent")
-    stats["core_http2_initiate_write_due_to_rst_stream"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_rst_stream")
-    stats["core_http2_initiate_write_due_to_close_from_api"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_close_from_api")
-    stats["core_http2_initiate_write_due_to_stream_flow_control"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_stream_flow_control")
-    stats["core_http2_initiate_write_due_to_transport_flow_control"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_transport_flow_control")
-    stats["core_http2_initiate_write_due_to_send_settings"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_send_settings")
-    stats["core_http2_initiate_write_due_to_bdp_estimator_ping"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_bdp_estimator_ping")
-    stats["core_http2_initiate_write_due_to_flow_control_unstalled_by_setting"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_flow_control_unstalled_by_setting")
-    stats["core_http2_initiate_write_due_to_flow_control_unstalled_by_update"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_flow_control_unstalled_by_update")
-    stats["core_http2_initiate_write_due_to_application_ping"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_application_ping")
-    stats["core_http2_initiate_write_due_to_keepalive_ping"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_keepalive_ping")
-    stats["core_http2_initiate_write_due_to_transport_flow_control_unstalled"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_transport_flow_control_unstalled")
-    stats["core_http2_initiate_write_due_to_ping_response"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_ping_response")
-    stats["core_http2_initiate_write_due_to_force_rst_stream"] = massage_qps_stats_helpers.counter(core_stats, "http2_initiate_write_due_to_force_rst_stream")
-    stats["core_http2_spurious_writes_begun"] = massage_qps_stats_helpers.counter(core_stats, "http2_spurious_writes_begun")
-    stats["core_hpack_recv_indexed"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_indexed")
-    stats["core_hpack_recv_lithdr_incidx"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_lithdr_incidx")
-    stats["core_hpack_recv_lithdr_incidx_v"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_lithdr_incidx_v")
-    stats["core_hpack_recv_lithdr_notidx"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_lithdr_notidx")
-    stats["core_hpack_recv_lithdr_notidx_v"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_lithdr_notidx_v")
-    stats["core_hpack_recv_lithdr_nvridx"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_lithdr_nvridx")
-    stats["core_hpack_recv_lithdr_nvridx_v"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_lithdr_nvridx_v")
-    stats["core_hpack_recv_uncompressed"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_uncompressed")
-    stats["core_hpack_recv_huffman"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_huffman")
-    stats["core_hpack_recv_binary"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_binary")
-    stats["core_hpack_recv_binary_base64"] = massage_qps_stats_helpers.counter(core_stats, "hpack_recv_binary_base64")
-    stats["core_hpack_send_indexed"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_indexed")
-    stats["core_hpack_send_lithdr_incidx"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_lithdr_incidx")
-    stats["core_hpack_send_lithdr_incidx_v"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_lithdr_incidx_v")
-    stats["core_hpack_send_lithdr_notidx"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_lithdr_notidx")
-    stats["core_hpack_send_lithdr_notidx_v"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_lithdr_notidx_v")
-    stats["core_hpack_send_lithdr_nvridx"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_lithdr_nvridx")
-    stats["core_hpack_send_lithdr_nvridx_v"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_lithdr_nvridx_v")
-    stats["core_hpack_send_uncompressed"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_uncompressed")
-    stats["core_hpack_send_huffman"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_huffman")
-    stats["core_hpack_send_binary"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_binary")
-    stats["core_hpack_send_binary_base64"] = massage_qps_stats_helpers.counter(core_stats, "hpack_send_binary_base64")
-    stats["core_combiner_locks_initiated"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_initiated")
-    stats["core_combiner_locks_scheduled_items"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_scheduled_items")
-    stats["core_combiner_locks_scheduled_final_items"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_scheduled_final_items")
-    stats["core_combiner_locks_offloaded"] = massage_qps_stats_helpers.counter(core_stats, "combiner_locks_offloaded")
-    stats["core_call_combiner_locks_initiated"] = massage_qps_stats_helpers.counter(core_stats, "call_combiner_locks_initiated")
-    stats["core_call_combiner_locks_scheduled_items"] = massage_qps_stats_helpers.counter(core_stats, "call_combiner_locks_scheduled_items")
-    stats["core_call_combiner_set_notify_on_cancel"] = massage_qps_stats_helpers.counter(core_stats, "call_combiner_set_notify_on_cancel")
-    stats["core_call_combiner_cancelled"] = massage_qps_stats_helpers.counter(core_stats, "call_combiner_cancelled")
-    stats["core_executor_scheduled_short_items"] = massage_qps_stats_helpers.counter(core_stats, "executor_scheduled_short_items")
-    stats["core_executor_scheduled_long_items"] = massage_qps_stats_helpers.counter(core_stats, "executor_scheduled_long_items")
-    stats["core_executor_scheduled_to_self"] = massage_qps_stats_helpers.counter(core_stats, "executor_scheduled_to_self")
-    stats["core_executor_wakeup_initiated"] = massage_qps_stats_helpers.counter(core_stats, "executor_wakeup_initiated")
-    stats["core_executor_queue_drained"] = massage_qps_stats_helpers.counter(core_stats, "executor_queue_drained")
-    stats["core_executor_push_retries"] = massage_qps_stats_helpers.counter(core_stats, "executor_push_retries")
-    stats["core_server_requested_calls"] = massage_qps_stats_helpers.counter(core_stats, "server_requested_calls")
-    stats["core_server_slowpath_requests_queued"] = massage_qps_stats_helpers.counter(core_stats, "server_slowpath_requests_queued")
-    stats["core_cq_ev_queue_trylock_failures"] = massage_qps_stats_helpers.counter(core_stats, "cq_ev_queue_trylock_failures")
-    stats["core_cq_ev_queue_trylock_successes"] = massage_qps_stats_helpers.counter(core_stats, "cq_ev_queue_trylock_successes")
-    stats["core_cq_ev_queue_transient_pop_failures"] = massage_qps_stats_helpers.counter(core_stats, "cq_ev_queue_transient_pop_failures")
-    h = massage_qps_stats_helpers.histogram(core_stats, "call_initial_size")
-    stats["core_call_initial_size"] = ",".join("%f" % x for x in h.buckets)
-    stats["core_call_initial_size_bkts"] = ",".join("%f" % x for x in h.boundaries)
-    stats["core_call_initial_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
-    stats["core_call_initial_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
-    stats["core_call_initial_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
-    h = massage_qps_stats_helpers.histogram(core_stats, "poll_events_returned")
-    stats["core_poll_events_returned"] = ",".join("%f" % x for x in h.buckets)
-    stats["core_poll_events_returned_bkts"] = ",".join("%f" % x for x in h.boundaries)
-    stats["core_poll_events_returned_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
-    stats["core_poll_events_returned_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
-    stats["core_poll_events_returned_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
-    h = massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size")
-    stats["core_tcp_write_size"] = ",".join("%f" % x for x in h.buckets)
-    stats["core_tcp_write_size_bkts"] = ",".join("%f" % x for x in h.boundaries)
-    stats["core_tcp_write_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
-    stats["core_tcp_write_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
-    stats["core_tcp_write_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
-    h = massage_qps_stats_helpers.histogram(core_stats, "tcp_write_iov_size")
-    stats["core_tcp_write_iov_size"] = ",".join("%f" % x for x in h.buckets)
-    stats["core_tcp_write_iov_size_bkts"] = ",".join("%f" % x for x in h.boundaries)
-    stats["core_tcp_write_iov_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
-    stats["core_tcp_write_iov_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
-    stats["core_tcp_write_iov_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
-    h = massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size")
-    stats["core_tcp_read_size"] = ",".join("%f" % x for x in h.buckets)
-    stats["core_tcp_read_size_bkts"] = ",".join("%f" % x for x in h.boundaries)
-    stats["core_tcp_read_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
-    stats["core_tcp_read_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
-    stats["core_tcp_read_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
-    h = massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer")
-    stats["core_tcp_read_offer"] = ",".join("%f" % x for x in h.buckets)
-    stats["core_tcp_read_offer_bkts"] = ",".join("%f" % x for x in h.boundaries)
-    stats["core_tcp_read_offer_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
-    stats["core_tcp_read_offer_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
-    stats["core_tcp_read_offer_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
-    h = massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer_iov_size")
-    stats["core_tcp_read_offer_iov_size"] = ",".join("%f" % x for x in h.buckets)
-    stats["core_tcp_read_offer_iov_size_bkts"] = ",".join("%f" % x for x in h.boundaries)
-    stats["core_tcp_read_offer_iov_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
-    stats["core_tcp_read_offer_iov_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
-    stats["core_tcp_read_offer_iov_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
-    h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_size")
-    stats["core_http2_send_message_size"] = ",".join("%f" % x for x in h.buckets)
-    stats["core_http2_send_message_size_bkts"] = ",".join("%f" % x for x in h.boundaries)
-    stats["core_http2_send_message_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
-    stats["core_http2_send_message_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
-    stats["core_http2_send_message_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
-    h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_initial_metadata_per_write")
-    stats["core_http2_send_initial_metadata_per_write"] = ",".join("%f" % x for x in h.buckets)
-    stats["core_http2_send_initial_metadata_per_write_bkts"] = ",".join("%f" % x for x in h.boundaries)
-    stats["core_http2_send_initial_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
-    stats["core_http2_send_initial_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
-    stats["core_http2_send_initial_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
-    h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_message_per_write")
-    stats["core_http2_send_message_per_write"] = ",".join("%f" % x for x in h.buckets)
-    stats["core_http2_send_message_per_write_bkts"] = ",".join("%f" % x for x in h.boundaries)
-    stats["core_http2_send_message_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
-    stats["core_http2_send_message_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
-    stats["core_http2_send_message_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
-    h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_trailing_metadata_per_write")
-    stats["core_http2_send_trailing_metadata_per_write"] = ",".join("%f" % x for x in h.buckets)
-    stats["core_http2_send_trailing_metadata_per_write_bkts"] = ",".join("%f" % x for x in h.boundaries)
-    stats["core_http2_send_trailing_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
-    stats["core_http2_send_trailing_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
-    stats["core_http2_send_trailing_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
-    h = massage_qps_stats_helpers.histogram(core_stats, "http2_send_flowctl_per_write")
-    stats["core_http2_send_flowctl_per_write"] = ",".join("%f" % x for x in h.buckets)
-    stats["core_http2_send_flowctl_per_write_bkts"] = ",".join("%f" % x for x in h.boundaries)
-    stats["core_http2_send_flowctl_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
-    stats["core_http2_send_flowctl_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
-    stats["core_http2_send_flowctl_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
-    h = massage_qps_stats_helpers.histogram(core_stats, "server_cqs_checked")
-    stats["core_server_cqs_checked"] = ",".join("%f" % x for x in h.buckets)
-    stats["core_server_cqs_checked_bkts"] = ",".join("%f" % x for x in h.boundaries)
-    stats["core_server_cqs_checked_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
-    stats["core_server_cqs_checked_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
-    stats["core_server_cqs_checked_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
+    for stats in scenario_result["serverStats"] + scenario_result[
+            "clientStats"]:
+        if "coreStats" not in stats: return
+        core_stats = stats["coreStats"]
+        del stats["coreStats"]
+        stats["core_client_calls_created"] = massage_qps_stats_helpers.counter(
+            core_stats, "client_calls_created")
+        stats["core_server_calls_created"] = massage_qps_stats_helpers.counter(
+            core_stats, "server_calls_created")
+        stats["core_cqs_created"] = massage_qps_stats_helpers.counter(
+            core_stats, "cqs_created")
+        stats[
+            "core_client_channels_created"] = massage_qps_stats_helpers.counter(
+                core_stats, "client_channels_created")
+        stats[
+            "core_client_subchannels_created"] = massage_qps_stats_helpers.counter(
+                core_stats, "client_subchannels_created")
+        stats[
+            "core_server_channels_created"] = massage_qps_stats_helpers.counter(
+                core_stats, "server_channels_created")
+        stats["core_syscall_poll"] = massage_qps_stats_helpers.counter(
+            core_stats, "syscall_poll")
+        stats["core_syscall_wait"] = massage_qps_stats_helpers.counter(
+            core_stats, "syscall_wait")
+        stats["core_pollset_kick"] = massage_qps_stats_helpers.counter(
+            core_stats, "pollset_kick")
+        stats[
+            "core_pollset_kicked_without_poller"] = massage_qps_stats_helpers.counter(
+                core_stats, "pollset_kicked_without_poller")
+        stats["core_pollset_kicked_again"] = massage_qps_stats_helpers.counter(
+            core_stats, "pollset_kicked_again")
+        stats[
+            "core_pollset_kick_wakeup_fd"] = massage_qps_stats_helpers.counter(
+                core_stats, "pollset_kick_wakeup_fd")
+        stats[
+            "core_pollset_kick_wakeup_cv"] = massage_qps_stats_helpers.counter(
+                core_stats, "pollset_kick_wakeup_cv")
+        stats[
+            "core_pollset_kick_own_thread"] = massage_qps_stats_helpers.counter(
+                core_stats, "pollset_kick_own_thread")
+        stats[
+            "core_histogram_slow_lookups"] = massage_qps_stats_helpers.counter(
+                core_stats, "histogram_slow_lookups")
+        stats["core_syscall_write"] = massage_qps_stats_helpers.counter(
+            core_stats, "syscall_write")
+        stats["core_syscall_read"] = massage_qps_stats_helpers.counter(
+            core_stats, "syscall_read")
+        stats[
+            "core_tcp_backup_pollers_created"] = massage_qps_stats_helpers.counter(
+                core_stats, "tcp_backup_pollers_created")
+        stats[
+            "core_tcp_backup_poller_polls"] = massage_qps_stats_helpers.counter(
+                core_stats, "tcp_backup_poller_polls")
+        stats["core_http2_op_batches"] = massage_qps_stats_helpers.counter(
+            core_stats, "http2_op_batches")
+        stats["core_http2_op_cancel"] = massage_qps_stats_helpers.counter(
+            core_stats, "http2_op_cancel")
+        stats[
+            "core_http2_op_send_initial_metadata"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_op_send_initial_metadata")
+        stats["core_http2_op_send_message"] = massage_qps_stats_helpers.counter(
+            core_stats, "http2_op_send_message")
+        stats[
+            "core_http2_op_send_trailing_metadata"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_op_send_trailing_metadata")
+        stats[
+            "core_http2_op_recv_initial_metadata"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_op_recv_initial_metadata")
+        stats["core_http2_op_recv_message"] = massage_qps_stats_helpers.counter(
+            core_stats, "http2_op_recv_message")
+        stats[
+            "core_http2_op_recv_trailing_metadata"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_op_recv_trailing_metadata")
+        stats["core_http2_settings_writes"] = massage_qps_stats_helpers.counter(
+            core_stats, "http2_settings_writes")
+        stats["core_http2_pings_sent"] = massage_qps_stats_helpers.counter(
+            core_stats, "http2_pings_sent")
+        stats["core_http2_writes_begun"] = massage_qps_stats_helpers.counter(
+            core_stats, "http2_writes_begun")
+        stats[
+            "core_http2_writes_offloaded"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_writes_offloaded")
+        stats[
+            "core_http2_writes_continued"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_writes_continued")
+        stats["core_http2_partial_writes"] = massage_qps_stats_helpers.counter(
+            core_stats, "http2_partial_writes")
+        stats[
+            "core_http2_initiate_write_due_to_initial_write"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_initiate_write_due_to_initial_write")
+        stats[
+            "core_http2_initiate_write_due_to_start_new_stream"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_initiate_write_due_to_start_new_stream")
+        stats[
+            "core_http2_initiate_write_due_to_send_message"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_initiate_write_due_to_send_message")
+        stats[
+            "core_http2_initiate_write_due_to_send_initial_metadata"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_initiate_write_due_to_send_initial_metadata")
+        stats[
+            "core_http2_initiate_write_due_to_send_trailing_metadata"] = massage_qps_stats_helpers.counter(
+                core_stats,
+                "http2_initiate_write_due_to_send_trailing_metadata")
+        stats[
+            "core_http2_initiate_write_due_to_retry_send_ping"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_initiate_write_due_to_retry_send_ping")
+        stats[
+            "core_http2_initiate_write_due_to_continue_pings"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_initiate_write_due_to_continue_pings")
+        stats[
+            "core_http2_initiate_write_due_to_goaway_sent"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_initiate_write_due_to_goaway_sent")
+        stats[
+            "core_http2_initiate_write_due_to_rst_stream"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_initiate_write_due_to_rst_stream")
+        stats[
+            "core_http2_initiate_write_due_to_close_from_api"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_initiate_write_due_to_close_from_api")
+        stats[
+            "core_http2_initiate_write_due_to_stream_flow_control"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_initiate_write_due_to_stream_flow_control")
+        stats[
+            "core_http2_initiate_write_due_to_transport_flow_control"] = massage_qps_stats_helpers.counter(
+                core_stats,
+                "http2_initiate_write_due_to_transport_flow_control")
+        stats[
+            "core_http2_initiate_write_due_to_send_settings"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_initiate_write_due_to_send_settings")
+        stats[
+            "core_http2_initiate_write_due_to_bdp_estimator_ping"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_initiate_write_due_to_bdp_estimator_ping")
+        stats[
+            "core_http2_initiate_write_due_to_flow_control_unstalled_by_setting"] = massage_qps_stats_helpers.counter(
+                core_stats,
+                "http2_initiate_write_due_to_flow_control_unstalled_by_setting")
+        stats[
+            "core_http2_initiate_write_due_to_flow_control_unstalled_by_update"] = massage_qps_stats_helpers.counter(
+                core_stats,
+                "http2_initiate_write_due_to_flow_control_unstalled_by_update")
+        stats[
+            "core_http2_initiate_write_due_to_application_ping"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_initiate_write_due_to_application_ping")
+        stats[
+            "core_http2_initiate_write_due_to_keepalive_ping"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_initiate_write_due_to_keepalive_ping")
+        stats[
+            "core_http2_initiate_write_due_to_transport_flow_control_unstalled"] = massage_qps_stats_helpers.counter(
+                core_stats,
+                "http2_initiate_write_due_to_transport_flow_control_unstalled")
+        stats[
+            "core_http2_initiate_write_due_to_ping_response"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_initiate_write_due_to_ping_response")
+        stats[
+            "core_http2_initiate_write_due_to_force_rst_stream"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_initiate_write_due_to_force_rst_stream")
+        stats[
+            "core_http2_spurious_writes_begun"] = massage_qps_stats_helpers.counter(
+                core_stats, "http2_spurious_writes_begun")
+        stats["core_hpack_recv_indexed"] = massage_qps_stats_helpers.counter(
+            core_stats, "hpack_recv_indexed")
+        stats[
+            "core_hpack_recv_lithdr_incidx"] = massage_qps_stats_helpers.counter(
+                core_stats, "hpack_recv_lithdr_incidx")
+        stats[
+            "core_hpack_recv_lithdr_incidx_v"] = massage_qps_stats_helpers.counter(
+                core_stats, "hpack_recv_lithdr_incidx_v")
+        stats[
+            "core_hpack_recv_lithdr_notidx"] = massage_qps_stats_helpers.counter(
+                core_stats, "hpack_recv_lithdr_notidx")
+        stats[
+            "core_hpack_recv_lithdr_notidx_v"] = massage_qps_stats_helpers.counter(
+                core_stats, "hpack_recv_lithdr_notidx_v")
+        stats[
+            "core_hpack_recv_lithdr_nvridx"] = massage_qps_stats_helpers.counter(
+                core_stats, "hpack_recv_lithdr_nvridx")
+        stats[
+            "core_hpack_recv_lithdr_nvridx_v"] = massage_qps_stats_helpers.counter(
+                core_stats, "hpack_recv_lithdr_nvridx_v")
+        stats[
+            "core_hpack_recv_uncompressed"] = massage_qps_stats_helpers.counter(
+                core_stats, "hpack_recv_uncompressed")
+        stats["core_hpack_recv_huffman"] = massage_qps_stats_helpers.counter(
+            core_stats, "hpack_recv_huffman")
+        stats["core_hpack_recv_binary"] = massage_qps_stats_helpers.counter(
+            core_stats, "hpack_recv_binary")
+        stats[
+            "core_hpack_recv_binary_base64"] = massage_qps_stats_helpers.counter(
+                core_stats, "hpack_recv_binary_base64")
+        stats["core_hpack_send_indexed"] = massage_qps_stats_helpers.counter(
+            core_stats, "hpack_send_indexed")
+        stats[
+            "core_hpack_send_lithdr_incidx"] = massage_qps_stats_helpers.counter(
+                core_stats, "hpack_send_lithdr_incidx")
+        stats[
+            "core_hpack_send_lithdr_incidx_v"] = massage_qps_stats_helpers.counter(
+                core_stats, "hpack_send_lithdr_incidx_v")
+        stats[
+            "core_hpack_send_lithdr_notidx"] = massage_qps_stats_helpers.counter(
+                core_stats, "hpack_send_lithdr_notidx")
+        stats[
+            "core_hpack_send_lithdr_notidx_v"] = massage_qps_stats_helpers.counter(
+                core_stats, "hpack_send_lithdr_notidx_v")
+        stats[
+            "core_hpack_send_lithdr_nvridx"] = massage_qps_stats_helpers.counter(
+                core_stats, "hpack_send_lithdr_nvridx")
+        stats[
+            "core_hpack_send_lithdr_nvridx_v"] = massage_qps_stats_helpers.counter(
+                core_stats, "hpack_send_lithdr_nvridx_v")
+        stats[
+            "core_hpack_send_uncompressed"] = massage_qps_stats_helpers.counter(
+                core_stats, "hpack_send_uncompressed")
+        stats["core_hpack_send_huffman"] = massage_qps_stats_helpers.counter(
+            core_stats, "hpack_send_huffman")
+        stats["core_hpack_send_binary"] = massage_qps_stats_helpers.counter(
+            core_stats, "hpack_send_binary")
+        stats[
+            "core_hpack_send_binary_base64"] = massage_qps_stats_helpers.counter(
+                core_stats, "hpack_send_binary_base64")
+        stats[
+            "core_combiner_locks_initiated"] = massage_qps_stats_helpers.counter(
+                core_stats, "combiner_locks_initiated")
+        stats[
+            "core_combiner_locks_scheduled_items"] = massage_qps_stats_helpers.counter(
+                core_stats, "combiner_locks_scheduled_items")
+        stats[
+            "core_combiner_locks_scheduled_final_items"] = massage_qps_stats_helpers.counter(
+                core_stats, "combiner_locks_scheduled_final_items")
+        stats[
+            "core_combiner_locks_offloaded"] = massage_qps_stats_helpers.counter(
+                core_stats, "combiner_locks_offloaded")
+        stats[
+            "core_call_combiner_locks_initiated"] = massage_qps_stats_helpers.counter(
+                core_stats, "call_combiner_locks_initiated")
+        stats[
+            "core_call_combiner_locks_scheduled_items"] = massage_qps_stats_helpers.counter(
+                core_stats, "call_combiner_locks_scheduled_items")
+        stats[
+            "core_call_combiner_set_notify_on_cancel"] = massage_qps_stats_helpers.counter(
+                core_stats, "call_combiner_set_notify_on_cancel")
+        stats[
+            "core_call_combiner_cancelled"] = massage_qps_stats_helpers.counter(
+                core_stats, "call_combiner_cancelled")
+        stats[
+            "core_executor_scheduled_short_items"] = massage_qps_stats_helpers.counter(
+                core_stats, "executor_scheduled_short_items")
+        stats[
+            "core_executor_scheduled_long_items"] = massage_qps_stats_helpers.counter(
+                core_stats, "executor_scheduled_long_items")
+        stats[
+            "core_executor_scheduled_to_self"] = massage_qps_stats_helpers.counter(
+                core_stats, "executor_scheduled_to_self")
+        stats[
+            "core_executor_wakeup_initiated"] = massage_qps_stats_helpers.counter(
+                core_stats, "executor_wakeup_initiated")
+        stats[
+            "core_executor_queue_drained"] = massage_qps_stats_helpers.counter(
+                core_stats, "executor_queue_drained")
+        stats["core_executor_push_retries"] = massage_qps_stats_helpers.counter(
+            core_stats, "executor_push_retries")
+        stats[
+            "core_server_requested_calls"] = massage_qps_stats_helpers.counter(
+                core_stats, "server_requested_calls")
+        stats[
+            "core_server_slowpath_requests_queued"] = massage_qps_stats_helpers.counter(
+                core_stats, "server_slowpath_requests_queued")
+        stats[
+            "core_cq_ev_queue_trylock_failures"] = massage_qps_stats_helpers.counter(
+                core_stats, "cq_ev_queue_trylock_failures")
+        stats[
+            "core_cq_ev_queue_trylock_successes"] = massage_qps_stats_helpers.counter(
+                core_stats, "cq_ev_queue_trylock_successes")
+        stats[
+            "core_cq_ev_queue_transient_pop_failures"] = massage_qps_stats_helpers.counter(
+                core_stats, "cq_ev_queue_transient_pop_failures")
+        h = massage_qps_stats_helpers.histogram(core_stats, "call_initial_size")
+        stats["core_call_initial_size"] = ",".join("%f" % x for x in h.buckets)
+        stats["core_call_initial_size_bkts"] = ",".join("%f" % x
+                                                        for x in h.boundaries)
+        stats[
+            "core_call_initial_size_50p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 50, h.boundaries)
+        stats[
+            "core_call_initial_size_95p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 95, h.boundaries)
+        stats[
+            "core_call_initial_size_99p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 99, h.boundaries)
+        h = massage_qps_stats_helpers.histogram(core_stats,
+                                                "poll_events_returned")
+        stats["core_poll_events_returned"] = ",".join("%f" % x
+                                                      for x in h.buckets)
+        stats["core_poll_events_returned_bkts"] = ",".join(
+            "%f" % x for x in h.boundaries)
+        stats[
+            "core_poll_events_returned_50p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 50, h.boundaries)
+        stats[
+            "core_poll_events_returned_95p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 95, h.boundaries)
+        stats[
+            "core_poll_events_returned_99p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 99, h.boundaries)
+        h = massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size")
+        stats["core_tcp_write_size"] = ",".join("%f" % x for x in h.buckets)
+        stats["core_tcp_write_size_bkts"] = ",".join("%f" % x
+                                                     for x in h.boundaries)
+        stats["core_tcp_write_size_50p"] = massage_qps_stats_helpers.percentile(
+            h.buckets, 50, h.boundaries)
+        stats["core_tcp_write_size_95p"] = massage_qps_stats_helpers.percentile(
+            h.buckets, 95, h.boundaries)
+        stats["core_tcp_write_size_99p"] = massage_qps_stats_helpers.percentile(
+            h.buckets, 99, h.boundaries)
+        h = massage_qps_stats_helpers.histogram(core_stats,
+                                                "tcp_write_iov_size")
+        stats["core_tcp_write_iov_size"] = ",".join("%f" % x for x in h.buckets)
+        stats["core_tcp_write_iov_size_bkts"] = ",".join("%f" % x
+                                                         for x in h.boundaries)
+        stats[
+            "core_tcp_write_iov_size_50p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 50, h.boundaries)
+        stats[
+            "core_tcp_write_iov_size_95p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 95, h.boundaries)
+        stats[
+            "core_tcp_write_iov_size_99p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 99, h.boundaries)
+        h = massage_qps_stats_helpers.histogram(core_stats, "tcp_read_size")
+        stats["core_tcp_read_size"] = ",".join("%f" % x for x in h.buckets)
+        stats["core_tcp_read_size_bkts"] = ",".join("%f" % x
+                                                    for x in h.boundaries)
+        stats["core_tcp_read_size_50p"] = massage_qps_stats_helpers.percentile(
+            h.buckets, 50, h.boundaries)
+        stats["core_tcp_read_size_95p"] = massage_qps_stats_helpers.percentile(
+            h.buckets, 95, h.boundaries)
+        stats["core_tcp_read_size_99p"] = massage_qps_stats_helpers.percentile(
+            h.buckets, 99, h.boundaries)
+        h = massage_qps_stats_helpers.histogram(core_stats, "tcp_read_offer")
+        stats["core_tcp_read_offer"] = ",".join("%f" % x for x in h.buckets)
+        stats["core_tcp_read_offer_bkts"] = ",".join("%f" % x
+                                                     for x in h.boundaries)
+        stats["core_tcp_read_offer_50p"] = massage_qps_stats_helpers.percentile(
+            h.buckets, 50, h.boundaries)
+        stats["core_tcp_read_offer_95p"] = massage_qps_stats_helpers.percentile(
+            h.buckets, 95, h.boundaries)
+        stats["core_tcp_read_offer_99p"] = massage_qps_stats_helpers.percentile(
+            h.buckets, 99, h.boundaries)
+        h = massage_qps_stats_helpers.histogram(core_stats,
+                                                "tcp_read_offer_iov_size")
+        stats["core_tcp_read_offer_iov_size"] = ",".join("%f" % x
+                                                         for x in h.buckets)
+        stats["core_tcp_read_offer_iov_size_bkts"] = ",".join(
+            "%f" % x for x in h.boundaries)
+        stats[
+            "core_tcp_read_offer_iov_size_50p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 50, h.boundaries)
+        stats[
+            "core_tcp_read_offer_iov_size_95p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 95, h.boundaries)
+        stats[
+            "core_tcp_read_offer_iov_size_99p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 99, h.boundaries)
+        h = massage_qps_stats_helpers.histogram(core_stats,
+                                                "http2_send_message_size")
+        stats["core_http2_send_message_size"] = ",".join("%f" % x
+                                                         for x in h.buckets)
+        stats["core_http2_send_message_size_bkts"] = ",".join(
+            "%f" % x for x in h.boundaries)
+        stats[
+            "core_http2_send_message_size_50p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 50, h.boundaries)
+        stats[
+            "core_http2_send_message_size_95p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 95, h.boundaries)
+        stats[
+            "core_http2_send_message_size_99p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 99, h.boundaries)
+        h = massage_qps_stats_helpers.histogram(
+            core_stats, "http2_send_initial_metadata_per_write")
+        stats["core_http2_send_initial_metadata_per_write"] = ",".join(
+            "%f" % x for x in h.buckets)
+        stats["core_http2_send_initial_metadata_per_write_bkts"] = ",".join(
+            "%f" % x for x in h.boundaries)
+        stats[
+            "core_http2_send_initial_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 50, h.boundaries)
+        stats[
+            "core_http2_send_initial_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 95, h.boundaries)
+        stats[
+            "core_http2_send_initial_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 99, h.boundaries)
+        h = massage_qps_stats_helpers.histogram(core_stats,
+                                                "http2_send_message_per_write")
+        stats["core_http2_send_message_per_write"] = ",".join(
+            "%f" % x for x in h.buckets)
+        stats["core_http2_send_message_per_write_bkts"] = ",".join(
+            "%f" % x for x in h.boundaries)
+        stats[
+            "core_http2_send_message_per_write_50p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 50, h.boundaries)
+        stats[
+            "core_http2_send_message_per_write_95p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 95, h.boundaries)
+        stats[
+            "core_http2_send_message_per_write_99p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 99, h.boundaries)
+        h = massage_qps_stats_helpers.histogram(
+            core_stats, "http2_send_trailing_metadata_per_write")
+        stats["core_http2_send_trailing_metadata_per_write"] = ",".join(
+            "%f" % x for x in h.buckets)
+        stats["core_http2_send_trailing_metadata_per_write_bkts"] = ",".join(
+            "%f" % x for x in h.boundaries)
+        stats[
+            "core_http2_send_trailing_metadata_per_write_50p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 50, h.boundaries)
+        stats[
+            "core_http2_send_trailing_metadata_per_write_95p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 95, h.boundaries)
+        stats[
+            "core_http2_send_trailing_metadata_per_write_99p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 99, h.boundaries)
+        h = massage_qps_stats_helpers.histogram(core_stats,
+                                                "http2_send_flowctl_per_write")
+        stats["core_http2_send_flowctl_per_write"] = ",".join(
+            "%f" % x for x in h.buckets)
+        stats["core_http2_send_flowctl_per_write_bkts"] = ",".join(
+            "%f" % x for x in h.boundaries)
+        stats[
+            "core_http2_send_flowctl_per_write_50p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 50, h.boundaries)
+        stats[
+            "core_http2_send_flowctl_per_write_95p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 95, h.boundaries)
+        stats[
+            "core_http2_send_flowctl_per_write_99p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 99, h.boundaries)
+        h = massage_qps_stats_helpers.histogram(core_stats,
+                                                "server_cqs_checked")
+        stats["core_server_cqs_checked"] = ",".join("%f" % x for x in h.buckets)
+        stats["core_server_cqs_checked_bkts"] = ",".join("%f" % x
+                                                         for x in h.boundaries)
+        stats[
+            "core_server_cqs_checked_50p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 50, h.boundaries)
+        stats[
+            "core_server_cqs_checked_95p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 95, h.boundaries)
+        stats[
+            "core_server_cqs_checked_99p"] = massage_qps_stats_helpers.percentile(
+                h.buckets, 99, h.boundaries)
diff --git a/tools/run_tests/performance/massage_qps_stats_helpers.py b/tools/run_tests/performance/massage_qps_stats_helpers.py
index a2fe4ae6c3..108451cd55 100644
--- a/tools/run_tests/performance/massage_qps_stats_helpers.py
+++ b/tools/run_tests/performance/massage_qps_stats_helpers.py
@@ -14,44 +14,49 @@
 
 import collections
 
+
 def _threshold_for_count_below(buckets, boundaries, count_below):
-  count_so_far = 0
-  for lower_idx in range(0, len(buckets)):
-    count_so_far += buckets[lower_idx]
-    if count_so_far >= count_below:
-      break
-  if count_so_far == count_below:
-    # this bucket hits the threshold exactly... we should be midway through
-    # any run of zero values following the bucket
-    for upper_idx in range(lower_idx + 1, len(buckets)):
-      if buckets[upper_idx] != 0:
-        break
-    return (boundaries[lower_idx] + boundaries[upper_idx]) / 2.0
-  else:
-    # treat values as uniform throughout the bucket, and find where this value
-    # should lie
-    lower_bound = boundaries[lower_idx]
-    upper_bound = boundaries[lower_idx + 1]
-    return (upper_bound -
-           (upper_bound - lower_bound) * (count_so_far - count_below) /
-               float(buckets[lower_idx]))
+    count_so_far = 0
+    for lower_idx in range(0, len(buckets)):
+        count_so_far += buckets[lower_idx]
+        if count_so_far >= count_below:
+            break
+    if count_so_far == count_below:
+        # this bucket hits the threshold exactly... we should be midway through
+        # any run of zero values following the bucket
+        for upper_idx in range(lower_idx + 1, len(buckets)):
+            if buckets[upper_idx] != 0:
+                break
+        return (boundaries[lower_idx] + boundaries[upper_idx]) / 2.0
+    else:
+        # treat values as uniform throughout the bucket, and find where this value
+        # should lie
+        lower_bound = boundaries[lower_idx]
+        upper_bound = boundaries[lower_idx + 1]
+        return (upper_bound - (upper_bound - lower_bound) *
+                (count_so_far - count_below) / float(buckets[lower_idx]))
+
 
 def percentile(buckets, pctl, boundaries):
-  return _threshold_for_count_below(
-      buckets, boundaries, sum(buckets) * pctl / 100.0)
+    return _threshold_for_count_below(buckets, boundaries,
+                                      sum(buckets) * pctl / 100.0)
+
 
 def counter(core_stats, name):
-  for stat in core_stats['metrics']:
-    if stat['name'] == name:
-      return int(stat.get('count', 0))
+    for stat in core_stats['metrics']:
+        if stat['name'] == name:
+            return int(stat.get('count', 0))
+
 
 Histogram = collections.namedtuple('Histogram', 'buckets boundaries')
+
+
 def histogram(core_stats, name):
-  for stat in core_stats['metrics']:
-    if stat['name'] == name:
-      buckets = []
-      boundaries = []
-      for b in stat['histogram']['buckets']:
-        buckets.append(int(b.get('count', 0)))
-        boundaries.append(int(b.get('start', 0)))
-  return Histogram(buckets=buckets, boundaries=boundaries)
+    for stat in core_stats['metrics']:
+        if stat['name'] == name:
+            buckets = []
+            boundaries = []
+            for b in stat['histogram']['buckets']:
+                buckets.append(int(b.get('count', 0)))
+                boundaries.append(int(b.get('start', 0)))
+    return Histogram(buckets=buckets, boundaries=boundaries)
diff --git a/tools/run_tests/performance/patch_scenario_results_schema.py b/tools/run_tests/performance/patch_scenario_results_schema.py
index 81ba5381b3..2a2aadc242 100755
--- a/tools/run_tests/performance/patch_scenario_results_schema.py
+++ b/tools/run_tests/performance/patch_scenario_results_schema.py
@@ -25,27 +25,32 @@ import sys
 import time
 import uuid
 
-
-gcp_utils_dir = os.path.abspath(os.path.join(
-    os.path.dirname(__file__), '../../gcp/utils'))
+gcp_utils_dir = os.path.abspath(
+    os.path.join(os.path.dirname(__file__), '../../gcp/utils'))
 sys.path.append(gcp_utils_dir)
 import big_query_utils
 
+_PROJECT_ID = 'grpc-testing'
 
-_PROJECT_ID='grpc-testing'
 
 def _patch_results_table(dataset_id, table_id):
-  bq = big_query_utils.create_big_query()
-  with open(os.path.dirname(__file__) + '/scenario_result_schema.json', 'r') as f:
-    table_schema = json.loads(f.read())
-  desc = 'Results of performance benchmarks.'
-  return big_query_utils.patch_table(bq, _PROJECT_ID, dataset_id,
-                                     table_id, table_schema)
-
-
-argp = argparse.ArgumentParser(description='Patch schema of scenario results table.')
-argp.add_argument('--bq_result_table', required=True, default=None, type=str,
-                  help='Bigquery "dataset.table" to patch.')
+    bq = big_query_utils.create_big_query()
+    with open(os.path.dirname(__file__) + '/scenario_result_schema.json',
+              'r') as f:
+        table_schema = json.loads(f.read())
+    desc = 'Results of performance benchmarks.'
+    return big_query_utils.patch_table(bq, _PROJECT_ID, dataset_id, table_id,
+                                       table_schema)
+
+
+argp = argparse.ArgumentParser(
+    description='Patch schema of scenario results table.')
+argp.add_argument(
+    '--bq_result_table',
+    required=True,
+    default=None,
+    type=str,
+    help='Bigquery "dataset.table" to patch.')
 
 args = argp.parse_args()
 
diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py
index cafac3dfad..e92de5898c 100644
--- a/tools/run_tests/performance/scenario_config.py
+++ b/tools/run_tests/performance/scenario_config.py
@@ -16,66 +16,64 @@
 
 import math
 
-WARMUP_SECONDS=5
-JAVA_WARMUP_SECONDS=15  # Java needs more warmup time for JIT to kick in.
-BENCHMARK_SECONDS=30
-
-SMOKETEST='smoketest'
-SCALABLE='scalable'
-INPROC='inproc'
-SWEEP='sweep'
-DEFAULT_CATEGORIES=[SCALABLE, SMOKETEST]
-
-SECURE_SECARGS = {'use_test_ca': True,
-                  'server_host_override': 'foo.test.google.fr'}
+WARMUP_SECONDS = 5
+JAVA_WARMUP_SECONDS = 15  # Java needs more warmup time for JIT to kick in.
+BENCHMARK_SECONDS = 30
+
+SMOKETEST = 'smoketest'
+SCALABLE = 'scalable'
+INPROC = 'inproc'
+SWEEP = 'sweep'
+DEFAULT_CATEGORIES = [SCALABLE, SMOKETEST]
+
+SECURE_SECARGS = {
+    'use_test_ca': True,
+    'server_host_override': 'foo.test.google.fr'
+}
 
 HISTOGRAM_PARAMS = {
-  'resolution': 0.01,
-  'max_possible': 60e9,
+    'resolution': 0.01,
+    'max_possible': 60e9,
 }
 
 # target number of RPCs outstanding on across all client channels in
 # non-ping-pong tests (since we can only specify per-channel numbers, the
 # actual target will be slightly higher)
-OUTSTANDING_REQUESTS={
-    'async': 6400,
-    'async-limited': 800,
-    'sync': 1000
-}
+OUTSTANDING_REQUESTS = {'async': 6400, 'async-limited': 800, 'sync': 1000}
 
 # wide is the number of client channels in multi-channel tests (1 otherwise)
-WIDE=64
+WIDE = 64
 
 
 def _get_secargs(is_secure):
-  if is_secure:
-    return SECURE_SECARGS
-  else:
-    return None
+    if is_secure:
+        return SECURE_SECARGS
+    else:
+        return None
 
 
 def remove_nonproto_fields(scenario):
-  """Remove special-purpose that contains some extra info about the scenario
+    """Remove special-purpose that contains some extra info about the scenario
   but don't belong to the ScenarioConfig protobuf message"""
-  scenario.pop('CATEGORIES', None)
-  scenario.pop('CLIENT_LANGUAGE', None)
-  scenario.pop('SERVER_LANGUAGE', None)
-  scenario.pop('EXCLUDED_POLL_ENGINES', None)
-  return scenario
+    scenario.pop('CATEGORIES', None)
+    scenario.pop('CLIENT_LANGUAGE', None)
+    scenario.pop('SERVER_LANGUAGE', None)
+    scenario.pop('EXCLUDED_POLL_ENGINES', None)
+    return scenario
 
 
 def geometric_progression(start, stop, step):
-  n = start
-  while n < stop:
-    yield int(round(n))
-    n *= step
+    n = start
+    while n < stop:
+        yield int(round(n))
+        n *= step
 
 
 def _payload_type(use_generic_payload, req_size, resp_size):
     r = {}
     sizes = {
-      'req_size': req_size,
-      'resp_size': resp_size,
+        'req_size': req_size,
+        'resp_size': resp_size,
     }
     if use_generic_payload:
         r['bytebuf_params'] = sizes
@@ -83,6 +81,7 @@ def _payload_type(use_generic_payload, req_size, resp_size):
         r['simple_params'] = sizes
     return r
 
+
 def _load_params(offered_load):
     r = {}
     if offered_load is None:
@@ -93,21 +92,25 @@ def _load_params(offered_load):
         r['poisson'] = load
     return r
 
+
 def _add_channel_arg(config, key, value):
-  if 'channel_args' in config:
-    channel_args = config['channel_args']
-  else:
-    channel_args = []
-    config['channel_args'] = channel_args
-  arg = {'name': key}
-  if isinstance(value, int):
-    arg['int_value'] = value
-  else:
-    arg['str_value'] = value
-  channel_args.append(arg)
-
-def _ping_pong_scenario(name, rpc_type,
-                        client_type, server_type,
+    if 'channel_args' in config:
+        channel_args = config['channel_args']
+    else:
+        channel_args = []
+        config['channel_args'] = channel_args
+    arg = {'name': key}
+    if isinstance(value, int):
+        arg['int_value'] = value
+    else:
+        arg['str_value'] = value
+    channel_args.append(arg)
+
+
+def _ping_pong_scenario(name,
+                        rpc_type,
+                        client_type,
+                        server_type,
                         secure=True,
                         use_generic_payload=False,
                         req_size=0,
@@ -128,824 +131,1029 @@ def _ping_pong_scenario(name, rpc_type,
                         excluded_poll_engines=[],
                         minimal_stack=False,
                         offered_load=None):
-  """Creates a basic ping pong scenario."""
-  scenario = {
-    'name': name,
-    'num_servers': 1,
-    'num_clients': 1,
-    'client_config': {
-      'client_type': client_type,
-      'security_params': _get_secargs(secure),
-      'outstanding_rpcs_per_channel': 1,
-      'client_channels': 1,
-      'async_client_threads': 1,
-      'threads_per_cq': client_threads_per_cq,
-      'rpc_type': rpc_type,
-      'histogram_params': HISTOGRAM_PARAMS,
-      'channel_args': [],
-    },
-    'server_config': {
-      'server_type': server_type,
-      'security_params': _get_secargs(secure),
-      'async_server_threads': async_server_threads,
-      'threads_per_cq': server_threads_per_cq,
-      'channel_args': [],
-    },
-    'warmup_seconds': warmup_seconds,
-    'benchmark_seconds': BENCHMARK_SECONDS
-  }
-  if resource_quota_size:
-    scenario['server_config']['resource_quota_size'] = resource_quota_size
-  if use_generic_payload:
-    if server_type != 'ASYNC_GENERIC_SERVER':
-      raise Exception('Use ASYNC_GENERIC_SERVER for generic payload.')
-    scenario['server_config']['payload_config'] = _payload_type(use_generic_payload, req_size, resp_size)
-
-  scenario['client_config']['payload_config'] = _payload_type(use_generic_payload, req_size, resp_size)
-
-  # Optimization target of 'throughput' does not work well with epoll1 polling
-  # engine. Use the default value of 'blend'
-  optimization_target = 'throughput'
-
-  if unconstrained_client:
-    outstanding_calls = outstanding if outstanding is not None else OUTSTANDING_REQUESTS[unconstrained_client]
-    # clamp buffer usage to something reasonable (16 gig for now)
-    MAX_MEMORY_USE = 16 * 1024 * 1024 * 1024
-    if outstanding_calls * max(req_size, resp_size) > MAX_MEMORY_USE:
-        outstanding_calls = max(1, MAX_MEMORY_USE / max(req_size, resp_size))
-    wide = channels if channels is not None else WIDE
-    deep = int(math.ceil(1.0 * outstanding_calls / wide))
-
-    scenario['num_clients'] = num_clients if num_clients is not None else 0  # use as many clients as available.
-    scenario['client_config']['outstanding_rpcs_per_channel'] = deep
-    scenario['client_config']['client_channels'] = wide
-    scenario['client_config']['async_client_threads'] = 0
-    if offered_load is not None:
+    """Creates a basic ping pong scenario."""
+    scenario = {
+        'name': name,
+        'num_servers': 1,
+        'num_clients': 1,
+        'client_config': {
+            'client_type': client_type,
+            'security_params': _get_secargs(secure),
+            'outstanding_rpcs_per_channel': 1,
+            'client_channels': 1,
+            'async_client_threads': 1,
+            'threads_per_cq': client_threads_per_cq,
+            'rpc_type': rpc_type,
+            'histogram_params': HISTOGRAM_PARAMS,
+            'channel_args': [],
+        },
+        'server_config': {
+            'server_type': server_type,
+            'security_params': _get_secargs(secure),
+            'async_server_threads': async_server_threads,
+            'threads_per_cq': server_threads_per_cq,
+            'channel_args': [],
+        },
+        'warmup_seconds': warmup_seconds,
+        'benchmark_seconds': BENCHMARK_SECONDS
+    }
+    if resource_quota_size:
+        scenario['server_config']['resource_quota_size'] = resource_quota_size
+    if use_generic_payload:
+        if server_type != 'ASYNC_GENERIC_SERVER':
+            raise Exception('Use ASYNC_GENERIC_SERVER for generic payload.')
+        scenario['server_config']['payload_config'] = _payload_type(
+            use_generic_payload, req_size, resp_size)
+
+    scenario['client_config']['payload_config'] = _payload_type(
+        use_generic_payload, req_size, resp_size)
+
+    # Optimization target of 'throughput' does not work well with epoll1 polling
+    # engine. Use the default value of 'blend'
+    optimization_target = 'throughput'
+
+    if unconstrained_client:
+        outstanding_calls = outstanding if outstanding is not None else OUTSTANDING_REQUESTS[
+            unconstrained_client]
+        # clamp buffer usage to something reasonable (16 gig for now)
+        MAX_MEMORY_USE = 16 * 1024 * 1024 * 1024
+        if outstanding_calls * max(req_size, resp_size) > MAX_MEMORY_USE:
+            outstanding_calls = max(1, MAX_MEMORY_USE / max(req_size,
+                                                            resp_size))
+        wide = channels if channels is not None else WIDE
+        deep = int(math.ceil(1.0 * outstanding_calls / wide))
+
+        scenario[
+            'num_clients'] = num_clients if num_clients is not None else 0  # use as many clients as available.
+        scenario['client_config']['outstanding_rpcs_per_channel'] = deep
+        scenario['client_config']['client_channels'] = wide
+        scenario['client_config']['async_client_threads'] = 0
+        if offered_load is not None:
+            optimization_target = 'latency'
+    else:
+        scenario['client_config']['outstanding_rpcs_per_channel'] = 1
+        scenario['client_config']['client_channels'] = 1
+        scenario['client_config']['async_client_threads'] = 1
         optimization_target = 'latency'
-  else:
-    scenario['client_config']['outstanding_rpcs_per_channel'] = 1
-    scenario['client_config']['client_channels'] = 1
-    scenario['client_config']['async_client_threads'] = 1
-    optimization_target = 'latency'
-
-  scenario['client_config']['load_params'] = _load_params(offered_load)
-
-  optimization_channel_arg = {
-    'name': 'grpc.optimization_target',
-    'str_value': optimization_target
-  }
-  scenario['client_config']['channel_args'].append(optimization_channel_arg)
-  scenario['server_config']['channel_args'].append(optimization_channel_arg)
-
-  if minimal_stack:
-    _add_channel_arg(scenario['client_config'], 'grpc.minimal_stack', 1)
-    _add_channel_arg(scenario['server_config'], 'grpc.minimal_stack', 1)
-
-  if messages_per_stream:
-    scenario['client_config']['messages_per_stream'] = messages_per_stream
-  if client_language:
-    # the CLIENT_LANGUAGE field is recognized by run_performance_tests.py
-    scenario['CLIENT_LANGUAGE'] = client_language
-  if server_language:
-    # the SERVER_LANGUAGE field is recognized by run_performance_tests.py
-    scenario['SERVER_LANGUAGE'] = server_language
-  if categories:
-    scenario['CATEGORIES'] = categories
-  if len(excluded_poll_engines):
-    # The polling engines for which this scenario is excluded
-    scenario['EXCLUDED_POLL_ENGINES'] = excluded_poll_engines
-  return scenario
+
+    scenario['client_config']['load_params'] = _load_params(offered_load)
+
+    optimization_channel_arg = {
+        'name': 'grpc.optimization_target',
+        'str_value': optimization_target
+    }
+    scenario['client_config']['channel_args'].append(optimization_channel_arg)
+    scenario['server_config']['channel_args'].append(optimization_channel_arg)
+
+    if minimal_stack:
+        _add_channel_arg(scenario['client_config'], 'grpc.minimal_stack', 1)
+        _add_channel_arg(scenario['server_config'], 'grpc.minimal_stack', 1)
+
+    if messages_per_stream:
+        scenario['client_config']['messages_per_stream'] = messages_per_stream
+    if client_language:
+        # the CLIENT_LANGUAGE field is recognized by run_performance_tests.py
+        scenario['CLIENT_LANGUAGE'] = client_language
+    if server_language:
+        # the SERVER_LANGUAGE field is recognized by run_performance_tests.py
+        scenario['SERVER_LANGUAGE'] = server_language
+    if categories:
+        scenario['CATEGORIES'] = categories
+    if len(excluded_poll_engines):
+        # The polling engines for which this scenario is excluded
+        scenario['EXCLUDED_POLL_ENGINES'] = excluded_poll_engines
+    return scenario
 
 
 class CXXLanguage:
 
-  def __init__(self):
-    self.safename = 'cxx'
-
-  def worker_cmdline(self):
-    return ['bins/opt/qps_worker']
-
-  def worker_port_offset(self):
-    return 0
-
-  def scenarios(self):
-    # TODO(ctiller): add 70% load latency test
-    yield _ping_pong_scenario(
-      'cpp_protobuf_async_unary_1channel_100rpcs_1MB', rpc_type='UNARY',
-      client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
-      req_size=1024*1024, resp_size=1024*1024,
-      unconstrained_client='async', outstanding=100, channels=1,
-      num_clients=1,
-      secure=False,
-      categories=[SMOKETEST] + [INPROC] + [SCALABLE])
-
-    yield _ping_pong_scenario(
-      'cpp_protobuf_async_streaming_from_client_1channel_1MB', rpc_type='STREAMING_FROM_CLIENT',
-      client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
-      req_size=1024*1024, resp_size=1024*1024,
-      unconstrained_client='async', outstanding=1, channels=1,
-      num_clients=1,
-      secure=False,
-      categories=[SMOKETEST] + [INPROC] + [SCALABLE])
-
-    yield _ping_pong_scenario(
-       'cpp_protobuf_async_unary_75Kqps_600channel_60Krpcs_300Breq_50Bresp',
-       rpc_type='UNARY', client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
-       req_size=300, resp_size=50,
-       unconstrained_client='async', outstanding=30000, channels=300,
-       offered_load=37500, secure=False,
-       async_server_threads=16, server_threads_per_cq=1,
-       categories=[SMOKETEST] + [SCALABLE])
-
-    for secure in [True, False]:
-      secstr = 'secure' if secure else 'insecure'
-      smoketest_categories = ([SMOKETEST] if secure else [INPROC]) + [SCALABLE]
-
-      yield _ping_pong_scenario(
-          'cpp_generic_async_streaming_ping_pong_%s' % secstr,
-          rpc_type='STREAMING',
-          client_type='ASYNC_CLIENT',
-          server_type='ASYNC_GENERIC_SERVER',
-          use_generic_payload=True, async_server_threads=1,
-          secure=secure,
-          categories=smoketest_categories)
-
-      yield _ping_pong_scenario(
-          'cpp_generic_async_streaming_qps_unconstrained_%s' % secstr,
-          rpc_type='STREAMING',
-          client_type='ASYNC_CLIENT',
-          server_type='ASYNC_GENERIC_SERVER',
-          unconstrained_client='async', use_generic_payload=True,
-          secure=secure,
-          minimal_stack=not secure,
-          categories=smoketest_categories+[SCALABLE])
-
-      for mps in geometric_progression(1, 20, 10):
-        yield _ping_pong_scenario(
-            'cpp_generic_async_streaming_qps_unconstrained_%smps_%s' % (mps, secstr),
-            rpc_type='STREAMING',
+    def __init__(self):
+        self.safename = 'cxx'
+
+    def worker_cmdline(self):
+        return ['bins/opt/qps_worker']
+
+    def worker_port_offset(self):
+        return 0
+
+    def scenarios(self):
+        # TODO(ctiller): add 70% load latency test
+        yield _ping_pong_scenario(
+            'cpp_protobuf_async_unary_1channel_100rpcs_1MB',
+            rpc_type='UNARY',
             client_type='ASYNC_CLIENT',
-            server_type='ASYNC_GENERIC_SERVER',
-            unconstrained_client='async', use_generic_payload=True,
-            secure=secure, messages_per_stream=mps,
-            minimal_stack=not secure,
-            categories=smoketest_categories+[SCALABLE])
+            server_type='ASYNC_SERVER',
+            req_size=1024 * 1024,
+            resp_size=1024 * 1024,
+            unconstrained_client='async',
+            outstanding=100,
+            channels=1,
+            num_clients=1,
+            secure=False,
+            categories=[SMOKETEST] + [INPROC] + [SCALABLE])
 
-      for mps in geometric_progression(1, 200, math.sqrt(10)):
         yield _ping_pong_scenario(
-            'cpp_generic_async_streaming_qps_unconstrained_%smps_%s' % (mps, secstr),
-            rpc_type='STREAMING',
+            'cpp_protobuf_async_streaming_from_client_1channel_1MB',
+            rpc_type='STREAMING_FROM_CLIENT',
             client_type='ASYNC_CLIENT',
-            server_type='ASYNC_GENERIC_SERVER',
-            unconstrained_client='async', use_generic_payload=True,
-            secure=secure, messages_per_stream=mps,
-            minimal_stack=not secure,
-            categories=[SWEEP])
-
-      yield _ping_pong_scenario(
-          'cpp_generic_async_streaming_qps_1channel_1MBmsg_%s' % secstr,
-          rpc_type='STREAMING',
-          req_size=1024*1024,
-          resp_size=1024*1024,
-          client_type='ASYNC_CLIENT',
-          server_type='ASYNC_GENERIC_SERVER',
-          unconstrained_client='async', use_generic_payload=True,
-          secure=secure,
-          minimal_stack=not secure,
-          categories=smoketest_categories+[SCALABLE],
-          channels=1, outstanding=100)
-
-      yield _ping_pong_scenario(
-          'cpp_generic_async_streaming_qps_unconstrained_64KBmsg_%s' % secstr,
-          rpc_type='STREAMING',
-          req_size=64*1024,
-          resp_size=64*1024,
-          client_type='ASYNC_CLIENT',
-          server_type='ASYNC_GENERIC_SERVER',
-          unconstrained_client='async', use_generic_payload=True,
-          secure=secure,
-          minimal_stack=not secure,
-          categories=smoketest_categories+[SCALABLE])
-
-      # TODO(https://github.com/grpc/grpc/issues/11500) Re-enable this test
-      #yield _ping_pong_scenario(
-      #    'cpp_generic_async_streaming_qps_unconstrained_1cq_%s' % secstr,
-      #    rpc_type='STREAMING',
-      #    client_type='ASYNC_CLIENT',
-      #    server_type='ASYNC_GENERIC_SERVER',
-      #    unconstrained_client='async-limited', use_generic_payload=True,
-      #    secure=secure,
-      #    client_threads_per_cq=1000000, server_threads_per_cq=1000000,
-      #    categories=smoketest_categories+[SCALABLE])
-
-      yield _ping_pong_scenario(
-          'cpp_generic_async_streaming_qps_unconstrained_2waysharedcq_%s' % secstr,
-          rpc_type='STREAMING',
-          client_type='ASYNC_CLIENT',
-          server_type='ASYNC_GENERIC_SERVER',
-          unconstrained_client='async', use_generic_payload=True,
-          secure=secure,
-          client_threads_per_cq=2, server_threads_per_cq=2,
-          categories=smoketest_categories+[SCALABLE])
-
-      #yield _ping_pong_scenario(
-      #    'cpp_protobuf_async_streaming_qps_unconstrained_1cq_%s' % secstr,
-      #    rpc_type='STREAMING',
-      #    client_type='ASYNC_CLIENT',
-      #    server_type='ASYNC_SERVER',
-      #    unconstrained_client='async-limited',
-      #    secure=secure,
-      #    client_threads_per_cq=1000000, server_threads_per_cq=1000000,
-      #    categories=smoketest_categories+[SCALABLE])
-
-      yield _ping_pong_scenario(
-          'cpp_protobuf_async_streaming_qps_unconstrained_2waysharedcq_%s' % secstr,
-          rpc_type='STREAMING',
-          client_type='ASYNC_CLIENT',
-          server_type='ASYNC_SERVER',
-          unconstrained_client='async',
-          secure=secure,
-          client_threads_per_cq=2, server_threads_per_cq=2,
-          categories=smoketest_categories+[SCALABLE])
-
-      #yield _ping_pong_scenario(
-      #    'cpp_protobuf_async_unary_qps_unconstrained_1cq_%s' % secstr,
-      #    rpc_type='UNARY',
-      #    client_type='ASYNC_CLIENT',
-      #    server_type='ASYNC_SERVER',
-      #    unconstrained_client='async-limited',
-      #    secure=secure,
-      #    client_threads_per_cq=1000000, server_threads_per_cq=1000000,
-      #    categories=smoketest_categories+[SCALABLE])
-
-      yield _ping_pong_scenario(
-          'cpp_protobuf_async_unary_qps_unconstrained_2waysharedcq_%s' % secstr,
-          rpc_type='UNARY',
-          client_type='ASYNC_CLIENT',
-          server_type='ASYNC_SERVER',
-          unconstrained_client='async',
-          secure=secure,
-          client_threads_per_cq=2, server_threads_per_cq=2,
-          categories=smoketest_categories+[SCALABLE])
-
-      yield _ping_pong_scenario(
-          'cpp_generic_async_streaming_qps_one_server_core_%s' % secstr,
-          rpc_type='STREAMING',
-          client_type='ASYNC_CLIENT',
-          server_type='ASYNC_GENERIC_SERVER',
-          unconstrained_client='async-limited', use_generic_payload=True,
-          async_server_threads=1,
-          minimal_stack=not secure,
-          secure=secure)
-
-      yield _ping_pong_scenario(
-          'cpp_protobuf_async_client_sync_server_unary_qps_unconstrained_%s' %
-          (secstr),
-          rpc_type='UNARY',
-          client_type='ASYNC_CLIENT',
-          server_type='SYNC_SERVER',
-          unconstrained_client='async',
-          secure=secure,
-          minimal_stack=not secure,
-          categories=smoketest_categories + [SCALABLE],
-          excluded_poll_engines = ['poll-cv'])
-
-      yield _ping_pong_scenario(
-          'cpp_protobuf_async_client_unary_1channel_64wide_128Breq_8MBresp_%s' %
-          (secstr),
-          rpc_type='UNARY',
-          client_type='ASYNC_CLIENT',
-          server_type='ASYNC_SERVER',
-          channels=1,
-          outstanding=64,
-          req_size=128,
-          resp_size=8*1024*1024,
-          secure=secure,
-          minimal_stack=not secure,
-          categories=smoketest_categories + [SCALABLE])
-
-      yield _ping_pong_scenario(
-          'cpp_protobuf_async_client_sync_server_streaming_qps_unconstrained_%s' % secstr,
-          rpc_type='STREAMING',
-          client_type='ASYNC_CLIENT',
-          server_type='SYNC_SERVER',
-          unconstrained_client='async',
-          secure=secure,
-          minimal_stack=not secure,
-          categories=smoketest_categories+[SCALABLE],
-          excluded_poll_engines = ['poll-cv'])
-
-      yield _ping_pong_scenario(
-        'cpp_protobuf_async_unary_ping_pong_%s_1MB' % secstr, rpc_type='UNARY',
-        client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
-        req_size=1024*1024, resp_size=1024*1024,
-        secure=secure,
-        minimal_stack=not secure,
-        categories=smoketest_categories + [SCALABLE])
-
-      for rpc_type in ['unary', 'streaming', 'streaming_from_client', 'streaming_from_server']:
-        for synchronicity in ['sync', 'async']:
-          yield _ping_pong_scenario(
-              'cpp_protobuf_%s_%s_ping_pong_%s' % (synchronicity, rpc_type, secstr),
-              rpc_type=rpc_type.upper(),
-              client_type='%s_CLIENT' % synchronicity.upper(),
-              server_type='%s_SERVER' % synchronicity.upper(),
-              async_server_threads=1,
-              minimal_stack=not secure,
-              secure=secure)
-
-          for size in geometric_progression(1, 1024*1024*1024+1, 8):
-              yield _ping_pong_scenario(
-                  'cpp_protobuf_%s_%s_qps_unconstrained_%s_%db' % (synchronicity, rpc_type, secstr, size),
-                  rpc_type=rpc_type.upper(),
-                  req_size=size,
-                  resp_size=size,
-                  client_type='%s_CLIENT' % synchronicity.upper(),
-                  server_type='%s_SERVER' % synchronicity.upper(),
-                  unconstrained_client=synchronicity,
-                  secure=secure,
-                  minimal_stack=not secure,
-                  categories=[SWEEP])
-
-          yield _ping_pong_scenario(
-              'cpp_protobuf_%s_%s_qps_unconstrained_%s' % (synchronicity, rpc_type, secstr),
-              rpc_type=rpc_type.upper(),
-              client_type='%s_CLIENT' % synchronicity.upper(),
-              server_type='%s_SERVER' % synchronicity.upper(),
-              unconstrained_client=synchronicity,
-              secure=secure,
-              minimal_stack=not secure,
-              server_threads_per_cq=3,
-              client_threads_per_cq=3,
-              categories=smoketest_categories+[SCALABLE])
-
-          # TODO(vjpai): Re-enable this test. It has a lot of timeouts
-          # and hasn't yet been conclusively identified as a test failure
-          # or race in the library
-          # yield _ping_pong_scenario(
-          #     'cpp_protobuf_%s_%s_qps_unconstrained_%s_500kib_resource_quota' % (synchronicity, rpc_type, secstr),
-          #     rpc_type=rpc_type.upper(),
-          #     client_type='%s_CLIENT' % synchronicity.upper(),
-          #     server_type='%s_SERVER' % synchronicity.upper(),
-          #     unconstrained_client=synchronicity,
-          #     secure=secure,
-          #     categories=smoketest_categories+[SCALABLE],
-          #     resource_quota_size=500*1024)
-
-          if rpc_type == 'streaming':
+            server_type='ASYNC_SERVER',
+            req_size=1024 * 1024,
+            resp_size=1024 * 1024,
+            unconstrained_client='async',
+            outstanding=1,
+            channels=1,
+            num_clients=1,
+            secure=False,
+            categories=[SMOKETEST] + [INPROC] + [SCALABLE])
+
+        yield _ping_pong_scenario(
+            'cpp_protobuf_async_unary_75Kqps_600channel_60Krpcs_300Breq_50Bresp',
+            rpc_type='UNARY',
+            client_type='ASYNC_CLIENT',
+            server_type='ASYNC_SERVER',
+            req_size=300,
+            resp_size=50,
+            unconstrained_client='async',
+            outstanding=30000,
+            channels=300,
+            offered_load=37500,
+            secure=False,
+            async_server_threads=16,
+            server_threads_per_cq=1,
+            categories=[SMOKETEST] + [SCALABLE])
+
+        for secure in [True, False]:
+            secstr = 'secure' if secure else 'insecure'
+            smoketest_categories = ([SMOKETEST]
+                                    if secure else [INPROC]) + [SCALABLE]
+
+            yield _ping_pong_scenario(
+                'cpp_generic_async_streaming_ping_pong_%s' % secstr,
+                rpc_type='STREAMING',
+                client_type='ASYNC_CLIENT',
+                server_type='ASYNC_GENERIC_SERVER',
+                use_generic_payload=True,
+                async_server_threads=1,
+                secure=secure,
+                categories=smoketest_categories)
+
+            yield _ping_pong_scenario(
+                'cpp_generic_async_streaming_qps_unconstrained_%s' % secstr,
+                rpc_type='STREAMING',
+                client_type='ASYNC_CLIENT',
+                server_type='ASYNC_GENERIC_SERVER',
+                unconstrained_client='async',
+                use_generic_payload=True,
+                secure=secure,
+                minimal_stack=not secure,
+                categories=smoketest_categories + [SCALABLE])
+
             for mps in geometric_progression(1, 20, 10):
-              yield _ping_pong_scenario(
-                  'cpp_protobuf_%s_%s_qps_unconstrained_%smps_%s' % (synchronicity, rpc_type, mps, secstr),
-                  rpc_type=rpc_type.upper(),
-                  client_type='%s_CLIENT' % synchronicity.upper(),
-                  server_type='%s_SERVER' % synchronicity.upper(),
-                  unconstrained_client=synchronicity,
-                  secure=secure, messages_per_stream=mps,
-                  minimal_stack=not secure,
-                  categories=smoketest_categories+[SCALABLE])
+                yield _ping_pong_scenario(
+                    'cpp_generic_async_streaming_qps_unconstrained_%smps_%s' %
+                    (mps, secstr),
+                    rpc_type='STREAMING',
+                    client_type='ASYNC_CLIENT',
+                    server_type='ASYNC_GENERIC_SERVER',
+                    unconstrained_client='async',
+                    use_generic_payload=True,
+                    secure=secure,
+                    messages_per_stream=mps,
+                    minimal_stack=not secure,
+                    categories=smoketest_categories + [SCALABLE])
 
             for mps in geometric_progression(1, 200, math.sqrt(10)):
-              yield _ping_pong_scenario(
-                  'cpp_protobuf_%s_%s_qps_unconstrained_%smps_%s' % (synchronicity, rpc_type, mps, secstr),
-                  rpc_type=rpc_type.upper(),
-                  client_type='%s_CLIENT' % synchronicity.upper(),
-                  server_type='%s_SERVER' % synchronicity.upper(),
-                  unconstrained_client=synchronicity,
-                  secure=secure, messages_per_stream=mps,
-                  minimal_stack=not secure,
-                  categories=[SWEEP])
-
-          for channels in geometric_progression(1, 20000, math.sqrt(10)):
-            for outstanding in geometric_progression(1, 200000, math.sqrt(10)):
-                if synchronicity == 'sync' and outstanding > 1200: continue
-                if outstanding < channels: continue
                 yield _ping_pong_scenario(
-                    'cpp_protobuf_%s_%s_qps_unconstrained_%s_%d_channels_%d_outstanding' % (synchronicity, rpc_type, secstr, channels, outstanding),
-                    rpc_type=rpc_type.upper(),
-                    client_type='%s_CLIENT' % synchronicity.upper(),
-                    server_type='%s_SERVER' % synchronicity.upper(),
-                    unconstrained_client=synchronicity, secure=secure,
+                    'cpp_generic_async_streaming_qps_unconstrained_%smps_%s' %
+                    (mps, secstr),
+                    rpc_type='STREAMING',
+                    client_type='ASYNC_CLIENT',
+                    server_type='ASYNC_GENERIC_SERVER',
+                    unconstrained_client='async',
+                    use_generic_payload=True,
+                    secure=secure,
+                    messages_per_stream=mps,
                     minimal_stack=not secure,
-                    categories=[SWEEP], channels=channels, outstanding=outstanding)
-
-  def __str__(self):
-    return 'c++'
+                    categories=[SWEEP])
+
+            yield _ping_pong_scenario(
+                'cpp_generic_async_streaming_qps_1channel_1MBmsg_%s' % secstr,
+                rpc_type='STREAMING',
+                req_size=1024 * 1024,
+                resp_size=1024 * 1024,
+                client_type='ASYNC_CLIENT',
+                server_type='ASYNC_GENERIC_SERVER',
+                unconstrained_client='async',
+                use_generic_payload=True,
+                secure=secure,
+                minimal_stack=not secure,
+                categories=smoketest_categories + [SCALABLE],
+                channels=1,
+                outstanding=100)
+
+            yield _ping_pong_scenario(
+                'cpp_generic_async_streaming_qps_unconstrained_64KBmsg_%s' %
+                secstr,
+                rpc_type='STREAMING',
+                req_size=64 * 1024,
+                resp_size=64 * 1024,
+                client_type='ASYNC_CLIENT',
+                server_type='ASYNC_GENERIC_SERVER',
+                unconstrained_client='async',
+                use_generic_payload=True,
+                secure=secure,
+                minimal_stack=not secure,
+                categories=smoketest_categories + [SCALABLE])
+
+            # TODO(https://github.com/grpc/grpc/issues/11500) Re-enable this test
+            #yield _ping_pong_scenario(
+            #    'cpp_generic_async_streaming_qps_unconstrained_1cq_%s' % secstr,
+            #    rpc_type='STREAMING',
+            #    client_type='ASYNC_CLIENT',
+            #    server_type='ASYNC_GENERIC_SERVER',
+            #    unconstrained_client='async-limited', use_generic_payload=True,
+            #    secure=secure,
+            #    client_threads_per_cq=1000000, server_threads_per_cq=1000000,
+            #    categories=smoketest_categories+[SCALABLE])
+
+            yield _ping_pong_scenario(
+                'cpp_generic_async_streaming_qps_unconstrained_2waysharedcq_%s'
+                % secstr,
+                rpc_type='STREAMING',
+                client_type='ASYNC_CLIENT',
+                server_type='ASYNC_GENERIC_SERVER',
+                unconstrained_client='async',
+                use_generic_payload=True,
+                secure=secure,
+                client_threads_per_cq=2,
+                server_threads_per_cq=2,
+                categories=smoketest_categories + [SCALABLE])
+
+            #yield _ping_pong_scenario(
+            #    'cpp_protobuf_async_streaming_qps_unconstrained_1cq_%s' % secstr,
+            #    rpc_type='STREAMING',
+            #    client_type='ASYNC_CLIENT',
+            #    server_type='ASYNC_SERVER',
+            #    unconstrained_client='async-limited',
+            #    secure=secure,
+            #    client_threads_per_cq=1000000, server_threads_per_cq=1000000,
+            #    categories=smoketest_categories+[SCALABLE])
+
+            yield _ping_pong_scenario(
+                'cpp_protobuf_async_streaming_qps_unconstrained_2waysharedcq_%s'
+                % secstr,
+                rpc_type='STREAMING',
+                client_type='ASYNC_CLIENT',
+                server_type='ASYNC_SERVER',
+                unconstrained_client='async',
+                secure=secure,
+                client_threads_per_cq=2,
+                server_threads_per_cq=2,
+                categories=smoketest_categories + [SCALABLE])
+
+            #yield _ping_pong_scenario(
+            #    'cpp_protobuf_async_unary_qps_unconstrained_1cq_%s' % secstr,
+            #    rpc_type='UNARY',
+            #    client_type='ASYNC_CLIENT',
+            #    server_type='ASYNC_SERVER',
+            #    unconstrained_client='async-limited',
+            #    secure=secure,
+            #    client_threads_per_cq=1000000, server_threads_per_cq=1000000,
+            #    categories=smoketest_categories+[SCALABLE])
+
+            yield _ping_pong_scenario(
+                'cpp_protobuf_async_unary_qps_unconstrained_2waysharedcq_%s' %
+                secstr,
+                rpc_type='UNARY',
+                client_type='ASYNC_CLIENT',
+                server_type='ASYNC_SERVER',
+                unconstrained_client='async',
+                secure=secure,
+                client_threads_per_cq=2,
+                server_threads_per_cq=2,
+                categories=smoketest_categories + [SCALABLE])
+
+            yield _ping_pong_scenario(
+                'cpp_generic_async_streaming_qps_one_server_core_%s' % secstr,
+                rpc_type='STREAMING',
+                client_type='ASYNC_CLIENT',
+                server_type='ASYNC_GENERIC_SERVER',
+                unconstrained_client='async-limited',
+                use_generic_payload=True,
+                async_server_threads=1,
+                minimal_stack=not secure,
+                secure=secure)
+
+            yield _ping_pong_scenario(
+                'cpp_protobuf_async_client_sync_server_unary_qps_unconstrained_%s'
+                % (secstr),
+                rpc_type='UNARY',
+                client_type='ASYNC_CLIENT',
+                server_type='SYNC_SERVER',
+                unconstrained_client='async',
+                secure=secure,
+                minimal_stack=not secure,
+                categories=smoketest_categories + [SCALABLE],
+                excluded_poll_engines=['poll-cv'])
+
+            yield _ping_pong_scenario(
+                'cpp_protobuf_async_client_unary_1channel_64wide_128Breq_8MBresp_%s'
+                % (secstr),
+                rpc_type='UNARY',
+                client_type='ASYNC_CLIENT',
+                server_type='ASYNC_SERVER',
+                channels=1,
+                outstanding=64,
+                req_size=128,
+                resp_size=8 * 1024 * 1024,
+                secure=secure,
+                minimal_stack=not secure,
+                categories=smoketest_categories + [SCALABLE])
+
+            yield _ping_pong_scenario(
+                'cpp_protobuf_async_client_sync_server_streaming_qps_unconstrained_%s'
+                % secstr,
+                rpc_type='STREAMING',
+                client_type='ASYNC_CLIENT',
+                server_type='SYNC_SERVER',
+                unconstrained_client='async',
+                secure=secure,
+                minimal_stack=not secure,
+                categories=smoketest_categories + [SCALABLE],
+                excluded_poll_engines=['poll-cv'])
+
+            yield _ping_pong_scenario(
+                'cpp_protobuf_async_unary_ping_pong_%s_1MB' % secstr,
+                rpc_type='UNARY',
+                client_type='ASYNC_CLIENT',
+                server_type='ASYNC_SERVER',
+                req_size=1024 * 1024,
+                resp_size=1024 * 1024,
+                secure=secure,
+                minimal_stack=not secure,
+                categories=smoketest_categories + [SCALABLE])
+
+            for rpc_type in [
+                    'unary', 'streaming', 'streaming_from_client',
+                    'streaming_from_server'
+            ]:
+                for synchronicity in ['sync', 'async']:
+                    yield _ping_pong_scenario(
+                        'cpp_protobuf_%s_%s_ping_pong_%s' %
+                        (synchronicity, rpc_type, secstr),
+                        rpc_type=rpc_type.upper(),
+                        client_type='%s_CLIENT' % synchronicity.upper(),
+                        server_type='%s_SERVER' % synchronicity.upper(),
+                        async_server_threads=1,
+                        minimal_stack=not secure,
+                        secure=secure)
+
+                    for size in geometric_progression(1, 1024 * 1024 * 1024 + 1,
+                                                      8):
+                        yield _ping_pong_scenario(
+                            'cpp_protobuf_%s_%s_qps_unconstrained_%s_%db' %
+                            (synchronicity, rpc_type, secstr, size),
+                            rpc_type=rpc_type.upper(),
+                            req_size=size,
+                            resp_size=size,
+                            client_type='%s_CLIENT' % synchronicity.upper(),
+                            server_type='%s_SERVER' % synchronicity.upper(),
+                            unconstrained_client=synchronicity,
+                            secure=secure,
+                            minimal_stack=not secure,
+                            categories=[SWEEP])
+
+                    yield _ping_pong_scenario(
+                        'cpp_protobuf_%s_%s_qps_unconstrained_%s' %
+                        (synchronicity, rpc_type, secstr),
+                        rpc_type=rpc_type.upper(),
+                        client_type='%s_CLIENT' % synchronicity.upper(),
+                        server_type='%s_SERVER' % synchronicity.upper(),
+                        unconstrained_client=synchronicity,
+                        secure=secure,
+                        minimal_stack=not secure,
+                        server_threads_per_cq=3,
+                        client_threads_per_cq=3,
+                        categories=smoketest_categories + [SCALABLE])
+
+                    # TODO(vjpai): Re-enable this test. It has a lot of timeouts
+                    # and hasn't yet been conclusively identified as a test failure
+                    # or race in the library
+                    # yield _ping_pong_scenario(
+                    #     'cpp_protobuf_%s_%s_qps_unconstrained_%s_500kib_resource_quota' % (synchronicity, rpc_type, secstr),
+                    #     rpc_type=rpc_type.upper(),
+                    #     client_type='%s_CLIENT' % synchronicity.upper(),
+                    #     server_type='%s_SERVER' % synchronicity.upper(),
+                    #     unconstrained_client=synchronicity,
+                    #     secure=secure,
+                    #     categories=smoketest_categories+[SCALABLE],
+                    #     resource_quota_size=500*1024)
+
+                    if rpc_type == 'streaming':
+                        for mps in geometric_progression(1, 20, 10):
+                            yield _ping_pong_scenario(
+                                'cpp_protobuf_%s_%s_qps_unconstrained_%smps_%s'
+                                % (synchronicity, rpc_type, mps, secstr),
+                                rpc_type=rpc_type.upper(),
+                                client_type='%s_CLIENT' % synchronicity.upper(),
+                                server_type='%s_SERVER' % synchronicity.upper(),
+                                unconstrained_client=synchronicity,
+                                secure=secure,
+                                messages_per_stream=mps,
+                                minimal_stack=not secure,
+                                categories=smoketest_categories + [SCALABLE])
+
+                        for mps in geometric_progression(1, 200, math.sqrt(10)):
+                            yield _ping_pong_scenario(
+                                'cpp_protobuf_%s_%s_qps_unconstrained_%smps_%s'
+                                % (synchronicity, rpc_type, mps, secstr),
+                                rpc_type=rpc_type.upper(),
+                                client_type='%s_CLIENT' % synchronicity.upper(),
+                                server_type='%s_SERVER' % synchronicity.upper(),
+                                unconstrained_client=synchronicity,
+                                secure=secure,
+                                messages_per_stream=mps,
+                                minimal_stack=not secure,
+                                categories=[SWEEP])
+
+                    for channels in geometric_progression(1, 20000,
+                                                          math.sqrt(10)):
+                        for outstanding in geometric_progression(1, 200000,
+                                                                 math.sqrt(10)):
+                            if synchronicity == 'sync' and outstanding > 1200:
+                                continue
+                            if outstanding < channels: continue
+                            yield _ping_pong_scenario(
+                                'cpp_protobuf_%s_%s_qps_unconstrained_%s_%d_channels_%d_outstanding'
+                                % (synchronicity, rpc_type, secstr, channels,
+                                   outstanding),
+                                rpc_type=rpc_type.upper(),
+                                client_type='%s_CLIENT' % synchronicity.upper(),
+                                server_type='%s_SERVER' % synchronicity.upper(),
+                                unconstrained_client=synchronicity,
+                                secure=secure,
+                                minimal_stack=not secure,
+                                categories=[SWEEP],
+                                channels=channels,
+                                outstanding=outstanding)
+
+    def __str__(self):
+        return 'c++'
 
 
 class CSharpLanguage:
 
-  def __init__(self):
-    self.safename = str(self)
-
-  def worker_cmdline(self):
-    return ['tools/run_tests/performance/run_worker_csharp.sh']
-
-  def worker_port_offset(self):
-    return 100
-
-  def scenarios(self):
-    yield _ping_pong_scenario(
-        'csharp_generic_async_streaming_ping_pong', rpc_type='STREAMING',
-        client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
-        use_generic_payload=True,
-        categories=[SMOKETEST, SCALABLE])
-
-    yield _ping_pong_scenario(
-        'csharp_generic_async_streaming_ping_pong_insecure_1MB', rpc_type='STREAMING',
-        client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
-        req_size=1024*1024, resp_size=1024*1024,
-        use_generic_payload=True,
-        secure=False,
-        categories=[SMOKETEST, SCALABLE])
-
-    yield _ping_pong_scenario(
-        'csharp_generic_async_streaming_qps_unconstrained_insecure', rpc_type='STREAMING',
-        client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
-        unconstrained_client='async', use_generic_payload=True,
-        secure=False,
-        categories=[SMOKETEST, SCALABLE])
-
-    yield _ping_pong_scenario(
-        'csharp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING',
-        client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER')
-
-    yield _ping_pong_scenario(
-        'csharp_protobuf_async_unary_ping_pong', rpc_type='UNARY',
-        client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
-        categories=[SMOKETEST, SCALABLE])
-
-    yield _ping_pong_scenario(
-        'csharp_protobuf_sync_to_async_unary_ping_pong', rpc_type='UNARY',
-        client_type='SYNC_CLIENT', server_type='ASYNC_SERVER')
-
-    yield _ping_pong_scenario(
-        'csharp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY',
-        client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
-        unconstrained_client='async',
-        categories=[SMOKETEST,SCALABLE])
-
-    yield _ping_pong_scenario(
-        'csharp_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING',
-        client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
-        unconstrained_client='async',
-        categories=[SCALABLE])
-
-    yield _ping_pong_scenario(
-        'csharp_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY',
-        client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
-        server_language='c++', async_server_threads=1,
-        categories=[SMOKETEST, SCALABLE])
-
-    yield _ping_pong_scenario(
-        'csharp_to_cpp_protobuf_async_streaming_ping_pong', rpc_type='STREAMING',
-        client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
-        server_language='c++', async_server_threads=1)
-
-    yield _ping_pong_scenario(
-        'csharp_to_cpp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY',
-        client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
-        unconstrained_client='async', server_language='c++',
-        categories=[SCALABLE])
-
-    yield _ping_pong_scenario(
-        'csharp_to_cpp_protobuf_sync_to_async_unary_qps_unconstrained', rpc_type='UNARY',
-        client_type='SYNC_CLIENT', server_type='ASYNC_SERVER',
-        unconstrained_client='sync', server_language='c++',
-        categories=[SCALABLE])
-
-    yield _ping_pong_scenario(
-        'cpp_to_csharp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY',
-        client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
-        unconstrained_client='async', client_language='c++',
-        categories=[SCALABLE])
-
-    yield _ping_pong_scenario(
-        'csharp_protobuf_async_unary_ping_pong_1MB', rpc_type='UNARY',
-        client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
-        req_size=1024*1024, resp_size=1024*1024,
-        categories=[SMOKETEST, SCALABLE])
-
-  def __str__(self):
-    return 'csharp'
+    def __init__(self):
+        self.safename = str(self)
+
+    def worker_cmdline(self):
+        return ['tools/run_tests/performance/run_worker_csharp.sh']
+
+    def worker_port_offset(self):
+        return 100
+
+    def scenarios(self):
+        yield _ping_pong_scenario(
+            'csharp_generic_async_streaming_ping_pong',
+            rpc_type='STREAMING',
+            client_type='ASYNC_CLIENT',
+            server_type='ASYNC_GENERIC_SERVER',
+            use_generic_payload=True,
+            categories=[SMOKETEST, SCALABLE])
+
+        yield _ping_pong_scenario(
+            'csharp_generic_async_streaming_ping_pong_insecure_1MB',
+            rpc_type='STREAMING',
+            client_type='ASYNC_CLIENT',
+            server_type='ASYNC_GENERIC_SERVER',
+            req_size=1024 * 1024,
+            resp_size=1024 * 1024,
+            use_generic_payload=True,
+            secure=False,
+            categories=[SMOKETEST, SCALABLE])
+
+        yield _ping_pong_scenario(
+            'csharp_generic_async_streaming_qps_unconstrained_insecure',
+            rpc_type='STREAMING',
+            client_type='ASYNC_CLIENT',
+            server_type='ASYNC_GENERIC_SERVER',
+            unconstrained_client='async',
+            use_generic_payload=True,
+            secure=False,
+            categories=[SMOKETEST, SCALABLE])
+
+        yield _ping_pong_scenario(
+            'csharp_protobuf_async_streaming_ping_pong',
+            rpc_type='STREAMING',
+            client_type='ASYNC_CLIENT',
+            server_type='ASYNC_SERVER')
+
+        yield _ping_pong_scenario(
+            'csharp_protobuf_async_unary_ping_pong',
+            rpc_type='UNARY',
+            client_type='ASYNC_CLIENT',
+            server_type='ASYNC_SERVER',
+            categories=[SMOKETEST, SCALABLE])
+
+        yield _ping_pong_scenario(
+            'csharp_protobuf_sync_to_async_unary_ping_pong',
+            rpc_type='UNARY',
+            client_type='SYNC_CLIENT',
+            server_type='ASYNC_SERVER')
+
+        yield _ping_pong_scenario(
+            'csharp_protobuf_async_unary_qps_unconstrained',
+            rpc_type='UNARY',
+            client_type='ASYNC_CLIENT',
+            server_type='ASYNC_SERVER',
+            unconstrained_client='async',
+            categories=[SMOKETEST, SCALABLE])
+
+        yield _ping_pong_scenario(
+            'csharp_protobuf_async_streaming_qps_unconstrained',
+            rpc_type='STREAMING',
+            client_type='ASYNC_CLIENT',
+            server_type='ASYNC_SERVER',
+            unconstrained_client='async',
+            categories=[SCALABLE])
+
+        yield _ping_pong_scenario(
+            'csharp_to_cpp_protobuf_sync_unary_ping_pong',
+            rpc_type='UNARY',
+            client_type='SYNC_CLIENT',
+            server_type='SYNC_SERVER',
+            server_language='c++',
+            async_server_threads=1,
+            categories=[SMOKETEST, SCALABLE])
+
+        yield _ping_pong_scenario(
+            'csharp_to_cpp_protobuf_async_streaming_ping_pong',
+            rpc_type='STREAMING',
+            client_type='ASYNC_CLIENT',
+            server_type='ASYNC_SERVER',
+            server_language='c++',
+            async_server_threads=1)
+
+        yield _ping_pong_scenario(
+            'csharp_to_cpp_protobuf_async_unary_qps_unconstrained',
+            rpc_type='UNARY',
+            client_type='ASYNC_CLIENT',
+            server_type='ASYNC_SERVER',
+            unconstrained_client='async',
+            server_language='c++',
+            categories=[SCALABLE])
+
+        yield _ping_pong_scenario(
+            'csharp_to_cpp_protobuf_sync_to_async_unary_qps_unconstrained',
+            rpc_type='UNARY',
+            client_type='SYNC_CLIENT',
+            server_type='ASYNC_SERVER',
+            unconstrained_client='sync',
+            server_language='c++',
+            categories=[SCALABLE])
+
+        yield _ping_pong_scenario(
+            'cpp_to_csharp_protobuf_async_unary_qps_unconstrained',
+            rpc_type='UNARY',
+            client_type='ASYNC_CLIENT',
+            server_type='ASYNC_SERVER',
+            unconstrained_client='async',
+            client_language='c++',
+            categories=[SCALABLE])
+
+        yield _ping_pong_scenario(
+            'csharp_protobuf_async_unary_ping_pong_1MB',
+            rpc_type='UNARY',
+            client_type='ASYNC_CLIENT',
+            server_type='ASYNC_SERVER',
+            req_size=1024 * 1024,
+            resp_size=1024 * 1024,
+            categories=[SMOKETEST, SCALABLE])
+
+    def __str__(self):
+        return 'csharp'
+
 
 class PythonLanguage:
 
-  def __init__(self):
-    self.safename = 'python'
-
-  def worker_cmdline(self):
-    return ['tools/run_tests/performance/run_worker_python.sh']
-
-  def worker_port_offset(self):
-    return 500
-
-  def scenarios(self):
-    yield _ping_pong_scenario(
-        'python_generic_sync_streaming_ping_pong', rpc_type='STREAMING',
-        client_type='SYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
-        use_generic_payload=True,
-        categories=[SMOKETEST, SCALABLE])
-
-    yield _ping_pong_scenario(
-        'python_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING',
-        client_type='SYNC_CLIENT', server_type='ASYNC_SERVER')
-
-    yield _ping_pong_scenario(
-        'python_protobuf_async_unary_ping_pong', rpc_type='UNARY',
-        client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER')
-
-    yield _ping_pong_scenario(
-        'python_protobuf_sync_unary_ping_pong', rpc_type='UNARY',
-        client_type='SYNC_CLIENT', server_type='ASYNC_SERVER',
-        categories=[SMOKETEST, SCALABLE])
-
-    yield _ping_pong_scenario(
-        'python_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY',
-        client_type='SYNC_CLIENT', server_type='ASYNC_SERVER',
-        unconstrained_client='sync')
-
-    yield _ping_pong_scenario(
-        'python_protobuf_sync_streaming_qps_unconstrained', rpc_type='STREAMING',
-        client_type='SYNC_CLIENT', server_type='ASYNC_SERVER',
-        unconstrained_client='sync')
-
-    yield _ping_pong_scenario(
-        'python_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY',
-        client_type='SYNC_CLIENT', server_type='ASYNC_SERVER',
-        server_language='c++', async_server_threads=1,
-        categories=[SMOKETEST, SCALABLE])
-
-    yield _ping_pong_scenario(
-        'python_to_cpp_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING',
-        client_type='SYNC_CLIENT', server_type='ASYNC_SERVER',
-        server_language='c++', async_server_threads=1)
-
-    yield _ping_pong_scenario(
-        'python_protobuf_sync_unary_ping_pong_1MB', rpc_type='UNARY',
-        client_type='SYNC_CLIENT', server_type='ASYNC_SERVER',
-        req_size=1024*1024, resp_size=1024*1024,
-        categories=[SMOKETEST, SCALABLE])
-
-  def __str__(self):
-    return 'python'
+    def __init__(self):
+        self.safename = 'python'
+
+    def worker_cmdline(self):
+        return ['tools/run_tests/performance/run_worker_python.sh']
+
+    def worker_port_offset(self):
+        return 500
+
+    def scenarios(self):
+        yield _ping_pong_scenario(
+            'python_generic_sync_streaming_ping_pong',
+            rpc_type='STREAMING',
+            client_type='SYNC_CLIENT',
+            server_type='ASYNC_GENERIC_SERVER',
+            use_generic_payload=True,
+            categories=[SMOKETEST, SCALABLE])
+
+        yield _ping_pong_scenario(
+            'python_protobuf_sync_streaming_ping_pong',
+            rpc_type='STREAMING',
+            client_type='SYNC_CLIENT',
+            server_type='ASYNC_SERVER')
+
+        yield _ping_pong_scenario(
+            'python_protobuf_async_unary_ping_pong',
+            rpc_type='UNARY',
+            client_type='ASYNC_CLIENT',
+            server_type='ASYNC_SERVER')
+
+        yield _ping_pong_scenario(
+            'python_protobuf_sync_unary_ping_pong',
+            rpc_type='UNARY',
+            client_type='SYNC_CLIENT',
+            server_type='ASYNC_SERVER',
+            categories=[SMOKETEST, SCALABLE])
+
+        yield _ping_pong_scenario(
+            'python_protobuf_sync_unary_qps_unconstrained',
+            rpc_type='UNARY',
+            client_type='SYNC_CLIENT',
+            server_type='ASYNC_SERVER',
+            unconstrained_client='sync')
+
+        yield _ping_pong_scenario(
+            'python_protobuf_sync_streaming_qps_unconstrained',
+            rpc_type='STREAMING',
+            client_type='SYNC_CLIENT',
+            server_type='ASYNC_SERVER',
+            unconstrained_client='sync')
+
+        yield _ping_pong_scenario(
+            'python_to_cpp_protobuf_sync_unary_ping_pong',
+            rpc_type='UNARY',
+            client_type='SYNC_CLIENT',
+            server_type='ASYNC_SERVER',
+            server_language='c++',
+            async_server_threads=1,
+            categories=[SMOKETEST, SCALABLE])
+
+        yield _ping_pong_scenario(
+            'python_to_cpp_protobuf_sync_streaming_ping_pong',
+            rpc_type='STREAMING',
+            client_type='SYNC_CLIENT',
+            server_type='ASYNC_SERVER',
+            server_language='c++',
+            async_server_threads=1)
+
+        yield _ping_pong_scenario(
+            'python_protobuf_sync_unary_ping_pong_1MB',
+            rpc_type='UNARY',
+            client_type='SYNC_CLIENT',
+            server_type='ASYNC_SERVER',
+            req_size=1024 * 1024,
+            resp_size=1024 * 1024,
+            categories=[SMOKETEST, SCALABLE])
+
+    def __str__(self):
+        return 'python'
+
 
 class RubyLanguage:
 
-  def __init__(self):
-    pass
-    self.safename = str(self)
+    def __init__(self):
+        pass
+        self.safename = str(self)
 
-  def worker_cmdline(self):
-    return ['tools/run_tests/performance/run_worker_ruby.sh']
+    def worker_cmdline(self):
+        return ['tools/run_tests/performance/run_worker_ruby.sh']
 
-  def worker_port_offset(self):
-    return 300
+    def worker_port_offset(self):
+        return 300
 
-  def scenarios(self):
-    yield _ping_pong_scenario(
-        'ruby_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING',
-        client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
-        categories=[SMOKETEST, SCALABLE])
+    def scenarios(self):
+        yield _ping_pong_scenario(
+            'ruby_protobuf_sync_streaming_ping_pong',
+            rpc_type='STREAMING',
+            client_type='SYNC_CLIENT',
+            server_type='SYNC_SERVER',
+            categories=[SMOKETEST, SCALABLE])
 
-    yield _ping_pong_scenario(
-        'ruby_protobuf_unary_ping_pong', rpc_type='UNARY',
-        client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
-        categories=[SMOKETEST, SCALABLE])
+        yield _ping_pong_scenario(
+            'ruby_protobuf_unary_ping_pong',
+            rpc_type='UNARY',
+            client_type='SYNC_CLIENT',
+            server_type='SYNC_SERVER',
+            categories=[SMOKETEST, SCALABLE])
 
-    yield _ping_pong_scenario(
-        'ruby_protobuf_sync_unary_qps_unconstrained', rpc_type='UNARY',
-        client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
-        unconstrained_client='sync')
+        yield _ping_pong_scenario(
+            'ruby_protobuf_sync_unary_qps_unconstrained',
+            rpc_type='UNARY',
+            client_type='SYNC_CLIENT',
+            server_type='SYNC_SERVER',
+            unconstrained_client='sync')
 
-    yield _ping_pong_scenario(
-        'ruby_protobuf_sync_streaming_qps_unconstrained', rpc_type='STREAMING',
-        client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
-        unconstrained_client='sync')
+        yield _ping_pong_scenario(
+            'ruby_protobuf_sync_streaming_qps_unconstrained',
+            rpc_type='STREAMING',
+            client_type='SYNC_CLIENT',
+            server_type='SYNC_SERVER',
+            unconstrained_client='sync')
 
-    yield _ping_pong_scenario(
-        'ruby_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY',
-        client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
-        server_language='c++', async_server_threads=1)
+        yield _ping_pong_scenario(
+            'ruby_to_cpp_protobuf_sync_unary_ping_pong',
+            rpc_type='UNARY',
+            client_type='SYNC_CLIENT',
+            server_type='SYNC_SERVER',
+            server_language='c++',
+            async_server_threads=1)
 
-    yield _ping_pong_scenario(
-        'ruby_to_cpp_protobuf_sync_streaming_ping_pong', rpc_type='STREAMING',
-        client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
-        server_language='c++', async_server_threads=1)
+        yield _ping_pong_scenario(
+            'ruby_to_cpp_protobuf_sync_streaming_ping_pong',
+            rpc_type='STREAMING',
+            client_type='SYNC_CLIENT',
+            server_type='SYNC_SERVER',
+            server_language='c++',
+            async_server_threads=1)
 
-    yield _ping_pong_scenario(
-        'ruby_protobuf_unary_ping_pong_1MB', rpc_type='UNARY',
-        client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
-        req_size=1024*1024, resp_size=1024*1024,
-        categories=[SMOKETEST, SCALABLE])
+        yield _ping_pong_scenario(
+            'ruby_protobuf_unary_ping_pong_1MB',
+            rpc_type='UNARY',
+            client_type='SYNC_CLIENT',
+            server_type='SYNC_SERVER',
+            req_size=1024 * 1024,
+            resp_size=1024 * 1024,
+            categories=[SMOKETEST, SCALABLE])
 
-  def __str__(self):
-    return 'ruby'
+    def __str__(self):
+        return 'ruby'
 
 
 class Php7Language:
 
-  def __init__(self, php7_protobuf_c=False):
-    pass
-    self.php7_protobuf_c=php7_protobuf_c
-    self.safename = str(self)
-
-  def worker_cmdline(self):
-    if self.php7_protobuf_c:
-        return ['tools/run_tests/performance/run_worker_php.sh', '--use_protobuf_c_extension']
-    return ['tools/run_tests/performance/run_worker_php.sh']
-
-  def worker_port_offset(self):
-    if self.php7_protobuf_c:
-        return 900
-    return 800
-
-  def scenarios(self):
-    php7_extension_mode='php7_protobuf_php_extension'
-    if self.php7_protobuf_c:
-        php7_extension_mode='php7_protobuf_c_extension'
-
-    yield _ping_pong_scenario(
-        '%s_to_cpp_protobuf_sync_unary_ping_pong' % php7_extension_mode,
-        rpc_type='UNARY', client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
-        server_language='c++', async_server_threads=1)
-
-    yield _ping_pong_scenario(
-        '%s_to_cpp_protobuf_sync_streaming_ping_pong' % php7_extension_mode,
-        rpc_type='STREAMING', client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
-        server_language='c++', async_server_threads=1)
-
-    # TODO(ddyihai): Investigate why when async_server_threads=1/CPU usage 340%, the QPS performs
-    # better than async_server_threads=0/CPU usage 490%.
-    yield _ping_pong_scenario(
-        '%s_to_cpp_protobuf_sync_unary_qps_unconstrained' % php7_extension_mode,
-        rpc_type='UNARY', client_type='SYNC_CLIENT', server_type='ASYNC_SERVER',
-        server_language='c++', outstanding=1, async_server_threads=1, unconstrained_client='sync')
-
-    yield _ping_pong_scenario(
-        '%s_to_cpp_protobuf_sync_streaming_qps_unconstrained' % php7_extension_mode,
-        rpc_type='STREAMING', client_type='SYNC_CLIENT', server_type='ASYNC_SERVER',
-        server_language='c++', outstanding=1, async_server_threads=1, unconstrained_client='sync')
-
-  def __str__(self):
-    if self.php7_protobuf_c:
-        return 'php7_protobuf_c'
-    return 'php7'
+    def __init__(self, php7_protobuf_c=False):
+        pass
+        self.php7_protobuf_c = php7_protobuf_c
+        self.safename = str(self)
+
+    def worker_cmdline(self):
+        if self.php7_protobuf_c:
+            return [
+                'tools/run_tests/performance/run_worker_php.sh',
+                '--use_protobuf_c_extension'
+            ]
+        return ['tools/run_tests/performance/run_worker_php.sh']
+
+    def worker_port_offset(self):
+        if self.php7_protobuf_c:
+            return 900
+        return 800
+
+    def scenarios(self):
+        php7_extension_mode = 'php7_protobuf_php_extension'
+        if self.php7_protobuf_c:
+            php7_extension_mode = 'php7_protobuf_c_extension'
+
+        yield _ping_pong_scenario(
+            '%s_to_cpp_protobuf_sync_unary_ping_pong' % php7_extension_mode,
+            rpc_type='UNARY',
+            client_type='SYNC_CLIENT',
+            server_type='SYNC_SERVER',
+            server_language='c++',
+            async_server_threads=1)
+
+        yield _ping_pong_scenario(
+            '%s_to_cpp_protobuf_sync_streaming_ping_pong' % php7_extension_mode,
+            rpc_type='STREAMING',
+            client_type='SYNC_CLIENT',
+            server_type='SYNC_SERVER',
+            server_language='c++',
+            async_server_threads=1)
+
+        # TODO(ddyihai): Investigate why when async_server_threads=1/CPU usage 340%, the QPS performs
+        # better than async_server_threads=0/CPU usage 490%.
+        yield _ping_pong_scenario(
+            '%s_to_cpp_protobuf_sync_unary_qps_unconstrained' %
+            php7_extension_mode,
+            rpc_type='UNARY',
+            client_type='SYNC_CLIENT',
+            server_type='ASYNC_SERVER',
+            server_language='c++',
+            outstanding=1,
+            async_server_threads=1,
+            unconstrained_client='sync')
+
+        yield _ping_pong_scenario(
+            '%s_to_cpp_protobuf_sync_streaming_qps_unconstrained' %
+            php7_extension_mode,
+            rpc_type='STREAMING',
+            client_type='SYNC_CLIENT',
+            server_type='ASYNC_SERVER',
+            server_language='c++',
+            outstanding=1,
+            async_server_threads=1,
+            unconstrained_client='sync')
+
+    def __str__(self):
+        if self.php7_protobuf_c:
+            return 'php7_protobuf_c'
+        return 'php7'
+
 
 class JavaLanguage:
 
-  def __init__(self):
-    pass
-    self.safename = str(self)
-
-  def worker_cmdline(self):
-    return ['tools/run_tests/performance/run_worker_java.sh']
-
-  def worker_port_offset(self):
-    return 400
-
-  def scenarios(self):
-    for secure in [True, False]:
-      secstr = 'secure' if secure else 'insecure'
-      smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE]
-
-      yield _ping_pong_scenario(
-          'java_generic_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING',
-          client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
-          use_generic_payload=True, async_server_threads=1,
-          secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS,
-          categories=smoketest_categories)
-
-      yield _ping_pong_scenario(
-          'java_protobuf_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING',
-          client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
-          async_server_threads=1,
-          secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS)
-
-      yield _ping_pong_scenario(
-          'java_protobuf_async_unary_ping_pong_%s' % secstr, rpc_type='UNARY',
-          client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
-          async_server_threads=1,
-          secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS,
-          categories=smoketest_categories)
-
-      yield _ping_pong_scenario(
-          'java_protobuf_unary_ping_pong_%s' % secstr, rpc_type='UNARY',
-          client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
-          async_server_threads=1,
-          secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS)
-
-      yield _ping_pong_scenario(
-          'java_protobuf_async_unary_qps_unconstrained_%s' % secstr, rpc_type='UNARY',
-          client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
-          unconstrained_client='async',
-          secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS,
-          categories=smoketest_categories+[SCALABLE])
-
-      yield _ping_pong_scenario(
-          'java_protobuf_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING',
-          client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
-          unconstrained_client='async',
-          secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS,
-          categories=[SCALABLE])
-
-      yield _ping_pong_scenario(
-          'java_generic_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING',
-          client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
-          unconstrained_client='async', use_generic_payload=True,
-          secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS,
-          categories=[SCALABLE])
-
-      yield _ping_pong_scenario(
-          'java_generic_async_streaming_qps_one_server_core_%s' % secstr, rpc_type='STREAMING',
-          client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
-          unconstrained_client='async-limited', use_generic_payload=True,
-          async_server_threads=1,
-          secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS)
-
-      # TODO(jtattermusch): add scenarios java vs C++
-
-  def __str__(self):
-    return 'java'
+    def __init__(self):
+        pass
+        self.safename = str(self)
+
+    def worker_cmdline(self):
+        return ['tools/run_tests/performance/run_worker_java.sh']
+
+    def worker_port_offset(self):
+        return 400
+
+    def scenarios(self):
+        for secure in [True, False]:
+            secstr = 'secure' if secure else 'insecure'
+            smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE]
+
+            yield _ping_pong_scenario(
+                'java_generic_async_streaming_ping_pong_%s' % secstr,
+                rpc_type='STREAMING',
+                client_type='ASYNC_CLIENT',
+                server_type='ASYNC_GENERIC_SERVER',
+                use_generic_payload=True,
+                async_server_threads=1,
+                secure=secure,
+                warmup_seconds=JAVA_WARMUP_SECONDS,
+                categories=smoketest_categories)
+
+            yield _ping_pong_scenario(
+                'java_protobuf_async_streaming_ping_pong_%s' % secstr,
+                rpc_type='STREAMING',
+                client_type='ASYNC_CLIENT',
+                server_type='ASYNC_SERVER',
+                async_server_threads=1,
+                secure=secure,
+                warmup_seconds=JAVA_WARMUP_SECONDS)
+
+            yield _ping_pong_scenario(
+                'java_protobuf_async_unary_ping_pong_%s' % secstr,
+                rpc_type='UNARY',
+                client_type='ASYNC_CLIENT',
+                server_type='ASYNC_SERVER',
+                async_server_threads=1,
+                secure=secure,
+                warmup_seconds=JAVA_WARMUP_SECONDS,
+                categories=smoketest_categories)
+
+            yield _ping_pong_scenario(
+                'java_protobuf_unary_ping_pong_%s' % secstr,
+                rpc_type='UNARY',
+                client_type='SYNC_CLIENT',
+                server_type='SYNC_SERVER',
+                async_server_threads=1,
+                secure=secure,
+                warmup_seconds=JAVA_WARMUP_SECONDS)
+
+            yield _ping_pong_scenario(
+                'java_protobuf_async_unary_qps_unconstrained_%s' % secstr,
+                rpc_type='UNARY',
+                client_type='ASYNC_CLIENT',
+                server_type='ASYNC_SERVER',
+                unconstrained_client='async',
+                secure=secure,
+                warmup_seconds=JAVA_WARMUP_SECONDS,
+                categories=smoketest_categories + [SCALABLE])
+
+            yield _ping_pong_scenario(
+                'java_protobuf_async_streaming_qps_unconstrained_%s' % secstr,
+                rpc_type='STREAMING',
+                client_type='ASYNC_CLIENT',
+                server_type='ASYNC_SERVER',
+                unconstrained_client='async',
+                secure=secure,
+                warmup_seconds=JAVA_WARMUP_SECONDS,
+                categories=[SCALABLE])
+
+            yield _ping_pong_scenario(
+                'java_generic_async_streaming_qps_unconstrained_%s' % secstr,
+                rpc_type='STREAMING',
+                client_type='ASYNC_CLIENT',
+                server_type='ASYNC_GENERIC_SERVER',
+                unconstrained_client='async',
+                use_generic_payload=True,
+                secure=secure,
+                warmup_seconds=JAVA_WARMUP_SECONDS,
+                categories=[SCALABLE])
+
+            yield _ping_pong_scenario(
+                'java_generic_async_streaming_qps_one_server_core_%s' % secstr,
+                rpc_type='STREAMING',
+                client_type='ASYNC_CLIENT',
+                server_type='ASYNC_GENERIC_SERVER',
+                unconstrained_client='async-limited',
+                use_generic_payload=True,
+                async_server_threads=1,
+                secure=secure,
+                warmup_seconds=JAVA_WARMUP_SECONDS)
+
+            # TODO(jtattermusch): add scenarios java vs C++
+
+    def __str__(self):
+        return 'java'
 
 
 class GoLanguage:
 
-  def __init__(self):
-    pass
-    self.safename = str(self)
-
-  def worker_cmdline(self):
-    return ['tools/run_tests/performance/run_worker_go.sh']
-
-  def worker_port_offset(self):
-    return 600
-
-  def scenarios(self):
-    for secure in [True, False]:
-      secstr = 'secure' if secure else 'insecure'
-      smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE]
-
-      # ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server,
-      # but that's mostly because of lack of better name of the enum value.
-      yield _ping_pong_scenario(
-          'go_generic_sync_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING',
-          client_type='SYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
-          use_generic_payload=True, async_server_threads=1,
-          secure=secure,
-          categories=smoketest_categories)
-
-      yield _ping_pong_scenario(
-          'go_protobuf_sync_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING',
-          client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
-          async_server_threads=1,
-          secure=secure)
-
-      yield _ping_pong_scenario(
-          'go_protobuf_sync_unary_ping_pong_%s' % secstr, rpc_type='UNARY',
-          client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
-          async_server_threads=1,
-          secure=secure,
-          categories=smoketest_categories)
-
-      # unconstrained_client='async' is intended (client uses goroutines)
-      yield _ping_pong_scenario(
-          'go_protobuf_sync_unary_qps_unconstrained_%s' % secstr, rpc_type='UNARY',
-          client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
-          unconstrained_client='async',
-          secure=secure,
-          categories=smoketest_categories+[SCALABLE])
-
-      # unconstrained_client='async' is intended (client uses goroutines)
-      yield _ping_pong_scenario(
-          'go_protobuf_sync_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING',
-          client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
-          unconstrained_client='async',
-          secure=secure,
-          categories=[SCALABLE])
-
-      # unconstrained_client='async' is intended (client uses goroutines)
-      # ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server,
-      # but that's mostly because of lack of better name of the enum value.
-      yield _ping_pong_scenario(
-          'go_generic_sync_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING',
-          client_type='SYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
-          unconstrained_client='async', use_generic_payload=True,
-          secure=secure,
-          categories=[SCALABLE])
-
-      # TODO(jtattermusch): add scenarios go vs C++
-
-  def __str__(self):
-    return 'go'
+    def __init__(self):
+        pass
+        self.safename = str(self)
+
+    def worker_cmdline(self):
+        return ['tools/run_tests/performance/run_worker_go.sh']
+
+    def worker_port_offset(self):
+        return 600
+
+    def scenarios(self):
+        for secure in [True, False]:
+            secstr = 'secure' if secure else 'insecure'
+            smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE]
+
+            # ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server,
+            # but that's mostly because of lack of better name of the enum value.
+            yield _ping_pong_scenario(
+                'go_generic_sync_streaming_ping_pong_%s' % secstr,
+                rpc_type='STREAMING',
+                client_type='SYNC_CLIENT',
+                server_type='ASYNC_GENERIC_SERVER',
+                use_generic_payload=True,
+                async_server_threads=1,
+                secure=secure,
+                categories=smoketest_categories)
+
+            yield _ping_pong_scenario(
+                'go_protobuf_sync_streaming_ping_pong_%s' % secstr,
+                rpc_type='STREAMING',
+                client_type='SYNC_CLIENT',
+                server_type='SYNC_SERVER',
+                async_server_threads=1,
+                secure=secure)
+
+            yield _ping_pong_scenario(
+                'go_protobuf_sync_unary_ping_pong_%s' % secstr,
+                rpc_type='UNARY',
+                client_type='SYNC_CLIENT',
+                server_type='SYNC_SERVER',
+                async_server_threads=1,
+                secure=secure,
+                categories=smoketest_categories)
+
+            # unconstrained_client='async' is intended (client uses goroutines)
+            yield _ping_pong_scenario(
+                'go_protobuf_sync_unary_qps_unconstrained_%s' % secstr,
+                rpc_type='UNARY',
+                client_type='SYNC_CLIENT',
+                server_type='SYNC_SERVER',
+                unconstrained_client='async',
+                secure=secure,
+                categories=smoketest_categories + [SCALABLE])
+
+            # unconstrained_client='async' is intended (client uses goroutines)
+            yield _ping_pong_scenario(
+                'go_protobuf_sync_streaming_qps_unconstrained_%s' % secstr,
+                rpc_type='STREAMING',
+                client_type='SYNC_CLIENT',
+                server_type='SYNC_SERVER',
+                unconstrained_client='async',
+                secure=secure,
+                categories=[SCALABLE])
+
+            # unconstrained_client='async' is intended (client uses goroutines)
+            # ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server,
+            # but that's mostly because of lack of better name of the enum value.
+            yield _ping_pong_scenario(
+                'go_generic_sync_streaming_qps_unconstrained_%s' % secstr,
+                rpc_type='STREAMING',
+                client_type='SYNC_CLIENT',
+                server_type='ASYNC_GENERIC_SERVER',
+                unconstrained_client='async',
+                use_generic_payload=True,
+                secure=secure,
+                categories=[SCALABLE])
+
+            # TODO(jtattermusch): add scenarios go vs C++
+
+    def __str__(self):
+        return 'go'
 
 
 LANGUAGES = {
-    'c++' : CXXLanguage(),
-    'csharp' : CSharpLanguage(),
-    'ruby' : RubyLanguage(),
-    'php7' : Php7Language(),
-    'php7_protobuf_c' : Php7Language(php7_protobuf_c=True),
-    'java' : JavaLanguage(),
-    'python' : PythonLanguage(),
-    'go' : GoLanguage(),
+    'c++': CXXLanguage(),
+    'csharp': CSharpLanguage(),
+    'ruby': RubyLanguage(),
+    'php7': Php7Language(),
+    'php7_protobuf_c': Php7Language(php7_protobuf_c=True),
+    'java': JavaLanguage(),
+    'python': PythonLanguage(),
+    'go': GoLanguage(),
 }