Merge github.com:grpc/grpc into flowctlN

5 files changed, 265 insertions, 8 deletions

diff --git a/tools/codegen/core/gen_static_metadata.py b/tools/codegen/core/gen_static_metadata.py
index 6ee8a7cace..af7521da36 100755
--- a/tools/codegen/core/gen_static_metadata.py
+++ b/tools/codegen/core/gen_static_metadata.py
@@ -370,8 +370,8 @@ for i, elem in enumerate(all_strs):
 
 
 def slice_def(i):
-  return ('{.refcount = &grpc_static_metadata_refcounts[%d], .data.refcounted ='
-          ' {g_bytes+%d, %d}}') % (
+  return ('{&grpc_static_metadata_refcounts[%d],'
+          ' {{g_bytes+%d, %d}}}') % (
       i, id2strofs[i], len(all_strs[i]))
 
 
diff --git a/tools/codegen/core/gen_stats_data.py b/tools/codegen/core/gen_stats_data.py
index 8359734c84..10ad0cc831 100755
--- a/tools/codegen/core/gen_stats_data.py
+++ b/tools/codegen/core/gen_stats_data.py
@@ -147,7 +147,8 @@ def gen_bucket_code(histogram):
     shift_data = find_ideal_shift(code_bounds[first_nontrivial:], 256 * histogram.buckets)
   #print first_nontrivial, shift_data, bounds
   #if shift_data is not None: print [hex(x >> shift_data[0]) for x in code_bounds[first_nontrivial:]]
-  code = 'value = GPR_CLAMP(value, 0, %d);\n' % histogram.max
+  code = '\n/* Automatically generated by tools/codegen/core/gen_stats_data.py */\n'
+  code += 'value = GPR_CLAMP(value, 0, %d);\n' % histogram.max
   map_table = gen_map_table(code_bounds[first_nontrivial:], shift_data)
   if first_nontrivial is None:
     code += ('GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_%s, value);\n'
@@ -407,4 +408,3 @@ with open('src/core/lib/debug/stats_data_bq_schema.sql', 'w') as S:
   for counter in inst_map['Counter']:
     columns.append(('%s_per_iteration' % counter.name, 'FLOAT'))
   print >>S, ',\n'.join('%s:%s' % x for x in columns)
-
diff --git a/tools/run_tests/performance/massage_qps_stats.py b/tools/run_tests/performance/massage_qps_stats.py
index 4d4c494405..3bdace1ac3 100644
--- a/tools/run_tests/performance/massage_qps_stats.py
+++ b/tools/run_tests/performance/massage_qps_stats.py
@@ -28,6 +28,12 @@ def massage_qps_stats(scenario_result):
     stats["core_server_channels_created"] = massage_qps_stats_helpers.counter(core_stats, "server_channels_created")
     stats["core_syscall_poll"] = massage_qps_stats_helpers.counter(core_stats, "syscall_poll")
     stats["core_syscall_wait"] = massage_qps_stats_helpers.counter(core_stats, "syscall_wait")
+    stats["core_pollset_kick"] = massage_qps_stats_helpers.counter(core_stats, "pollset_kick")
+    stats["core_pollset_kicked_without_poller"] = massage_qps_stats_helpers.counter(core_stats, "pollset_kicked_without_poller")
+    stats["core_pollset_kicked_again"] = massage_qps_stats_helpers.counter(core_stats, "pollset_kicked_again")
+    stats["core_pollset_kick_wakeup_fd"] = massage_qps_stats_helpers.counter(core_stats, "pollset_kick_wakeup_fd")
+    stats["core_pollset_kick_wakeup_cv"] = massage_qps_stats_helpers.counter(core_stats, "pollset_kick_wakeup_cv")
+    stats["core_pollset_kick_own_thread"] = massage_qps_stats_helpers.counter(core_stats, "pollset_kick_own_thread")
     stats["core_histogram_slow_lookups"] = massage_qps_stats_helpers.counter(core_stats, "histogram_slow_lookups")
     stats["core_syscall_write"] = massage_qps_stats_helpers.counter(core_stats, "syscall_write")
     stats["core_syscall_read"] = massage_qps_stats_helpers.counter(core_stats, "syscall_read")
@@ -79,8 +85,22 @@ def massage_qps_stats(scenario_result):
     stats["core_executor_wakeup_initiated"] = massage_qps_stats_helpers.counter(core_stats, "executor_wakeup_initiated")
     stats["core_executor_queue_drained"] = massage_qps_stats_helpers.counter(core_stats, "executor_queue_drained")
     stats["core_executor_push_retries"] = massage_qps_stats_helpers.counter(core_stats, "executor_push_retries")
+    stats["core_executor_threads_created"] = massage_qps_stats_helpers.counter(core_stats, "executor_threads_created")
+    stats["core_executor_threads_used"] = massage_qps_stats_helpers.counter(core_stats, "executor_threads_used")
     stats["core_server_requested_calls"] = massage_qps_stats_helpers.counter(core_stats, "server_requested_calls")
     stats["core_server_slowpath_requests_queued"] = massage_qps_stats_helpers.counter(core_stats, "server_slowpath_requests_queued")
+    h = massage_qps_stats_helpers.histogram(core_stats, "call_initial_size")
+    stats["core_call_initial_size"] = ",".join("%f" % x for x in h.buckets)
+    stats["core_call_initial_size_bkts"] = ",".join("%f" % x for x in h.boundaries)
+    stats["core_call_initial_size_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
+    stats["core_call_initial_size_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
+    stats["core_call_initial_size_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
+    h = massage_qps_stats_helpers.histogram(core_stats, "poll_events_returned")
+    stats["core_poll_events_returned"] = ",".join("%f" % x for x in h.buckets)
+    stats["core_poll_events_returned_bkts"] = ",".join("%f" % x for x in h.boundaries)
+    stats["core_poll_events_returned_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
+    stats["core_poll_events_returned_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
+    stats["core_poll_events_returned_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
     h = massage_qps_stats_helpers.histogram(core_stats, "tcp_write_size")
     stats["core_tcp_write_size"] = ",".join("%f" % x for x in h.buckets)
     stats["core_tcp_write_size_bkts"] = ",".join("%f" % x for x in h.boundaries)
@@ -141,6 +161,12 @@ def massage_qps_stats(scenario_result):
     stats["core_http2_send_flowctl_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
     stats["core_http2_send_flowctl_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
     stats["core_http2_send_flowctl_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
+    h = massage_qps_stats_helpers.histogram(core_stats, "executor_closures_per_wakeup")
+    stats["core_executor_closures_per_wakeup"] = ",".join("%f" % x for x in h.buckets)
+    stats["core_executor_closures_per_wakeup_bkts"] = ",".join("%f" % x for x in h.boundaries)
+    stats["core_executor_closures_per_wakeup_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
+    stats["core_executor_closures_per_wakeup_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
+    stats["core_executor_closures_per_wakeup_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
     h = massage_qps_stats_helpers.histogram(core_stats, "server_cqs_checked")
     stats["core_server_cqs_checked"] = ",".join("%f" % x for x in h.buckets)
     stats["core_server_cqs_checked_bkts"] = ",".join("%f" % x for x in h.boundaries)
diff --git a/tools/run_tests/performance/scenario_result_schema.json b/tools/run_tests/performance/scenario_result_schema.json
index 725b3a7056..583bce1fff 100644
--- a/tools/run_tests/performance/scenario_result_schema.json
+++ b/tools/run_tests/performance/scenario_result_schema.json
@@ -152,6 +152,36 @@
       }, 
       {
         "mode": "NULLABLE", 
+        "name": "core_pollset_kick", 
+        "type": "INTEGER"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_pollset_kicked_without_poller", 
+        "type": "INTEGER"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_pollset_kicked_again", 
+        "type": "INTEGER"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_pollset_kick_wakeup_fd", 
+        "type": "INTEGER"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_pollset_kick_wakeup_cv", 
+        "type": "INTEGER"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_pollset_kick_own_thread", 
+        "type": "INTEGER"
+      }, 
+      {
+        "mode": "NULLABLE", 
         "name": "core_histogram_slow_lookups", 
         "type": "INTEGER"
       }, 
@@ -407,6 +437,16 @@
       }, 
       {
         "mode": "NULLABLE", 
+        "name": "core_executor_threads_created", 
+        "type": "INTEGER"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_executor_threads_used", 
+        "type": "INTEGER"
+      }, 
+      {
+        "mode": "NULLABLE", 
         "name": "core_server_requested_calls", 
         "type": "INTEGER"
       }, 
@@ -417,6 +457,56 @@
       }, 
       {
         "mode": "NULLABLE", 
+        "name": "core_call_initial_size", 
+        "type": "STRING"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_call_initial_size_bkts", 
+        "type": "STRING"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_call_initial_size_50p", 
+        "type": "FLOAT"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_call_initial_size_95p", 
+        "type": "FLOAT"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_call_initial_size_99p", 
+        "type": "FLOAT"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_poll_events_returned", 
+        "type": "STRING"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_poll_events_returned_bkts", 
+        "type": "STRING"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_poll_events_returned_50p", 
+        "type": "FLOAT"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_poll_events_returned_95p", 
+        "type": "FLOAT"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_poll_events_returned_99p", 
+        "type": "FLOAT"
+      }, 
+      {
+        "mode": "NULLABLE", 
         "name": "core_tcp_write_size", 
         "type": "STRING"
       }, 
@@ -667,6 +757,31 @@
       }, 
       {
         "mode": "NULLABLE", 
+        "name": "core_executor_closures_per_wakeup", 
+        "type": "STRING"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_executor_closures_per_wakeup_bkts", 
+        "type": "STRING"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_executor_closures_per_wakeup_50p", 
+        "type": "FLOAT"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_executor_closures_per_wakeup_95p", 
+        "type": "FLOAT"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_executor_closures_per_wakeup_99p", 
+        "type": "FLOAT"
+      }, 
+      {
+        "mode": "NULLABLE", 
         "name": "core_server_cqs_checked", 
         "type": "STRING"
       }, 
@@ -759,6 +874,36 @@
       }, 
       {
         "mode": "NULLABLE", 
+        "name": "core_pollset_kick", 
+        "type": "INTEGER"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_pollset_kicked_without_poller", 
+        "type": "INTEGER"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_pollset_kicked_again", 
+        "type": "INTEGER"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_pollset_kick_wakeup_fd", 
+        "type": "INTEGER"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_pollset_kick_wakeup_cv", 
+        "type": "INTEGER"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_pollset_kick_own_thread", 
+        "type": "INTEGER"
+      }, 
+      {
+        "mode": "NULLABLE", 
         "name": "core_histogram_slow_lookups", 
         "type": "INTEGER"
       }, 
@@ -1014,6 +1159,16 @@
       }, 
       {
         "mode": "NULLABLE", 
+        "name": "core_executor_threads_created", 
+        "type": "INTEGER"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_executor_threads_used", 
+        "type": "INTEGER"
+      }, 
+      {
+        "mode": "NULLABLE", 
         "name": "core_server_requested_calls", 
         "type": "INTEGER"
       }, 
@@ -1024,6 +1179,56 @@
       }, 
       {
         "mode": "NULLABLE", 
+        "name": "core_call_initial_size", 
+        "type": "STRING"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_call_initial_size_bkts", 
+        "type": "STRING"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_call_initial_size_50p", 
+        "type": "FLOAT"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_call_initial_size_95p", 
+        "type": "FLOAT"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_call_initial_size_99p", 
+        "type": "FLOAT"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_poll_events_returned", 
+        "type": "STRING"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_poll_events_returned_bkts", 
+        "type": "STRING"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_poll_events_returned_50p", 
+        "type": "FLOAT"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_poll_events_returned_95p", 
+        "type": "FLOAT"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_poll_events_returned_99p", 
+        "type": "FLOAT"
+      }, 
+      {
+        "mode": "NULLABLE", 
         "name": "core_tcp_write_size", 
         "type": "STRING"
       }, 
@@ -1274,6 +1479,31 @@
       }, 
       {
         "mode": "NULLABLE", 
+        "name": "core_executor_closures_per_wakeup", 
+        "type": "STRING"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_executor_closures_per_wakeup_bkts", 
+        "type": "STRING"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_executor_closures_per_wakeup_50p", 
+        "type": "FLOAT"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_executor_closures_per_wakeup_95p", 
+        "type": "FLOAT"
+      }, 
+      {
+        "mode": "NULLABLE", 
+        "name": "core_executor_closures_per_wakeup_99p", 
+        "type": "FLOAT"
+      }, 
+      {
+        "mode": "NULLABLE", 
         "name": "core_server_cqs_checked", 
         "type": "STRING"
       }, 
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index 176c1097ee..b38108d456 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -350,11 +350,12 @@ class CLanguage(object):
                                                 environ=env))
           else:
             cmdline = [binary] + target['args']
+            shortname = target.get('shortname', ' '.join(
+                          pipes.quote(arg)
+                          for arg in cmdline))
+            shortname += shortname_ext
             out.append(self.config.job_spec(cmdline,
-                                            shortname=' '.join(
-                                                          pipes.quote(arg)
-                                                          for arg in cmdline) +
-                                                      shortname_ext,
+                                            shortname=shortname,
                                             cpu_cost=cpu_cost,
                                             flaky=target.get('flaky', False),
                                             timeout_seconds=target.get('timeout_seconds', _DEFAULT_TIMEOUT_SECONDS) * timeout_scaling,