Merge remote-tracking branch 'upstream/master' into fix-stream-compression-config-interface

2 files changed, 98 insertions, 3 deletions

diff --git a/tools/codegen/core/gen_static_metadata.py b/tools/codegen/core/gen_static_metadata.py
index 36d951ff70..b7edbbb454 100755
--- a/tools/codegen/core/gen_static_metadata.py
+++ b/tools/codegen/core/gen_static_metadata.py
@@ -374,8 +374,8 @@ for i, elem in enumerate(all_strs):
 
 
 def slice_def(i):
-  return ('{.refcount = &grpc_static_metadata_refcounts[%d], .data.refcounted ='
-          ' {g_bytes+%d, %d}}') % (
+  return ('{&grpc_static_metadata_refcounts[%d],'
+          ' {{g_bytes+%d, %d}}}') % (
       i, id2strofs[i], len(all_strs[i]))
 
 
diff --git a/tools/codegen/core/gen_stats_data.py b/tools/codegen/core/gen_stats_data.py
index 8e4ef594af..10ad0cc831 100755
--- a/tools/codegen/core/gen_stats_data.py
+++ b/tools/codegen/core/gen_stats_data.py
@@ -147,7 +147,8 @@ def gen_bucket_code(histogram):
     shift_data = find_ideal_shift(code_bounds[first_nontrivial:], 256 * histogram.buckets)
   #print first_nontrivial, shift_data, bounds
   #if shift_data is not None: print [hex(x >> shift_data[0]) for x in code_bounds[first_nontrivial:]]
-  code = 'value = GPR_CLAMP(value, 0, %d);\n' % histogram.max
+  code = '\n/* Automatically generated by tools/codegen/core/gen_stats_data.py */\n'
+  code += 'value = GPR_CLAMP(value, 0, %d);\n' % histogram.max
   map_table = gen_map_table(code_bounds[first_nontrivial:], shift_data)
   if first_nontrivial is None:
     code += ('GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_%s, value);\n'
@@ -313,3 +314,97 @@ with open('src/core/lib/debug/stats_data.c', 'w') as C:
       len(inst_map['Histogram']), ','.join('grpc_stats_table_%d' % x for x in histo_bucket_boundaries))
   print >>C, "void (*const grpc_stats_inc_histogram[%d])(grpc_exec_ctx *exec_ctx, int x) = {%s};" % (
       len(inst_map['Histogram']), ','.join('grpc_stats_inc_%s' % histogram.name.lower() for histogram in inst_map['Histogram']))
+
+# patch qps_test bigquery schema
+RECORD_EXPLICIT_PERCENTILES = [50, 95, 99]
+
+with open('tools/run_tests/performance/scenario_result_schema.json', 'r') as f:
+  qps_schema = json.loads(f.read())
+
+def FindNamed(js, name):
+  for el in js:
+    if el['name'] == name:
+      return el
+
+def RemoveCoreFields(js):
+  new_fields = []
+  for field in js['fields']:
+    if not field['name'].startswith('core_'):
+      new_fields.append(field)
+  js['fields'] = new_fields
+
+RemoveCoreFields(FindNamed(qps_schema, 'clientStats'))
+RemoveCoreFields(FindNamed(qps_schema, 'serverStats'))
+
+def AddCoreFields(js):
+  for counter in inst_map['Counter']:
+    js['fields'].append({
+      'name': 'core_%s' % counter.name,
+      'type': 'INTEGER',
+      'mode': 'NULLABLE'
+    })
+  for histogram in inst_map['Histogram']:
+    js['fields'].append({
+      'name': 'core_%s' % histogram.name,
+      'type': 'STRING',
+      'mode': 'NULLABLE'
+    })
+    js['fields'].append({
+      'name': 'core_%s_bkts' % histogram.name,
+      'type': 'STRING',
+      'mode': 'NULLABLE'
+    })
+    for pctl in RECORD_EXPLICIT_PERCENTILES:
+      js['fields'].append({
+        'name': 'core_%s_%dp' % (histogram.name, pctl),
+        'type': 'FLOAT',
+        'mode': 'NULLABLE'
+      })
+
+AddCoreFields(FindNamed(qps_schema, 'clientStats'))
+AddCoreFields(FindNamed(qps_schema, 'serverStats'))
+
+with open('tools/run_tests/performance/scenario_result_schema.json', 'w') as f:
+  f.write(json.dumps(qps_schema, indent=2, sort_keys=True))
+
+# and generate a helper script to massage scenario results into the format we'd
+# like to query
+with open('tools/run_tests/performance/massage_qps_stats.py', 'w') as P:
+  with open(sys.argv[0]) as my_source:
+    for line in my_source:
+      if line[0] != '#': break
+    for line in my_source:
+      if line[0] == '#':
+        print >>P, line.rstrip()
+        break
+    for line in my_source:
+      if line[0] != '#':
+        break
+      print >>P, line.rstrip()
+
+  print >>P
+  print >>P, '# Autogenerated by tools/codegen/core/gen_stats_data.py'
+  print >>P
+
+  print >>P, 'import massage_qps_stats_helpers'
+
+  print >>P, 'def massage_qps_stats(scenario_result):'
+  print >>P, '  for stats in scenario_result["serverStats"] + scenario_result["clientStats"]:'
+  print >>P, '    if "coreStats" not in stats: return'
+  print >>P, '    core_stats = stats["coreStats"]'
+  print >>P, '    del stats["coreStats"]'
+  for counter in inst_map['Counter']:
+    print >>P, '    stats["core_%s"] = massage_qps_stats_helpers.counter(core_stats, "%s")' % (counter.name, counter.name)
+  for i, histogram in enumerate(inst_map['Histogram']):
+    print >>P, '    h = massage_qps_stats_helpers.histogram(core_stats, "%s")' % histogram.name
+    print >>P, '    stats["core_%s"] = ",".join("%%f" %% x for x in h.buckets)' % histogram.name
+    print >>P, '    stats["core_%s_bkts"] = ",".join("%%f" %% x for x in h.boundaries)' % histogram.name
+    for pctl in RECORD_EXPLICIT_PERCENTILES:
+      print >>P, '    stats["core_%s_%dp"] = massage_qps_stats_helpers.percentile(h.buckets, %d, h.boundaries)' % (
+          histogram.name, pctl, pctl)
+
+with open('src/core/lib/debug/stats_data_bq_schema.sql', 'w') as S:
+  columns = []
+  for counter in inst_map['Counter']:
+    columns.append(('%s_per_iteration' % counter.name, 'FLOAT'))
+  print >>S, ',\n'.join('%s:%s' % x for x in columns)