Merge github.com:grpc/grpc into count_now

3 files changed, 287 insertions, 66 deletions

diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py
index 14ea24f65e..09b62ae1c9 100755
--- a/tools/profiling/microbenchmarks/bm_diff.py
+++ b/tools/profiling/microbenchmarks/bm_diff.py
@@ -33,6 +33,30 @@ import json
 import bm_json
 import tabulate
 import argparse
+from scipy import stats
+import subprocess
+import multiprocessing
+import collections
+import pipes
+import os
+sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils'))
+import comment_on_pr
+import jobset
+import itertools
+import speedup
+import random
+import shutil
+import errno
+
+_INTERESTING = (
+  'cpu_time',
+  'real_time',
+  'locks_per_iteration',
+  'allocs_per_iteration',
+  'writes_per_iteration',
+  'atm_cas_per_iteration',
+  'atm_add_per_iteration',
+)
 
 def changed_ratio(n, o):
   if float(o) <= .0001: o = 0
@@ -41,78 +65,185 @@ def changed_ratio(n, o):
   if o == 0: return 100
   return (float(n)-float(o))/float(o)
 
+def median(ary):
+  ary = sorted(ary)
+  n = len(ary)
+  if n%2 == 0:
+    return (ary[n/2] + ary[n/2+1]) / 2.0
+  else:
+    return ary[n/2]
+
 def min_change(pct):
   return lambda n, o: abs(changed_ratio(n,o)) > pct/100.0
 
-_INTERESTING = {
-  'cpu_time': min_change(10),
-  'real_time': min_change(10),
-  'locks_per_iteration': min_change(5),
-  'allocs_per_iteration': min_change(5),
-  'writes_per_iteration': min_change(5),
-  'atm_cas_per_iteration': min_change(1),
-  'atm_add_per_iteration': min_change(5),
-}
+_AVAILABLE_BENCHMARK_TESTS = ['bm_fullstack_unary_ping_pong',
+                              'bm_fullstack_streaming_ping_pong',
+                              'bm_fullstack_streaming_pump',
+                              'bm_closure',
+                              'bm_cq',
+                              'bm_call_create',
+                              'bm_error',
+                              'bm_chttp2_hpack',
+                              'bm_chttp2_transport',
+                              'bm_pollset',
+                              'bm_metadata',
+                              'bm_fullstack_trickle']
 
 argp = argparse.ArgumentParser(description='Perform diff on microbenchmarks')
 argp.add_argument('-t', '--track',
-                  choices=sorted(_INTERESTING.keys()),
+                  choices=sorted(_INTERESTING),
                   nargs='+',
-                  default=sorted(_INTERESTING.keys()),
+                  default=sorted(_INTERESTING),
                   help='Which metrics to track')
-argp.add_argument('files', metavar='bm_file.json', type=str, nargs=4,
-                    help='files to diff. ')
+argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_cq'])
+argp.add_argument('-d', '--diff_base', type=str)
+argp.add_argument('-r', '--repetitions', type=int, default=1)
+argp.add_argument('-l', '--loops', type=int, default=20)
+argp.add_argument('-j', '--jobs', type=int, default=multiprocessing.cpu_count())
 args = argp.parse_args()
 
-with open(args.files[0]) as f:
-  js_new_ctr = json.loads(f.read())
-with open(args.files[1]) as f:
-  js_new_opt = json.loads(f.read())
-with open(args.files[2]) as f:
-  js_old_ctr = json.loads(f.read())
-with open(args.files[3]) as f:
-  js_old_opt = json.loads(f.read())
-
-new = {}
-old = {}
-
-for row in bm_json.expand_json(js_new_ctr, js_new_opt):
-  new[row['cpp_name']] = row
-for row in bm_json.expand_json(js_old_ctr, js_old_opt):
-  old[row['cpp_name']] = row
-
-changed = []
-for fld in args.track:
-  chk = _INTERESTING[fld]
-  for bm in new.keys():
-    if bm not in old: continue
-    n = new[bm]
-    o = old[bm]
-    if fld not in n or fld not in o: continue
-    if chk(n[fld], o[fld]):
-      changed.append((fld, chk))
-      break
-
-headers = ['Benchmark'] + [c[0] for c in changed] + ['Details']
-rows = []
-for bm in sorted(new.keys()):
-  if bm not in old: continue
-  row = [bm]
-  any_changed = False
-  n = new[bm]
-  o = old[bm]
-  details = ''
-  for fld in args.track:
-    chk = _INTERESTING[fld]
-    if fld not in n or fld not in o: continue
-    if chk(n[fld], o[fld]):
-      row.append(changed_ratio(n[fld], o[fld]))
-      if details: details += ', '
-      details += '%s:%r-->%r' % (fld, float(o[fld]), float(n[fld]))
-      any_changed = True
-    else:
-      row.append('')
-  if any_changed:
-    row.append(details)
-    rows.append(row)
-print tabulate.tabulate(rows, headers=headers, floatfmt='+.2f')
+assert args.diff_base
+
+def avg(lst):
+  sum = 0.0
+  n = 0.0
+  for el in lst:
+    sum += el
+    n += 1
+  return sum / n
+
+def make_cmd(cfg):
+  return ['make'] + args.benchmarks + [
+      'CONFIG=%s' % cfg, '-j', '%d' % args.jobs]
+
+def build(dest):
+  shutil.rmtree('bm_diff_%s' % dest, ignore_errors=True)
+  subprocess.check_call(['git', 'submodule', 'update'])
+  try:
+    subprocess.check_call(make_cmd('opt'))
+    subprocess.check_call(make_cmd('counters'))
+  except subprocess.CalledProcessError, e:
+    subprocess.check_call(['make', 'clean'])
+    subprocess.check_call(make_cmd('opt'))
+    subprocess.check_call(make_cmd('counters'))
+  os.rename('bins', 'bm_diff_%s' % dest)
+
+def collect1(bm, cfg, ver, idx):
+  cmd = ['bm_diff_%s/%s/%s' % (ver, cfg, bm),
+         '--benchmark_out=%s.%s.%s.%d.json' % (bm, cfg, ver, idx),
+         '--benchmark_out_format=json',
+         '--benchmark_repetitions=%d' % (args.repetitions)
+         ]
+  return jobset.JobSpec(cmd, shortname='%s %s %s %d/%d' % (bm, cfg, ver, idx+1, args.loops),
+                             verbose_success=True, timeout_seconds=None)
+
+build('new')
+
+where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
+subprocess.check_call(['git', 'checkout', args.diff_base])
+try:
+  build('old')
+finally:
+  subprocess.check_call(['git', 'checkout', where_am_i])
+  subprocess.check_call(['git', 'submodule', 'update'])
+
+jobs = []
+for loop in range(0, args.loops):
+  jobs.extend(x for x in itertools.chain(
+    (collect1(bm, 'opt', 'new', loop) for bm in args.benchmarks),
+    (collect1(bm, 'counters', 'new', loop) for bm in args.benchmarks),
+    (collect1(bm, 'opt', 'old', loop) for bm in args.benchmarks),
+    (collect1(bm, 'counters', 'old', loop) for bm in args.benchmarks),
+  ))
+random.shuffle(jobs, random.SystemRandom().random)
+
+jobset.run(jobs, maxjobs=args.jobs)
+
+class Benchmark:
+
+  def __init__(self):
+    self.samples = {
+      True: collections.defaultdict(list),
+      False: collections.defaultdict(list)
+    }
+    self.final = {}
+
+  def add_sample(self, data, new):
+    for f in args.track:
+      if f in data:
+        self.samples[new][f].append(float(data[f]))
+
+  def process(self):
+    for f in sorted(args.track):
+      new = self.samples[True][f]
+      old = self.samples[False][f]
+      if not new or not old: continue
+      mdn_diff = abs(median(new) - median(old))
+      print '%s: new=%r old=%r mdn_diff=%r' % (f, new, old, mdn_diff)
+      s = speedup.speedup(new, old)
+      if abs(s) > 3 and mdn_diff > 0.5:
+        self.final[f] = '%+d%%' % s
+    return self.final.keys()
+
+  def skip(self):
+    return not self.final
+
+  def row(self, flds):
+    return [self.final[f] if f in self.final else '' for f in flds]
+
+
+def eintr_be_gone(fn):
+  """Run fn until it doesn't stop because of EINTR"""
+  while True:
+    try:
+      return fn()
+    except IOError, e:
+      if e.errno != errno.EINTR:
+        raise
+
+
+def read_json(filename):
+  with open(filename) as f: return json.loads(f.read())
+
+
+def finalize():
+  benchmarks = collections.defaultdict(Benchmark)
+
+  for bm in args.benchmarks:
+    for loop in range(0, args.loops):
+      js_new_ctr = read_json('%s.counters.new.%d.json' % (bm, loop))
+      js_new_opt = read_json('%s.opt.new.%d.json' % (bm, loop))
+      js_old_ctr = read_json('%s.counters.old.%d.json' % (bm, loop))
+      js_old_opt = read_json('%s.opt.old.%d.json' % (bm, loop))
+
+      for row in bm_json.expand_json(js_new_ctr, js_new_opt):
+        print row
+        name = row['cpp_name']
+        if name.endswith('_mean') or name.endswith('_stddev'): continue
+        benchmarks[name].add_sample(row, True)
+      for row in bm_json.expand_json(js_old_ctr, js_old_opt):
+        print row
+        name = row['cpp_name']
+        if name.endswith('_mean') or name.endswith('_stddev'): continue
+        benchmarks[name].add_sample(row, False)
+
+  really_interesting = set()
+  for name, bm in benchmarks.items():
+    print name
+    really_interesting.update(bm.process())
+  fields = [f for f in args.track if f in really_interesting]
+
+  headers = ['Benchmark'] + fields
+  rows = []
+  for name in sorted(benchmarks.keys()):
+    if benchmarks[name].skip(): continue
+    rows.append([name] + benchmarks[name].row(fields))
+  if rows:
+    text = 'Performance differences noted:\n' + tabulate.tabulate(rows, headers=headers, floatfmt='+.2f')
+  else:
+    text = 'No significant performance differences'
+  comment_on_pr.comment_on_pr('```\n%s\n```' % text)
+  print text
+
+
+eintr_be_gone(finalize)
diff --git a/tools/profiling/microbenchmarks/bm_json.py b/tools/profiling/microbenchmarks/bm_json.py
index 4695f829f4..917269823d 100644
--- a/tools/profiling/microbenchmarks/bm_json.py
+++ b/tools/profiling/microbenchmarks/bm_json.py
@@ -101,6 +101,26 @@ _BM_SPECS = {
   'BM_PollEmptyPollset_SpeedOfLight': {
     'tpl': [],
     'dyn': ['request_size', 'request_count'],
+  },
+  'BM_StreamCreateSendInitialMetadataDestroy': {
+    'tpl': ['fixture'],
+    'dyn': [],
+  },
+  'BM_TransportStreamSend': {
+    'tpl': [],
+    'dyn': ['request_size'],
+  },
+  'BM_TransportStreamRecv': {
+    'tpl': [],
+    'dyn': ['request_size'],
+  },
+  'BM_StreamingPingPongWithCoalescingApi': {
+    'tpl': ['fixture', 'client_mutator', 'server_mutator'],
+    'dyn': ['request_size', 'request_count', 'end_of_stream'],
+  },
+  'BM_Base16SomeStuff': {
+    'tpl': [],
+    'dyn': ['request_size'],
   }
 }
 
@@ -159,6 +179,7 @@ def parse_name(name):
 
 def expand_json(js, js2 = None):
   for bm in js['benchmarks']:
+    if bm['name'].endswith('_stddev') or bm['name'].endswith('_mean'): continue
     context = js['context']
     if 'label' in bm:
       labels_list = [s.split(':') for s in bm['label'].strip().split(' ') if len(s) and s[0] != '#']
@@ -177,8 +198,10 @@ def expand_json(js, js2 = None):
     row.update(labels)
     if js2:
       for bm2 in js2['benchmarks']:
-        if bm['name'] == bm2['name']:
+        if bm['name'] == bm2['name'] and 'already_used' not in bm2:
           row['cpu_time'] = bm2['cpu_time']
           row['real_time'] = bm2['real_time']
           row['iterations'] = bm2['iterations']
+          bm2['already_used'] = True
+          break
     yield row
diff --git a/tools/profiling/microbenchmarks/speedup.py b/tools/profiling/microbenchmarks/speedup.py
new file mode 100644
index 0000000000..8af0066c9d
--- /dev/null
+++ b/tools/profiling/microbenchmarks/speedup.py
@@ -0,0 +1,67 @@
+# Copyright 2017, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from scipy import stats
+import math
+
+_THRESHOLD = 1e-10
+
+def scale(a, mul):
+  return [x*mul for x in a]
+
+def cmp(a, b):
+  return stats.ttest_ind(a, b)
+
+def speedup(new, old):
+  s0, p0 = cmp(new, old)
+  if math.isnan(p0): return 0
+  if s0 == 0: return 0
+  if p0 > _THRESHOLD: return 0
+  if s0 < 0:
+    pct = 1
+    while pct < 101:
+      sp, pp = cmp(new, scale(old, 1 - pct/100.0))
+      if sp > 0: break
+      if pp > _THRESHOLD: break
+      pct += 1
+    return -(pct - 1)
+  else:
+    pct = 1
+    while pct < 100000:
+      sp, pp = cmp(new, scale(old, 1 + pct/100.0))
+      if sp < 0: break
+      if pp > _THRESHOLD: break
+      pct += 1
+    return pct - 1
+
+if __name__ == "__main__":
+  new=[66034560.0, 126765693.0, 99074674.0, 98588433.0, 96731372.0, 110179725.0, 103802110.0, 101139800.0, 102357205.0, 99016353.0, 98840824.0, 99585632.0, 98791720.0, 96171521.0, 95327098.0, 95629704.0, 98209772.0, 99779411.0, 100182488.0, 98354192.0, 99644781.0, 98546709.0, 99019176.0, 99543014.0, 99077269.0, 98046601.0, 99319039.0, 98542572.0, 98886614.0, 72560968.0]
+  old=[60423464.0, 71249570.0, 73213089.0, 73200055.0, 72911768.0, 72347798.0, 72494672.0, 72756976.0, 72116565.0, 71541342.0, 73442538.0, 74817383.0, 73007780.0, 72499062.0, 72404945.0, 71843504.0, 73245405.0, 72778304.0, 74004519.0, 73694464.0, 72919931.0, 72955481.0, 71583857.0, 71350467.0, 71836817.0, 70064115.0, 70355345.0, 72516202.0, 71716777.0, 71532266.0]
+  print speedup(new, old)
+  print speedup(old, new)