Add driver for bm_*.py pipeline

5 files changed, 133 insertions, 32 deletions

diff --git a/tools/profiling/microbenchmarks/README.md b/tools/profiling/microbenchmarks/README.md
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/tools/profiling/microbenchmarks/README.md
diff --git a/tools/profiling/microbenchmarks/bm_build.py b/tools/profiling/microbenchmarks/bm_build.py
index 13ca1dfd1b..a5d1ec3447 100755
--- a/tools/profiling/microbenchmarks/bm_build.py
+++ b/tools/profiling/microbenchmarks/bm_build.py
@@ -45,24 +45,24 @@ def _args():
   argp.add_argument('-n', '--name', type=str, help='Unique name of this build')
   return argp.parse_args()
 
-def _make_cmd(cfg, jobs, benchmarks):
+def _make_cmd(cfg, benchmarks, jobs):
   return ['make'] + benchmarks + [
       'CONFIG=%s' % cfg, '-j', '%d' % jobs]
 
-def build(name, jobs, benchmarks):
+def build(name, benchmarks, jobs):
   shutil.rmtree('bm_diff_%s' % name, ignore_errors=True)
   subprocess.check_call(['git', 'submodule', 'update'])
   try:
-    subprocess.check_call(_make_cmd('opt', jobs, benchmarks))
-    subprocess.check_call(_make_cmd('counters', jobs, benchmarks))
+    subprocess.check_call(_make_cmd('opt', benchmarks, jobs))
+    subprocess.check_call(_make_cmd('counters', benchmarks, jobs))
   except subprocess.CalledProcessError, e:
     subprocess.check_call(['make', 'clean'])
-    subprocess.check_call(_make_cmd('opt', jobs, benchmarks))
-    subprocess.check_call(_make_cmd('counters', jobs, benchmarks))
+    subprocess.check_call(_make_cmd('opt', benchmarks, jobs))
+    subprocess.check_call(_make_cmd('counters', benchmarks, jobs))
   os.rename('bins', 'bm_diff_%s' % name, )
 
 if __name__ == '__main__':
   args = _args()
-  build(args.name, args.jobs, args.benchmarks)
+  build(args.name, args.benchmarks, args.jobs)
 
 
diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py
index 4de318fec7..f68dff840e 100755
--- a/tools/profiling/microbenchmarks/bm_diff.py
+++ b/tools/profiling/microbenchmarks/bm_diff.py
@@ -41,7 +41,7 @@ import collections
 
 verbose = False
 
-def median(ary):
+def _median(ary):
   ary = sorted(ary)
   n = len(ary)
   if n%2 == 0:
@@ -68,7 +68,7 @@ def _args():
   assert args.old
   return args
 
-def maybe_print(str):
+def _maybe_print(str):
   if verbose: print str
 
 class Benchmark:
@@ -85,14 +85,15 @@ class Benchmark:
       if f in data:
         self.samples[new][f].append(float(data[f]))
 
-  def process(self, track):
+  def process(self, track, new_name, old_name):
     for f in sorted(track):
       new = self.samples[True][f]
       old = self.samples[False][f]
       if not new or not old: continue
-      mdn_diff = abs(median(new) - median(old))
-      maybe_print('%s: new=%r old=%r mdn_diff=%r' % (f, new, old, mdn_diff))
-      s = speedup.speedup(new, old)
+      mdn_diff = abs(_median(new) - _median(old))
+      _maybe_print('%s: %s=%r %s=%r mdn_diff=%r' % 
+          (f, new_name, new, old_name, old, mdn_diff))
+      s = bm_speedup.speedup(new, old)
       if abs(s) > 3 and mdn_diff > 0.5:
         self.final[f] = '%+d%%' % s
     return self.final.keys()
@@ -103,21 +104,21 @@ class Benchmark:
   def row(self, flds):
     return [self.final[f] if f in self.final else '' for f in flds]
 
-def read_json(filename):
+def _read_json(filename):
   try:
     with open(filename) as f: return json.loads(f.read())
   except ValueError, e:
     return None
 
-def finalize(bms, loops, track):
+def diff(bms, loops, track, old, new):
   benchmarks = collections.defaultdict(Benchmark)
 
   for bm in bms:
     for loop in range(0, loops):
-      js_new_ctr = read_json('%s.counters.new.%d.json' % (bm, loop))
-      js_new_opt = read_json('%s.opt.new.%d.json' % (bm, loop))
-      js_old_ctr = read_json('%s.counters.old.%d.json' % (bm, loop))
-      js_old_opt = read_json('%s.opt.old.%d.json' % (bm, loop))
+      js_new_ctr = _read_json('%s.counters.%s.%d.json' % (bm, new, loop))
+      js_new_opt = _read_json('%s.opt.%s.%d.json' % (bm, new, loop))
+      js_old_ctr = _read_json('%s.counters.%s.%d.json' % (bm, old, loop))
+      js_old_opt = _read_json('%s.opt.%s.%d.json' % (bm, old, loop))
 
       if js_new_ctr:
         for row in bm_json.expand_json(js_new_ctr, js_new_opt):
@@ -132,8 +133,8 @@ def finalize(bms, loops, track):
 
   really_interesting = set()
   for name, bm in benchmarks.items():
-    maybe_print(name)
-    really_interesting.update(bm.process(track))
+    _maybe_print(name)
+    really_interesting.update(bm.process(track, new, old))
   fields = [f for f in track if f in really_interesting]
 
   headers = ['Benchmark'] + fields
@@ -148,6 +149,6 @@ def finalize(bms, loops, track):
 
 if __name__ == '__main__':
   args = _args()
-  print finalize(args.benchmarks, args.loops, args.track)
+  print diff(args.benchmarks, args.loops, args.track, args.old, args.new)
 
 
diff --git a/tools/profiling/microbenchmarks/bm_main.py b/tools/profiling/microbenchmarks/bm_main.py
new file mode 100755
index 0000000000..1a46b17015
--- /dev/null
+++ b/tools/profiling/microbenchmarks/bm_main.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python2.7
+# Copyright 2017, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+""" Runs the entire bm_*.py pipeline, and possible comments on the PR """
+
+import bm_constants
+import bm_build
+import bm_run
+import bm_diff
+
+import sys
+import os
+import argparse
+import multiprocessing
+import subprocess
+
+sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils'))
+import comment_on_pr
+
+def _args():
+  argp = argparse.ArgumentParser(description='Perform diff on microbenchmarks')
+  argp.add_argument('-t', '--track',
+                    choices=sorted(bm_constants._INTERESTING),
+                    nargs='+',
+                    default=sorted(bm_constants._INTERESTING),
+                    help='Which metrics to track')
+  argp.add_argument('-b', '--benchmarks', nargs='+', choices=bm_constants._AVAILABLE_BENCHMARK_TESTS, default=bm_constants._AVAILABLE_BENCHMARK_TESTS)
+  argp.add_argument('-d', '--diff_base', type=str)
+  argp.add_argument('-r', '--repetitions', type=int, default=1)
+  argp.add_argument('-l', '--loops', type=int, default=20)
+  argp.add_argument('-j', '--jobs', type=int, default=multiprocessing.cpu_count())
+  args = argp.parse_args()
+  assert args.diff_base
+  return args
+
+
+def eintr_be_gone(fn):
+  """Run fn until it doesn't stop because of EINTR"""
+  def inner(*args):
+    while True:
+      try:
+        return fn(*args)
+      except IOError, e:
+        if e.errno != errno.EINTR:
+          raise
+  return inner
+
+def main(args):
+
+  bm_build.build('new', args.benchmarks, args.jobs)
+
+  where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
+  subprocess.check_call(['git', 'checkout', args.diff_base])
+  try:
+    bm_build.build('old', args.benchmarks, args.jobs)
+  finally:
+    subprocess.check_call(['git', 'checkout', where_am_i])
+    subprocess.check_call(['git', 'submodule', 'update'])
+
+  bm_run.run('new', args.benchmarks, args.jobs, args.loops, args.repetitions)
+  bm_run.run('old', args.benchmarks, args.jobs, args.loops, args.repetitions)
+
+  diff = bm_diff.diff(args.benchmarks, args.loops, args.track, 'old', 'new')
+  if diff:
+    text = 'Performance differences noted:\n' + diff
+  else:
+    text = 'No significant performance differences'
+  print text
+  comment_on_pr.comment_on_pr('```\n%s\n```' % text)
+
+if __name__ == '__main__':
+  args = _args()
+  main(args)
diff --git a/tools/profiling/microbenchmarks/bm_run.py b/tools/profiling/microbenchmarks/bm_run.py
index 458e319440..a476a7e19f 100755
--- a/tools/profiling/microbenchmarks/bm_run.py
+++ b/tools/profiling/microbenchmarks/bm_run.py
@@ -51,27 +51,27 @@ def _args():
   argp.add_argument('-l', '--loops', type=int, default=20)
   return argp.parse_args()
 
-def _collect_bm_data(bm, cfg, name, reps, idx):
+def _collect_bm_data(bm, cfg, name, reps, idx, loops):
   cmd = ['bm_diff_%s/%s/%s' % (name, cfg, bm),
          '--benchmark_out=%s.%s.%s.%d.json' % (bm, cfg, name, idx),
          '--benchmark_out_format=json',
          '--benchmark_repetitions=%d' % (reps)
          ]
-  return jobset.JobSpec(cmd, shortname='%s %s %s %d/%d' % (bm, cfg, name, idx+1, args.loops),
+  return jobset.JobSpec(cmd, shortname='%s %s %s %d/%d' % (bm, cfg, name, idx+1, loops),
                              verbose_success=True, timeout_seconds=None)
 
-def _run_bms(benchmarks, name, loops, reps):
-  jobs = []
+def run(name, benchmarks, jobs, loops, reps):
+  jobs_list = []
   for loop in range(0, loops):
-    jobs.extend(x for x in itertools.chain(
-      (_collect_bm_data(bm, 'opt', name, reps, loop) for bm in benchmarks),
-      (_collect_bm_data(bm, 'counters', name, reps, loop) for bm in benchmarks),
+    jobs_list.extend(x for x in itertools.chain(
+      (_collect_bm_data(bm, 'opt', name, reps, loop, loops) for bm in benchmarks),
+      (_collect_bm_data(bm, 'counters', name, reps, loop, loops) for bm in benchmarks),
     ))
-  random.shuffle(jobs, random.SystemRandom().random)
+  random.shuffle(jobs_list, random.SystemRandom().random)
 
-  jobset.run(jobs, maxjobs=args.jobs)
+  jobset.run(jobs_list, maxjobs=jobs)
 
 if __name__ == '__main__':
   args = _args()
   assert args.name
-  _run_bms(args.benchmarks, args.name, args.loops, args.repetitions)
+  run(args.name, args.benchmarks, args.jobs, args.loops, args.repetitions)