1 files changed, 207 insertions, 0 deletions
diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff/bm_diff.py
new file mode 100755
index 0000000000..b049f41ca0
--- /dev/null
+++ b/tools/profiling/microbenchmarks/bm_diff/bm_diff.py
@@ -0,0 +1,207 @@
+#!/usr/bin/env python2.7
+# Copyright 2017, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+""" Computes the diff between two bm runs and outputs significant results """
+
+import bm_constants
+import bm_speedup
+
+import sys
+import os
+
+sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..'))
+import bm_json
+
+import json
+import tabulate
+import argparse
+import collections
+import subprocess
+
+verbose = False
+
+
+def _median(ary):
+    ary = sorted(ary)
+    n = len(ary)
+    if n % 2 == 0:
+        return (ary[n / 2] + ary[n / 2 + 1]) / 2.0
+    else:
+        return ary[n / 2]
+
+
+def _args():
+    argp = argparse.ArgumentParser(
+        description='Perform diff on microbenchmarks')
+    argp.add_argument(
+        '-t',
+        '--track',
+        choices=sorted(bm_constants._INTERESTING),
+        nargs='+',
+        default=sorted(bm_constants._INTERESTING),
+        help='Which metrics to track')
+    argp.add_argument(
+        '-b',
+        '--benchmarks',
+        nargs='+',
+        choices=bm_constants._AVAILABLE_BENCHMARK_TESTS,
+        default=bm_constants._AVAILABLE_BENCHMARK_TESTS,
+        help='Which benchmarks to run')
+    argp.add_argument(
+        '-l',
+        '--loops',
+        type=int,
+        default=20,
+        help='Number of times to loops the benchmarks. Must match what was passed to bm_run.py'
+    )
+    argp.add_argument('-n', '--new', type=str, help='New benchmark name')
+    argp.add_argument('-o', '--old', type=str, help='Old benchmark name')
+    argp.add_argument(
+        '-v', '--verbose', type=bool, help='print details of before/after')
+    args = argp.parse_args()
+    global verbose
+    if args.verbose: verbose = True
+    assert args.new
+    assert args.old
+    return args
+
+
+def _maybe_print(str):
+    if verbose: print str
+
+
+class Benchmark:
+
+    def __init__(self):
+        self.samples = {
+            True: collections.defaultdict(list),
+            False: collections.defaultdict(list)
+        }
+        self.final = {}
+
+    def add_sample(self, track, data, new):
+        for f in track:
+            if f in data:
+                self.samples[new][f].append(float(data[f]))
+
+    def process(self, track, new_name, old_name):
+        for f in sorted(track):
+            new = self.samples[True][f]
+            old = self.samples[False][f]
+            if not new or not old: continue
+            mdn_diff = abs(_median(new) - _median(old))
+            _maybe_print('%s: %s=%r %s=%r mdn_diff=%r' %
+                         (f, new_name, new, old_name, old, mdn_diff))
+            s = bm_speedup.speedup(new, old)
+            if abs(s) > 3 and mdn_diff > 0.5:
+                self.final[f] = '%+d%%' % s
+        return self.final.keys()
+
+    def skip(self):
+        return not self.final
+
+    def row(self, flds):
+        return [self.final[f] if f in self.final else '' for f in flds]
+
+
+def _read_json(filename, badfiles):
+    stripped = ".".join(filename.split(".")[:-2])
+    try:
+        with open(filename) as f:
+            return json.loads(f.read())
+    except ValueError, e:
+        if stripped in badfiles:
+            badfiles[stripped] += 1
+        else:
+            badfiles[stripped] = 1
+        return None
+
+
+def diff(bms, loops, track, old, new):
+    benchmarks = collections.defaultdict(Benchmark)
+
+    badfiles = {}
+    for bm in bms:
+        for loop in range(0, loops):
+            for line in subprocess.check_output(
+                ['bm_diff_%s/opt/%s' % (old, bm),
+                 '--benchmark_list_tests']).splitlines():
+                stripped_line = line.strip().replace("/", "_").replace(
+                    "<", "_").replace(">", "_").replace(", ", "_")
+                js_new_ctr = _read_json('%s.%s.counters.%s.%d.json' %
+                                        (bm, stripped_line, new, loop),
+                                        badfiles)
+                js_new_opt = _read_json('%s.%s.opt.%s.%d.json' %
+                                        (bm, stripped_line, new, loop),
+                                        badfiles)
+                js_old_ctr = _read_json('%s.%s.counters.%s.%d.json' %
+                                        (bm, stripped_line, old, loop),
+                                        badfiles)
+                js_old_opt = _read_json('%s.%s.opt.%s.%d.json' %
+                                        (bm, stripped_line, old, loop),
+                                        badfiles)
+
+                if js_new_ctr:
+                    for row in bm_json.expand_json(js_new_ctr, js_new_opt):
+                        name = row['cpp_name']
+                        if name.endswith('_mean') or name.endswith('_stddev'):
+                            continue
+                        benchmarks[name].add_sample(track, row, True)
+                if js_old_ctr:
+                    for row in bm_json.expand_json(js_old_ctr, js_old_opt):
+                        name = row['cpp_name']
+                        if name.endswith('_mean') or name.endswith('_stddev'):
+                            continue
+                        benchmarks[name].add_sample(track, row, False)
+
+    really_interesting = set()
+    for name, bm in benchmarks.items():
+        _maybe_print(name)
+        really_interesting.update(bm.process(track, new, old))
+    fields = [f for f in track if f in really_interesting]
+
+    headers = ['Benchmark'] + fields
+    rows = []
+    for name in sorted(benchmarks.keys()):
+        if benchmarks[name].skip(): continue
+        rows.append([name] + benchmarks[name].row(fields))
+    note = 'flakiness data = %s' % str(badfiles)
+    if rows:
+        return tabulate.tabulate(rows, headers=headers, floatfmt='+.2f'), note
+    else:
+        return None, note
+
+
+if __name__ == '__main__':
+    args = _args()
+    diff, note = diff(args.benchmarks, args.loops, args.track, args.old,
+                      args.new)
+    print note
+    print ""
+    print diff