diff options
Diffstat (limited to 'tools/profiling/microbenchmarks/bm_diff.py')
-rwxr-xr-x | tools/profiling/microbenchmarks/bm_diff.py | 259 |
1 files changed, 0 insertions, 259 deletions
diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py deleted file mode 100755 index 299abb5fdb..0000000000 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ /dev/null @@ -1,259 +0,0 @@ -#!/usr/bin/env python2.7 -# Copyright 2017, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import sys -import json -import bm_json -import tabulate -import argparse -from scipy import stats -import subprocess -import multiprocessing -import collections -import pipes -import os -sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils')) -import comment_on_pr -import jobset -import itertools -import speedup -import random -import shutil -import errno - -_INTERESTING = ( - 'cpu_time', - 'real_time', - 'locks_per_iteration', - 'allocs_per_iteration', - 'writes_per_iteration', - 'atm_cas_per_iteration', - 'atm_add_per_iteration', - 'cli_transport_stalls_per_iteration', - 'cli_stream_stalls_per_iteration', - 'svr_transport_stalls_per_iteration', - 'svr_stream_stalls_per_iteration' - 'nows_per_iteration', -) - -def changed_ratio(n, o): - if float(o) <= .0001: o = 0 - if float(n) <= .0001: n = 0 - if o == 0 and n == 0: return 0 - if o == 0: return 100 - return (float(n)-float(o))/float(o) - -def median(ary): - ary = sorted(ary) - n = len(ary) - if n%2 == 0: - return (ary[n/2] + ary[n/2+1]) / 2.0 - else: - return ary[n/2] - -def min_change(pct): - return lambda n, o: abs(changed_ratio(n,o)) > pct/100.0 - -_AVAILABLE_BENCHMARK_TESTS = ['bm_fullstack_unary_ping_pong', - 'bm_fullstack_streaming_ping_pong', - 'bm_fullstack_streaming_pump', - 'bm_closure', - 'bm_cq', - 'bm_call_create', - 'bm_error', - 'bm_chttp2_hpack', - 'bm_chttp2_transport', - 'bm_pollset', - 'bm_metadata', - 'bm_fullstack_trickle'] - -argp = argparse.ArgumentParser(description='Perform diff on microbenchmarks') -argp.add_argument('-t', '--track', - choices=sorted(_INTERESTING), - nargs='+', - default=sorted(_INTERESTING), - help='Which metrics to track') -argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_cq']) -argp.add_argument('-d', '--diff_base', type=str) -argp.add_argument('-r', '--repetitions', type=int, default=1) -argp.add_argument('-l', '--loops', type=int, default=20) -argp.add_argument('-j', '--jobs', type=int, default=multiprocessing.cpu_count()) -args = argp.parse_args() - -assert args.diff_base - -def avg(lst): - sum = 0.0 - n = 0.0 - for el in lst: - sum += el - n += 1 - return sum / n - -def make_cmd(cfg): - return ['make'] + args.benchmarks + [ - 'CONFIG=%s' % cfg, '-j', '%d' % args.jobs] - -def build(dest): - shutil.rmtree('bm_diff_%s' % dest, ignore_errors=True) - subprocess.check_call(['git', 'submodule', 'update']) - try: - subprocess.check_call(make_cmd('opt')) - subprocess.check_call(make_cmd('counters')) - except subprocess.CalledProcessError, e: - subprocess.check_call(['make', 'clean']) - subprocess.check_call(make_cmd('opt')) - subprocess.check_call(make_cmd('counters')) - os.rename('bins', 'bm_diff_%s' % dest) - -def collect1(bm, cfg, ver, idx): - cmd = ['bm_diff_%s/%s/%s' % (ver, cfg, bm), - '--benchmark_out=%s.%s.%s.%d.json' % (bm, cfg, ver, idx), - '--benchmark_out_format=json', - '--benchmark_repetitions=%d' % (args.repetitions) - ] - return jobset.JobSpec(cmd, shortname='%s %s %s %d/%d' % (bm, cfg, ver, idx+1, args.loops), - verbose_success=True, timeout_seconds=None) - -build('new') - -where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() -subprocess.check_call(['git', 'checkout', args.diff_base]) -try: - build('old') -finally: - subprocess.check_call(['git', 'checkout', where_am_i]) - subprocess.check_call(['git', 'submodule', 'update']) - -jobs = [] -for loop in range(0, args.loops): - jobs.extend(x for x in itertools.chain( - (collect1(bm, 'opt', 'new', loop) for bm in args.benchmarks), - (collect1(bm, 'counters', 'new', loop) for bm in args.benchmarks), - (collect1(bm, 'opt', 'old', loop) for bm in args.benchmarks), - (collect1(bm, 'counters', 'old', loop) for bm in args.benchmarks), - )) -random.shuffle(jobs, random.SystemRandom().random) - -jobset.run(jobs, maxjobs=args.jobs) - -class Benchmark: - - def __init__(self): - self.samples = { - True: collections.defaultdict(list), - False: collections.defaultdict(list) - } - self.final = {} - - def add_sample(self, data, new): - for f in args.track: - if f in data: - self.samples[new][f].append(float(data[f])) - - def process(self): - for f in sorted(args.track): - new = self.samples[True][f] - old = self.samples[False][f] - if not new or not old: continue - mdn_diff = abs(median(new) - median(old)) - print '%s: new=%r old=%r mdn_diff=%r' % (f, new, old, mdn_diff) - s = speedup.speedup(new, old) - if abs(s) > 3 and mdn_diff > 0.5: - self.final[f] = '%+d%%' % s - return self.final.keys() - - def skip(self): - return not self.final - - def row(self, flds): - return [self.final[f] if f in self.final else '' for f in flds] - - -def eintr_be_gone(fn): - """Run fn until it doesn't stop because of EINTR""" - while True: - try: - return fn() - except IOError, e: - if e.errno != errno.EINTR: - raise - - -def read_json(filename): - try: - with open(filename) as f: return json.loads(f.read()) - except ValueError, e: - return None - - -def finalize(): - benchmarks = collections.defaultdict(Benchmark) - - for bm in args.benchmarks: - for loop in range(0, args.loops): - js_new_ctr = read_json('%s.counters.new.%d.json' % (bm, loop)) - js_new_opt = read_json('%s.opt.new.%d.json' % (bm, loop)) - js_old_ctr = read_json('%s.counters.old.%d.json' % (bm, loop)) - js_old_opt = read_json('%s.opt.old.%d.json' % (bm, loop)) - - if js_new_ctr: - for row in bm_json.expand_json(js_new_ctr, js_new_opt): - print row - name = row['cpp_name'] - if name.endswith('_mean') or name.endswith('_stddev'): continue - benchmarks[name].add_sample(row, True) - if js_old_ctr: - for row in bm_json.expand_json(js_old_ctr, js_old_opt): - print row - name = row['cpp_name'] - if name.endswith('_mean') or name.endswith('_stddev'): continue - benchmarks[name].add_sample(row, False) - - really_interesting = set() - for name, bm in benchmarks.items(): - print name - really_interesting.update(bm.process()) - fields = [f for f in args.track if f in really_interesting] - - headers = ['Benchmark'] + fields - rows = [] - for name in sorted(benchmarks.keys()): - if benchmarks[name].skip(): continue - rows.append([name] + benchmarks[name].row(fields)) - if rows: - text = 'Performance differences noted:\n' + tabulate.tabulate(rows, headers=headers, floatfmt='+.2f') - else: - text = 'No significant performance differences' - print text - comment_on_pr.comment_on_pr('```\n%s\n```' % text) - - -eintr_be_gone(finalize) |