From 25e3c6da76a95bcd35fc675ad8a176060ebd59dc Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 18 Apr 2017 13:57:38 -0700 Subject: Fix EINTR forever --- tools/profiling/microbenchmarks/bm_diff.py | 91 ++++++++++++++++-------------- 1 file changed, 49 insertions(+), 42 deletions(-) (limited to 'tools/profiling') diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index ba0c225f6c..f1b6ef1ab9 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -192,52 +192,59 @@ class Benchmark: return [self.final[f] if f in self.final else '' for f in flds] -def read_file(filename): +def eintr_be_gone(fn): + """Run fn until it doesn't stop because of EINTR""" while True: try: - with open(filename) as f: - return f.read() + return fn() except IOError, e: if e.errno != errno.EINTR: raise + def read_json(filename): - return json.loads(read_file(filename)) - -benchmarks = collections.defaultdict(Benchmark) - -for bm in args.benchmarks: - for loop in range(0, args.loops): - js_new_ctr = read_json('%s.counters.new.%d.json' % (bm, loop)) - js_new_opt = read_json('%s.opt.new.%d.json' % (bm, loop)) - js_old_ctr = read_json('%s.counters.old.%d.json' % (bm, loop)) - js_old_opt = read_json('%s.opt.old.%d.json' % (bm, loop)) - - for row in bm_json.expand_json(js_new_ctr, js_new_opt): - print row - name = row['cpp_name'] - if name.endswith('_mean') or name.endswith('_stddev'): continue - benchmarks[name].add_sample(row, True) - for row in bm_json.expand_json(js_old_ctr, js_old_opt): - print row - name = row['cpp_name'] - if name.endswith('_mean') or name.endswith('_stddev'): continue - benchmarks[name].add_sample(row, False) - -really_interesting = set() -for name, bm in benchmarks.items(): - print name - really_interesting.update(bm.process()) -fields = [f for f in args.track if f in really_interesting] - -headers = ['Benchmark'] + fields -rows = [] -for name in sorted(benchmarks.keys()): - if benchmarks[name].skip(): continue - rows.append([name] + benchmarks[name].row(fields)) -if rows: - text = 'Performance differences noted:\n' + tabulate.tabulate(rows, headers=headers, floatfmt='+.2f') -else: - text = 'No significant performance differences' -comment_on_pr.comment_on_pr('```\n%s\n```' % text) -print text + with open(filename) as f: return json.loads(f.read()) + + +def finalize(): + benchmarks = collections.defaultdict(Benchmark) + + for bm in args.benchmarks: + for loop in range(0, args.loops): + js_new_ctr = read_json('%s.counters.new.%d.json' % (bm, loop)) + js_new_opt = read_json('%s.opt.new.%d.json' % (bm, loop)) + js_old_ctr = read_json('%s.counters.old.%d.json' % (bm, loop)) + js_old_opt = read_json('%s.opt.old.%d.json' % (bm, loop)) + + for row in bm_json.expand_json(js_new_ctr, js_new_opt): + print row + name = row['cpp_name'] + if name.endswith('_mean') or name.endswith('_stddev'): continue + benchmarks[name].add_sample(row, True) + for row in bm_json.expand_json(js_old_ctr, js_old_opt): + print row + name = row['cpp_name'] + if name.endswith('_mean') or name.endswith('_stddev'): continue + benchmarks[name].add_sample(row, False) + + really_interesting = set() + for name, bm in benchmarks.items(): + print name + really_interesting.update(bm.process()) + fields = [f for f in args.track if f in really_interesting] + + headers = ['Benchmark'] + fields + rows = [] + for name in sorted(benchmarks.keys()): + if benchmarks[name].skip(): continue + rows.append([name] + benchmarks[name].row(fields)) + if rows: + text = 'Performance differences noted:\n' + tabulate.tabulate(rows, headers=headers, floatfmt='+.2f') + else: + text = 'No significant performance differences' + comment_on_pr.comment_on_pr('```\n%s\n```' % text) + print text + + +eintr_be_gone(finalize) + -- cgit v1.2.3 From ad7f066b16bcd62cf36bf78f78ae0ae0f8ca242e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 19 Apr 2017 09:16:45 -0700 Subject: Allow >100% increases, add more jobs to reduce noise --- tools/profiling/microbenchmarks/bm_diff.py | 3 +-- tools/profiling/microbenchmarks/speedup.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) (limited to 'tools/profiling') diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index f1b6ef1ab9..09b62ae1c9 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -98,7 +98,7 @@ argp.add_argument('-t', '--track', argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_cq']) argp.add_argument('-d', '--diff_base', type=str) argp.add_argument('-r', '--repetitions', type=int, default=1) -argp.add_argument('-l', '--loops', type=int, default=12) +argp.add_argument('-l', '--loops', type=int, default=20) argp.add_argument('-j', '--jobs', type=int, default=multiprocessing.cpu_count()) args = argp.parse_args() @@ -247,4 +247,3 @@ def finalize(): eintr_be_gone(finalize) - diff --git a/tools/profiling/microbenchmarks/speedup.py b/tools/profiling/microbenchmarks/speedup.py index 8f9023d2c8..b3888e5ea8 100644 --- a/tools/profiling/microbenchmarks/speedup.py +++ b/tools/profiling/microbenchmarks/speedup.py @@ -53,7 +53,7 @@ def speedup(new, old): return -(pct - 1) else: pct = 1 - while pct < 101: + while pct < 100000: sp, pp = cmp(new, scale(old, 1 + pct/100.0)) if sp < 0: break if pp > _THRESHOLD: break -- cgit v1.2.3 From e530845d91c118e3176d116d58420c9e3afa44b0 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 19 Apr 2017 09:18:57 -0700 Subject: Also increase threshold for significant differences --- tools/profiling/microbenchmarks/speedup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools/profiling') diff --git a/tools/profiling/microbenchmarks/speedup.py b/tools/profiling/microbenchmarks/speedup.py index b3888e5ea8..5a1ed3f2cf 100644 --- a/tools/profiling/microbenchmarks/speedup.py +++ b/tools/profiling/microbenchmarks/speedup.py @@ -30,7 +30,7 @@ from scipy import stats import math -_THRESHOLD = 0.0001 +_THRESHOLD = 0.00001 def scale(a, mul): return [x*mul for x in a] -- cgit v1.2.3 From f2ea1d19f5d9abd358ae5c5fe33d5491c0b20556 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 20 Apr 2017 10:48:57 -0700 Subject: Increase threshold probability for reporting differences --- tools/profiling/microbenchmarks/speedup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools/profiling') diff --git a/tools/profiling/microbenchmarks/speedup.py b/tools/profiling/microbenchmarks/speedup.py index 5a1ed3f2cf..8af0066c9d 100644 --- a/tools/profiling/microbenchmarks/speedup.py +++ b/tools/profiling/microbenchmarks/speedup.py @@ -30,7 +30,7 @@ from scipy import stats import math -_THRESHOLD = 0.00001 +_THRESHOLD = 1e-10 def scale(a, mul): return [x*mul for x in a] -- cgit v1.2.3