diff options
author | mtklein <mtklein@chromium.org> | 2014-11-24 12:39:59 -0800 |
---|---|---|
committer | Commit bot <commit-bot@chromium.org> | 2014-11-24 12:39:59 -0800 |
commit | 7ba39cb9a6c97f07eb392a1cf99ce65c1f23ded0 (patch) | |
tree | 96b0b9b2003ca83897a25c1354f4c3defee6d904 /bin/compare | |
parent | 19cd0f1813c4050e7604e1c5b9b5c7ca3de85e7b (diff) |
Add nanobench stats scripts to Skia repo.
These are the scripts I've been homegrowing for measuring perf impact. I think we found them useful today as a way of sifting through the noise.
BUG=skia:
Review URL: https://codereview.chromium.org/703713002
Diffstat (limited to 'bin/compare')
-rwxr-xr-x | bin/compare | 39 |
1 files changed, 39 insertions, 0 deletions
diff --git a/bin/compare b/bin/compare new file mode 100755 index 0000000000..fe489abadf --- /dev/null +++ b/bin/compare @@ -0,0 +1,39 @@ +#!/usr/bin/env python + +import sys +from scipy.stats import mannwhitneyu + +SIGNIFICANCE_THRESHOLD = 0.0001 + +a,b = {},{} +for (path, d) in [(sys.argv[1], a), (sys.argv[2], b)]: + for line in open(path): + try: + tokens = line.split() + samples = tokens[:-1] + label = tokens[-1] + d[label] = map(float, samples) + except: + pass + +common = set(a.keys()).intersection(b.keys()) + +ps = [] +for key in common: + _, p = mannwhitneyu(a[key], b[key]) # Non-parametric t-test. Doesn't assume normal dist. + am, bm = min(a[key]), min(b[key]) + ps.append((bm/am, p, key, am, bm)) +ps.sort(reverse=True) + +def humanize(ns): + for threshold, suffix in [(1e9, 's'), (1e6, 'ms'), (1e3, 'us'), (1e0, 'ns')]: + if ns > threshold: + return "%.3g%s" % (ns/threshold, suffix) + +maxlen = max(map(len, common)) + +# We print only signficant changes in benchmark timing distribution. +bonferroni = SIGNIFICANCE_THRESHOLD / len(ps) # Adjust for the fact we've run multiple tests. +for ratio, p, key, am, bm in ps: + if p < bonferroni: + print '%*s\t%6s -> %6s\t%.2gx' % (maxlen, key, humanize(am), humanize(bm), ratio) |