diff options
| -rwxr-xr-x | tools/jenkins/run_performance.sh | 2 | ||||
| -rwxr-xr-x | tools/profiling/microbenchmarks/bm_diff.py | 16 |
2 files changed, 12 insertions, 6 deletions
diff --git a/tools/jenkins/run_performance.sh b/tools/jenkins/run_performance.sh index dee033a951..544e31dcbd 100755 --- a/tools/jenkins/run_performance.sh +++ b/tools/jenkins/run_performance.sh @@ -32,7 +32,7 @@ set -ex # List of benchmarks that provide good signal for analyzing performance changes in pull requests -BENCHMARKS_TO_RUN="bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_chttp2_transport bm_pollset bm_metadata" +BENCHMARKS_TO_RUN="bm_fullstack_unary_ping_pong bm_fullstack_streaming_ping_pong bm_fullstack_streaming_pump bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_chttp2_transport bm_pollset bm_metadata" # Enter the gRPC repo root cd $(dirname $0)/../.. diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 339e959518..1dff3335ee 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -65,8 +65,8 @@ nanos = { 'pct_diff': 5, } counter = { - 'abs_diff': 1, - 'pct_diff': 1, + 'abs_diff': 0.5, + 'pct_diff': 5, } _INTERESTING = { @@ -102,7 +102,7 @@ argp.add_argument('-t', '--track', argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_cq']) argp.add_argument('-d', '--diff_base', type=str) argp.add_argument('-r', '--repetitions', type=int, default=4) -argp.add_argument('-p', '--p_threshold', type=float, default=0.05) +argp.add_argument('-p', '--p_threshold', type=float, default=0.03) args = argp.parse_args() assert args.diff_base @@ -176,7 +176,7 @@ class Benchmark: self.samples[new][f].append(float(data[f])) def process(self): - for f in args.track: + for f in sorted(args.track): new = self.samples[True][f] old = self.samples[False][f] if not new or not old: continue @@ -185,7 +185,10 @@ class Benchmark: old_mdn = median(old) delta = new_mdn - old_mdn ratio = changed_ratio(new_mdn, old_mdn) - if p < args.p_threshold and abs(delta) > _INTERESTING[f]['abs_diff'] and abs(ratio) > _INTERESTING[f]['pct_diff']: + print '%s: new=%r old=%r new_mdn=%f old_mdn=%f delta=%f(%f:%f) ratio=%f(%f:%f) p=%f' % ( + f, new, old, new_mdn, old_mdn, delta, abs(delta), _INTERESTING[f]['abs_diff'], ratio, abs(ratio), _INTERESTING[f]['pct_diff']/100.0, p + ) + if p < args.p_threshold and abs(delta) > _INTERESTING[f]['abs_diff'] and abs(ratio) > _INTERESTING[f]['pct_diff']/100.0: self.final[f] = delta return self.final.keys() @@ -209,16 +212,19 @@ for bm in comparables: js_old_opt = json.loads(f.read()) for row in bm_json.expand_json(js_new_ctr, js_new_opt): + print row name = row['cpp_name'] if name.endswith('_mean') or name.endswith('_stddev'): continue benchmarks[name].add_sample(row, True) for row in bm_json.expand_json(js_old_ctr, js_old_opt): + print row name = row['cpp_name'] if name.endswith('_mean') or name.endswith('_stddev'): continue benchmarks[name].add_sample(row, False) really_interesting = set() for name, bm in benchmarks.items(): + print name really_interesting.update(bm.process()) fields = [f for f in args.track if f in args.track] |