diff options
| -rwxr-xr-x | tools/jenkins/run_performance.sh | 2 | ||||
| -rwxr-xr-x | tools/jenkins/run_trickle_diff.sh | 23 | ||||
| -rwxr-xr-x | tools/profiling/microbenchmarks/bm_diff/bm_build.py | 13 | ||||
| -rw-r--r-- | tools/profiling/microbenchmarks/bm_diff/bm_constants.py | 3 | ||||
| -rwxr-xr-x | tools/profiling/microbenchmarks/bm_diff/bm_diff.py | 35 | ||||
| -rwxr-xr-x | tools/profiling/microbenchmarks/bm_diff/bm_main.py | 22 | ||||
| -rwxr-xr-x | tools/profiling/microbenchmarks/bm_diff/bm_run.py | 14 |
7 files changed, 82 insertions, 30 deletions
diff --git a/tools/jenkins/run_performance.sh b/tools/jenkins/run_performance.sh index 177b076650..3ce05cc7f1 100755 --- a/tools/jenkins/run_performance.sh +++ b/tools/jenkins/run_performance.sh @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# This script is invoked by Jenkins and runs performance smoke test. +# This script is invoked by Jenkins and runs a diff on the microbenchmarks set -ex # List of benchmarks that provide good signal for analyzing performance changes in pull requests diff --git a/tools/jenkins/run_trickle_diff.sh b/tools/jenkins/run_trickle_diff.sh new file mode 100755 index 0000000000..da905d0249 --- /dev/null +++ b/tools/jenkins/run_trickle_diff.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This script is invoked by Jenkins and runs a diff on bm_fullstack_trickle +set -ex + +# Enter the gRPC repo root +cd $(dirname $0)/../.. + +tools/run_tests/start_port_server.py +tools/profiling/microbenchmarks/bm_diff/bm_main.py -d origin/$ghprbTargetBranch -b bm_fullstack_trickle -l 4 -t cli_transport_stalls cli_stream_stalls svr_transport_stalls svr_stream_stalls --no-counters --pr_comment_name trickle diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_build.py b/tools/profiling/microbenchmarks/bm_diff/bm_build.py index 650ccdc2b2..ce62c09d72 100755 --- a/tools/profiling/microbenchmarks/bm_diff/bm_build.py +++ b/tools/profiling/microbenchmarks/bm_diff/bm_build.py @@ -46,6 +46,9 @@ def _args(): type=str, help='Unique name of this build. To be used as a handle to pass to the other bm* scripts' ) + argp.add_argument('--counters', dest='counters', action='store_true') + argp.add_argument('--no-counters', dest='counters', action='store_false') + argp.set_defaults(counters=True) args = argp.parse_args() assert args.name return args @@ -55,16 +58,18 @@ def _make_cmd(cfg, benchmarks, jobs): return ['make'] + benchmarks + ['CONFIG=%s' % cfg, '-j', '%d' % jobs] -def build(name, benchmarks, jobs): +def build(name, benchmarks, jobs, counters): shutil.rmtree('bm_diff_%s' % name, ignore_errors=True) subprocess.check_call(['git', 'submodule', 'update']) try: subprocess.check_call(_make_cmd('opt', benchmarks, jobs)) - subprocess.check_call(_make_cmd('counters', benchmarks, jobs)) + if counters: + subprocess.check_call(_make_cmd('counters', benchmarks, jobs)) except subprocess.CalledProcessError, e: subprocess.check_call(['make', 'clean']) subprocess.check_call(_make_cmd('opt', benchmarks, jobs)) - subprocess.check_call(_make_cmd('counters', benchmarks, jobs)) + if counters: + subprocess.check_call(_make_cmd('counters', benchmarks, jobs)) os.rename( 'bins', 'bm_diff_%s' % name,) @@ -72,4 +77,4 @@ def build(name, benchmarks, jobs): if __name__ == '__main__': args = _args() - build(args.name, args.benchmarks, args.jobs) + build(args.name, args.benchmarks, args.jobs, args.counters) diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_constants.py b/tools/profiling/microbenchmarks/bm_diff/bm_constants.py index 2bbd987b20..4cd65867c3 100644 --- a/tools/profiling/microbenchmarks/bm_diff/bm_constants.py +++ b/tools/profiling/microbenchmarks/bm_diff/bm_constants.py @@ -26,4 +26,5 @@ _AVAILABLE_BENCHMARK_TESTS = [ _INTERESTING = ('cpu_time', 'real_time', 'locks_per_iteration', 'allocs_per_iteration', 'writes_per_iteration', 'atm_cas_per_iteration', 'atm_add_per_iteration', - 'nows_per_iteration',) + 'nows_per_iteration', 'cli_transport_stalls', 'cli_stream_stalls', + 'svr_transport_stalls', 'svr_stream_stalls',) diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff/bm_diff.py index b8e803749a..73abf90ff5 100755 --- a/tools/profiling/microbenchmarks/bm_diff/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff/bm_diff.py @@ -67,6 +67,9 @@ def _args(): default=20, help='Number of times to loops the benchmarks. Must match what was passed to bm_run.py' ) + argp.add_argument('--counters', dest='counters', action='store_true') + argp.add_argument('--no-counters', dest='counters', action='store_false') + argp.set_defaults(counters=True) argp.add_argument('-n', '--new', type=str, help='New benchmark name') argp.add_argument('-o', '--old', type=str, help='Old benchmark name') argp.add_argument( @@ -121,7 +124,8 @@ def _read_json(filename, badjson_files, nonexistant_files): stripped = ".".join(filename.split(".")[:-2]) try: with open(filename) as f: - return json.loads(f.read()) + r = f.read(); + return json.loads(r) except IOError, e: if stripped in nonexistant_files: nonexistant_files[stripped] += 1 @@ -129,14 +133,17 @@ def _read_json(filename, badjson_files, nonexistant_files): nonexistant_files[stripped] = 1 return None except ValueError, e: + print r if stripped in badjson_files: badjson_files[stripped] += 1 else: badjson_files[stripped] = 1 return None +def fmt_dict(d): + return ''.join([" " + k + ": " + str(d[k]) + "\n" for k in d]) -def diff(bms, loops, track, old, new): +def diff(bms, loops, track, old, new, counters): benchmarks = collections.defaultdict(Benchmark) badjson_files = {} @@ -148,18 +155,22 @@ def diff(bms, loops, track, old, new): '--benchmark_list_tests']).splitlines(): stripped_line = line.strip().replace("/", "_").replace( "<", "_").replace(">", "_").replace(", ", "_") - js_new_ctr = _read_json('%s.%s.counters.%s.%d.json' % - (bm, stripped_line, new, loop), - badjson_files, nonexistant_files) js_new_opt = _read_json('%s.%s.opt.%s.%d.json' % (bm, stripped_line, new, loop), badjson_files, nonexistant_files) - js_old_ctr = _read_json('%s.%s.counters.%s.%d.json' % - (bm, stripped_line, old, loop), - badjson_files, nonexistant_files) js_old_opt = _read_json('%s.%s.opt.%s.%d.json' % (bm, stripped_line, old, loop), badjson_files, nonexistant_files) + if counters: + js_new_ctr = _read_json('%s.%s.counters.%s.%d.json' % + (bm, stripped_line, new, loop), + badjson_files, nonexistant_files) + js_old_ctr = _read_json('%s.%s.counters.%s.%d.json' % + (bm, stripped_line, old, loop), + badjson_files, nonexistant_files) + else: + js_new_ctr = None + js_old_ctr = None if js_new_ctr: for row in bm_json.expand_json(js_new_ctr, js_new_opt): @@ -187,12 +198,12 @@ def diff(bms, loops, track, old, new): rows.append([name] + benchmarks[name].row(fields)) note = None if len(badjson_files): - note = 'Corrupt JSON data (indicates timeout or crash) = %s' % str(badjson_files) + note = 'Corrupt JSON data (indicates timeout or crash): \n%s' % fmt_dict(badjson_files) if len(nonexistant_files): if note: - note += '\n\nMissing files (indicates new benchmark) = %s' % str(nonexistant_files) + note += '\n\nMissing files (indicates new benchmark): \n%s' % fmt_dict(nonexistant_files) else: - note = '\n\nMissing files (indicates new benchmark) = %s' % str(nonexistant_files) + note = '\n\nMissing files (indicates new benchmark): \n%s' % fmt_dict(nonexistant_files) if rows: return tabulate.tabulate(rows, headers=headers, floatfmt='+.2f'), note else: @@ -202,5 +213,5 @@ def diff(bms, loops, track, old, new): if __name__ == '__main__': args = _args() diff, note = diff(args.benchmarks, args.loops, args.track, args.old, - args.new) + args.new, args.counters) print('%s\n%s' % (note, diff if diff else "No performance differences")) diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_main.py b/tools/profiling/microbenchmarks/bm_diff/bm_main.py index 8a54f198ab..8b4e0cb69a 100755 --- a/tools/profiling/microbenchmarks/bm_diff/bm_main.py +++ b/tools/profiling/microbenchmarks/bm_diff/bm_main.py @@ -80,6 +80,14 @@ def _args(): type=int, default=multiprocessing.cpu_count(), help='Number of CPUs to use') + argp.add_argument( + '--pr_comment_name', + type=str, + default="microbenchmarks", + help='Name that Jenkins will use to commen on the PR') + argp.add_argument('--counters', dest='counters', action='store_true') + argp.add_argument('--no-counters', dest='counters', action='store_false') + argp.set_defaults(counters=True) args = argp.parse_args() assert args.diff_base or args.old, "One of diff_base or old must be set!" if args.loops < 3: @@ -103,7 +111,7 @@ def eintr_be_gone(fn): def main(args): - bm_build.build('new', args.benchmarks, args.jobs) + bm_build.build('new', args.benchmarks, args.jobs, args.counters) old = args.old if args.diff_base: @@ -112,20 +120,20 @@ def main(args): ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() subprocess.check_call(['git', 'checkout', args.diff_base]) try: - bm_build.build('old', args.benchmarks, args.jobs) + bm_build.build(old, args.benchmarks, args.jobs, args.counters) finally: subprocess.check_call(['git', 'checkout', where_am_i]) subprocess.check_call(['git', 'submodule', 'update']) - bm_run.run('new', args.benchmarks, args.jobs, args.loops, args.repetitions) - bm_run.run(old, args.benchmarks, args.jobs, args.loops, args.repetitions) + bm_run.run('new', args.benchmarks, args.jobs, args.loops, args.repetitions, args.counters) + bm_run.run(old, args.benchmarks, args.jobs, args.loops, args.repetitions, args.counters) diff, note = bm_diff.diff(args.benchmarks, args.loops, args.track, old, - 'new') + 'new', args.counters) if diff: - text = 'Performance differences noted:\n' + diff + text = '[%s] Performance differences noted:\n%s' % (args.pr_comment_name, diff) else: - text = 'No significant performance differences' + text = '[%s] No significant performance differences' % args.pr_comment_name if note: text = note + '\n\n' + text print('%s' % text) diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_run.py b/tools/profiling/microbenchmarks/bm_diff/bm_run.py index 3457af916b..72b3d3cf10 100755 --- a/tools/profiling/microbenchmarks/bm_diff/bm_run.py +++ b/tools/profiling/microbenchmarks/bm_diff/bm_run.py @@ -67,6 +67,9 @@ def _args(): default=20, help='Number of times to loops the benchmarks. More loops cuts down on noise' ) + argp.add_argument('--counters', dest='counters', action='store_true') + argp.add_argument('--no-counters', dest='counters', action='store_false') + argp.set_defaults(counters=True) args = argp.parse_args() assert args.name if args.loops < 3: @@ -93,21 +96,22 @@ def _collect_bm_data(bm, cfg, name, reps, idx, loops): shortname='%s %s %s %s %d/%d' % (bm, line, cfg, name, idx + 1, loops), verbose_success=True, - timeout_seconds=60 * 2)) + timeout_seconds=60 * 60)) # one hour return jobs_list -def run(name, benchmarks, jobs, loops, reps): +def run(name, benchmarks, jobs, loops, reps, counters): jobs_list = [] for loop in range(0, loops): for bm in benchmarks: jobs_list += _collect_bm_data(bm, 'opt', name, reps, loop, loops) - jobs_list += _collect_bm_data(bm, 'counters', name, reps, loop, - loops) + if counters: + jobs_list += _collect_bm_data(bm, 'counters', name, reps, loop, + loops) random.shuffle(jobs_list, random.SystemRandom().random) jobset.run(jobs_list, maxjobs=jobs) if __name__ == '__main__': args = _args() - run(args.name, args.benchmarks, args.jobs, args.loops, args.repetitions) + run(args.name, args.benchmarks, args.jobs, args.loops, args.repetitions, args.counters) |