From 173c477bd077e9081505e44af17bc2d442ae11b0 Mon Sep 17 00:00:00 2001 From: ncteisen Date: Mon, 11 Dec 2017 16:52:44 -0800 Subject: tools/profiling --- tools/profiling/bloat/bloat_diff.py | 76 ++--- .../profiling/latency_profile/profile_analyzer.py | 345 +++++++++---------- tools/profiling/microbenchmarks/bm2bq.py | 43 +-- .../profiling/microbenchmarks/bm_diff/bm_build.py | 87 +++-- .../microbenchmarks/bm_diff/bm_constants.py | 22 +- tools/profiling/microbenchmarks/bm_diff/bm_diff.py | 335 +++++++++---------- tools/profiling/microbenchmarks/bm_diff/bm_main.py | 218 ++++++------ tools/profiling/microbenchmarks/bm_diff/bm_run.py | 158 ++++----- .../microbenchmarks/bm_diff/bm_speedup.py | 61 ++-- tools/profiling/microbenchmarks/bm_json.py | 365 +++++++++++---------- tools/profiling/qps/qps_diff.py | 208 ++++++------ tools/profiling/qps/qps_scenarios.py | 6 +- 12 files changed, 975 insertions(+), 949 deletions(-) (limited to 'tools/profiling') diff --git a/tools/profiling/bloat/bloat_diff.py b/tools/profiling/bloat/bloat_diff.py index 9b40685e90..91611c2ca4 100755 --- a/tools/profiling/bloat/bloat_diff.py +++ b/tools/profiling/bloat/bloat_diff.py @@ -23,12 +23,11 @@ import subprocess import sys sys.path.append( - os.path.join( - os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils')) + os.path.join( + os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils')) import comment_on_pr -argp = argparse.ArgumentParser( - description='Perform diff on microbenchmarks') +argp = argparse.ArgumentParser(description='Perform diff on microbenchmarks') argp.add_argument( '-d', @@ -36,64 +35,59 @@ argp.add_argument( type=str, help='Commit or branch to compare the current one to') -argp.add_argument( - '-j', - '--jobs', - type=int, - default=multiprocessing.cpu_count()) +argp.add_argument('-j', '--jobs', type=int, default=multiprocessing.cpu_count()) args = argp.parse_args() LIBS = [ - 'libgrpc.so', - 'libgrpc++.so', + 'libgrpc.so', + 'libgrpc++.so', ] + def build(where): - subprocess.check_call('make -j%d' % args.jobs, - shell=True, cwd='.') - shutil.rmtree('bloat_diff_%s' % where, ignore_errors=True) - os.rename('libs', 'bloat_diff_%s' % where) + subprocess.check_call('make -j%d' % args.jobs, shell=True, cwd='.') + shutil.rmtree('bloat_diff_%s' % where, ignore_errors=True) + os.rename('libs', 'bloat_diff_%s' % where) + build('new') if args.diff_base: old = 'old' where_am_i = subprocess.check_output( - ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() + ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() subprocess.check_call(['git', 'checkout', args.diff_base]) subprocess.check_call(['git', 'submodule', 'update']) try: - try: - build('old') - except subprocess.CalledProcessError, e: - subprocess.check_call(['make', 'clean']) - build('old') + try: + build('old') + except subprocess.CalledProcessError, e: + subprocess.check_call(['make', 'clean']) + build('old') finally: - subprocess.check_call(['git', 'checkout', where_am_i]) - subprocess.check_call(['git', 'submodule', 'update']) + subprocess.check_call(['git', 'checkout', where_am_i]) + subprocess.check_call(['git', 'submodule', 'update']) -subprocess.check_call('make -j%d' % args.jobs, - shell=True, cwd='third_party/bloaty') +subprocess.check_call( + 'make -j%d' % args.jobs, shell=True, cwd='third_party/bloaty') text = '' for lib in LIBS: - text += '****************************************************************\n\n' - text += lib + '\n\n' - old_version = glob.glob('bloat_diff_old/opt/%s' % lib) - new_version = glob.glob('bloat_diff_new/opt/%s' % lib) - assert len(new_version) == 1 - cmd = 'third_party/bloaty/bloaty -d compileunits,symbols' - if old_version: - assert len(old_version) == 1 - text += subprocess.check_output('%s %s -- %s' % - (cmd, new_version[0], old_version[0]), - shell=True) - else: - text += subprocess.check_output('%s %s' % - (cmd, new_version[0]), - shell=True) - text += '\n\n' + text += '****************************************************************\n\n' + text += lib + '\n\n' + old_version = glob.glob('bloat_diff_old/opt/%s' % lib) + new_version = glob.glob('bloat_diff_new/opt/%s' % lib) + assert len(new_version) == 1 + cmd = 'third_party/bloaty/bloaty -d compileunits,symbols' + if old_version: + assert len(old_version) == 1 + text += subprocess.check_output( + '%s %s -- %s' % (cmd, new_version[0], old_version[0]), shell=True) + else: + text += subprocess.check_output( + '%s %s' % (cmd, new_version[0]), shell=True) + text += '\n\n' print text comment_on_pr.comment_on_pr('```\n%s\n```' % text) diff --git a/tools/profiling/latency_profile/profile_analyzer.py b/tools/profiling/latency_profile/profile_analyzer.py index 8a19afb761..e3d33574c2 100755 --- a/tools/profiling/latency_profile/profile_analyzer.py +++ b/tools/profiling/latency_profile/profile_analyzer.py @@ -23,7 +23,6 @@ import sys import tabulate import time - SELF_TIME = object() TIME_FROM_SCOPE_START = object() TIME_TO_SCOPE_END = object() @@ -31,124 +30,129 @@ TIME_FROM_STACK_START = object() TIME_TO_STACK_END = object() TIME_FROM_LAST_IMPORTANT = object() - -argp = argparse.ArgumentParser(description='Process output of basic_prof builds') +argp = argparse.ArgumentParser( + description='Process output of basic_prof builds') argp.add_argument('--source', default='latency_trace.txt', type=str) argp.add_argument('--fmt', choices=tabulate.tabulate_formats, default='simple') argp.add_argument('--out', default='-', type=str) args = argp.parse_args() + class LineItem(object): - def __init__(self, line, indent): - self.tag = line['tag'] - self.indent = indent - self.start_time = line['t'] - self.end_time = None - self.important = line['imp'] - self.filename = line['file'] - self.fileline = line['line'] - self.times = {} + def __init__(self, line, indent): + self.tag = line['tag'] + self.indent = indent + self.start_time = line['t'] + self.end_time = None + self.important = line['imp'] + self.filename = line['file'] + self.fileline = line['line'] + self.times = {} class ScopeBuilder(object): - def __init__(self, call_stack_builder, line): - self.call_stack_builder = call_stack_builder - self.indent = len(call_stack_builder.stk) - self.top_line = LineItem(line, self.indent) - call_stack_builder.lines.append(self.top_line) - self.first_child_pos = len(call_stack_builder.lines) - - def mark(self, line): - line_item = LineItem(line, self.indent + 1) - line_item.end_time = line_item.start_time - self.call_stack_builder.lines.append(line_item) - - def finish(self, line): - assert line['tag'] == self.top_line.tag, ( - 'expected %s, got %s; thread=%s; t0=%f t1=%f' % - (self.top_line.tag, line['tag'], line['thd'], self.top_line.start_time, - line['t'])) - final_time_stamp = line['t'] - assert self.top_line.end_time is None - self.top_line.end_time = final_time_stamp - self.top_line.important = self.top_line.important or line['imp'] - assert SELF_TIME not in self.top_line.times - self.top_line.times[SELF_TIME] = final_time_stamp - self.top_line.start_time - for line in self.call_stack_builder.lines[self.first_child_pos:]: - if TIME_FROM_SCOPE_START not in line.times: - line.times[TIME_FROM_SCOPE_START] = line.start_time - self.top_line.start_time - line.times[TIME_TO_SCOPE_END] = final_time_stamp - line.end_time + def __init__(self, call_stack_builder, line): + self.call_stack_builder = call_stack_builder + self.indent = len(call_stack_builder.stk) + self.top_line = LineItem(line, self.indent) + call_stack_builder.lines.append(self.top_line) + self.first_child_pos = len(call_stack_builder.lines) + + def mark(self, line): + line_item = LineItem(line, self.indent + 1) + line_item.end_time = line_item.start_time + self.call_stack_builder.lines.append(line_item) + + def finish(self, line): + assert line['tag'] == self.top_line.tag, ( + 'expected %s, got %s; thread=%s; t0=%f t1=%f' % + (self.top_line.tag, line['tag'], line['thd'], + self.top_line.start_time, line['t'])) + final_time_stamp = line['t'] + assert self.top_line.end_time is None + self.top_line.end_time = final_time_stamp + self.top_line.important = self.top_line.important or line['imp'] + assert SELF_TIME not in self.top_line.times + self.top_line.times[ + SELF_TIME] = final_time_stamp - self.top_line.start_time + for line in self.call_stack_builder.lines[self.first_child_pos:]: + if TIME_FROM_SCOPE_START not in line.times: + line.times[ + TIME_FROM_SCOPE_START] = line.start_time - self.top_line.start_time + line.times[TIME_TO_SCOPE_END] = final_time_stamp - line.end_time class CallStackBuilder(object): - def __init__(self): - self.stk = [] - self.signature = hashlib.md5() - self.lines = [] - - def finish(self): - start_time = self.lines[0].start_time - end_time = self.lines[0].end_time - self.signature = self.signature.hexdigest() - last_important = start_time - for line in self.lines: - line.times[TIME_FROM_STACK_START] = line.start_time - start_time - line.times[TIME_TO_STACK_END] = end_time - line.end_time - line.times[TIME_FROM_LAST_IMPORTANT] = line.start_time - last_important - if line.important: - last_important = line.end_time - last_important = end_time - - def add(self, line): - line_type = line['type'] - self.signature.update(line_type) - self.signature.update(line['tag']) - if line_type == '{': - self.stk.append(ScopeBuilder(self, line)) - return False - elif line_type == '}': - assert self.stk, ( - 'expected non-empty stack for closing %s; thread=%s; t=%f' % - (line['tag'], line['thd'], line['t'])) - self.stk.pop().finish(line) - if not self.stk: - self.finish() - return True - return False - elif line_type == '.' or line_type == '!': - self.stk[-1].mark(line) - return False - else: - raise Exception('Unknown line type: \'%s\'' % line_type) + def __init__(self): + self.stk = [] + self.signature = hashlib.md5() + self.lines = [] + + def finish(self): + start_time = self.lines[0].start_time + end_time = self.lines[0].end_time + self.signature = self.signature.hexdigest() + last_important = start_time + for line in self.lines: + line.times[TIME_FROM_STACK_START] = line.start_time - start_time + line.times[TIME_TO_STACK_END] = end_time - line.end_time + line.times[ + TIME_FROM_LAST_IMPORTANT] = line.start_time - last_important + if line.important: + last_important = line.end_time + last_important = end_time + + def add(self, line): + line_type = line['type'] + self.signature.update(line_type) + self.signature.update(line['tag']) + if line_type == '{': + self.stk.append(ScopeBuilder(self, line)) + return False + elif line_type == '}': + assert self.stk, ( + 'expected non-empty stack for closing %s; thread=%s; t=%f' % + (line['tag'], line['thd'], line['t'])) + self.stk.pop().finish(line) + if not self.stk: + self.finish() + return True + return False + elif line_type == '.' or line_type == '!': + self.stk[-1].mark(line) + return False + else: + raise Exception('Unknown line type: \'%s\'' % line_type) class CallStack(object): - def __init__(self, initial_call_stack_builder): - self.count = 1 - self.signature = initial_call_stack_builder.signature - self.lines = initial_call_stack_builder.lines - for line in self.lines: - for key, val in line.times.items(): - line.times[key] = [val] - - def add(self, call_stack_builder): - assert self.signature == call_stack_builder.signature - self.count += 1 - assert len(self.lines) == len(call_stack_builder.lines) - for lsum, line in itertools.izip(self.lines, call_stack_builder.lines): - assert lsum.tag == line.tag - assert lsum.times.keys() == line.times.keys() - for k, lst in lsum.times.iteritems(): - lst.append(line.times[k]) - - def finish(self): - for line in self.lines: - for lst in line.times.itervalues(): - lst.sort() + def __init__(self, initial_call_stack_builder): + self.count = 1 + self.signature = initial_call_stack_builder.signature + self.lines = initial_call_stack_builder.lines + for line in self.lines: + for key, val in line.times.items(): + line.times[key] = [val] + + def add(self, call_stack_builder): + assert self.signature == call_stack_builder.signature + self.count += 1 + assert len(self.lines) == len(call_stack_builder.lines) + for lsum, line in itertools.izip(self.lines, call_stack_builder.lines): + assert lsum.tag == line.tag + assert lsum.times.keys() == line.times.keys() + for k, lst in lsum.times.iteritems(): + lst.append(line.times[k]) + + def finish(self): + for line in self.lines: + for lst in line.times.itervalues(): + lst.sort() + builder = collections.defaultdict(CallStackBuilder) call_stacks = collections.defaultdict(CallStack) @@ -156,26 +160,28 @@ call_stacks = collections.defaultdict(CallStack) lines = 0 start = time.time() with open(args.source) as f: - for line in f: - lines += 1 - inf = json.loads(line) - thd = inf['thd'] - cs = builder[thd] - if cs.add(inf): - if cs.signature in call_stacks: - call_stacks[cs.signature].add(cs) - else: - call_stacks[cs.signature] = CallStack(cs) - del builder[thd] + for line in f: + lines += 1 + inf = json.loads(line) + thd = inf['thd'] + cs = builder[thd] + if cs.add(inf): + if cs.signature in call_stacks: + call_stacks[cs.signature].add(cs) + else: + call_stacks[cs.signature] = CallStack(cs) + del builder[thd] time_taken = time.time() - start -call_stacks = sorted(call_stacks.values(), key=lambda cs: cs.count, reverse=True) +call_stacks = sorted( + call_stacks.values(), key=lambda cs: cs.count, reverse=True) total_stacks = 0 for cs in call_stacks: - total_stacks += cs.count - cs.finish() + total_stacks += cs.count + cs.finish() + -def percentile(N, percent, key=lambda x:x): +def percentile(N, percent, key=lambda x: x): """ Find the percentile of a list of values. @@ -187,80 +193,83 @@ def percentile(N, percent, key=lambda x:x): """ if not N: return None - k = (len(N)-1) * percent + k = (len(N) - 1) * percent f = math.floor(k) c = math.ceil(k) if f == c: return key(N[int(k)]) - d0 = key(N[int(f)]) * (c-k) - d1 = key(N[int(c)]) * (k-f) - return d0+d1 + d0 = key(N[int(f)]) * (c - k) + d1 = key(N[int(c)]) * (k - f) + return d0 + d1 + def tidy_tag(tag): - if tag[0:10] == 'GRPC_PTAG_': - return tag[10:] - return tag + if tag[0:10] == 'GRPC_PTAG_': + return tag[10:] + return tag + def time_string(values): - num_values = len(values) - return '%.1f/%.1f/%.1f' % ( - 1e6 * percentile(values, 0.5), - 1e6 * percentile(values, 0.9), - 1e6 * percentile(values, 0.99)) + num_values = len(values) + return '%.1f/%.1f/%.1f' % (1e6 * percentile(values, 0.5), + 1e6 * percentile(values, 0.9), + 1e6 * percentile(values, 0.99)) + def time_format(idx): - def ent(line, idx=idx): - if idx in line.times: - return time_string(line.times[idx]) - return '' - return ent -BANNER = { - 'simple': 'Count: %(count)d', - 'html': '

Count: %(count)d

' -} + def ent(line, idx=idx): + if idx in line.times: + return time_string(line.times[idx]) + return '' + + return ent + + +BANNER = {'simple': 'Count: %(count)d', 'html': '

Count: %(count)d

'} FORMAT = [ - ('TAG', lambda line: '..'*line.indent + tidy_tag(line.tag)), - ('LOC', lambda line: '%s:%d' % (line.filename[line.filename.rfind('/')+1:], line.fileline)), - ('IMP', lambda line: '*' if line.important else ''), - ('FROM_IMP', time_format(TIME_FROM_LAST_IMPORTANT)), - ('FROM_STACK_START', time_format(TIME_FROM_STACK_START)), - ('SELF', time_format(SELF_TIME)), - ('TO_STACK_END', time_format(TIME_TO_STACK_END)), - ('FROM_SCOPE_START', time_format(TIME_FROM_SCOPE_START)), - ('SELF', time_format(SELF_TIME)), - ('TO_SCOPE_END', time_format(TIME_TO_SCOPE_END)), + ('TAG', lambda line: '..' * line.indent + tidy_tag(line.tag)), + ('LOC', + lambda line: '%s:%d' % (line.filename[line.filename.rfind('/') + 1:], line.fileline) + ), + ('IMP', lambda line: '*' if line.important else ''), + ('FROM_IMP', time_format(TIME_FROM_LAST_IMPORTANT)), + ('FROM_STACK_START', time_format(TIME_FROM_STACK_START)), + ('SELF', time_format(SELF_TIME)), + ('TO_STACK_END', time_format(TIME_TO_STACK_END)), + ('FROM_SCOPE_START', time_format(TIME_FROM_SCOPE_START)), + ('SELF', time_format(SELF_TIME)), + ('TO_SCOPE_END', time_format(TIME_TO_SCOPE_END)), ] out = sys.stdout if args.out != '-': - out = open(args.out, 'w') + out = open(args.out, 'w') if args.fmt == 'html': - print >>out, '' - print >>out, '' - print >>out, 'Profile Report' - print >>out, '' + print >> out, '' + print >> out, '' + print >> out, 'Profile Report' + print >> out, '' accounted_for = 0 for cs in call_stacks: - if args.fmt in BANNER: - print >>out, BANNER[args.fmt] % { - 'count': cs.count, - } - header, _ = zip(*FORMAT) - table = [] - for line in cs.lines: - fields = [] - for _, fn in FORMAT: - fields.append(fn(line)) - table.append(fields) - print >>out, tabulate.tabulate(table, header, tablefmt=args.fmt) - accounted_for += cs.count - if accounted_for > .99 * total_stacks: - break + if args.fmt in BANNER: + print >> out, BANNER[args.fmt] % { + 'count': cs.count, + } + header, _ = zip(*FORMAT) + table = [] + for line in cs.lines: + fields = [] + for _, fn in FORMAT: + fields.append(fn(line)) + table.append(fields) + print >> out, tabulate.tabulate(table, header, tablefmt=args.fmt) + accounted_for += cs.count + if accounted_for > .99 * total_stacks: + break if args.fmt == 'html': - print '' - + print '' diff --git a/tools/profiling/microbenchmarks/bm2bq.py b/tools/profiling/microbenchmarks/bm2bq.py index 9f9b672f75..746b643b43 100755 --- a/tools/profiling/microbenchmarks/bm2bq.py +++ b/tools/profiling/microbenchmarks/bm2bq.py @@ -28,37 +28,38 @@ import subprocess columns = [] for row in json.loads( - subprocess.check_output([ - 'bq','--format=json','show','microbenchmarks.microbenchmarks']))['schema']['fields']: - columns.append((row['name'], row['type'].lower())) + subprocess.check_output([ + 'bq', '--format=json', 'show', 'microbenchmarks.microbenchmarks' + ]))['schema']['fields']: + columns.append((row['name'], row['type'].lower())) SANITIZE = { - 'integer': int, - 'float': float, - 'boolean': bool, - 'string': str, - 'timestamp': str, + 'integer': int, + 'float': float, + 'boolean': bool, + 'string': str, + 'timestamp': str, } if sys.argv[1] == '--schema': - print ',\n'.join('%s:%s' % (k, t.upper()) for k, t in columns) - sys.exit(0) + print ',\n'.join('%s:%s' % (k, t.upper()) for k, t in columns) + sys.exit(0) with open(sys.argv[1]) as f: - js = json.loads(f.read()) + js = json.loads(f.read()) if len(sys.argv) > 2: - with open(sys.argv[2]) as f: - js2 = json.loads(f.read()) + with open(sys.argv[2]) as f: + js2 = json.loads(f.read()) else: - js2 = None + js2 = None -writer = csv.DictWriter(sys.stdout, [c for c,t in columns]) +writer = csv.DictWriter(sys.stdout, [c for c, t in columns]) for row in bm_json.expand_json(js, js2): - sane_row = {} - for name, sql_type in columns: - if name in row: - if row[name] == '': continue - sane_row[name] = SANITIZE[sql_type](row[name]) - writer.writerow(sane_row) + sane_row = {} + for name, sql_type in columns: + if name in row: + if row[name] == '': continue + sane_row[name] = SANITIZE[sql_type](row[name]) + writer.writerow(sane_row) diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_build.py b/tools/profiling/microbenchmarks/bm_diff/bm_build.py index ce62c09d72..a4cd61707d 100755 --- a/tools/profiling/microbenchmarks/bm_diff/bm_build.py +++ b/tools/profiling/microbenchmarks/bm_diff/bm_build.py @@ -13,7 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ Python utility to build opt and counters benchmarks """ import bm_constants @@ -26,55 +25,55 @@ import shutil def _args(): - argp = argparse.ArgumentParser(description='Builds microbenchmarks') - argp.add_argument( - '-b', - '--benchmarks', - nargs='+', - choices=bm_constants._AVAILABLE_BENCHMARK_TESTS, - default=bm_constants._AVAILABLE_BENCHMARK_TESTS, - help='Which benchmarks to build') - argp.add_argument( - '-j', - '--jobs', - type=int, - default=multiprocessing.cpu_count(), - help='How many CPUs to dedicate to this task') - argp.add_argument( - '-n', - '--name', - type=str, - help='Unique name of this build. To be used as a handle to pass to the other bm* scripts' - ) - argp.add_argument('--counters', dest='counters', action='store_true') - argp.add_argument('--no-counters', dest='counters', action='store_false') - argp.set_defaults(counters=True) - args = argp.parse_args() - assert args.name - return args + argp = argparse.ArgumentParser(description='Builds microbenchmarks') + argp.add_argument( + '-b', + '--benchmarks', + nargs='+', + choices=bm_constants._AVAILABLE_BENCHMARK_TESTS, + default=bm_constants._AVAILABLE_BENCHMARK_TESTS, + help='Which benchmarks to build') + argp.add_argument( + '-j', + '--jobs', + type=int, + default=multiprocessing.cpu_count(), + help='How many CPUs to dedicate to this task') + argp.add_argument( + '-n', + '--name', + type=str, + help='Unique name of this build. To be used as a handle to pass to the other bm* scripts' + ) + argp.add_argument('--counters', dest='counters', action='store_true') + argp.add_argument('--no-counters', dest='counters', action='store_false') + argp.set_defaults(counters=True) + args = argp.parse_args() + assert args.name + return args def _make_cmd(cfg, benchmarks, jobs): - return ['make'] + benchmarks + ['CONFIG=%s' % cfg, '-j', '%d' % jobs] + return ['make'] + benchmarks + ['CONFIG=%s' % cfg, '-j', '%d' % jobs] def build(name, benchmarks, jobs, counters): - shutil.rmtree('bm_diff_%s' % name, ignore_errors=True) - subprocess.check_call(['git', 'submodule', 'update']) - try: - subprocess.check_call(_make_cmd('opt', benchmarks, jobs)) - if counters: - subprocess.check_call(_make_cmd('counters', benchmarks, jobs)) - except subprocess.CalledProcessError, e: - subprocess.check_call(['make', 'clean']) - subprocess.check_call(_make_cmd('opt', benchmarks, jobs)) - if counters: - subprocess.check_call(_make_cmd('counters', benchmarks, jobs)) - os.rename( - 'bins', - 'bm_diff_%s' % name,) + shutil.rmtree('bm_diff_%s' % name, ignore_errors=True) + subprocess.check_call(['git', 'submodule', 'update']) + try: + subprocess.check_call(_make_cmd('opt', benchmarks, jobs)) + if counters: + subprocess.check_call(_make_cmd('counters', benchmarks, jobs)) + except subprocess.CalledProcessError, e: + subprocess.check_call(['make', 'clean']) + subprocess.check_call(_make_cmd('opt', benchmarks, jobs)) + if counters: + subprocess.check_call(_make_cmd('counters', benchmarks, jobs)) + os.rename( + 'bins', + 'bm_diff_%s' % name,) if __name__ == '__main__': - args = _args() - build(args.name, args.benchmarks, args.jobs, args.counters) + args = _args() + build(args.name, args.benchmarks, args.jobs, args.counters) diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_constants.py b/tools/profiling/microbenchmarks/bm_diff/bm_constants.py index 0ec17fa17e..cff29dbe08 100644 --- a/tools/profiling/microbenchmarks/bm_diff/bm_constants.py +++ b/tools/profiling/microbenchmarks/bm_diff/bm_constants.py @@ -13,19 +13,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ Configurable constants for the bm_*.py family """ _AVAILABLE_BENCHMARK_TESTS = [ - 'bm_fullstack_unary_ping_pong', 'bm_fullstack_streaming_ping_pong', - 'bm_fullstack_streaming_pump', 'bm_closure', 'bm_cq', 'bm_call_create', - 'bm_error', 'bm_chttp2_hpack', 'bm_chttp2_transport', 'bm_pollset', - 'bm_metadata', 'bm_fullstack_trickle' + 'bm_fullstack_unary_ping_pong', 'bm_fullstack_streaming_ping_pong', + 'bm_fullstack_streaming_pump', 'bm_closure', 'bm_cq', 'bm_call_create', + 'bm_error', 'bm_chttp2_hpack', 'bm_chttp2_transport', 'bm_pollset', + 'bm_metadata', 'bm_fullstack_trickle' ] -_INTERESTING = ('cpu_time', 'real_time', 'call_initial_size-median', 'locks_per_iteration', - 'allocs_per_iteration', 'writes_per_iteration', - 'atm_cas_per_iteration', 'atm_add_per_iteration', - 'nows_per_iteration', 'cli_transport_stalls_per_iteration', - 'cli_stream_stalls_per_iteration', 'svr_transport_stalls_per_iteration', - 'svr_stream_stalls_per_iteration', 'http2_pings_sent_per_iteration') +_INTERESTING = ( + 'cpu_time', 'real_time', 'call_initial_size-median', 'locks_per_iteration', + 'allocs_per_iteration', 'writes_per_iteration', 'atm_cas_per_iteration', + 'atm_add_per_iteration', 'nows_per_iteration', + 'cli_transport_stalls_per_iteration', 'cli_stream_stalls_per_iteration', + 'svr_transport_stalls_per_iteration', 'svr_stream_stalls_per_iteration', + 'http2_pings_sent_per_iteration') diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff/bm_diff.py index a41d0f0552..b8a3b22861 100755 --- a/tools/profiling/microbenchmarks/bm_diff/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff/bm_diff.py @@ -34,190 +34,195 @@ verbose = False def _median(ary): - assert (len(ary)) - ary = sorted(ary) - n = len(ary) - if n % 2 == 0: - return (ary[(n - 1) / 2] + ary[(n - 1) / 2 + 1]) / 2.0 - else: - return ary[n / 2] + assert (len(ary)) + ary = sorted(ary) + n = len(ary) + if n % 2 == 0: + return (ary[(n - 1) / 2] + ary[(n - 1) / 2 + 1]) / 2.0 + else: + return ary[n / 2] def _args(): - argp = argparse.ArgumentParser( - description='Perform diff on microbenchmarks') - argp.add_argument( - '-t', - '--track', - choices=sorted(bm_constants._INTERESTING), - nargs='+', - default=sorted(bm_constants._INTERESTING), - help='Which metrics to track') - argp.add_argument( - '-b', - '--benchmarks', - nargs='+', - choices=bm_constants._AVAILABLE_BENCHMARK_TESTS, - default=bm_constants._AVAILABLE_BENCHMARK_TESTS, - help='Which benchmarks to run') - argp.add_argument( - '-l', - '--loops', - type=int, - default=20, - help='Number of times to loops the benchmarks. Must match what was passed to bm_run.py' - ) - argp.add_argument( - '-r', - '--regex', - type=str, - default="", - help='Regex to filter benchmarks run') - argp.add_argument('--counters', dest='counters', action='store_true') - argp.add_argument('--no-counters', dest='counters', action='store_false') - argp.set_defaults(counters=True) - argp.add_argument('-n', '--new', type=str, help='New benchmark name') - argp.add_argument('-o', '--old', type=str, help='Old benchmark name') - argp.add_argument( - '-v', '--verbose', type=bool, help='Print details of before/after') - args = argp.parse_args() - global verbose - if args.verbose: verbose = True - assert args.new - assert args.old - return args + argp = argparse.ArgumentParser( + description='Perform diff on microbenchmarks') + argp.add_argument( + '-t', + '--track', + choices=sorted(bm_constants._INTERESTING), + nargs='+', + default=sorted(bm_constants._INTERESTING), + help='Which metrics to track') + argp.add_argument( + '-b', + '--benchmarks', + nargs='+', + choices=bm_constants._AVAILABLE_BENCHMARK_TESTS, + default=bm_constants._AVAILABLE_BENCHMARK_TESTS, + help='Which benchmarks to run') + argp.add_argument( + '-l', + '--loops', + type=int, + default=20, + help='Number of times to loops the benchmarks. Must match what was passed to bm_run.py' + ) + argp.add_argument( + '-r', + '--regex', + type=str, + default="", + help='Regex to filter benchmarks run') + argp.add_argument('--counters', dest='counters', action='store_true') + argp.add_argument('--no-counters', dest='counters', action='store_false') + argp.set_defaults(counters=True) + argp.add_argument('-n', '--new', type=str, help='New benchmark name') + argp.add_argument('-o', '--old', type=str, help='Old benchmark name') + argp.add_argument( + '-v', '--verbose', type=bool, help='Print details of before/after') + args = argp.parse_args() + global verbose + if args.verbose: verbose = True + assert args.new + assert args.old + return args def _maybe_print(str): - if verbose: print str + if verbose: print str class Benchmark: - def __init__(self): - self.samples = { - True: collections.defaultdict(list), - False: collections.defaultdict(list) - } - self.final = {} - - def add_sample(self, track, data, new): - for f in track: - if f in data: - self.samples[new][f].append(float(data[f])) - - def process(self, track, new_name, old_name): - for f in sorted(track): - new = self.samples[True][f] - old = self.samples[False][f] - if not new or not old: continue - mdn_diff = abs(_median(new) - _median(old)) - _maybe_print('%s: %s=%r %s=%r mdn_diff=%r' % - (f, new_name, new, old_name, old, mdn_diff)) - s = bm_speedup.speedup(new, old, 1e-5) - if abs(s) > 3: - if mdn_diff > 0.5 or 'trickle' in f: - self.final[f] = '%+d%%' % s - return self.final.keys() - - def skip(self): - return not self.final - - def row(self, flds): - return [self.final[f] if f in self.final else '' for f in flds] + def __init__(self): + self.samples = { + True: collections.defaultdict(list), + False: collections.defaultdict(list) + } + self.final = {} + + def add_sample(self, track, data, new): + for f in track: + if f in data: + self.samples[new][f].append(float(data[f])) + + def process(self, track, new_name, old_name): + for f in sorted(track): + new = self.samples[True][f] + old = self.samples[False][f] + if not new or not old: continue + mdn_diff = abs(_median(new) - _median(old)) + _maybe_print('%s: %s=%r %s=%r mdn_diff=%r' % + (f, new_name, new, old_name, old, mdn_diff)) + s = bm_speedup.speedup(new, old, 1e-5) + if abs(s) > 3: + if mdn_diff > 0.5 or 'trickle' in f: + self.final[f] = '%+d%%' % s + return self.final.keys() + + def skip(self): + return not self.final + + def row(self, flds): + return [self.final[f] if f in self.final else '' for f in flds] def _read_json(filename, badjson_files, nonexistant_files): - stripped = ".".join(filename.split(".")[:-2]) - try: - with open(filename) as f: - r = f.read(); - return json.loads(r) - except IOError, e: - if stripped in nonexistant_files: - nonexistant_files[stripped] += 1 - else: - nonexistant_files[stripped] = 1 - return None - except ValueError, e: - print r - if stripped in badjson_files: - badjson_files[stripped] += 1 - else: - badjson_files[stripped] = 1 - return None + stripped = ".".join(filename.split(".")[:-2]) + try: + with open(filename) as f: + r = f.read() + return json.loads(r) + except IOError, e: + if stripped in nonexistant_files: + nonexistant_files[stripped] += 1 + else: + nonexistant_files[stripped] = 1 + return None + except ValueError, e: + print r + if stripped in badjson_files: + badjson_files[stripped] += 1 + else: + badjson_files[stripped] = 1 + return None + def fmt_dict(d): - return ''.join([" " + k + ": " + str(d[k]) + "\n" for k in d]) + return ''.join([" " + k + ": " + str(d[k]) + "\n" for k in d]) + def diff(bms, loops, regex, track, old, new, counters): - benchmarks = collections.defaultdict(Benchmark) - - badjson_files = {} - nonexistant_files = {} - for bm in bms: - for loop in range(0, loops): - for line in subprocess.check_output( - ['bm_diff_%s/opt/%s' % (old, bm), - '--benchmark_list_tests', - '--benchmark_filter=%s' % regex]).splitlines(): - stripped_line = line.strip().replace("/", "_").replace( - "<", "_").replace(">", "_").replace(", ", "_") - js_new_opt = _read_json('%s.%s.opt.%s.%d.json' % - (bm, stripped_line, new, loop), - badjson_files, nonexistant_files) - js_old_opt = _read_json('%s.%s.opt.%s.%d.json' % - (bm, stripped_line, old, loop), - badjson_files, nonexistant_files) - if counters: - js_new_ctr = _read_json('%s.%s.counters.%s.%d.json' % - (bm, stripped_line, new, loop), - badjson_files, nonexistant_files) - js_old_ctr = _read_json('%s.%s.counters.%s.%d.json' % - (bm, stripped_line, old, loop), - badjson_files, nonexistant_files) + benchmarks = collections.defaultdict(Benchmark) + + badjson_files = {} + nonexistant_files = {} + for bm in bms: + for loop in range(0, loops): + for line in subprocess.check_output([ + 'bm_diff_%s/opt/%s' % (old, bm), '--benchmark_list_tests', + '--benchmark_filter=%s' % regex + ]).splitlines(): + stripped_line = line.strip().replace("/", "_").replace( + "<", "_").replace(">", "_").replace(", ", "_") + js_new_opt = _read_json('%s.%s.opt.%s.%d.json' % + (bm, stripped_line, new, loop), + badjson_files, nonexistant_files) + js_old_opt = _read_json('%s.%s.opt.%s.%d.json' % + (bm, stripped_line, old, loop), + badjson_files, nonexistant_files) + if counters: + js_new_ctr = _read_json('%s.%s.counters.%s.%d.json' % + (bm, stripped_line, new, loop), + badjson_files, nonexistant_files) + js_old_ctr = _read_json('%s.%s.counters.%s.%d.json' % + (bm, stripped_line, old, loop), + badjson_files, nonexistant_files) + else: + js_new_ctr = None + js_old_ctr = None + + for row in bm_json.expand_json(js_new_ctr, js_new_opt): + name = row['cpp_name'] + if name.endswith('_mean') or name.endswith('_stddev'): + continue + benchmarks[name].add_sample(track, row, True) + for row in bm_json.expand_json(js_old_ctr, js_old_opt): + name = row['cpp_name'] + if name.endswith('_mean') or name.endswith('_stddev'): + continue + benchmarks[name].add_sample(track, row, False) + + really_interesting = set() + for name, bm in benchmarks.items(): + _maybe_print(name) + really_interesting.update(bm.process(track, new, old)) + fields = [f for f in track if f in really_interesting] + + headers = ['Benchmark'] + fields + rows = [] + for name in sorted(benchmarks.keys()): + if benchmarks[name].skip(): continue + rows.append([name] + benchmarks[name].row(fields)) + note = None + if len(badjson_files): + note = 'Corrupt JSON data (indicates timeout or crash): \n%s' % fmt_dict( + badjson_files) + if len(nonexistant_files): + if note: + note += '\n\nMissing files (indicates new benchmark): \n%s' % fmt_dict( + nonexistant_files) else: - js_new_ctr = None - js_old_ctr = None - - for row in bm_json.expand_json(js_new_ctr, js_new_opt): - name = row['cpp_name'] - if name.endswith('_mean') or name.endswith('_stddev'): - continue - benchmarks[name].add_sample(track, row, True) - for row in bm_json.expand_json(js_old_ctr, js_old_opt): - name = row['cpp_name'] - if name.endswith('_mean') or name.endswith('_stddev'): - continue - benchmarks[name].add_sample(track, row, False) - - really_interesting = set() - for name, bm in benchmarks.items(): - _maybe_print(name) - really_interesting.update(bm.process(track, new, old)) - fields = [f for f in track if f in really_interesting] - - headers = ['Benchmark'] + fields - rows = [] - for name in sorted(benchmarks.keys()): - if benchmarks[name].skip(): continue - rows.append([name] + benchmarks[name].row(fields)) - note = None - if len(badjson_files): - note = 'Corrupt JSON data (indicates timeout or crash): \n%s' % fmt_dict(badjson_files) - if len(nonexistant_files): - if note: - note += '\n\nMissing files (indicates new benchmark): \n%s' % fmt_dict(nonexistant_files) + note = '\n\nMissing files (indicates new benchmark): \n%s' % fmt_dict( + nonexistant_files) + if rows: + return tabulate.tabulate(rows, headers=headers, floatfmt='+.2f'), note else: - note = '\n\nMissing files (indicates new benchmark): \n%s' % fmt_dict(nonexistant_files) - if rows: - return tabulate.tabulate(rows, headers=headers, floatfmt='+.2f'), note - else: - return None, note + return None, note if __name__ == '__main__': - args = _args() - diff, note = diff(args.benchmarks, args.loops, args.regex, args.track, args.old, - args.new, args.counters) - print('%s\n%s' % (note, diff if diff else "No performance differences")) + args = _args() + diff, note = diff(args.benchmarks, args.loops, args.regex, args.track, + args.old, args.new, args.counters) + print('%s\n%s' % (note, diff if diff else "No performance differences")) diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_main.py b/tools/profiling/microbenchmarks/bm_diff/bm_main.py index 74b7174f5d..137c22bf8e 100755 --- a/tools/profiling/microbenchmarks/bm_diff/bm_main.py +++ b/tools/profiling/microbenchmarks/bm_diff/bm_main.py @@ -13,7 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ Runs the entire bm_*.py pipeline, and possible comments on the PR """ import bm_constants @@ -29,129 +28,132 @@ import multiprocessing import subprocess sys.path.append( - os.path.join( - os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils')) + os.path.join( + os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils')) import comment_on_pr sys.path.append( - os.path.join( - os.path.dirname(sys.argv[0]), '..', '..', '..', 'run_tests', - 'python_utils')) + os.path.join( + os.path.dirname(sys.argv[0]), '..', '..', '..', 'run_tests', + 'python_utils')) import jobset def _args(): - argp = argparse.ArgumentParser( - description='Perform diff on microbenchmarks') - argp.add_argument( - '-t', - '--track', - choices=sorted(bm_constants._INTERESTING), - nargs='+', - default=sorted(bm_constants._INTERESTING), - help='Which metrics to track') - argp.add_argument( - '-b', - '--benchmarks', - nargs='+', - choices=bm_constants._AVAILABLE_BENCHMARK_TESTS, - default=bm_constants._AVAILABLE_BENCHMARK_TESTS, - help='Which benchmarks to run') - argp.add_argument( - '-d', - '--diff_base', - type=str, - help='Commit or branch to compare the current one to') - argp.add_argument( - '-o', - '--old', - default='old', - type=str, - help='Name of baseline run to compare to. Ususally just called "old"') - argp.add_argument( - '-r', - '--regex', - type=str, - default="", - help='Regex to filter benchmarks run') - argp.add_argument( - '-l', - '--loops', - type=int, - default=10, - help='Number of times to loops the benchmarks. More loops cuts down on noise' - ) - argp.add_argument( - '-j', - '--jobs', - type=int, - default=multiprocessing.cpu_count(), - help='Number of CPUs to use') - argp.add_argument( - '--pr_comment_name', - type=str, - default="microbenchmarks", - help='Name that Jenkins will use to commen on the PR') - argp.add_argument('--counters', dest='counters', action='store_true') - argp.add_argument('--no-counters', dest='counters', action='store_false') - argp.set_defaults(counters=True) - args = argp.parse_args() - assert args.diff_base or args.old, "One of diff_base or old must be set!" - if args.loops < 3: - print "WARNING: This run will likely be noisy. Increase loops." - return args + argp = argparse.ArgumentParser( + description='Perform diff on microbenchmarks') + argp.add_argument( + '-t', + '--track', + choices=sorted(bm_constants._INTERESTING), + nargs='+', + default=sorted(bm_constants._INTERESTING), + help='Which metrics to track') + argp.add_argument( + '-b', + '--benchmarks', + nargs='+', + choices=bm_constants._AVAILABLE_BENCHMARK_TESTS, + default=bm_constants._AVAILABLE_BENCHMARK_TESTS, + help='Which benchmarks to run') + argp.add_argument( + '-d', + '--diff_base', + type=str, + help='Commit or branch to compare the current one to') + argp.add_argument( + '-o', + '--old', + default='old', + type=str, + help='Name of baseline run to compare to. Ususally just called "old"') + argp.add_argument( + '-r', + '--regex', + type=str, + default="", + help='Regex to filter benchmarks run') + argp.add_argument( + '-l', + '--loops', + type=int, + default=10, + help='Number of times to loops the benchmarks. More loops cuts down on noise' + ) + argp.add_argument( + '-j', + '--jobs', + type=int, + default=multiprocessing.cpu_count(), + help='Number of CPUs to use') + argp.add_argument( + '--pr_comment_name', + type=str, + default="microbenchmarks", + help='Name that Jenkins will use to commen on the PR') + argp.add_argument('--counters', dest='counters', action='store_true') + argp.add_argument('--no-counters', dest='counters', action='store_false') + argp.set_defaults(counters=True) + args = argp.parse_args() + assert args.diff_base or args.old, "One of diff_base or old must be set!" + if args.loops < 3: + print "WARNING: This run will likely be noisy. Increase loops." + return args def eintr_be_gone(fn): - """Run fn until it doesn't stop because of EINTR""" + """Run fn until it doesn't stop because of EINTR""" - def inner(*args): - while True: - try: - return fn(*args) - except IOError, e: - if e.errno != errno.EINTR: - raise + def inner(*args): + while True: + try: + return fn(*args) + except IOError, e: + if e.errno != errno.EINTR: + raise - return inner + return inner def main(args): - bm_build.build('new', args.benchmarks, args.jobs, args.counters) - - old = args.old - if args.diff_base: - old = 'old' - where_am_i = subprocess.check_output( - ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() - subprocess.check_call(['git', 'checkout', args.diff_base]) - try: - bm_build.build(old, args.benchmarks, args.jobs, args.counters) - finally: - subprocess.check_call(['git', 'checkout', where_am_i]) - subprocess.check_call(['git', 'submodule', 'update']) - - jobs_list = [] - jobs_list += bm_run.create_jobs('new', args.benchmarks, args.loops, args.regex, args.counters) - jobs_list += bm_run.create_jobs(old, args.benchmarks, args.loops, args.regex, args.counters) - - # shuffle all jobs to eliminate noise from GCE CPU drift - random.shuffle(jobs_list, random.SystemRandom().random) - jobset.run(jobs_list, maxjobs=args.jobs) - - diff, note = bm_diff.diff(args.benchmarks, args.loops, args.regex, args.track, old, - 'new', args.counters) - if diff: - text = '[%s] Performance differences noted:\n%s' % (args.pr_comment_name, diff) - else: - text = '[%s] No significant performance differences' % args.pr_comment_name - if note: - text = note + '\n\n' + text - print('%s' % text) - comment_on_pr.comment_on_pr('```\n%s\n```' % text) + bm_build.build('new', args.benchmarks, args.jobs, args.counters) + + old = args.old + if args.diff_base: + old = 'old' + where_am_i = subprocess.check_output( + ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() + subprocess.check_call(['git', 'checkout', args.diff_base]) + try: + bm_build.build(old, args.benchmarks, args.jobs, args.counters) + finally: + subprocess.check_call(['git', 'checkout', where_am_i]) + subprocess.check_call(['git', 'submodule', 'update']) + + jobs_list = [] + jobs_list += bm_run.create_jobs('new', args.benchmarks, args.loops, + args.regex, args.counters) + jobs_list += bm_run.create_jobs(old, args.benchmarks, args.loops, + args.regex, args.counters) + + # shuffle all jobs to eliminate noise from GCE CPU drift + random.shuffle(jobs_list, random.SystemRandom().random) + jobset.run(jobs_list, maxjobs=args.jobs) + + diff, note = bm_diff.diff(args.benchmarks, args.loops, args.regex, + args.track, old, 'new', args.counters) + if diff: + text = '[%s] Performance differences noted:\n%s' % ( + args.pr_comment_name, diff) + else: + text = '[%s] No significant performance differences' % args.pr_comment_name + if note: + text = note + '\n\n' + text + print('%s' % text) + comment_on_pr.comment_on_pr('```\n%s\n```' % text) if __name__ == '__main__': - args = _args() - main(args) + args = _args() + main(args) diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_run.py b/tools/profiling/microbenchmarks/bm_diff/bm_run.py index 81db5a226a..08894bbe4d 100755 --- a/tools/profiling/microbenchmarks/bm_diff/bm_run.py +++ b/tools/profiling/microbenchmarks/bm_diff/bm_run.py @@ -13,7 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ Python utility to run opt and counters benchmarks and save json output """ import bm_constants @@ -27,93 +26,96 @@ import sys import os sys.path.append( - os.path.join( - os.path.dirname(sys.argv[0]), '..', '..', '..', 'run_tests', - 'python_utils')) + os.path.join( + os.path.dirname(sys.argv[0]), '..', '..', '..', 'run_tests', + 'python_utils')) import jobset def _args(): - argp = argparse.ArgumentParser(description='Runs microbenchmarks') - argp.add_argument( - '-b', - '--benchmarks', - nargs='+', - choices=bm_constants._AVAILABLE_BENCHMARK_TESTS, - default=bm_constants._AVAILABLE_BENCHMARK_TESTS, - help='Benchmarks to run') - argp.add_argument( - '-j', - '--jobs', - type=int, - default=multiprocessing.cpu_count(), - help='Number of CPUs to use') - argp.add_argument( - '-n', - '--name', - type=str, - help='Unique name of the build to run. Needs to match the handle passed to bm_build.py' - ) - argp.add_argument( - '-r', - '--regex', - type=str, - default="", - help='Regex to filter benchmarks run') - argp.add_argument( - '-l', - '--loops', - type=int, - default=20, - help='Number of times to loops the benchmarks. More loops cuts down on noise' - ) - argp.add_argument('--counters', dest='counters', action='store_true') - argp.add_argument('--no-counters', dest='counters', action='store_false') - argp.set_defaults(counters=True) - args = argp.parse_args() - assert args.name - if args.loops < 3: - print "WARNING: This run will likely be noisy. Increase loops to at least 3." - return args + argp = argparse.ArgumentParser(description='Runs microbenchmarks') + argp.add_argument( + '-b', + '--benchmarks', + nargs='+', + choices=bm_constants._AVAILABLE_BENCHMARK_TESTS, + default=bm_constants._AVAILABLE_BENCHMARK_TESTS, + help='Benchmarks to run') + argp.add_argument( + '-j', + '--jobs', + type=int, + default=multiprocessing.cpu_count(), + help='Number of CPUs to use') + argp.add_argument( + '-n', + '--name', + type=str, + help='Unique name of the build to run. Needs to match the handle passed to bm_build.py' + ) + argp.add_argument( + '-r', + '--regex', + type=str, + default="", + help='Regex to filter benchmarks run') + argp.add_argument( + '-l', + '--loops', + type=int, + default=20, + help='Number of times to loops the benchmarks. More loops cuts down on noise' + ) + argp.add_argument('--counters', dest='counters', action='store_true') + argp.add_argument('--no-counters', dest='counters', action='store_false') + argp.set_defaults(counters=True) + args = argp.parse_args() + assert args.name + if args.loops < 3: + print "WARNING: This run will likely be noisy. Increase loops to at least 3." + return args def _collect_bm_data(bm, cfg, name, regex, idx, loops): - jobs_list = [] - for line in subprocess.check_output( - ['bm_diff_%s/%s/%s' % (name, cfg, bm), - '--benchmark_list_tests', '--benchmark_filter=%s' % regex]).splitlines(): - stripped_line = line.strip().replace("/", "_").replace( - "<", "_").replace(">", "_").replace(", ", "_") - cmd = [ - 'bm_diff_%s/%s/%s' % (name, cfg, bm), '--benchmark_filter=^%s$' % - line, '--benchmark_out=%s.%s.%s.%s.%d.json' % - (bm, stripped_line, cfg, name, idx), '--benchmark_out_format=json', - ] - jobs_list.append( - jobset.JobSpec( - cmd, - shortname='%s %s %s %s %d/%d' % (bm, line, cfg, name, idx + 1, - loops), - verbose_success=True, - cpu_cost=2, - timeout_seconds=60 * 60)) # one hour - return jobs_list + jobs_list = [] + for line in subprocess.check_output([ + 'bm_diff_%s/%s/%s' % (name, cfg, bm), '--benchmark_list_tests', + '--benchmark_filter=%s' % regex + ]).splitlines(): + stripped_line = line.strip().replace("/", "_").replace( + "<", "_").replace(">", "_").replace(", ", "_") + cmd = [ + 'bm_diff_%s/%s/%s' % (name, cfg, bm), + '--benchmark_filter=^%s$' % line, + '--benchmark_out=%s.%s.%s.%s.%d.json' % + (bm, stripped_line, cfg, name, idx), + '--benchmark_out_format=json', + ] + jobs_list.append( + jobset.JobSpec( + cmd, + shortname='%s %s %s %s %d/%d' % (bm, line, cfg, name, idx + 1, + loops), + verbose_success=True, + cpu_cost=2, + timeout_seconds=60 * 60)) # one hour + return jobs_list def create_jobs(name, benchmarks, loops, regex, counters): - jobs_list = [] - for loop in range(0, loops): - for bm in benchmarks: - jobs_list += _collect_bm_data(bm, 'opt', name, regex, loop, loops) - if counters: - jobs_list += _collect_bm_data(bm, 'counters', name, regex, loop, - loops) - random.shuffle(jobs_list, random.SystemRandom().random) - return jobs_list + jobs_list = [] + for loop in range(0, loops): + for bm in benchmarks: + jobs_list += _collect_bm_data(bm, 'opt', name, regex, loop, loops) + if counters: + jobs_list += _collect_bm_data(bm, 'counters', name, regex, loop, + loops) + random.shuffle(jobs_list, random.SystemRandom().random) + return jobs_list if __name__ == '__main__': - args = _args() - jobs_list = create_jobs(args.name, args.benchmarks, args.loops, - args.regex, args.counters) - jobset.run(jobs_list, maxjobs=args.jobs) + args = _args() + jobs_list = create_jobs(args.name, args.benchmarks, args.loops, args.regex, + args.counters) + jobset.run(jobs_list, maxjobs=args.jobs) diff --git a/tools/profiling/microbenchmarks/bm_diff/bm_speedup.py b/tools/profiling/microbenchmarks/bm_diff/bm_speedup.py index 63e691af02..2a77040360 100755 --- a/tools/profiling/microbenchmarks/bm_diff/bm_speedup.py +++ b/tools/profiling/microbenchmarks/bm_diff/bm_speedup.py @@ -19,40 +19,41 @@ import math _DEFAULT_THRESHOLD = 1e-10 + def scale(a, mul): - return [x * mul for x in a] + return [x * mul for x in a] def cmp(a, b): - return stats.ttest_ind(a, b) - - -def speedup(new, old, threshold = _DEFAULT_THRESHOLD): - if (len(set(new))) == 1 and new == old: return 0 - s0, p0 = cmp(new, old) - if math.isnan(p0): return 0 - if s0 == 0: return 0 - if p0 > threshold: return 0 - if s0 < 0: - pct = 1 - while pct < 100: - sp, pp = cmp(new, scale(old, 1 - pct / 100.0)) - if sp > 0: break - if pp > threshold: break - pct += 1 - return -(pct - 1) - else: - pct = 1 - while pct < 10000: - sp, pp = cmp(new, scale(old, 1 + pct / 100.0)) - if sp < 0: break - if pp > threshold: break - pct += 1 - return pct - 1 + return stats.ttest_ind(a, b) + + +def speedup(new, old, threshold=_DEFAULT_THRESHOLD): + if (len(set(new))) == 1 and new == old: return 0 + s0, p0 = cmp(new, old) + if math.isnan(p0): return 0 + if s0 == 0: return 0 + if p0 > threshold: return 0 + if s0 < 0: + pct = 1 + while pct < 100: + sp, pp = cmp(new, scale(old, 1 - pct / 100.0)) + if sp > 0: break + if pp > threshold: break + pct += 1 + return -(pct - 1) + else: + pct = 1 + while pct < 10000: + sp, pp = cmp(new, scale(old, 1 + pct / 100.0)) + if sp < 0: break + if pp > threshold: break + pct += 1 + return pct - 1 if __name__ == "__main__": - new = [0.0, 0.0, 0.0, 0.0] - old = [2.96608e-06, 3.35076e-06, 3.45384e-06, 3.34407e-06] - print speedup(new, old, 1e-5) - print speedup(old, new, 1e-5) + new = [0.0, 0.0, 0.0, 0.0] + old = [2.96608e-06, 3.35076e-06, 3.45384e-06, 3.34407e-06] + print speedup(new, old, 1e-5) + print speedup(old, new, 1e-5) diff --git a/tools/profiling/microbenchmarks/bm_json.py b/tools/profiling/microbenchmarks/bm_json.py index eb450ee6ad..1dd9f65dbf 100644 --- a/tools/profiling/microbenchmarks/bm_json.py +++ b/tools/profiling/microbenchmarks/bm_json.py @@ -15,187 +15,196 @@ import os _BM_SPECS = { - 'BM_UnaryPingPong': { - 'tpl': ['fixture', 'client_mutator', 'server_mutator'], - 'dyn': ['request_size', 'response_size'], - }, - 'BM_PumpStreamClientToServer': { - 'tpl': ['fixture'], - 'dyn': ['request_size'], - }, - 'BM_PumpStreamServerToClient': { - 'tpl': ['fixture'], - 'dyn': ['request_size'], - }, - 'BM_StreamingPingPong': { - 'tpl': ['fixture', 'client_mutator', 'server_mutator'], - 'dyn': ['request_size', 'request_count'], - }, - 'BM_StreamingPingPongMsgs': { - 'tpl': ['fixture', 'client_mutator', 'server_mutator'], - 'dyn': ['request_size'], - }, - 'BM_PumpStreamServerToClient_Trickle': { - 'tpl': [], - 'dyn': ['request_size', 'bandwidth_kilobits'], - }, - 'BM_PumpUnbalancedUnary_Trickle': { - 'tpl': [], - 'dyn': ['cli_req_size', 'svr_req_size', 'bandwidth_kilobits'], - }, - 'BM_ErrorStringOnNewError': { - 'tpl': ['fixture'], - 'dyn': [], - }, - 'BM_ErrorStringRepeatedly': { - 'tpl': ['fixture'], - 'dyn': [], - }, - 'BM_ErrorGetStatus': { - 'tpl': ['fixture'], - 'dyn': [], - }, - 'BM_ErrorGetStatusCode': { - 'tpl': ['fixture'], - 'dyn': [], - }, - 'BM_ErrorHttpError': { - 'tpl': ['fixture'], - 'dyn': [], - }, - 'BM_HasClearGrpcStatus': { - 'tpl': ['fixture'], - 'dyn': [], - }, - 'BM_IsolatedFilter': { - 'tpl': ['fixture', 'client_mutator'], - 'dyn': [], - }, - 'BM_HpackEncoderEncodeHeader': { - 'tpl': ['fixture'], - 'dyn': ['end_of_stream', 'request_size'], - }, - 'BM_HpackParserParseHeader': { - 'tpl': ['fixture', 'on_header'], - 'dyn': [], - }, - 'BM_CallCreateDestroy': { - 'tpl': ['fixture'], - 'dyn': [], - }, - 'BM_Zalloc': { - 'tpl': [], - 'dyn': ['request_size'], - }, - 'BM_PollEmptyPollset_SpeedOfLight': { - 'tpl': [], - 'dyn': ['request_size', 'request_count'], - }, - 'BM_StreamCreateSendInitialMetadataDestroy': { - 'tpl': ['fixture'], - 'dyn': [], - }, - 'BM_TransportStreamSend': { - 'tpl': [], - 'dyn': ['request_size'], - }, - 'BM_TransportStreamRecv': { - 'tpl': [], - 'dyn': ['request_size'], - }, - 'BM_StreamingPingPongWithCoalescingApi': { - 'tpl': ['fixture', 'client_mutator', 'server_mutator'], - 'dyn': ['request_size', 'request_count', 'end_of_stream'], - }, - 'BM_Base16SomeStuff': { - 'tpl': [], - 'dyn': ['request_size'], - } + 'BM_UnaryPingPong': { + 'tpl': ['fixture', 'client_mutator', 'server_mutator'], + 'dyn': ['request_size', 'response_size'], + }, + 'BM_PumpStreamClientToServer': { + 'tpl': ['fixture'], + 'dyn': ['request_size'], + }, + 'BM_PumpStreamServerToClient': { + 'tpl': ['fixture'], + 'dyn': ['request_size'], + }, + 'BM_StreamingPingPong': { + 'tpl': ['fixture', 'client_mutator', 'server_mutator'], + 'dyn': ['request_size', 'request_count'], + }, + 'BM_StreamingPingPongMsgs': { + 'tpl': ['fixture', 'client_mutator', 'server_mutator'], + 'dyn': ['request_size'], + }, + 'BM_PumpStreamServerToClient_Trickle': { + 'tpl': [], + 'dyn': ['request_size', 'bandwidth_kilobits'], + }, + 'BM_PumpUnbalancedUnary_Trickle': { + 'tpl': [], + 'dyn': ['cli_req_size', 'svr_req_size', 'bandwidth_kilobits'], + }, + 'BM_ErrorStringOnNewError': { + 'tpl': ['fixture'], + 'dyn': [], + }, + 'BM_ErrorStringRepeatedly': { + 'tpl': ['fixture'], + 'dyn': [], + }, + 'BM_ErrorGetStatus': { + 'tpl': ['fixture'], + 'dyn': [], + }, + 'BM_ErrorGetStatusCode': { + 'tpl': ['fixture'], + 'dyn': [], + }, + 'BM_ErrorHttpError': { + 'tpl': ['fixture'], + 'dyn': [], + }, + 'BM_HasClearGrpcStatus': { + 'tpl': ['fixture'], + 'dyn': [], + }, + 'BM_IsolatedFilter': { + 'tpl': ['fixture', 'client_mutator'], + 'dyn': [], + }, + 'BM_HpackEncoderEncodeHeader': { + 'tpl': ['fixture'], + 'dyn': ['end_of_stream', 'request_size'], + }, + 'BM_HpackParserParseHeader': { + 'tpl': ['fixture', 'on_header'], + 'dyn': [], + }, + 'BM_CallCreateDestroy': { + 'tpl': ['fixture'], + 'dyn': [], + }, + 'BM_Zalloc': { + 'tpl': [], + 'dyn': ['request_size'], + }, + 'BM_PollEmptyPollset_SpeedOfLight': { + 'tpl': [], + 'dyn': ['request_size', 'request_count'], + }, + 'BM_StreamCreateSendInitialMetadataDestroy': { + 'tpl': ['fixture'], + 'dyn': [], + }, + 'BM_TransportStreamSend': { + 'tpl': [], + 'dyn': ['request_size'], + }, + 'BM_TransportStreamRecv': { + 'tpl': [], + 'dyn': ['request_size'], + }, + 'BM_StreamingPingPongWithCoalescingApi': { + 'tpl': ['fixture', 'client_mutator', 'server_mutator'], + 'dyn': ['request_size', 'request_count', 'end_of_stream'], + }, + 'BM_Base16SomeStuff': { + 'tpl': [], + 'dyn': ['request_size'], + } } + def numericalize(s): - if not s: return '' - if s[-1] == 'k': - return float(s[:-1]) * 1024 - if s[-1] == 'M': - return float(s[:-1]) * 1024 * 1024 - if 0 <= (ord(s[-1]) - ord('0')) <= 9: - return float(s) - assert 'not a number: %s' % s + if not s: return '' + if s[-1] == 'k': + return float(s[:-1]) * 1024 + if s[-1] == 'M': + return float(s[:-1]) * 1024 * 1024 + if 0 <= (ord(s[-1]) - ord('0')) <= 9: + return float(s) + assert 'not a number: %s' % s + def parse_name(name): - cpp_name = name - if '<' not in name and '/' not in name and name not in _BM_SPECS: - return {'name': name, 'cpp_name': name} - rest = name - out = {} - tpl_args = [] - dyn_args = [] - if '<' in rest: - tpl_bit = rest[rest.find('<') + 1 : rest.rfind('>')] - arg = '' - nesting = 0 - for c in tpl_bit: - if c == '<': - nesting += 1 - arg += c - elif c == '>': - nesting -= 1 - arg += c - elif c == ',': - if nesting == 0: - tpl_args.append(arg.strip()) - arg = '' - else: - arg += c - else: - arg += c - tpl_args.append(arg.strip()) - rest = rest[:rest.find('<')] + rest[rest.rfind('>') + 1:] - if '/' in rest: - s = rest.split('/') - rest = s[0] - dyn_args = s[1:] - name = rest - print (name) - print (dyn_args, _BM_SPECS[name]['dyn']) - print (tpl_args, _BM_SPECS[name]['tpl']) - assert name in _BM_SPECS, '_BM_SPECS needs to be expanded for %s' % name - assert len(dyn_args) == len(_BM_SPECS[name]['dyn']) - assert len(tpl_args) == len(_BM_SPECS[name]['tpl']) - out['name'] = name - out['cpp_name'] = cpp_name - out.update(dict((k, numericalize(v)) for k, v in zip(_BM_SPECS[name]['dyn'], dyn_args))) - out.update(dict(zip(_BM_SPECS[name]['tpl'], tpl_args))) - return out + cpp_name = name + if '<' not in name and '/' not in name and name not in _BM_SPECS: + return {'name': name, 'cpp_name': name} + rest = name + out = {} + tpl_args = [] + dyn_args = [] + if '<' in rest: + tpl_bit = rest[rest.find('<') + 1:rest.rfind('>')] + arg = '' + nesting = 0 + for c in tpl_bit: + if c == '<': + nesting += 1 + arg += c + elif c == '>': + nesting -= 1 + arg += c + elif c == ',': + if nesting == 0: + tpl_args.append(arg.strip()) + arg = '' + else: + arg += c + else: + arg += c + tpl_args.append(arg.strip()) + rest = rest[:rest.find('<')] + rest[rest.rfind('>') + 1:] + if '/' in rest: + s = rest.split('/') + rest = s[0] + dyn_args = s[1:] + name = rest + print(name) + print(dyn_args, _BM_SPECS[name]['dyn']) + print(tpl_args, _BM_SPECS[name]['tpl']) + assert name in _BM_SPECS, '_BM_SPECS needs to be expanded for %s' % name + assert len(dyn_args) == len(_BM_SPECS[name]['dyn']) + assert len(tpl_args) == len(_BM_SPECS[name]['tpl']) + out['name'] = name + out['cpp_name'] = cpp_name + out.update( + dict((k, numericalize(v)) + for k, v in zip(_BM_SPECS[name]['dyn'], dyn_args))) + out.update(dict(zip(_BM_SPECS[name]['tpl'], tpl_args))) + return out -def expand_json(js, js2 = None): - if not js and not js2: raise StopIteration() - if not js: js = js2 - for bm in js['benchmarks']: - if bm['name'].endswith('_stddev') or bm['name'].endswith('_mean'): continue - context = js['context'] - if 'label' in bm: - labels_list = [s.split(':') for s in bm['label'].strip().split(' ') if len(s) and s[0] != '#'] - for el in labels_list: - el[0] = el[0].replace('/iter', '_per_iteration') - labels = dict(labels_list) - else: - labels = {} - row = { - 'jenkins_build': os.environ.get('BUILD_NUMBER', ''), - 'jenkins_job': os.environ.get('JOB_NAME', ''), - } - row.update(context) - row.update(bm) - row.update(parse_name(row['name'])) - row.update(labels) - if js2: - for bm2 in js2['benchmarks']: - if bm['name'] == bm2['name'] and 'already_used' not in bm2: - row['cpu_time'] = bm2['cpu_time'] - row['real_time'] = bm2['real_time'] - row['iterations'] = bm2['iterations'] - bm2['already_used'] = True - break - yield row + +def expand_json(js, js2=None): + if not js and not js2: raise StopIteration() + if not js: js = js2 + for bm in js['benchmarks']: + if bm['name'].endswith('_stddev') or bm['name'].endswith('_mean'): + continue + context = js['context'] + if 'label' in bm: + labels_list = [ + s.split(':') for s in bm['label'].strip().split(' ') + if len(s) and s[0] != '#' + ] + for el in labels_list: + el[0] = el[0].replace('/iter', '_per_iteration') + labels = dict(labels_list) + else: + labels = {} + row = { + 'jenkins_build': os.environ.get('BUILD_NUMBER', ''), + 'jenkins_job': os.environ.get('JOB_NAME', ''), + } + row.update(context) + row.update(bm) + row.update(parse_name(row['name'])) + row.update(labels) + if js2: + for bm2 in js2['benchmarks']: + if bm['name'] == bm2['name'] and 'already_used' not in bm2: + row['cpu_time'] = bm2['cpu_time'] + row['real_time'] = bm2['real_time'] + row['iterations'] = bm2['iterations'] + bm2['already_used'] = True + break + yield row diff --git a/tools/profiling/qps/qps_diff.py b/tools/profiling/qps/qps_diff.py index 0654f45666..55a81f034d 100755 --- a/tools/profiling/qps/qps_diff.py +++ b/tools/profiling/qps/qps_diff.py @@ -26,144 +26,146 @@ import sys import tabulate sys.path.append( - os.path.join( - os.path.dirname(sys.argv[0]), '..', 'microbenchmarks', 'bm_diff')) + os.path.join( + os.path.dirname(sys.argv[0]), '..', 'microbenchmarks', 'bm_diff')) import bm_speedup sys.path.append( - os.path.join( - os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils')) + os.path.join( + os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils')) import comment_on_pr def _args(): - argp = argparse.ArgumentParser( - description='Perform diff on QPS Driver') - argp.add_argument( - '-d', - '--diff_base', - type=str, - help='Commit or branch to compare the current one to') - argp.add_argument( - '-l', - '--loops', - type=int, - default=4, - help='Number of loops for each benchmark. More loops cuts down on noise' - ) - argp.add_argument( - '-j', - '--jobs', - type=int, - default=multiprocessing.cpu_count(), - help='Number of CPUs to use') - args = argp.parse_args() - assert args.diff_base, "diff_base must be set" - return args + argp = argparse.ArgumentParser(description='Perform diff on QPS Driver') + argp.add_argument( + '-d', + '--diff_base', + type=str, + help='Commit or branch to compare the current one to') + argp.add_argument( + '-l', + '--loops', + type=int, + default=4, + help='Number of loops for each benchmark. More loops cuts down on noise') + argp.add_argument( + '-j', + '--jobs', + type=int, + default=multiprocessing.cpu_count(), + help='Number of CPUs to use') + args = argp.parse_args() + assert args.diff_base, "diff_base must be set" + return args def _make_cmd(jobs): - return ['make', '-j', '%d' % jobs, 'qps_json_driver', 'qps_worker'] + return ['make', '-j', '%d' % jobs, 'qps_json_driver', 'qps_worker'] def build(name, jobs): - shutil.rmtree('qps_diff_%s' % name, ignore_errors=True) - subprocess.check_call(['git', 'submodule', 'update']) - try: - subprocess.check_call(_make_cmd(jobs)) - except subprocess.CalledProcessError, e: - subprocess.check_call(['make', 'clean']) - subprocess.check_call(_make_cmd(jobs)) - os.rename('bins', 'qps_diff_%s' % name) + shutil.rmtree('qps_diff_%s' % name, ignore_errors=True) + subprocess.check_call(['git', 'submodule', 'update']) + try: + subprocess.check_call(_make_cmd(jobs)) + except subprocess.CalledProcessError, e: + subprocess.check_call(['make', 'clean']) + subprocess.check_call(_make_cmd(jobs)) + os.rename('bins', 'qps_diff_%s' % name) def _run_cmd(name, scenario, fname): - return ['qps_diff_%s/opt/qps_json_driver' % name, '--scenarios_json', scenario, '--json_file_out', fname] + return [ + 'qps_diff_%s/opt/qps_json_driver' % name, '--scenarios_json', scenario, + '--json_file_out', fname + ] def run(name, scenarios, loops): - for sn in scenarios: - for i in range(0, loops): - fname = "%s.%s.%d.json" % (sn, name, i) - subprocess.check_call(_run_cmd(name, scenarios[sn], fname)) + for sn in scenarios: + for i in range(0, loops): + fname = "%s.%s.%d.json" % (sn, name, i) + subprocess.check_call(_run_cmd(name, scenarios[sn], fname)) def _load_qps(fname): - try: - with open(fname) as f: - return json.loads(f.read())['qps'] - except IOError, e: - print("IOError occurred reading file: %s" % fname) - return None - except ValueError, e: - print("ValueError occurred reading file: %s" % fname) - return None + try: + with open(fname) as f: + return json.loads(f.read())['qps'] + except IOError, e: + print("IOError occurred reading file: %s" % fname) + return None + except ValueError, e: + print("ValueError occurred reading file: %s" % fname) + return None def _median(ary): - assert (len(ary)) - ary = sorted(ary) - n = len(ary) - if n % 2 == 0: - return (ary[(n - 1) / 2] + ary[(n - 1) / 2 + 1]) / 2.0 - else: - return ary[n / 2] + assert (len(ary)) + ary = sorted(ary) + n = len(ary) + if n % 2 == 0: + return (ary[(n - 1) / 2] + ary[(n - 1) / 2 + 1]) / 2.0 + else: + return ary[n / 2] def diff(scenarios, loops, old, new): - old_data = {} - new_data = {} - - # collect data - for sn in scenarios: - old_data[sn] = [] - new_data[sn] = [] - for i in range(loops): - old_data[sn].append(_load_qps("%s.%s.%d.json" % (sn, old, i))) - new_data[sn].append(_load_qps("%s.%s.%d.json" % (sn, new, i))) - - # crunch data - headers = ['Benchmark', 'qps'] - rows = [] - for sn in scenarios: - mdn_diff = abs(_median(new_data[sn]) - _median(old_data[sn])) - print('%s: %s=%r %s=%r mdn_diff=%r' % (sn, new, new_data[sn], old, old_data[sn], mdn_diff)) - s = bm_speedup.speedup(new_data[sn], old_data[sn], 10e-5) - if abs(s) > 3 and mdn_diff > 0.5: - rows.append([sn, '%+d%%' % s]) - - if rows: - return tabulate.tabulate(rows, headers=headers, floatfmt='+.2f') - else: - return None + old_data = {} + new_data = {} + + # collect data + for sn in scenarios: + old_data[sn] = [] + new_data[sn] = [] + for i in range(loops): + old_data[sn].append(_load_qps("%s.%s.%d.json" % (sn, old, i))) + new_data[sn].append(_load_qps("%s.%s.%d.json" % (sn, new, i))) + + # crunch data + headers = ['Benchmark', 'qps'] + rows = [] + for sn in scenarios: + mdn_diff = abs(_median(new_data[sn]) - _median(old_data[sn])) + print('%s: %s=%r %s=%r mdn_diff=%r' % + (sn, new, new_data[sn], old, old_data[sn], mdn_diff)) + s = bm_speedup.speedup(new_data[sn], old_data[sn], 10e-5) + if abs(s) > 3 and mdn_diff > 0.5: + rows.append([sn, '%+d%%' % s]) + + if rows: + return tabulate.tabulate(rows, headers=headers, floatfmt='+.2f') + else: + return None def main(args): - build('new', args.jobs) + build('new', args.jobs) - if args.diff_base: - where_am_i = subprocess.check_output( - ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() - subprocess.check_call(['git', 'checkout', args.diff_base]) - try: - build('old', args.jobs) - finally: - subprocess.check_call(['git', 'checkout', where_am_i]) - subprocess.check_call(['git', 'submodule', 'update']) + if args.diff_base: + where_am_i = subprocess.check_output( + ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() + subprocess.check_call(['git', 'checkout', args.diff_base]) + try: + build('old', args.jobs) + finally: + subprocess.check_call(['git', 'checkout', where_am_i]) + subprocess.check_call(['git', 'submodule', 'update']) - run('new', qps_scenarios._SCENARIOS, args.loops) - run('old', qps_scenarios._SCENARIOS, args.loops) + run('new', qps_scenarios._SCENARIOS, args.loops) + run('old', qps_scenarios._SCENARIOS, args.loops) - diff_output = diff(qps_scenarios._SCENARIOS, args.loops, 'old', 'new') + diff_output = diff(qps_scenarios._SCENARIOS, args.loops, 'old', 'new') - if diff_output: - text = '[qps] Performance differences noted:\n%s' % diff_output - else: - text = '[qps] No significant performance differences' - print('%s' % text) - comment_on_pr.comment_on_pr('```\n%s\n```' % text) + if diff_output: + text = '[qps] Performance differences noted:\n%s' % diff_output + else: + text = '[qps] No significant performance differences' + print('%s' % text) + comment_on_pr.comment_on_pr('```\n%s\n```' % text) if __name__ == '__main__': - args = _args() - main(args) + args = _args() + main(args) diff --git a/tools/profiling/qps/qps_scenarios.py b/tools/profiling/qps/qps_scenarios.py index 4fbbdefc4d..532acc915c 100644 --- a/tools/profiling/qps/qps_scenarios.py +++ b/tools/profiling/qps/qps_scenarios.py @@ -14,6 +14,8 @@ """ QPS Scenarios to run """ _SCENARIOS = { - 'large-message-throughput': '{"scenarios":[{"name":"large-message-throughput", "spawn_local_worker_count": -2, "warmup_seconds": 30, "benchmark_seconds": 270, "num_servers": 1, "server_config": {"async_server_threads": 1, "security_params": null, "server_type": "ASYNC_SERVER"}, "num_clients": 1, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": null, "payload_config": {"simple_params": {"resp_size": 1048576, "req_size": 1048576}}, "client_channels": 1, "async_client_threads": 1, "outstanding_rpcs_per_channel": 1, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"max_possible": 60000000000.0, "resolution": 0.01}}}]}', - 'multi-channel-64-KiB': '{"scenarios":[{"name":"multi-channel-64-KiB", "spawn_local_worker_count": -3, "warmup_seconds": 30, "benchmark_seconds": 270, "num_servers": 1, "server_config": {"async_server_threads": 31, "security_params": null, "server_type": "ASYNC_SERVER"}, "num_clients": 2, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": null, "payload_config": {"simple_params": {"resp_size": 65536, "req_size": 65536}}, "client_channels": 32, "async_client_threads": 31, "outstanding_rpcs_per_channel": 100, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"max_possible": 60000000000.0, "resolution": 0.01}}}]}' + 'large-message-throughput': + '{"scenarios":[{"name":"large-message-throughput", "spawn_local_worker_count": -2, "warmup_seconds": 30, "benchmark_seconds": 270, "num_servers": 1, "server_config": {"async_server_threads": 1, "security_params": null, "server_type": "ASYNC_SERVER"}, "num_clients": 1, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": null, "payload_config": {"simple_params": {"resp_size": 1048576, "req_size": 1048576}}, "client_channels": 1, "async_client_threads": 1, "outstanding_rpcs_per_channel": 1, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"max_possible": 60000000000.0, "resolution": 0.01}}}]}', + 'multi-channel-64-KiB': + '{"scenarios":[{"name":"multi-channel-64-KiB", "spawn_local_worker_count": -3, "warmup_seconds": 30, "benchmark_seconds": 270, "num_servers": 1, "server_config": {"async_server_threads": 31, "security_params": null, "server_type": "ASYNC_SERVER"}, "num_clients": 2, "client_config": {"client_type": "ASYNC_CLIENT", "security_params": null, "payload_config": {"simple_params": {"resp_size": 65536, "req_size": 65536}}, "client_channels": 32, "async_client_threads": 31, "outstanding_rpcs_per_channel": 100, "rpc_type": "UNARY", "load_params": {"closed_loop": {}}, "histogram_params": {"max_possible": 60000000000.0, "resolution": 0.01}}}]}' } -- cgit v1.2.3