From 7dc4ea66f0688c30aa7ddd865074a808fd20f076 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 2 Feb 2017 16:08:05 -0800 Subject: Make the microbenchmark profile gatherer run some in parallel --- .../profiling/latency_profile/profile_analyzer.py | 18 ++-- tools/profiling/microbenchmark/bm.py | 109 --------------------- 2 files changed, 12 insertions(+), 115 deletions(-) delete mode 100755 tools/profiling/microbenchmark/bm.py (limited to 'tools/profiling') diff --git a/tools/profiling/latency_profile/profile_analyzer.py b/tools/profiling/latency_profile/profile_analyzer.py index 48b8e9b950..2087cd2793 100755 --- a/tools/profiling/latency_profile/profile_analyzer.py +++ b/tools/profiling/latency_profile/profile_analyzer.py @@ -34,6 +34,7 @@ import hashlib import itertools import json import math +import sys import tabulate import time @@ -49,6 +50,7 @@ TIME_FROM_LAST_IMPORTANT = object() argp = argparse.ArgumentParser(description='Process output of basic_prof builds') argp.add_argument('--source', default='latency_trace.txt', type=str) argp.add_argument('--fmt', choices=tabulate.tabulate_formats, default='simple') +argp.add_argument('--out', default='-', type=str) args = argp.parse_args() class LineItem(object): @@ -246,16 +248,20 @@ FORMAT = [ ('TO_SCOPE_END', time_format(TIME_TO_SCOPE_END)), ] +out = sys.stdout +if args.out != '-': + out = open(args.out, 'w') + if args.fmt == 'html': - print '' - print '' - print 'Profile Report' - print '' + print >>out, '' + print >>out, '' + print >>out, 'Profile Report' + print >>out, '' accounted_for = 0 for cs in call_stacks: if args.fmt in BANNER: - print BANNER[args.fmt] % { + print >>out, BANNER[args.fmt] % { 'count': cs.count, } header, _ = zip(*FORMAT) @@ -265,7 +271,7 @@ for cs in call_stacks: for _, fn in FORMAT: fields.append(fn(line)) table.append(fields) - print tabulate.tabulate(table, header, tablefmt=args.fmt) + print >>out, tabulate.tabulate(table, header, tablefmt=args.fmt) accounted_for += cs.count if accounted_for > .99 * total_stacks: break diff --git a/tools/profiling/microbenchmark/bm.py b/tools/profiling/microbenchmark/bm.py deleted file mode 100755 index b4f0de841f..0000000000 --- a/tools/profiling/microbenchmark/bm.py +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env python2.7 -# Copyright 2017, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import multiprocessing -import os -import subprocess -import sys - -flamegraph_dir = os.path.join(os.path.expanduser('~'), 'FlameGraph') - -def fnize(s): - out = '' - for c in s: - if c in '<>, /': - if len(out) and out[-1] == '_': continue - out += '_' - else: - out += c - return out - -os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../../..')) -if not os.path.exists('reports'): - os.makedirs('reports') - -# index html -index_html = """ - - -Microbenchmark Results - - -""" - -def heading(name): - global index_html - index_html += "

%s

\n" % name - -def link(txt, tgt): - global index_html - index_html += "

\n" % (tgt, txt) - -for bm_name in sys.argv[1:]: - # generate latency profiles - heading('Latency Profiles: %s' % bm_name) - subprocess.check_call( - ['make', bm_name, - 'CONFIG=basicprof', '-j', '%d' % multiprocessing.cpu_count()]) - for line in subprocess.check_output(['bins/basicprof/%s' % bm_name, - '--benchmark_list_tests']).splitlines(): - link(line, '%s.txt' % fnize(line)) - with open('reports/%s.txt' % fnize(line), 'w') as f: - f.write(subprocess.check_output(['bins/basicprof/%s' % bm_name, - '--benchmark_filter=^%s$' % line])) - f.write('\n***********************************************************\n') - f.write(subprocess.check_output([ - sys.executable, 'tools/profiling/latency_profile/profile_analyzer.py', - '--source', 'latency_trace.txt', '--fmt', 'simple'])) - - # generate flamegraphs - heading('Flamegraphs: %s' % bm_name) - subprocess.check_call( - ['make', bm_name, - 'CONFIG=mutrace', '-j', '%d' % multiprocessing.cpu_count()]) - for line in subprocess.check_output(['bins/mutrace/%s' % bm_name, - '--benchmark_list_tests']).splitlines(): - subprocess.check_call(['sudo', 'perf', 'record', '-g', '-c', '1000', - 'bins/mutrace/%s' % bm_name, - '--benchmark_filter=^%s$' % line, - '--benchmark_min_time=20']) - with open('/tmp/bm.perf', 'w') as f: - f.write(subprocess.check_output(['sudo', 'perf', 'script'])) - with open('/tmp/bm.folded', 'w') as f: - f.write(subprocess.check_output([ - '%s/stackcollapse-perf.pl' % flamegraph_dir, '/tmp/bm.perf'])) - link(line, '%s.svg' % fnize(line)) - with open('reports/%s.svg' % fnize(line), 'w') as f: - f.write(subprocess.check_output([ - '%s/flamegraph.pl' % flamegraph_dir, '/tmp/bm.folded'])) - -index_html += "\n\n" -with open('reports/index.html', 'w') as f: - w.write(index_html) -- cgit v1.2.3