aboutsummaryrefslogtreecommitdiffhomepage
path: root/tools/run_tests/run_microbenchmark.py
diff options
context:
space:
mode:
authorGravatar ncteisen <ncteisen@gmail.com>2017-12-11 18:00:40 -0800
committerGravatar ncteisen <ncteisen@gmail.com>2017-12-11 18:08:35 -0800
commit888093c6ed0d24eed699173b5fb35235fe7a6069 (patch)
treeffbbc9fbfaacff17c3b94a2f2d780ece6abbfa36 /tools/run_tests/run_microbenchmark.py
parenta69c6901f9c4e375ad543cb4ee6ce747d953bf18 (diff)
yapf run_tests
Diffstat (limited to 'tools/run_tests/run_microbenchmark.py')
-rwxr-xr-xtools/run_tests/run_microbenchmark.py346
1 files changed, 192 insertions, 154 deletions
diff --git a/tools/run_tests/run_microbenchmark.py b/tools/run_tests/run_microbenchmark.py
index c136af58cb..561217ceb1 100755
--- a/tools/run_tests/run_microbenchmark.py
+++ b/tools/run_tests/run_microbenchmark.py
@@ -23,26 +23,31 @@ import argparse
import python_utils.jobset as jobset
import python_utils.start_port_server as start_port_server
-sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', 'profiling', 'microbenchmarks', 'bm_diff'))
+sys.path.append(
+ os.path.join(
+ os.path.dirname(sys.argv[0]), '..', 'profiling', 'microbenchmarks',
+ 'bm_diff'))
import bm_constants
flamegraph_dir = os.path.join(os.path.expanduser('~'), 'FlameGraph')
os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
if not os.path.exists('reports'):
- os.makedirs('reports')
+ os.makedirs('reports')
start_port_server.start_port_server()
+
def fnize(s):
- out = ''
- for c in s:
- if c in '<>, /':
- if len(out) and out[-1] == '_': continue
- out += '_'
- else:
- out += c
- return out
+ out = ''
+ for c in s:
+ if c in '<>, /':
+ if len(out) and out[-1] == '_': continue
+ out += '_'
+ else:
+ out += c
+ return out
+
# index html
index_html = """
@@ -53,169 +58,202 @@ index_html = """
<body>
"""
+
def heading(name):
- global index_html
- index_html += "<h1>%s</h1>\n" % name
+ global index_html
+ index_html += "<h1>%s</h1>\n" % name
+
def link(txt, tgt):
- global index_html
- index_html += "<p><a href=\"%s\">%s</a></p>\n" % (
- cgi.escape(tgt, quote=True), cgi.escape(txt))
+ global index_html
+ index_html += "<p><a href=\"%s\">%s</a></p>\n" % (
+ cgi.escape(tgt, quote=True), cgi.escape(txt))
+
def text(txt):
- global index_html
- index_html += "<p><pre>%s</pre></p>\n" % cgi.escape(txt)
+ global index_html
+ index_html += "<p><pre>%s</pre></p>\n" % cgi.escape(txt)
+
def collect_latency(bm_name, args):
- """generate latency profiles"""
- benchmarks = []
- profile_analysis = []
- cleanup = []
-
- heading('Latency Profiles: %s' % bm_name)
- subprocess.check_call(
- ['make', bm_name,
- 'CONFIG=basicprof', '-j', '%d' % multiprocessing.cpu_count()])
- for line in subprocess.check_output(['bins/basicprof/%s' % bm_name,
- '--benchmark_list_tests']).splitlines():
- link(line, '%s.txt' % fnize(line))
- benchmarks.append(
- jobset.JobSpec(['bins/basicprof/%s' % bm_name,
- '--benchmark_filter=^%s$' % line,
- '--benchmark_min_time=0.05'],
- environ={'LATENCY_TRACE': '%s.trace' % fnize(line)},
- shortname='profile-%s' % fnize(line)))
- profile_analysis.append(
- jobset.JobSpec([sys.executable,
- 'tools/profiling/latency_profile/profile_analyzer.py',
- '--source', '%s.trace' % fnize(line), '--fmt', 'simple',
- '--out', 'reports/%s.txt' % fnize(line)], timeout_seconds=20*60,
- shortname='analyze-%s' % fnize(line)))
- cleanup.append(jobset.JobSpec(['rm', '%s.trace' % fnize(line)]))
- # periodically flush out the list of jobs: profile_analysis jobs at least
- # consume upwards of five gigabytes of ram in some cases, and so analysing
- # hundreds of them at once is impractical -- but we want at least some
- # concurrency or the work takes too long
- if len(benchmarks) >= min(16, multiprocessing.cpu_count()):
- # run up to half the cpu count: each benchmark can use up to two cores
- # (one for the microbenchmark, one for the data flush)
- jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2))
- jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
- jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
- benchmarks = []
- profile_analysis = []
- cleanup = []
- # run the remaining benchmarks that weren't flushed
- if len(benchmarks):
- jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2))
- jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
- jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
+ """generate latency profiles"""
+ benchmarks = []
+ profile_analysis = []
+ cleanup = []
+
+ heading('Latency Profiles: %s' % bm_name)
+ subprocess.check_call([
+ 'make', bm_name, 'CONFIG=basicprof', '-j',
+ '%d' % multiprocessing.cpu_count()
+ ])
+ for line in subprocess.check_output(
+ ['bins/basicprof/%s' % bm_name, '--benchmark_list_tests']).splitlines():
+ link(line, '%s.txt' % fnize(line))
+ benchmarks.append(
+ jobset.JobSpec(
+ [
+ 'bins/basicprof/%s' % bm_name, '--benchmark_filter=^%s$' %
+ line, '--benchmark_min_time=0.05'
+ ],
+ environ={'LATENCY_TRACE': '%s.trace' % fnize(line)},
+ shortname='profile-%s' % fnize(line)))
+ profile_analysis.append(
+ jobset.JobSpec(
+ [
+ sys.executable,
+ 'tools/profiling/latency_profile/profile_analyzer.py',
+ '--source', '%s.trace' % fnize(line), '--fmt', 'simple',
+ '--out', 'reports/%s.txt' % fnize(line)
+ ],
+ timeout_seconds=20 * 60,
+ shortname='analyze-%s' % fnize(line)))
+ cleanup.append(jobset.JobSpec(['rm', '%s.trace' % fnize(line)]))
+ # periodically flush out the list of jobs: profile_analysis jobs at least
+ # consume upwards of five gigabytes of ram in some cases, and so analysing
+ # hundreds of them at once is impractical -- but we want at least some
+ # concurrency or the work takes too long
+ if len(benchmarks) >= min(16, multiprocessing.cpu_count()):
+ # run up to half the cpu count: each benchmark can use up to two cores
+ # (one for the microbenchmark, one for the data flush)
+ jobset.run(
+ benchmarks, maxjobs=max(1, multiprocessing.cpu_count() / 2))
+ jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
+ jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
+ benchmarks = []
+ profile_analysis = []
+ cleanup = []
+ # run the remaining benchmarks that weren't flushed
+ if len(benchmarks):
+ jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count() / 2))
+ jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
+ jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
+
def collect_perf(bm_name, args):
- """generate flamegraphs"""
- heading('Flamegraphs: %s' % bm_name)
- subprocess.check_call(
- ['make', bm_name,
- 'CONFIG=mutrace', '-j', '%d' % multiprocessing.cpu_count()])
- benchmarks = []
- profile_analysis = []
- cleanup = []
- for line in subprocess.check_output(['bins/mutrace/%s' % bm_name,
- '--benchmark_list_tests']).splitlines():
- link(line, '%s.svg' % fnize(line))
- benchmarks.append(
- jobset.JobSpec(['perf', 'record', '-o', '%s-perf.data' % fnize(line),
- '-g', '-F', '997',
- 'bins/mutrace/%s' % bm_name,
- '--benchmark_filter=^%s$' % line,
- '--benchmark_min_time=10'],
- shortname='perf-%s' % fnize(line)))
- profile_analysis.append(
- jobset.JobSpec(['tools/run_tests/performance/process_local_perf_flamegraphs.sh'],
- environ = {
- 'PERF_BASE_NAME': fnize(line),
- 'OUTPUT_DIR': 'reports',
- 'OUTPUT_FILENAME': fnize(line),
- },
- shortname='flame-%s' % fnize(line)))
- cleanup.append(jobset.JobSpec(['rm', '%s-perf.data' % fnize(line)]))
- cleanup.append(jobset.JobSpec(['rm', '%s-out.perf' % fnize(line)]))
- # periodically flush out the list of jobs: temporary space required for this
- # processing is large
- if len(benchmarks) >= 20:
- # run up to half the cpu count: each benchmark can use up to two cores
- # (one for the microbenchmark, one for the data flush)
- jobset.run(benchmarks, maxjobs=1)
- jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
- jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
- benchmarks = []
- profile_analysis = []
- cleanup = []
- # run the remaining benchmarks that weren't flushed
- if len(benchmarks):
- jobset.run(benchmarks, maxjobs=1)
- jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
- jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
+ """generate flamegraphs"""
+ heading('Flamegraphs: %s' % bm_name)
+ subprocess.check_call([
+ 'make', bm_name, 'CONFIG=mutrace', '-j',
+ '%d' % multiprocessing.cpu_count()
+ ])
+ benchmarks = []
+ profile_analysis = []
+ cleanup = []
+ for line in subprocess.check_output(
+ ['bins/mutrace/%s' % bm_name, '--benchmark_list_tests']).splitlines():
+ link(line, '%s.svg' % fnize(line))
+ benchmarks.append(
+ jobset.JobSpec(
+ [
+ 'perf', 'record', '-o', '%s-perf.data' % fnize(
+ line), '-g', '-F', '997', 'bins/mutrace/%s' % bm_name,
+ '--benchmark_filter=^%s$' % line, '--benchmark_min_time=10'
+ ],
+ shortname='perf-%s' % fnize(line)))
+ profile_analysis.append(
+ jobset.JobSpec(
+ [
+ 'tools/run_tests/performance/process_local_perf_flamegraphs.sh'
+ ],
+ environ={
+ 'PERF_BASE_NAME': fnize(line),
+ 'OUTPUT_DIR': 'reports',
+ 'OUTPUT_FILENAME': fnize(line),
+ },
+ shortname='flame-%s' % fnize(line)))
+ cleanup.append(jobset.JobSpec(['rm', '%s-perf.data' % fnize(line)]))
+ cleanup.append(jobset.JobSpec(['rm', '%s-out.perf' % fnize(line)]))
+ # periodically flush out the list of jobs: temporary space required for this
+ # processing is large
+ if len(benchmarks) >= 20:
+ # run up to half the cpu count: each benchmark can use up to two cores
+ # (one for the microbenchmark, one for the data flush)
+ jobset.run(benchmarks, maxjobs=1)
+ jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
+ jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
+ benchmarks = []
+ profile_analysis = []
+ cleanup = []
+ # run the remaining benchmarks that weren't flushed
+ if len(benchmarks):
+ jobset.run(benchmarks, maxjobs=1)
+ jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
+ jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
+
def run_summary(bm_name, cfg, base_json_name):
- subprocess.check_call(
- ['make', bm_name,
- 'CONFIG=%s' % cfg, '-j', '%d' % multiprocessing.cpu_count()])
- cmd = ['bins/%s/%s' % (cfg, bm_name),
- '--benchmark_out=%s.%s.json' % (base_json_name, cfg),
- '--benchmark_out_format=json']
- if args.summary_time is not None:
- cmd += ['--benchmark_min_time=%d' % args.summary_time]
- return subprocess.check_output(cmd)
+ subprocess.check_call([
+ 'make', bm_name, 'CONFIG=%s' % cfg, '-j',
+ '%d' % multiprocessing.cpu_count()
+ ])
+ cmd = [
+ 'bins/%s/%s' % (cfg, bm_name), '--benchmark_out=%s.%s.json' %
+ (base_json_name, cfg), '--benchmark_out_format=json'
+ ]
+ if args.summary_time is not None:
+ cmd += ['--benchmark_min_time=%d' % args.summary_time]
+ return subprocess.check_output(cmd)
+
def collect_summary(bm_name, args):
- heading('Summary: %s [no counters]' % bm_name)
- text(run_summary(bm_name, 'opt', bm_name))
- heading('Summary: %s [with counters]' % bm_name)
- text(run_summary(bm_name, 'counters', bm_name))
- if args.bigquery_upload:
- with open('%s.csv' % bm_name, 'w') as f:
- f.write(subprocess.check_output(['tools/profiling/microbenchmarks/bm2bq.py',
- '%s.counters.json' % bm_name,
- '%s.opt.json' % bm_name]))
- subprocess.check_call(['bq', 'load', 'microbenchmarks.microbenchmarks', '%s.csv' % bm_name])
+ heading('Summary: %s [no counters]' % bm_name)
+ text(run_summary(bm_name, 'opt', bm_name))
+ heading('Summary: %s [with counters]' % bm_name)
+ text(run_summary(bm_name, 'counters', bm_name))
+ if args.bigquery_upload:
+ with open('%s.csv' % bm_name, 'w') as f:
+ f.write(
+ subprocess.check_output([
+ 'tools/profiling/microbenchmarks/bm2bq.py',
+ '%s.counters.json' % bm_name, '%s.opt.json' % bm_name
+ ]))
+ subprocess.check_call([
+ 'bq', 'load', 'microbenchmarks.microbenchmarks', '%s.csv' % bm_name
+ ])
+
collectors = {
- 'latency': collect_latency,
- 'perf': collect_perf,
- 'summary': collect_summary,
+ 'latency': collect_latency,
+ 'perf': collect_perf,
+ 'summary': collect_summary,
}
argp = argparse.ArgumentParser(description='Collect data from microbenchmarks')
-argp.add_argument('-c', '--collect',
- choices=sorted(collectors.keys()),
- nargs='*',
- default=sorted(collectors.keys()),
- help='Which collectors should be run against each benchmark')
-argp.add_argument('-b', '--benchmarks',
- choices=bm_constants._AVAILABLE_BENCHMARK_TESTS,
- default=bm_constants._AVAILABLE_BENCHMARK_TESTS,
- nargs='+',
- type=str,
- help='Which microbenchmarks should be run')
-argp.add_argument('--bigquery_upload',
- default=False,
- action='store_const',
- const=True,
- help='Upload results from summary collection to bigquery')
-argp.add_argument('--summary_time',
- default=None,
- type=int,
- help='Minimum time to run benchmarks for the summary collection')
+argp.add_argument(
+ '-c',
+ '--collect',
+ choices=sorted(collectors.keys()),
+ nargs='*',
+ default=sorted(collectors.keys()),
+ help='Which collectors should be run against each benchmark')
+argp.add_argument(
+ '-b',
+ '--benchmarks',
+ choices=bm_constants._AVAILABLE_BENCHMARK_TESTS,
+ default=bm_constants._AVAILABLE_BENCHMARK_TESTS,
+ nargs='+',
+ type=str,
+ help='Which microbenchmarks should be run')
+argp.add_argument(
+ '--bigquery_upload',
+ default=False,
+ action='store_const',
+ const=True,
+ help='Upload results from summary collection to bigquery')
+argp.add_argument(
+ '--summary_time',
+ default=None,
+ type=int,
+ help='Minimum time to run benchmarks for the summary collection')
args = argp.parse_args()
try:
- for collect in args.collect:
- for bm_name in args.benchmarks:
- collectors[collect](bm_name, args)
+ for collect in args.collect:
+ for bm_name in args.benchmarks:
+ collectors[collect](bm_name, args)
finally:
- if not os.path.exists('reports'):
- os.makedirs('reports')
- index_html += "</body>\n</html>\n"
- with open('reports/index.html', 'w') as f:
- f.write(index_html)
+ if not os.path.exists('reports'):
+ os.makedirs('reports')
+ index_html += "</body>\n</html>\n"
+ with open('reports/index.html', 'w') as f:
+ f.write(index_html)
generated by cgit v1.2.3 (git 2.39.1) at 2024-06-28 17:24:54 +0000