From 9c2249f51c861f3d0db089fbba38b8ac1d63d160 Mon Sep 17 00:00:00 2001 From: Kevin Lubick Date: Thu, 10 Nov 2016 14:19:00 -0500 Subject: skpbench: add utility to format results for Skia Perf Adds skiaperf.py for formatting skpbench.py outputs for Skia Perf. Also renames parseskpbench.py to sheet.py. BUG=skia: GOLD_TRYBOT_URL= https://gold.skia.org/search?issue=4657 Change-Id: I758678e1c589b15ec2d07c43e4921663e919b47b Reviewed-on: https://skia-review.googlesource.com/4657 Commit-Queue: Kevin Lubick Reviewed-by: Kevin Lubick --- tools/skpbench/parseskpbench.py | 166 ---------------------------------------- tools/skpbench/sheet.py | 166 ++++++++++++++++++++++++++++++++++++++++ tools/skpbench/skiaperf.py | 71 +++++++++++++++++ tools/skpbench/skpbench.py | 36 ++++----- 4 files changed, 255 insertions(+), 184 deletions(-) delete mode 100755 tools/skpbench/parseskpbench.py create mode 100755 tools/skpbench/sheet.py create mode 100755 tools/skpbench/skiaperf.py (limited to 'tools/skpbench') diff --git a/tools/skpbench/parseskpbench.py b/tools/skpbench/parseskpbench.py deleted file mode 100755 index 800c1ca124..0000000000 --- a/tools/skpbench/parseskpbench.py +++ /dev/null @@ -1,166 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2016 Google Inc. -# -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -from __future__ import print_function -from _benchresult import BenchResult -from argparse import ArgumentParser -from collections import defaultdict, namedtuple -from datetime import datetime -import operator -import os -import sys -import tempfile -import urllib -import urlparse -import webbrowser - -__argparse = ArgumentParser(description=""" - -Parses output files from skpbench.py into csv. - -This script can also be used to generate a Google sheet: - -(1) Install the "Office Editing for Docs, Sheets & Slides" Chrome extension: - https://chrome.google.com/webstore/detail/office-editing-for-docs-s/gbkeegbaiigmenfmjfclcdgdpimamgkj - -(2) Update your global OS file associations to use Chrome for .csv files. - -(3) Run parseskpbench.py with the --open flag. - -""") - -__argparse.add_argument('-r', '--result', - choices=['accum', 'median', 'max', 'min'], default='accum', - help="result to use for cell values") -__argparse.add_argument('-f', '--force', - action='store_true', help='silently ignore warnings') -__argparse.add_argument('-o', '--open', - action='store_true', - help="generate a temp file and open it (theoretically in a web browser)") -__argparse.add_argument('-n', '--name', - default='skpbench_%s' % datetime.now().strftime('%Y-%m-%d_%H.%M.%S.csv'), - help="if using --open, a name for the temp file") -__argparse.add_argument('sources', - nargs='+', help="source files with skpbench results ('-' for stdin)") - -FLAGS = __argparse.parse_args() - -RESULT_QUALIFIERS = ('sample_ms', 'clock', 'metric') - -class FullConfig(namedtuple('fullconfig', ('config',) + RESULT_QUALIFIERS)): - def qualified_name(self, qualifiers=RESULT_QUALIFIERS): - return get_qualified_name(self.config.replace(',', ' '), - {x:getattr(self, x) for x in qualifiers}) - -def get_qualified_name(name, qualifiers): - if not qualifiers: - return name - else: - args = ('%s=%s' % (k,v) for k,v in qualifiers.iteritems()) - return '%s (%s)' % (name, ' '.join(args)) - -class Parser: - def __init__(self): - self.sheet_qualifiers = {x:None for x in RESULT_QUALIFIERS} - self.config_qualifiers = set() - self.fullconfigs = list() # use list to preserve the order. - self.rows = defaultdict(dict) - self.cols = defaultdict(dict) - - def parse_file(self, infile): - for line in infile: - match = BenchResult.match(line) - if not match: - continue - - fullconfig = FullConfig(*(match.get_string(x) - for x in FullConfig._fields)) - if not fullconfig in self.fullconfigs: - self.fullconfigs.append(fullconfig) - - for qualifier, value in self.sheet_qualifiers.items(): - if value is None: - self.sheet_qualifiers[qualifier] = match.get_string(qualifier) - elif value != match.get_string(qualifier): - del self.sheet_qualifiers[qualifier] - self.config_qualifiers.add(qualifier) - - self.rows[match.bench][fullconfig] = match.get_string(FLAGS.result) - self.cols[fullconfig][match.bench] = getattr(match, FLAGS.result) - - def print_csv(self, outfile=sys.stdout): - # Write the title. - print(get_qualified_name(FLAGS.result, self.sheet_qualifiers), file=outfile) - - # Write the header. - outfile.write('bench,') - for fullconfig in self.fullconfigs: - outfile.write('%s,' % fullconfig.qualified_name(self.config_qualifiers)) - outfile.write('\n') - - # Write the rows. - for bench, row in self.rows.iteritems(): - outfile.write('%s,' % bench) - for fullconfig in self.fullconfigs: - if fullconfig in row: - outfile.write('%s,' % row[fullconfig]) - elif FLAGS.force: - outfile.write('NULL,') - else: - raise ValueError("%s: missing value for %s. (use --force to ignore)" % - (bench, - fullconfig.qualified_name(self.config_qualifiers))) - outfile.write('\n') - - # Add simple, literal averages. - if len(self.rows) > 1: - outfile.write('\n') - self._print_computed_row('MEAN', - lambda col: reduce(operator.add, col.values()) / len(col), - outfile=outfile) - self._print_computed_row('GEOMEAN', - lambda col: reduce(operator.mul, col.values()) ** (1.0 / len(col)), - outfile=outfile) - - def _print_computed_row(self, name, func, outfile=sys.stdout): - outfile.write('%s,' % name) - for fullconfig in self.fullconfigs: - if len(self.cols[fullconfig]) != len(self.rows): - outfile.write('NULL,') - continue - outfile.write('%.4g,' % func(self.cols[fullconfig])) - outfile.write('\n') - -def main(): - parser = Parser() - - # Parse the input files. - for src in FLAGS.sources: - if src == '-': - parser.parse_file(sys.stdin) - else: - with open(src, mode='r') as infile: - parser.parse_file(infile) - - # Print the csv. - if not FLAGS.open: - parser.print_csv() - else: - dirname = tempfile.mkdtemp() - basename = FLAGS.name - if os.path.splitext(basename)[1] != '.csv': - basename += '.csv'; - pathname = os.path.join(dirname, basename) - with open(pathname, mode='w') as tmpfile: - parser.print_csv(outfile=tmpfile) - fileuri = urlparse.urljoin('file:', urllib.pathname2url(pathname)) - print('opening %s' % fileuri) - webbrowser.open(fileuri) - - -if __name__ == '__main__': - main() diff --git a/tools/skpbench/sheet.py b/tools/skpbench/sheet.py new file mode 100755 index 0000000000..d7cf1e2fca --- /dev/null +++ b/tools/skpbench/sheet.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. +# +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from __future__ import print_function +from _benchresult import BenchResult +from argparse import ArgumentParser +from collections import defaultdict, namedtuple +from datetime import datetime +import operator +import os +import sys +import tempfile +import urllib +import urlparse +import webbrowser + +__argparse = ArgumentParser(description=""" + +Formats skpbench.py outputs as csv. + +This script can also be used to generate a Google sheet: + +(1) Install the "Office Editing for Docs, Sheets & Slides" Chrome extension: + https://chrome.google.com/webstore/detail/office-editing-for-docs-s/gbkeegbaiigmenfmjfclcdgdpimamgkj + +(2) Update your global OS file associations to use Chrome for .csv files. + +(3) Run parseskpbench.py with the --open flag. + +""") + +__argparse.add_argument('-r', '--result', + choices=['accum', 'median', 'max', 'min'], default='accum', + help="result to use for cell values") +__argparse.add_argument('-f', '--force', + action='store_true', help='silently ignore warnings') +__argparse.add_argument('-o', '--open', + action='store_true', + help="generate a temp file and open it (theoretically in a web browser)") +__argparse.add_argument('-n', '--name', + default='skpbench_%s' % datetime.now().strftime('%Y-%m-%d_%H.%M.%S.csv'), + help="if using --open, a name for the temp file") +__argparse.add_argument('sources', + nargs='+', help="source files that contain skpbench results ('-' for stdin)") + +FLAGS = __argparse.parse_args() + +RESULT_QUALIFIERS = ('sample_ms', 'clock', 'metric') + +class FullConfig(namedtuple('fullconfig', ('config',) + RESULT_QUALIFIERS)): + def qualified_name(self, qualifiers=RESULT_QUALIFIERS): + return get_qualified_name(self.config.replace(',', ' '), + {x:getattr(self, x) for x in qualifiers}) + +def get_qualified_name(name, qualifiers): + if not qualifiers: + return name + else: + args = ('%s=%s' % (k,v) for k,v in qualifiers.iteritems()) + return '%s (%s)' % (name, ' '.join(args)) + +class Parser: + def __init__(self): + self.sheet_qualifiers = {x:None for x in RESULT_QUALIFIERS} + self.config_qualifiers = set() + self.fullconfigs = list() # use list to preserve the order. + self.rows = defaultdict(dict) + self.cols = defaultdict(dict) + + def parse_file(self, infile): + for line in infile: + match = BenchResult.match(line) + if not match: + continue + + fullconfig = FullConfig(*(match.get_string(x) + for x in FullConfig._fields)) + if not fullconfig in self.fullconfigs: + self.fullconfigs.append(fullconfig) + + for qualifier, value in self.sheet_qualifiers.items(): + if value is None: + self.sheet_qualifiers[qualifier] = match.get_string(qualifier) + elif value != match.get_string(qualifier): + del self.sheet_qualifiers[qualifier] + self.config_qualifiers.add(qualifier) + + self.rows[match.bench][fullconfig] = match.get_string(FLAGS.result) + self.cols[fullconfig][match.bench] = getattr(match, FLAGS.result) + + def print_csv(self, outfile=sys.stdout): + # Write the title. + print(get_qualified_name(FLAGS.result, self.sheet_qualifiers), file=outfile) + + # Write the header. + outfile.write('bench,') + for fullconfig in self.fullconfigs: + outfile.write('%s,' % fullconfig.qualified_name(self.config_qualifiers)) + outfile.write('\n') + + # Write the rows. + for bench, row in self.rows.iteritems(): + outfile.write('%s,' % bench) + for fullconfig in self.fullconfigs: + if fullconfig in row: + outfile.write('%s,' % row[fullconfig]) + elif FLAGS.force: + outfile.write('NULL,') + else: + raise ValueError("%s: missing value for %s. (use --force to ignore)" % + (bench, + fullconfig.qualified_name(self.config_qualifiers))) + outfile.write('\n') + + # Add simple, literal averages. + if len(self.rows) > 1: + outfile.write('\n') + self._print_computed_row('MEAN', + lambda col: reduce(operator.add, col.values()) / len(col), + outfile=outfile) + self._print_computed_row('GEOMEAN', + lambda col: reduce(operator.mul, col.values()) ** (1.0 / len(col)), + outfile=outfile) + + def _print_computed_row(self, name, func, outfile=sys.stdout): + outfile.write('%s,' % name) + for fullconfig in self.fullconfigs: + if len(self.cols[fullconfig]) != len(self.rows): + outfile.write('NULL,') + continue + outfile.write('%.4g,' % func(self.cols[fullconfig])) + outfile.write('\n') + +def main(): + parser = Parser() + + # Parse the input files. + for src in FLAGS.sources: + if src == '-': + parser.parse_file(sys.stdin) + else: + with open(src, mode='r') as infile: + parser.parse_file(infile) + + # Print the csv. + if not FLAGS.open: + parser.print_csv() + else: + dirname = tempfile.mkdtemp() + basename = FLAGS.name + if os.path.splitext(basename)[1] != '.csv': + basename += '.csv'; + pathname = os.path.join(dirname, basename) + with open(pathname, mode='w') as tmpfile: + parser.print_csv(outfile=tmpfile) + fileuri = urlparse.urljoin('file:', urllib.pathname2url(pathname)) + print('opening %s' % fileuri) + webbrowser.open(fileuri) + + +if __name__ == '__main__': + main() diff --git a/tools/skpbench/skiaperf.py b/tools/skpbench/skiaperf.py new file mode 100755 index 0000000000..cfbd49d6c1 --- /dev/null +++ b/tools/skpbench/skiaperf.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. +# +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +from __future__ import print_function +from _benchresult import BenchResult +from argparse import ArgumentTypeError, ArgumentParser +from collections import defaultdict +import json +import sys + +__argparse = ArgumentParser(description=""" + +Formats skpbench.py outputs for Skia Perf. + +""") + +__argparse.add_argument('sources', + nargs='+', help="source files that contain skpbench results") +__argparse.add_argument('--properties', + nargs='*', default=list(), + help="space-separated key/value pairs identifying the run") +__argparse.add_argument('--key', + nargs='*', default=list(), + help="space-separated key/value pairs identifying the builder") +__argparse.add_argument('-o', '--outfile', + default='-', help="output file ('-' for stdout)") + +FLAGS = __argparse.parse_args() + +def parse_key_value_pairs(args): + if not args: + return dict() + if len(args) % 2: + raise ArgumentTypeError("uneven number of key/value arguments.") + return {k:v for k,v in zip(args[::2], args[1::2])} + +def skiaperf_result(benchresult): + result = {x:benchresult.get_string(x) for x in ('accum', 'median')} + result['options'] = {x:benchresult.get_string(x) + for x in ('clock', 'metric', 'sample_ms')} + return result + +def emit_as_json(data, outfile): + json.dump(data, outfile, indent=4, separators=(',', ' : '), sort_keys=True) + print('', file=outfile) + +def main(): + data = parse_key_value_pairs( + FLAGS.properties + [ + 'key', parse_key_value_pairs(FLAGS.key), + 'results', defaultdict(dict)]) + + for src in FLAGS.sources: + with open(src, mode='r') as infile: + for line in infile: + match = BenchResult.match(line) + if match: + data['results'][match.bench][match.config] = skiaperf_result(match) + + if FLAGS.outfile != '-': + with open(FLAGS.outfile, 'w+') as outfile: + emit_as_json(data, outfile) + else: + emit_as_json(data, sys.stdout) + +if __name__ == '__main__': + main() diff --git a/tools/skpbench/skpbench.py b/tools/skpbench/skpbench.py index a21312365f..7cc0a231e9 100755 --- a/tools/skpbench/skpbench.py +++ b/tools/skpbench/skpbench.py @@ -30,37 +30,37 @@ unacceptable stddev. """) __argparse.add_argument('skpbench', - help="path to the skpbench binary") + help="path to the skpbench binary") __argparse.add_argument('--adb', - action='store_true', help="execute skpbench over adb") + action='store_true', help="execute skpbench over adb") __argparse.add_argument('-s', '--device-serial', - help="if using adb, ID of the specific device to target " - "(only required if more than 1 device is attached)") + help="if using adb, ID of the specific device to target " + "(only required if more than 1 device is attached)") __argparse.add_argument('-m', '--max-stddev', - type=float, default=4, - help="initial max allowable relative standard deviation") + type=float, default=4, + help="initial max allowable relative standard deviation") __argparse.add_argument('-x', '--suffix', - help="suffix to append on config (e.g. '_before', '_after')") + help="suffix to append on config (e.g. '_before', '_after')") __argparse.add_argument('-w','--write-path', - help="directory to save .png proofs to disk.") + help="directory to save .png proofs to disk.") __argparse.add_argument('-v','--verbosity', - type=int, default=1, help="level of verbosity (0=none to 5=debug)") + type=int, default=1, help="level of verbosity (0=none to 5=debug)") __argparse.add_argument('-d', '--duration', - type=int, help="number of milliseconds to run each benchmark") + type=int, help="number of milliseconds to run each benchmark") __argparse.add_argument('-l', '--sample-ms', - type=int, help="duration of a sample (minimum)") + type=int, help="duration of a sample (minimum)") __argparse.add_argument('--gpu', - action='store_true', - help="perform timing on the gpu clock instead of cpu (gpu work only)") + action='store_true', + help="perform timing on the gpu clock instead of cpu (gpu work only)") __argparse.add_argument('--fps', - action='store_true', help="use fps instead of ms") + action='store_true', help="use fps instead of ms") __argparse.add_argument('-c', '--config', - default='gpu', help="comma- or space-separated list of GPU configs") + default='gpu', help="comma- or space-separated list of GPU configs") __argparse.add_argument('-a', '--resultsfile', - help="optional file to append results into") + help="optional file to append results into") __argparse.add_argument('skps', - nargs='+', - help=".skp files or directories to expand for .skp files") + nargs='+', + help=".skp files or directories to expand for .skp files") FLAGS = __argparse.parse_args() if FLAGS.adb: -- cgit v1.2.3