#!/usr/bin/env python # Copyright (c) 2012 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be found # in the LICENSE file. """ Analyze recent SkPicture or Microbench data, and output suggested ranges. The outputs can be edited and pasted to bench_expectations.txt to trigger buildbot alerts if the actual benches are out of range. Details are documented in the .txt file. Currently the easiest way to batch update bench_expectations.txt is to delete all bench lines, run this script, and redirect outputs (">>") to be added to the .txt file. You can also just manually change a few lines of interest, of course. Note: since input data are stored in Google Storage, you will need to set up the corresponding library. See http://developers.google.com/storage/docs/gspythonlibrary for details. """ __author__ = 'bensong@google.com (Ben Chen)' import bench_util import boto import cStringIO import optparse import re import shutil from oauth2_plugin import oauth2_plugin # Ratios for calculating suggested picture bench upper and lower bounds. BENCH_UB = 1.1 # Allow for 10% room for normal variance on the up side. BENCH_LB = 0.9 # Further allow for a fixed amount of noise. This is especially useful for # benches of smaller absolute value. Keeping this value small will not affect # performance tunings. BENCH_ALLOWED_NOISE = 10 # Name prefix for benchmark builders. BENCH_BUILDER_PREFIX = 'Perf-' # List of platforms to track. Feel free to change it to meet your needs. PLATFORMS = ['Perf-Mac10.8-MacMini4.1-GeForce320M-x86-Release', 'Perf-Android-Nexus7-Tegra3-Arm7-Release', 'Perf-Ubuntu12-ShuttleA-ATI5770-x86-Release', 'Perf-Win7-ShuttleA-HD2000-x86-Release', ] # Filter for configs of no interest. They are old config names replaced by more # specific ones. CONFIGS_TO_FILTER = ['gpu', 'raster'] # Template for gsutil uri. GOOGLE_STORAGE_URI_SCHEME = 'gs' URI_BUCKET = 'chromium-skia-gm' # Constants for optparse. USAGE_STRING = 'USAGE: %s [options]' HOWTO_STRING = """ Feel free to revise PLATFORMS for your own needs. The default is the most common combination that we care most about. Platforms that did not run bench_pictures or benchmain in the given revision range will not have corresponding outputs. Please check http://go/skpbench to choose a range that fits your needs. BENCH_UB, BENCH_LB and BENCH_ALLOWED_NOISE can be changed to expand or narrow the permitted bench ranges without triggering buidbot alerts. """ HELP_STRING = """ Outputs expectation picture bench ranges for the latest revisions for the given revision range. For instance, --rev_range=6000:6000 will return only bench ranges for the bots that ran benches at rev 6000; --rev-range=6000:7000 may have multiple bench data points for each bench configuration, and the code returns bench data for the latest revision of all available (closer to 7000). """ + HOWTO_STRING OPTION_REVISION_RANGE = '--rev-range' OPTION_REVISION_RANGE_SHORT = '-r' # Bench representation algorithm flag. OPTION_REPRESENTATION_ALG = '--algorithm' OPTION_REPRESENTATION_ALG_SHORT = '-a' # Bench type to examine. Either 'micro' or 'skp'. OPTION_BENCH_TYPE = '--bench-type' OPTION_BENCH_TYPE_SHORT = '-b' # List of valid bench types. BENCH_TYPES = ['micro', 'skp'] # List of valid representation algorithms. REPRESENTATION_ALGS = ['avg', 'min', 'med', '25th'] def OutputBenchExpectations(bench_type, rev_min, rev_max, representation_alg): """Reads bench data from google storage, and outputs expectations. Ignores data with revisions outside [rev_min, rev_max] integer range. For bench data with multiple revisions, we use higher revisions to calculate expected bench values. bench_type is either 'micro' or 'skp', according to the flag '-b'. Uses the provided representation_alg for calculating bench representations. """ if bench_type not in BENCH_TYPES: raise Exception('Not valid bench_type! (%s)' % BENCH_TYPES) expectation_dic = {} uri = boto.storage_uri(URI_BUCKET, GOOGLE_STORAGE_URI_SCHEME) for obj in uri.get_bucket(): # Filters out non-bench files. if ((not obj.name.startswith('perfdata/%s' % BENCH_BUILDER_PREFIX) and not obj.name.startswith( 'playback/perfdata/%s' % BENCH_BUILDER_PREFIX)) or obj.name.find('_data') < 0): continue if ((bench_type == 'micro' and obj.name.find('_data_skp_') > 0) or (bench_type == 'skp' and obj.name.find('_skp_') < 0)): # Skips wrong bench type. continue # Ignores uninterested platforms. platform = obj.name.split('/')[1] if not platform.startswith(BENCH_BUILDER_PREFIX): platform = obj.name.split('/')[2] if not platform.startswith(BENCH_BUILDER_PREFIX): continue # Ignores non-platform object if platform not in PLATFORMS: continue # Filters by revision. to_filter = True for rev in range(rev_min, rev_max + 1): if '_r%s_' % rev in obj.name: to_filter = False break if to_filter: continue contents = cStringIO.StringIO() obj.get_file(contents) for point in bench_util.parse('', contents.getvalue().split('\n'), representation_alg): if point.config in CONFIGS_TO_FILTER: continue key = '%s_%s_%s,%s-%s' % (point.bench, point.config, point.time_type, platform, representation_alg) # It is fine to have later revisions overwrite earlier benches, since we # only use the latest bench within revision range to set expectations. expectation_dic[key] = point.time keys = expectation_dic.keys() keys.sort() for key in keys: bench_val = expectation_dic[key] # Prints out expectation lines. print '%s,%.3f,%.3f,%.3f' % (key, bench_val, bench_val * BENCH_LB - BENCH_ALLOWED_NOISE, bench_val * BENCH_UB + BENCH_ALLOWED_NOISE) def main(): """Parses flags and outputs expected Skia bench results.""" parser = optparse.OptionParser(USAGE_STRING % '%prog' + HELP_STRING) parser.add_option(OPTION_REVISION_RANGE_SHORT, OPTION_REVISION_RANGE, dest='rev_range', help='(Mandatory) revision range separated by ":", e.g., 6000:6005') parser.add_option(OPTION_BENCH_TYPE_SHORT, OPTION_BENCH_TYPE, dest='bench_type', default='skp', help=('Bench type, either "skp" or "micro". Default to "skp".')) parser.add_option(OPTION_REPRESENTATION_ALG_SHORT, OPTION_REPRESENTATION_ALG, dest='alg', default='25th', help=('Bench representation algorithm. One of ' '%s. Default to "25th".' % str(REPRESENTATION_ALGS))) (options, args) = parser.parse_args() if options.rev_range: range_match = re.search('(\d+)\:(\d+)', options.rev_range) if not range_match: parser.error('Wrong format for rev-range [%s]' % options.rev_range) else: rev_min = int(range_match.group(1)) rev_max = int(range_match.group(2)) OutputBenchExpectations(options.bench_type, rev_min, rev_max, options.alg) else: parser.error('Please provide mandatory flag %s' % OPTION_REVISION_RANGE) if '__main__' == __name__: main()