''' Created on May 16, 2011 @author: bungeman ''' import bench_util import getopt import httplib import itertools import json import os import re import sys import urllib import urllib2 import xml.sax.saxutils # Maximum expected number of characters we expect in an svn revision. MAX_SVN_REV_LENGTH = 5 def usage(): """Prints simple usage information.""" print '-a bench representation algorithm to use. ' print ' Defaults to "25th". See bench_util.py for details.' print '-b name of the builder whose bench data we are checking.' print '-d a directory containing bench__ files.' print '-e file containing expected bench builder values/ranges.' print ' Will raise exception if actual bench values are out of range.' print ' See bench_expectations_.txt for data format / examples.' print '-r the git commit hash or svn revision for checking ' print ' bench values.' class Label: """The information in a label. (str, str, str, str, {str:str})""" def __init__(self, bench, config, time_type, settings): self.bench = bench self.config = config self.time_type = time_type self.settings = settings def __repr__(self): return "Label(%s, %s, %s, %s)" % ( str(self.bench), str(self.config), str(self.time_type), str(self.settings), ) def __str__(self): return "%s_%s_%s_%s" % ( str(self.bench), str(self.config), str(self.time_type), str(self.settings), ) def __eq__(self, other): return (self.bench == other.bench and self.config == other.config and self.time_type == other.time_type and self.settings == other.settings) def __hash__(self): return (hash(self.bench) ^ hash(self.config) ^ hash(self.time_type) ^ hash(frozenset(self.settings.iteritems()))) def parse_dir(directory, default_settings, revision, rep): """Parses bench data from bench logs files. revision can be either svn revision or git commit hash. """ revision_data_points = [] # list of BenchDataPoint file_list = os.listdir(directory) file_list.sort() for bench_file in file_list: scalar_type = None # Scalar type, if any, is in the bench filename after revision if (len(revision) > MAX_SVN_REV_LENGTH and bench_file.startswith('bench_' + revision + '_')): # The revision is GIT commit hash. scalar_type = bench_file[len(revision) + len('bench_') + 1:] elif (bench_file.startswith('bench_r' + revision + '_') and revision.isdigit()): # The revision is SVN number scalar_type = bench_file[len(revision) + len('bench_r') + 1:] else: continue file_handle = open(directory + '/' + bench_file, 'r') default_settings['scalar'] = scalar_type revision_data_points.extend( bench_util.parse(default_settings, file_handle, rep)) file_handle.close() return revision_data_points def create_bench_dict(revision_data_points): """Convert current revision data into a dictionary of line data. Args: revision_data_points: a list of bench data points Returns: a dictionary of this form: keys = Label objects values = the corresponding bench value """ bench_dict = {} for point in revision_data_points: point_name = Label(point.bench,point.config,point.time_type, point.settings) if point_name not in bench_dict: bench_dict[point_name] = point.time else: raise Exception('Duplicate expectation entry: ' + str(point_name)) return bench_dict def read_expectations(expectations, filename): """Reads expectations data from file and put in expectations dict.""" for expectation in open(filename).readlines(): elements = expectation.strip().split(',') if not elements[0] or elements[0].startswith('#'): continue if len(elements) != 5: raise Exception("Invalid expectation line format: %s" % expectation) bench_entry = elements[0] + ',' + elements[1] if bench_entry in expectations: raise Exception("Dup entries for bench expectation %s" % bench_entry) # [,] -> (LB, UB) expectations[bench_entry] = (float(elements[-2]), float(elements[-1])) def check_expectations(lines, expectations, revision, key_suffix): """Check if there are benches in the given revising out of range. """ # The platform for this bot, to pass to the dashboard plot. platform = key_suffix[ : key_suffix.rfind('-')] exceptions = [] for line in lines: line_str = str(line) line_str = line_str[ : line_str.find('_{')] bench_platform_key = line_str + ',' + key_suffix if bench_platform_key not in expectations: continue this_bench_value = lines[line] this_min, this_max = expectations[bench_platform_key] if this_bench_value < this_min or this_bench_value > this_max: exception = 'Bench %s value %s out of range [%s, %s].' % ( bench_platform_key, this_bench_value, this_min, this_max) exceptions.append(exception) if exceptions: raise Exception('Bench values out of range:\n' + '\n'.join(exceptions)) def main(): """Parses command line and checks bench expectations.""" try: opts, _ = getopt.getopt(sys.argv[1:], "a:b:d:e:r:", "default-setting=") except getopt.GetoptError, err: print str(err) usage() sys.exit(2) directory = None bench_expectations = {} rep = '25th' # bench representation algorithm, default to 25th rev = None # git commit hash or svn revision number bot = None try: for option, value in opts: if option == "-a": rep = value elif option == "-b": bot = value elif option == "-d": directory = value elif option == "-e": read_expectations(bench_expectations, value) elif option == "-r": rev = value else: usage() assert False, "unhandled option" except ValueError: usage() sys.exit(2) if directory is None or bot is None or rev is None: usage() sys.exit(2) platform_and_alg = bot + '-' + rep data_points = parse_dir(directory, {}, # Sets default settings to empty. rev, rep) bench_dict = create_bench_dict(data_points) if bench_expectations: check_expectations(bench_dict, bench_expectations, rev, platform_and_alg) if __name__ == "__main__": main()