#!/usr/bin/python ''' Copyright 2012 Google Inc. Use of this source code is governed by a BSD-style license that can be found in the LICENSE file. ''' ''' Rebaselines the given GM tests, on all bots and all configurations. ''' # System-level imports import argparse import json import os import re import subprocess import sys import urllib2 # Imports from within Skia # # We need to add the 'gm' directory, so that we can import gm_json.py within # that directory. That script allows us to parse the actual-results.json file # written out by the GM tool. # Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end* # so any dirs that are already in the PYTHONPATH will be preferred. # # This assumes that the 'gm' directory has been checked out as a sibling of # the 'tools' directory containing this script, which will be the case if # 'trunk' was checked out as a single unit. GM_DIRECTORY = os.path.realpath( os.path.join(os.path.dirname(os.path.dirname(__file__)), 'gm')) if GM_DIRECTORY not in sys.path: sys.path.append(GM_DIRECTORY) import gm_json # TODO(epoger): In the long run, we want to build this list automatically, # but for now we hard-code it until we can properly address # https://code.google.com/p/skia/issues/detail?id=1544 # ('live query of builder list makes rebaseline.py slow to start up') TEST_BUILDERS = [ 'Test-Android-GalaxyNexus-SGX540-Arm7-Debug', 'Test-Android-GalaxyNexus-SGX540-Arm7-Release', 'Test-Android-IntelRhb-SGX544-x86-Debug', 'Test-Android-IntelRhb-SGX544-x86-Release', 'Test-Android-Nexus10-MaliT604-Arm7-Debug', 'Test-Android-Nexus10-MaliT604-Arm7-Release', 'Test-Android-Nexus4-Adreno320-Arm7-Debug', 'Test-Android-Nexus4-Adreno320-Arm7-Release', 'Test-Android-Nexus7-Tegra3-Arm7-Debug', 'Test-Android-Nexus7-Tegra3-Arm7-Release', 'Test-Android-NexusS-SGX540-Arm7-Debug', 'Test-Android-NexusS-SGX540-Arm7-Release', 'Test-Android-Xoom-Tegra2-Arm7-Debug', 'Test-Android-Xoom-Tegra2-Arm7-Release', 'Test-ChromeOS-Alex-GMA3150-x86-Debug', 'Test-ChromeOS-Alex-GMA3150-x86-Release', 'Test-ChromeOS-Daisy-MaliT604-Arm7-Debug', 'Test-ChromeOS-Daisy-MaliT604-Arm7-Release', 'Test-ChromeOS-Link-HD4000-x86_64-Debug', 'Test-ChromeOS-Link-HD4000-x86_64-Release', 'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug', 'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release', 'Test-Mac10.6-MacMini4.1-GeForce320M-x86_64-Debug', 'Test-Mac10.6-MacMini4.1-GeForce320M-x86_64-Release', 'Test-Mac10.7-MacMini4.1-GeForce320M-x86-Debug', 'Test-Mac10.7-MacMini4.1-GeForce320M-x86-Release', 'Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug', 'Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Release', 'Test-Mac10.8-MacMini4.1-GeForce320M-x86-Debug', 'Test-Mac10.8-MacMini4.1-GeForce320M-x86-Release', 'Test-Mac10.8-MacMini4.1-GeForce320M-x86_64-Debug', 'Test-Mac10.8-MacMini4.1-GeForce320M-x86_64-Release', 'Test-Ubuntu12-ShuttleA-ATI5770-x86-Debug', 'Test-Ubuntu12-ShuttleA-ATI5770-x86-Release', 'Test-Ubuntu12-ShuttleA-ATI5770-x86_64-Debug', 'Test-Ubuntu12-ShuttleA-ATI5770-x86_64-Release', 'Test-Ubuntu12-ShuttleA-HD2000-x86_64-Release-Valgrind', 'Test-Ubuntu12-ShuttleA-NoGPU-x86_64-Debug', 'Test-Ubuntu13-ShuttleA-HD2000-x86_64-Debug-ASAN', 'Test-Win7-ShuttleA-HD2000-x86-Debug', 'Test-Win7-ShuttleA-HD2000-x86-Debug-ANGLE', 'Test-Win7-ShuttleA-HD2000-x86-Debug-DirectWrite', 'Test-Win7-ShuttleA-HD2000-x86-Release', 'Test-Win7-ShuttleA-HD2000-x86-Release-ANGLE', 'Test-Win7-ShuttleA-HD2000-x86-Release-DirectWrite', 'Test-Win7-ShuttleA-HD2000-x86_64-Debug', 'Test-Win7-ShuttleA-HD2000-x86_64-Release', 'Test-Win8-ShuttleA-GTX660-x86-Debug', 'Test-Win8-ShuttleA-GTX660-x86-Release', 'Test-Win8-ShuttleA-GTX660-x86-Release-NVPR', 'Test-Win8-ShuttleA-GTX660-x86_64-Debug', 'Test-Win8-ShuttleA-GTX660-x86_64-Release', 'Test-Win8-ShuttleA-HD7770-x86-Debug', 'Test-Win8-ShuttleA-HD7770-x86-Release', 'Test-Win8-ShuttleA-HD7770-x86_64-Debug', 'Test-Win8-ShuttleA-HD7770-x86_64-Release', ] # TODO: Get this from builder_name_schema in buildbot. TRYBOT_SUFFIX = '-Trybot' class _InternalException(Exception): pass class ExceptionHandler(object): """ Object that handles exceptions, either raising them immediately or collecting them to display later on.""" # params: def __init__(self, keep_going_on_failure=False): """ params: keep_going_on_failure: if False, report failures and quit right away; if True, collect failures until ReportAllFailures() is called """ self._keep_going_on_failure = keep_going_on_failure self._failures_encountered = [] def RaiseExceptionOrContinue(self): """ We have encountered an exception; either collect the info and keep going, or exit the program right away.""" # Get traceback information about the most recently raised exception. exc_info = sys.exc_info() if self._keep_going_on_failure: print >> sys.stderr, ('WARNING: swallowing exception %s' % repr(exc_info[1])) self._failures_encountered.append(exc_info) else: print >> sys.stderr, ( '\nHalting at first exception.\n' + 'Please file a bug to epoger@google.com at ' + 'https://code.google.com/p/skia/issues/entry, containing the ' + 'command you ran and the following stack trace.\n\n' + 'Afterwards, you can re-run with the --keep-going-on-failure ' + 'option set.\n') raise exc_info[1], None, exc_info[2] def ReportAllFailures(self): if self._failures_encountered: print >> sys.stderr, ('Encountered %d failures (see above).' % len(self._failures_encountered)) sys.exit(1) # Object that rebaselines a JSON expectations file (not individual image files). class JsonRebaseliner(object): # params: # expectations_root: root directory of all expectations JSON files # expectations_input_filename: filename (under expectations_root) of JSON # expectations file to read; typically # "expected-results.json" # expectations_output_filename: filename (under expectations_root) to # which updated expectations should be # written; typically the same as # expectations_input_filename, to overwrite # the old content # actuals_base_url: base URL from which to read actual-result JSON files # actuals_filename: filename (under actuals_base_url) from which to read a # summary of results; typically "actual-results.json" # exception_handler: reference to rebaseline.ExceptionHandler object # tests: list of tests to rebaseline, or None if we should rebaseline # whatever files the JSON results summary file tells us to # configs: which configs to run for each test, or None if we should # rebaseline whatever configs the JSON results summary file tells # us to # add_new: if True, add expectations for tests which don't have any yet # add_ignored: if True, add expectations for tests for which failures are # currently ignored # bugs: optional list of bug numbers which pertain to these expectations # notes: free-form text notes to add to all updated expectations # mark_unreviewed: if True, mark these expectations as NOT having been # reviewed by a human; otherwise, leave that field blank. # Currently, there is no way to make this script mark # expectations as reviewed-by-human=True. # TODO(epoger): Add that capability to a review tool. # mark_ignore_failure: if True, mark failures of a given test as being # ignored. # from_trybot: if True, read actual-result JSON files generated from a # trybot run rather than a waterfall run. def __init__(self, expectations_root, expectations_input_filename, expectations_output_filename, actuals_base_url, actuals_filename, exception_handler, tests=None, configs=None, add_new=False, add_ignored=False, bugs=None, notes=None, mark_unreviewed=None, mark_ignore_failure=False, from_trybot=False): self._expectations_root = expectations_root self._expectations_input_filename = expectations_input_filename self._expectations_output_filename = expectations_output_filename self._tests = tests self._configs = configs self._actuals_base_url = actuals_base_url self._actuals_filename = actuals_filename self._exception_handler = exception_handler self._add_new = add_new self._add_ignored = add_ignored self._bugs = bugs self._notes = notes self._mark_unreviewed = mark_unreviewed self._mark_ignore_failure = mark_ignore_failure; if self._tests or self._configs: self._image_filename_re = re.compile(gm_json.IMAGE_FILENAME_PATTERN) else: self._image_filename_re = None self._using_svn = os.path.isdir(os.path.join(expectations_root, '.svn')) self._from_trybot = from_trybot # Executes subprocess.call(cmd). # Raises an Exception if the command fails. def _Call(self, cmd): if subprocess.call(cmd) != 0: raise _InternalException('error running command: ' + ' '.join(cmd)) # Returns the full contents of filepath, as a single string. # If filepath looks like a URL, try to read it that way instead of as # a path on local storage. # # Raises _InternalException if there is a problem. def _GetFileContents(self, filepath): if filepath.startswith('http:') or filepath.startswith('https:'): try: return urllib2.urlopen(filepath).read() except urllib2.HTTPError as e: raise _InternalException('unable to read URL %s: %s' % ( filepath, e)) else: return open(filepath, 'r').read() # Returns a dictionary of actual results from actual-results.json file. # # The dictionary returned has this format: # { # u'imageblur_565.png': [u'bitmap-64bitMD5', 3359963596899141322], # u'imageblur_8888.png': [u'bitmap-64bitMD5', 4217923806027861152], # u'shadertext3_8888.png': [u'bitmap-64bitMD5', 3713708307125704716] # } # # If the JSON actual result summary file cannot be loaded, logs a warning # message and returns None. # If the JSON actual result summary file can be loaded, but we have # trouble parsing it, raises an Exception. # # params: # json_url: URL pointing to a JSON actual result summary file # sections: a list of section names to include in the results, e.g. # [gm_json.JSONKEY_ACTUALRESULTS_FAILED, # gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON] ; # if None, then include ALL sections. def _GetActualResults(self, json_url, sections=None): try: json_contents = self._GetFileContents(json_url) except _InternalException: print >> sys.stderr, ( 'could not read json_url %s ; skipping this platform.' % json_url) return None json_dict = gm_json.LoadFromString(json_contents) results_to_return = {} actual_results = json_dict[gm_json.JSONKEY_ACTUALRESULTS] if not sections: sections = actual_results.keys() for section in sections: section_results = actual_results[section] if section_results: results_to_return.update(section_results) return results_to_return # Rebaseline all tests/types we specified in the constructor, # within this builder's subdirectory in expectations/gm . # # params: # builder : e.g. 'Test-Win7-ShuttleA-HD2000-x86-Release' def RebaselineSubdir(self, builder): # Read in the actual result summary, and extract all the tests whose # results we need to update. results_builder = str(builder) if self._from_trybot: results_builder = results_builder + TRYBOT_SUFFIX actuals_url = '/'.join([self._actuals_base_url, results_builder, self._actuals_filename]) # Only update results for tests that are currently failing. # We don't want to rewrite results for tests that are already succeeding, # because we don't want to add annotation fields (such as # JSONKEY_EXPECTEDRESULTS_BUGS) except for tests whose expectations we # are actually modifying. sections = [gm_json.JSONKEY_ACTUALRESULTS_FAILED] if self._add_new: sections.append(gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON) if self._add_ignored: sections.append(gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED) results_to_update = self._GetActualResults(json_url=actuals_url, sections=sections) # Read in current expectations. expectations_input_filepath = os.path.join( self._expectations_root, builder, self._expectations_input_filename) expectations_dict = gm_json.LoadFromFile(expectations_input_filepath) expected_results = expectations_dict.get(gm_json.JSONKEY_EXPECTEDRESULTS) if not expected_results: expected_results = {} expectations_dict[gm_json.JSONKEY_EXPECTEDRESULTS] = expected_results # Update the expectations in memory, skipping any tests/configs that # the caller asked to exclude. skipped_images = [] if results_to_update: for (image_name, image_results) in results_to_update.iteritems(): if self._image_filename_re: (test, config) = self._image_filename_re.match(image_name).groups() if self._tests: if test not in self._tests: skipped_images.append(image_name) continue if self._configs: if config not in self._configs: skipped_images.append(image_name) continue if not expected_results.get(image_name): expected_results[image_name] = {} expected_results[image_name]\ [gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS]\ = [image_results] if self._mark_unreviewed: expected_results[image_name]\ [gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED]\ = False if self._mark_ignore_failure: expected_results[image_name]\ [gm_json.JSONKEY_EXPECTEDRESULTS_IGNOREFAILURE]\ = True if self._bugs: expected_results[image_name]\ [gm_json.JSONKEY_EXPECTEDRESULTS_BUGS]\ = self._bugs if self._notes: expected_results[image_name]\ [gm_json.JSONKEY_EXPECTEDRESULTS_NOTES]\ = self._notes # Write out updated expectations. expectations_output_filepath = os.path.join( self._expectations_root, builder, self._expectations_output_filename) gm_json.WriteToFile(expectations_dict, expectations_output_filepath) # Mark the JSON file as plaintext, so text-style diffs can be applied. # Fixes https://code.google.com/p/skia/issues/detail?id=1442 if self._using_svn: self._Call(['svn', 'propset', '--quiet', 'svn:mime-type', 'text/x-json', expectations_output_filepath]) # main... parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, epilog='Here is the full set of builders we know about:' + '\n '.join([''] + sorted(TEST_BUILDERS))) parser.add_argument('--actuals-base-url', help=('base URL from which to read files containing JSON ' 'summaries of actual GM results; defaults to ' '%(default)s. To get a specific revision (useful for ' 'trybots) replace "svn" with "svn-history/r123". ' 'If SKIMAGE is True, defaults to ' + gm_json.SKIMAGE_ACTUALS_BASE_URL), default='http://skia-autogen.googlecode.com/svn/gm-actual') parser.add_argument('--actuals-filename', help=('filename (within builder-specific subdirectories ' 'of ACTUALS_BASE_URL) to read a summary of results ' 'from; defaults to %(default)s'), default='actual-results.json') parser.add_argument('--add-new', action='store_true', help=('in addition to the standard behavior of ' 'updating expectations for failing tests, add ' 'expectations for tests which don\'t have ' 'expectations yet.')) parser.add_argument('--add-ignored', action='store_true', help=('in addition to the standard behavior of ' 'updating expectations for failing tests, add ' 'expectations for tests for which failures are ' 'currently ignored.')) parser.add_argument('--bugs', metavar='BUG', type=int, nargs='+', help=('Skia bug numbers (under ' 'https://code.google.com/p/skia/issues/list ) which ' 'pertain to this set of rebaselines.')) parser.add_argument('--builders', metavar='BUILDER', nargs='+', help=('which platforms to rebaseline; ' 'if unspecified, rebaseline all known platforms ' '(see below for a list)')) # TODO(epoger): Add test that exercises --configs argument. parser.add_argument('--configs', metavar='CONFIG', nargs='+', help=('which configurations to rebaseline, e.g. ' '"--configs 565 8888", as a filter over the full set ' 'of results in ACTUALS_FILENAME; if unspecified, ' 'rebaseline *all* configs that are available.')) parser.add_argument('--expectations-filename', help=('filename (under EXPECTATIONS_ROOT) to read ' 'current expectations from, and to write new ' 'expectations into (unless a separate ' 'EXPECTATIONS_FILENAME_OUTPUT has been specified); ' 'defaults to %(default)s'), default='expected-results.json') parser.add_argument('--expectations-filename-output', help=('filename (under EXPECTATIONS_ROOT) to write ' 'updated expectations into; by default, overwrites ' 'the input file (EXPECTATIONS_FILENAME)'), default='') parser.add_argument('--expectations-root', help=('root of expectations directory to update-- should ' 'contain one or more builder subdirectories. ' 'Defaults to %(default)s. If SKIMAGE is set, ' ' defaults to ' + gm_json.SKIMAGE_EXPECTATIONS_ROOT), default=os.path.join('expectations', 'gm')) parser.add_argument('--keep-going-on-failure', action='store_true', help=('instead of halting at the first error encountered, ' 'keep going and rebaseline as many tests as ' 'possible, and then report the full set of errors ' 'at the end')) parser.add_argument('--notes', help=('free-form text notes to add to all updated ' 'expectations')) # TODO(epoger): Add test that exercises --tests argument. parser.add_argument('--tests', metavar='TEST', nargs='+', help=('which tests to rebaseline, e.g. ' '"--tests aaclip bigmatrix", as a filter over the ' 'full set of results in ACTUALS_FILENAME; if ' 'unspecified, rebaseline *all* tests that are ' 'available.')) parser.add_argument('--unreviewed', action='store_true', help=('mark all expectations modified by this run as ' '"%s": False' % gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED)) parser.add_argument('--ignore-failure', action='store_true', help=('mark all expectations modified by this run as ' '"%s": True' % gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED)) parser.add_argument('--from-trybot', action='store_true', help=('pull the actual-results.json file from the ' 'corresponding trybot, rather than the main builder')) parser.add_argument('--skimage', action='store_true', help=('Rebaseline skimage results instead of gm. Defaults ' 'to False. If True, TESTS and CONFIGS are ignored, ' 'and ACTUALS_BASE_URL and EXPECTATIONS_ROOT are set ' 'to alternate defaults, specific to skimage.')) args = parser.parse_args() exception_handler = ExceptionHandler( keep_going_on_failure=args.keep_going_on_failure) if args.builders: builders = args.builders missing_json_is_fatal = True else: builders = sorted(TEST_BUILDERS) missing_json_is_fatal = False if args.skimage: # Use a different default if --skimage is specified. if args.actuals_base_url == parser.get_default('actuals_base_url'): args.actuals_base_url = gm_json.SKIMAGE_ACTUALS_BASE_URL if args.expectations_root == parser.get_default('expectations_root'): args.expectations_root = gm_json.SKIMAGE_EXPECTATIONS_ROOT for builder in builders: if not builder in TEST_BUILDERS: raise Exception(('unrecognized builder "%s"; ' + 'should be one of %s') % ( builder, TEST_BUILDERS)) expectations_json_file = os.path.join(args.expectations_root, builder, args.expectations_filename) if os.path.isfile(expectations_json_file): rebaseliner = JsonRebaseliner( expectations_root=args.expectations_root, expectations_input_filename=args.expectations_filename, expectations_output_filename=(args.expectations_filename_output or args.expectations_filename), tests=args.tests, configs=args.configs, actuals_base_url=args.actuals_base_url, actuals_filename=args.actuals_filename, exception_handler=exception_handler, add_new=args.add_new, add_ignored=args.add_ignored, bugs=args.bugs, notes=args.notes, mark_unreviewed=args.unreviewed, mark_ignore_failure=args.ignore_failure, from_trybot=args.from_trybot) try: rebaseliner.RebaselineSubdir(builder=builder) except: exception_handler.RaiseExceptionOrContinue() else: try: raise _InternalException('expectations_json_file %s not found' % expectations_json_file) except: exception_handler.RaiseExceptionOrContinue() exception_handler.ReportAllFailures()