path: root/tools/rebaseline.py

#!/usr/bin/python

'''
Copyright 2012 Google Inc.

Use of this source code is governed by a BSD-style license that can be
found in the LICENSE file.
'''

'''
Rebaselines the given GM tests, on all bots and all configurations.
'''

# System-level imports
import argparse
import json
import os
import re
import subprocess
import sys
import urllib2

# Imports from within Skia
#
# We need to add the 'gm' directory, so that we can import gm_json.py within
# that directory.  That script allows us to parse the actual-results.json file
# written out by the GM tool.
# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end*
# so any dirs that are already in the PYTHONPATH will be preferred.
#
# This assumes that the 'gm' directory has been checked out as a sibling of
# the 'tools' directory containing this script, which will be the case if
# 'trunk' was checked out as a single unit.
GM_DIRECTORY = os.path.realpath(
    os.path.join(os.path.dirname(os.path.dirname(__file__)), 'gm'))
if GM_DIRECTORY not in sys.path:
  sys.path.append(GM_DIRECTORY)
import gm_json

# TODO(epoger): In the long run, we want to build this list automatically,
# but for now we hard-code it until we can properly address
# https://code.google.com/p/skia/issues/detail?id=1544
# ('live query of builder list makes rebaseline.py slow to start up')
TEST_BUILDERS = [
    'Test-Android-GalaxyNexus-SGX540-Arm7-Debug',
    'Test-Android-GalaxyNexus-SGX540-Arm7-Release',
    'Test-Android-IntelRhb-SGX544-x86-Debug',
    'Test-Android-IntelRhb-SGX544-x86-Release',
    'Test-Android-Nexus10-MaliT604-Arm7-Debug',
    'Test-Android-Nexus10-MaliT604-Arm7-Release',
    'Test-Android-Nexus4-Adreno320-Arm7-Debug',
    'Test-Android-Nexus4-Adreno320-Arm7-Release',
    'Test-Android-Nexus7-Tegra3-Arm7-Debug',
    'Test-Android-Nexus7-Tegra3-Arm7-Release',
    'Test-Android-NexusS-SGX540-Arm7-Debug',
    'Test-Android-NexusS-SGX540-Arm7-Release',
    'Test-Android-Xoom-Tegra2-Arm7-Debug',
    'Test-Android-Xoom-Tegra2-Arm7-Release',
    'Test-ChromeOS-Alex-GMA3150-x86-Debug',
    'Test-ChromeOS-Alex-GMA3150-x86-Release',
    'Test-ChromeOS-Daisy-MaliT604-Arm7-Debug',
    'Test-ChromeOS-Daisy-MaliT604-Arm7-Release',
    'Test-ChromeOS-Link-HD4000-x86_64-Debug',
    'Test-ChromeOS-Link-HD4000-x86_64-Release',
    'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug',
    'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release',
    'Test-Mac10.6-MacMini4.1-GeForce320M-x86_64-Debug',
    'Test-Mac10.6-MacMini4.1-GeForce320M-x86_64-Release',
    'Test-Mac10.7-MacMini4.1-GeForce320M-x86-Debug',
    'Test-Mac10.7-MacMini4.1-GeForce320M-x86-Release',
    'Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Debug',
    'Test-Mac10.7-MacMini4.1-GeForce320M-x86_64-Release',
    'Test-Mac10.8-MacMini4.1-GeForce320M-x86-Debug',
    'Test-Mac10.8-MacMini4.1-GeForce320M-x86-Release',
    'Test-Mac10.8-MacMini4.1-GeForce320M-x86_64-Debug',
    'Test-Mac10.8-MacMini4.1-GeForce320M-x86_64-Release',
    'Test-Ubuntu12-ShuttleA-ATI5770-x86-Debug',
    'Test-Ubuntu12-ShuttleA-ATI5770-x86-Release',
    'Test-Ubuntu12-ShuttleA-ATI5770-x86_64-Debug',
    'Test-Ubuntu12-ShuttleA-ATI5770-x86_64-Release',
    'Test-Ubuntu12-ShuttleA-HD2000-x86_64-Release-Valgrind',
    'Test-Ubuntu12-ShuttleA-NoGPU-x86_64-Debug',
    'Test-Ubuntu13-ShuttleA-HD2000-x86_64-Debug-ASAN',
    'Test-Win7-ShuttleA-HD2000-x86-Debug',
    'Test-Win7-ShuttleA-HD2000-x86-Debug-ANGLE',
    'Test-Win7-ShuttleA-HD2000-x86-Debug-DirectWrite',
    'Test-Win7-ShuttleA-HD2000-x86-Release',
    'Test-Win7-ShuttleA-HD2000-x86-Release-ANGLE',
    'Test-Win7-ShuttleA-HD2000-x86-Release-DirectWrite',
    'Test-Win7-ShuttleA-HD2000-x86_64-Debug',
    'Test-Win7-ShuttleA-HD2000-x86_64-Release',
]

# TODO: Get this from builder_name_schema in buildbot.
TRYBOT_SUFFIX = '-Trybot'


class _InternalException(Exception):
  pass

class ExceptionHandler(object):
  """ Object that handles exceptions, either raising them immediately or
  collecting them to display later on."""

  # params:
  def __init__(self, keep_going_on_failure=False):
    """
    params:
      keep_going_on_failure: if False, report failures and quit right away;
                             if True, collect failures until
                             ReportAllFailures() is called
    """
    self._keep_going_on_failure = keep_going_on_failure
    self._failures_encountered = []

  def RaiseExceptionOrContinue(self):
    """ We have encountered an exception; either collect the info and keep
    going, or exit the program right away."""
    # Get traceback information about the most recently raised exception.
    exc_info = sys.exc_info()

    if self._keep_going_on_failure:
      print >> sys.stderr, ('WARNING: swallowing exception %s' %
                            repr(exc_info[1]))
      self._failures_encountered.append(exc_info)
    else:
      print >> sys.stderr, (
          '\nHalting at first exception.\n' +
          'Please file a bug to epoger@google.com at ' +
          'https://code.google.com/p/skia/issues/entry, containing the ' +
          'command you ran and the following stack trace.\n\n' +
          'Afterwards, you can re-run with the --keep-going-on-failure ' +
          'option set.\n')
      raise exc_info[1], None, exc_info[2]

  def ReportAllFailures(self):
    if self._failures_encountered:
      print >> sys.stderr, ('Encountered %d failures (see above).' %
                            len(self._failures_encountered))
      sys.exit(1)


# Object that rebaselines a JSON expectations file (not individual image files).
class JsonRebaseliner(object):

  # params:
  #  expectations_root: root directory of all expectations JSON files
  #  expectations_input_filename: filename (under expectations_root) of JSON
  #                               expectations file to read; typically
  #                               "expected-results.json"
  #  expectations_output_filename: filename (under expectations_root) to
  #                                which updated expectations should be
  #                                written; typically the same as
  #                                expectations_input_filename, to overwrite
  #                                the old content
  #  actuals_base_url: base URL from which to read actual-result JSON files
  #  actuals_filename: filename (under actuals_base_url) from which to read a
  #                    summary of results; typically "actual-results.json"
  #  exception_handler: reference to rebaseline.ExceptionHandler object
  #  tests: list of tests to rebaseline, or None if we should rebaseline
  #         whatever files the JSON results summary file tells us to
  #  configs: which configs to run for each test, or None if we should
  #           rebaseline whatever configs the JSON results summary file tells
  #           us to
  #  add_new: if True, add expectations for tests which don't have any yet
  #  add_ignored: if True, add expectations for tests for which failures are
  #               currently ignored
  #  bugs: optional list of bug numbers which pertain to these expectations
  #  notes: free-form text notes to add to all updated expectations
  #  mark_unreviewed: if True, mark these expectations as NOT having been
  #                   reviewed by a human; otherwise, leave that field blank.
  #                   Currently, there is no way to make this script mark
  #                   expectations as reviewed-by-human=True.
  #                   TODO(epoger): Add that capability to a review tool.
  #  mark_ignore_failure: if True, mark failures of a given test as being
  #                       ignored.
  #  from_trybot: if True, read actual-result JSON files generated from a
  #               trybot run rather than a waterfall run.
  def __init__(self, expectations_root, expectations_input_filename,
               expectations_output_filename, actuals_base_url,
               actuals_filename, exception_handler,
               tests=None, configs=None, add_new=False, add_ignored=False,
               bugs=None, notes=None, mark_unreviewed=None,
               mark_ignore_failure=False, from_trybot=False):
    self._expectations_root = expectations_root
    self._expectations_input_filename = expectations_input_filename
    self._expectations_output_filename = expectations_output_filename
    self._tests = tests
    self._configs = configs
    self._actuals_base_url = actuals_base_url
    self._actuals_filename = actuals_filename
    self._exception_handler = exception_handler
    self._add_new = add_new
    self._add_ignored = add_ignored
    self._bugs = bugs
    self._notes = notes
    self._mark_unreviewed = mark_unreviewed
    self._mark_ignore_failure = mark_ignore_failure;
    if self._tests or self._configs:
      self._image_filename_re = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
    else:
      self._image_filename_re = None
    self._using_svn = os.path.isdir(os.path.join(expectations_root, '.svn'))
    self._from_trybot = from_trybot

  # Executes subprocess.call(cmd).
  # Raises an Exception if the command fails.
  def _Call(self, cmd):
    if subprocess.call(cmd) != 0:
      raise _InternalException('error running command: ' + ' '.join(cmd))

  # Returns the full contents of filepath, as a single string.
  # If filepath looks like a URL, try to read it that way instead of as
  # a path on local storage.
  #
  # Raises _InternalException if there is a problem.
  def _GetFileContents(self, filepath):
    if filepath.startswith('http:') or filepath.startswith('https:'):
      try:
        return urllib2.urlopen(filepath).read()
      except urllib2.HTTPError as e:
        raise _InternalException('unable to read URL %s: %s' % (
            filepath, e))
    else:
      return open(filepath, 'r').read()

  # Returns a dictionary of actual results from actual-results.json file.
  #
  # The dictionary returned has this format:
  # {
  #  u'imageblur_565.png': [u'bitmap-64bitMD5', 3359963596899141322],
  #  u'imageblur_8888.png': [u'bitmap-64bitMD5', 4217923806027861152],
  #  u'shadertext3_8888.png': [u'bitmap-64bitMD5', 3713708307125704716]
  # }
  #
  # If the JSON actual result summary file cannot be loaded, logs a warning
  # message and returns None.
  # If the JSON actual result summary file can be loaded, but we have
  # trouble parsing it, raises an Exception.
  #
  # params:
  #  json_url: URL pointing to a JSON actual result summary file
  #  sections: a list of section names to include in the results, e.g.
  #            [gm_json.JSONKEY_ACTUALRESULTS_FAILED,
  #             gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON] ;
  #            if None, then include ALL sections.
  def _GetActualResults(self, json_url, sections=None):
    try:
      json_contents = self._GetFileContents(json_url)
    except _InternalException:
      print >> sys.stderr, (
          'could not read json_url %s ; skipping this platform.' %
          json_url)
      return None
    json_dict = gm_json.LoadFromString(json_contents)
    results_to_return = {}
    actual_results = json_dict[gm_json.JSONKEY_ACTUALRESULTS]
    if not sections:
      sections = actual_results.keys()
    for section in sections:
      section_results = actual_results[section]
      if section_results:
        results_to_return.update(section_results)
    return results_to_return

  # Rebaseline all tests/types we specified in the constructor,
  # within this builder's subdirectory in expectations/gm .
  #
  # params:
  #  builder : e.g. 'Test-Win7-ShuttleA-HD2000-x86-Release'
  def RebaselineSubdir(self, builder):
    # Read in the actual result summary, and extract all the tests whose
    # results we need to update.
    results_builder = str(builder)
    if self._from_trybot:
      results_builder = results_builder + TRYBOT_SUFFIX
    actuals_url = '/'.join([self._actuals_base_url, results_builder,
                            self._actuals_filename])
    # Only update results for tests that are currently failing.
    # We don't want to rewrite results for tests that are already succeeding,
    # because we don't want to add annotation fields (such as
    # JSONKEY_EXPECTEDRESULTS_BUGS) except for tests whose expectations we
    # are actually modifying.
    sections = [gm_json.JSONKEY_ACTUALRESULTS_FAILED]
    if self._add_new:
      sections.append(gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON)
    if self._add_ignored:
      sections.append(gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED)
    results_to_update = self._GetActualResults(json_url=actuals_url,
                                               sections=sections)

    # Read in current expectations.
    expectations_input_filepath = os.path.join(
        self._expectations_root, builder, self._expectations_input_filename)
    expectations_dict = gm_json.LoadFromFile(expectations_input_filepath)
    expected_results = expectations_dict.get(gm_json.JSONKEY_EXPECTEDRESULTS)
    if not expected_results:
      expected_results = {}
      expectations_dict[gm_json.JSONKEY_EXPECTEDRESULTS] = expected_results

    # Update the expectations in memory, skipping any tests/configs that
    # the caller asked to exclude.
    skipped_images = []
    if results_to_update:
      for (image_name, image_results) in results_to_update.iteritems():
        if self._image_filename_re:
          (test, config) = self._image_filename_re.match(image_name).groups()
          if self._tests:
            if test not in self._tests:
              skipped_images.append(image_name)
              continue
          if self._configs:
            if config not in self._configs:
              skipped_images.append(image_name)
              continue
        if not expected_results.get(image_name):
          expected_results[image_name] = {}
        expected_results[image_name]\
                        [gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS]\
                        = [image_results]
        if self._mark_unreviewed:
          expected_results[image_name]\
                          [gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED]\
                          = False
        if self._mark_ignore_failure:
          expected_results[image_name]\
                          [gm_json.JSONKEY_EXPECTEDRESULTS_IGNOREFAILURE]\
                          = True
        if self._bugs:
          expected_results[image_name]\
                          [gm_json.JSONKEY_EXPECTEDRESULTS_BUGS]\
                          = self._bugs
        if self._notes:
          expected_results[image_name]\
                          [gm_json.JSONKEY_EXPECTEDRESULTS_NOTES]\
                          = self._notes

    # Write out updated expectations.
    expectations_output_filepath = os.path.join(
        self._expectations_root, builder, self._expectations_output_filename)
    gm_json.WriteToFile(expectations_dict, expectations_output_filepath)

    # Mark the JSON file as plaintext, so text-style diffs can be applied.
    # Fixes https://code.google.com/p/skia/issues/detail?id=1442
    if self._using_svn:
      self._Call(['svn', 'propset', '--quiet', 'svn:mime-type',
                  'text/x-json', expectations_output_filepath])

# main...

parser = argparse.ArgumentParser(
    formatter_class=argparse.RawDescriptionHelpFormatter,
    epilog='Here is the full set of builders we know about:' +
           '\n    '.join([''] + sorted(TEST_BUILDERS)))
parser.add_argument('--actuals-base-url',
                    help=('base URL from which to read files containing JSON '
                          'summaries of actual GM results; defaults to '
                          '%(default)s. To get a specific revision (useful for '
                          'trybots) replace "svn" with "svn-history/r123". '
                          'If SKIMAGE is True, defaults to ' +
                          gm_json.SKIMAGE_ACTUALS_BASE_URL),
                    default='http://skia-autogen.googlecode.com/svn/gm-actual')
parser.add_argument('--actuals-filename',
                    help=('filename (within builder-specific subdirectories '
                          'of ACTUALS_BASE_URL) to read a summary of results '
                          'from; defaults to %(default)s'),
                    default='actual-results.json')
parser.add_argument('--add-new', action='store_true',
                    help=('in addition to the standard behavior of '
                          'updating expectations for failing tests, add '
                          'expectations for tests which don\'t have '
                          'expectations yet.'))
parser.add_argument('--add-ignored', action='store_true',
                    help=('in addition to the standard behavior of '
                          'updating expectations for failing tests, add '
                          'expectations for tests for which failures are '
                          'currently ignored.'))
parser.add_argument('--bugs', metavar='BUG', type=int, nargs='+',
                    help=('Skia bug numbers (under '
                          'https://code.google.com/p/skia/issues/list ) which '
                          'pertain to this set of rebaselines.'))
parser.add_argument('--builders', metavar='BUILDER', nargs='+',
                    help=('which platforms to rebaseline; '
                          'if unspecified, rebaseline all known platforms '
                          '(see below for a list)'))
# TODO(epoger): Add test that exercises --configs argument.
parser.add_argument('--configs', metavar='CONFIG', nargs='+',
                    help=('which configurations to rebaseline, e.g. '
                          '"--configs 565 8888", as a filter over the full set '
                          'of results in ACTUALS_FILENAME; if unspecified, '
                          'rebaseline *all* configs that are available.'))
parser.add_argument('--expectations-filename',
                    help=('filename (under EXPECTATIONS_ROOT) to read '
                          'current expectations from, and to write new '
                          'expectations into (unless a separate '
                          'EXPECTATIONS_FILENAME_OUTPUT has been specified); '
                          'defaults to %(default)s'),
                    default='expected-results.json')
parser.add_argument('--expectations-filename-output',
                    help=('filename (under EXPECTATIONS_ROOT) to write '
                          'updated expectations into; by default, overwrites '
                          'the input file (EXPECTATIONS_FILENAME)'),
                    default='')
parser.add_argument('--expectations-root',
                    help=('root of expectations directory to update-- should '
                          'contain one or more builder subdirectories. '
                          'Defaults to %(default)s. If SKIMAGE is set, '
                          ' defaults to ' + gm_json.SKIMAGE_EXPECTATIONS_ROOT),
                    default=os.path.join('expectations', 'gm'))
parser.add_argument('--keep-going-on-failure', action='store_true',
                    help=('instead of halting at the first error encountered, '
                          'keep going and rebaseline as many tests as '
                          'possible, and then report the full set of errors '
                          'at the end'))
parser.add_argument('--notes',
                    help=('free-form text notes to add to all updated '
                          'expectations'))
# TODO(epoger): Add test that exercises --tests argument.
parser.add_argument('--tests', metavar='TEST', nargs='+',
                    help=('which tests to rebaseline, e.g. '
                          '"--tests aaclip bigmatrix", as a filter over the '
                          'full set of results in ACTUALS_FILENAME; if '
                          'unspecified, rebaseline *all* tests that are '
                          'available.'))
parser.add_argument('--unreviewed', action='store_true',
                    help=('mark all expectations modified by this run as '
                          '"%s": False' %
                          gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED))
parser.add_argument('--ignore-failure', action='store_true',
                    help=('mark all expectations modified by this run as '
                          '"%s": True' %
                          gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED))
parser.add_argument('--from-trybot', action='store_true',
                    help=('pull the actual-results.json file from the '
                          'corresponding trybot, rather than the main builder'))
parser.add_argument('--skimage', action='store_true',
                    help=('Rebaseline skimage results instead of gm. Defaults '
                          'to False. If True, TESTS and CONFIGS are ignored, '
                          'and ACTUALS_BASE_URL and EXPECTATIONS_ROOT are set '
                          'to alternate defaults, specific to skimage.'))
args = parser.parse_args()
exception_handler = ExceptionHandler(
    keep_going_on_failure=args.keep_going_on_failure)
if args.builders:
  builders = args.builders
  missing_json_is_fatal = True
else:
  builders = sorted(TEST_BUILDERS)
  missing_json_is_fatal = False
if args.skimage:
  # Use a different default if --skimage is specified.
  if args.actuals_base_url == parser.get_default('actuals_base_url'):
    args.actuals_base_url = gm_json.SKIMAGE_ACTUALS_BASE_URL
  if args.expectations_root == parser.get_default('expectations_root'):
    args.expectations_root = gm_json.SKIMAGE_EXPECTATIONS_ROOT
for builder in builders:
  if not builder in TEST_BUILDERS:
    raise Exception(('unrecognized builder "%s"; ' +
                     'should be one of %s') % (
                         builder, TEST_BUILDERS))

  expectations_json_file = os.path.join(args.expectations_root, builder,
                                        args.expectations_filename)
  if os.path.isfile(expectations_json_file):
    rebaseliner = JsonRebaseliner(
        expectations_root=args.expectations_root,
        expectations_input_filename=args.expectations_filename,
        expectations_output_filename=(args.expectations_filename_output or
                                      args.expectations_filename),
        tests=args.tests, configs=args.configs,
        actuals_base_url=args.actuals_base_url,
        actuals_filename=args.actuals_filename,
        exception_handler=exception_handler,
        add_new=args.add_new, add_ignored=args.add_ignored,
        bugs=args.bugs, notes=args.notes,
        mark_unreviewed=args.unreviewed,
        mark_ignore_failure=args.ignore_failure,
        from_trybot=args.from_trybot)
    try:
      rebaseliner.RebaselineSubdir(builder=builder)
    except:
      exception_handler.RaiseExceptionOrContinue()
  else:
    try:
      raise _InternalException('expectations_json_file %s not found' %
                               expectations_json_file)
    except:
      exception_handler.RaiseExceptionOrContinue()

exception_handler.ReportAllFailures()