path: root/gm/rebaseline_server/compare_to_expectations.py

#!/usr/bin/python

"""
Copyright 2013 Google Inc.

Use of this source code is governed by a BSD-style license that can be
found in the LICENSE file.

Repackage expected/actual GM results as needed by our HTML rebaseline viewer.
"""

# System-level imports
import argparse
import fnmatch
import logging
import os
import time

# Must fix up PYTHONPATH before importing from within Skia
# pylint: disable=W0611
import fix_pythonpath
# pylint: enable=W0611

# Imports from within Skia
from pyutils import url_utils
import column
import gm_json
import imagediffdb
import imagepair
import imagepairset
import results

EXPECTATION_FIELDS_PASSED_THRU_VERBATIM = [
    results.KEY__EXPECTATIONS__BUGS,
    results.KEY__EXPECTATIONS__IGNOREFAILURE,
    results.KEY__EXPECTATIONS__REVIEWED,
]
FREEFORM_COLUMN_IDS = [
    results.KEY__EXTRACOLUMNS__BUILDER,
    results.KEY__EXTRACOLUMNS__TEST,
]
ORDERED_COLUMN_IDS = [
    results.KEY__EXTRACOLUMNS__RESULT_TYPE,
    results.KEY__EXTRACOLUMNS__BUILDER,
    results.KEY__EXTRACOLUMNS__TEST,
    results.KEY__EXTRACOLUMNS__CONFIG,
]

TRUNK_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
DEFAULT_EXPECTATIONS_DIR = os.path.join(TRUNK_DIRECTORY, 'expectations', 'gm')
DEFAULT_IGNORE_FAILURES_FILE = 'ignored-tests.txt'

IMAGEPAIR_SET_DESCRIPTIONS = ('expected image', 'actual image')


class ExpectationComparisons(results.BaseComparisons):
  """Loads actual and expected GM results into an ImagePairSet.

  Loads actual and expected results from all builders, except for those skipped
  by _ignore_builder().

  Once this object has been constructed, the results (in self._results[])
  are immutable.  If you want to update the results based on updated JSON
  file contents, you will need to create a new ExpectationComparisons object."""

  def __init__(self, image_diff_db, actuals_root=results.DEFAULT_ACTUALS_DIR,
               expected_root=DEFAULT_EXPECTATIONS_DIR,
               ignore_failures_file=DEFAULT_IGNORE_FAILURES_FILE,
               diff_base_url=None, builder_regex_list=None):
    """
    Args:
      image_diff_db: instance of ImageDiffDB we use to cache the image diffs
      actuals_root: root directory containing all actual-results.json files
      expected_root: root directory containing all expected-results.json files
      ignore_failures_file: if a file with this name is found within
          expected_root, ignore failures for any tests listed in the file
      diff_base_url: base URL within which the client should look for diff
          images; if not specified, defaults to a "file:///" URL representation
          of image_diff_db's storage_root
      builder_regex_list: List of regular expressions specifying which builders
          we will process. If None, process all builders.
    """
    time_start = int(time.time())
    if builder_regex_list != None:
      self.set_match_builders_pattern_list(builder_regex_list)
    self._image_diff_db = image_diff_db
    self._diff_base_url = (
        diff_base_url or
        url_utils.create_filepath_url(image_diff_db.storage_root))
    self._actuals_root = actuals_root
    self._expected_root = expected_root
    self._ignore_failures_on_these_tests = []
    if ignore_failures_file:
      self._ignore_failures_on_these_tests = (
          ExpectationComparisons._read_noncomment_lines(
              os.path.join(expected_root, ignore_failures_file)))
    self._load_actual_and_expected()
    self._timestamp = int(time.time())
    logging.info('Results complete; took %d seconds.' %
                 (self._timestamp - time_start))

  def edit_expectations(self, modifications):
    """Edit the expectations stored within this object and write them back
    to disk.

    Note that this will NOT update the results stored in self._results[] ;
    in order to see those updates, you must instantiate a new
    ExpectationComparisons object based on the (now updated) files on disk.

    Args:
      modifications: a list of dictionaries, one for each expectation to update:

         [
           {
             imagepair.KEY__IMAGEPAIRS__EXPECTATIONS: {
               results.KEY__EXPECTATIONS__BUGS: [123, 456],
               results.KEY__EXPECTATIONS__IGNOREFAILURE: false,
               results.KEY__EXPECTATIONS__REVIEWED: true,
             },
             imagepair.KEY__IMAGEPAIRS__EXTRACOLUMNS: {
               results.KEY__EXTRACOLUMNS__BUILDER: 'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug',
               results.KEY__EXTRACOLUMNS__CONFIG: '8888',
               results.KEY__EXTRACOLUMNS__TEST: 'bigmatrix',
             },
             results.KEY__IMAGEPAIRS__IMAGE_B_URL: 'bitmap-64bitMD5/bigmatrix/10894408024079689926.png',
           },
           ...
         ]

    """
    expected_builder_dicts = self._read_builder_dicts_from_root(
        self._expected_root)
    for mod in modifications:
      image_name = results.IMAGE_FILENAME_FORMATTER % (
          mod[imagepair.KEY__IMAGEPAIRS__EXTRACOLUMNS]
             [results.KEY__EXTRACOLUMNS__TEST],
          mod[imagepair.KEY__IMAGEPAIRS__EXTRACOLUMNS]
             [results.KEY__EXTRACOLUMNS__CONFIG])
      _, hash_type, hash_digest = gm_json.SplitGmRelativeUrl(
          mod[imagepair.KEY__IMAGEPAIRS__IMAGE_B_URL])
      allowed_digests = [[hash_type, int(hash_digest)]]
      new_expectations = {
          gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS: allowed_digests,
      }
      for field in EXPECTATION_FIELDS_PASSED_THRU_VERBATIM:
        value = mod[imagepair.KEY__IMAGEPAIRS__EXPECTATIONS].get(field)
        if value is not None:
          new_expectations[field] = value
      builder_dict = expected_builder_dicts[
          mod[imagepair.KEY__IMAGEPAIRS__EXTRACOLUMNS]
             [results.KEY__EXTRACOLUMNS__BUILDER]]
      builder_expectations = builder_dict.get(gm_json.JSONKEY_EXPECTEDRESULTS)
      if not builder_expectations:
        builder_expectations = {}
        builder_dict[gm_json.JSONKEY_EXPECTEDRESULTS] = builder_expectations
      builder_expectations[image_name] = new_expectations
    ExpectationComparisons._write_dicts_to_root(
        expected_builder_dicts, self._expected_root)

  @staticmethod
  def _write_dicts_to_root(meta_dict, root, pattern='*.json'):
    """Write all per-builder dictionaries within meta_dict to files under
    the root path.

    Security note: this will only write to files that already exist within
    the root path (as found by os.walk() within root), so we don't need to
    worry about malformed content writing to disk outside of root.
    However, the data written to those files is not double-checked, so it
    could contain poisonous data.

    Args:
      meta_dict: a builder-keyed meta-dictionary containing all the JSON
                 dictionaries we want to write out
      root: path to root of directory tree within which to write files
      pattern: which files to write within root (fnmatch-style pattern)

    Raises:
      IOError if root does not refer to an existing directory
      KeyError if the set of per-builder dictionaries written out was
               different than expected
    """
    if not os.path.isdir(root):
      raise IOError('no directory found at path %s' % root)
    actual_builders_written = []
    for dirpath, _, filenames in os.walk(root):
      for matching_filename in fnmatch.filter(filenames, pattern):
        builder = os.path.basename(dirpath)
        per_builder_dict = meta_dict.get(builder)
        if per_builder_dict is not None:
          fullpath = os.path.join(dirpath, matching_filename)
          gm_json.WriteToFile(per_builder_dict, fullpath)
          actual_builders_written.append(builder)

    # Check: did we write out the set of per-builder dictionaries we
    # expected to?
    expected_builders_written = sorted(meta_dict.keys())
    actual_builders_written.sort()
    if expected_builders_written != actual_builders_written:
      raise KeyError(
          'expected to write dicts for builders %s, but actually wrote them '
          'for builders %s' % (
              expected_builders_written, actual_builders_written))

  def _load_actual_and_expected(self):
    """Loads the results of all tests, across all builders (based on the
    files within self._actuals_root and self._expected_root),
    and stores them in self._results.
    """
    logging.info('Reading actual-results JSON files from %s...' %
                 self._actuals_root)
    actual_builder_dicts = self._read_builder_dicts_from_root(
        self._actuals_root)
    logging.info('Reading expected-results JSON files from %s...' %
                 self._expected_root)
    expected_builder_dicts = self._read_builder_dicts_from_root(
        self._expected_root)

    all_image_pairs = imagepairset.ImagePairSet(
        descriptions=IMAGEPAIR_SET_DESCRIPTIONS,
        diff_base_url=self._diff_base_url)
    failing_image_pairs = imagepairset.ImagePairSet(
        descriptions=IMAGEPAIR_SET_DESCRIPTIONS,
        diff_base_url=self._diff_base_url)

    # Override settings for columns that should be filtered using freeform text.
    for column_id in FREEFORM_COLUMN_IDS:
      factory = column.ColumnHeaderFactory(
          header_text=column_id, use_freeform_filter=True)
      all_image_pairs.set_column_header_factory(
          column_id=column_id, column_header_factory=factory)
      failing_image_pairs.set_column_header_factory(
          column_id=column_id, column_header_factory=factory)

    all_image_pairs.ensure_extra_column_values_in_summary(
        column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[
            results.KEY__RESULT_TYPE__FAILED,
            results.KEY__RESULT_TYPE__FAILUREIGNORED,
            results.KEY__RESULT_TYPE__NOCOMPARISON,
            results.KEY__RESULT_TYPE__SUCCEEDED,
        ])
    failing_image_pairs.ensure_extra_column_values_in_summary(
        column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[
            results.KEY__RESULT_TYPE__FAILED,
            results.KEY__RESULT_TYPE__FAILUREIGNORED,
            results.KEY__RESULT_TYPE__NOCOMPARISON,
        ])

    # Only consider builders we have both expected and actual results for.
    # Fixes http://skbug.com/2486 ('rebaseline_server shows actual results
    # (but not expectations) for Test-Ubuntu12-ShuttleA-NoGPU-x86_64-Debug
    # builder')
    actual_builder_set = set(actual_builder_dicts.keys())
    expected_builder_set = set(expected_builder_dicts.keys())
    builders = sorted(actual_builder_set.intersection(expected_builder_set))

    num_builders = len(builders)
    builder_num = 0
    for builder in builders:
      builder_num += 1
      logging.info('Generating pixel diffs for builder #%d of %d, "%s"...' %
                   (builder_num, num_builders, builder))
      actual_results_for_this_builder = (
          actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
      for result_type in sorted(actual_results_for_this_builder.keys()):
        results_of_this_type = actual_results_for_this_builder[result_type]
        if not results_of_this_type:
          continue
        for image_name in sorted(results_of_this_type.keys()):
          (test, config) = results.IMAGE_FILENAME_RE.match(image_name).groups()
          actual_image_relative_url = (
              ExpectationComparisons._create_relative_url(
                  hashtype_and_digest=results_of_this_type[image_name],
                  test_name=test))

          # Default empty expectations; overwrite these if we find any real ones
          expectations_per_test = None
          expected_image_relative_url = None
          expectations_dict = None
          try:
            expectations_per_test = (
                expected_builder_dicts
                [builder][gm_json.JSONKEY_EXPECTEDRESULTS][image_name])
            # TODO(epoger): assumes a single allowed digest per test, which is
            # fine; see https://code.google.com/p/skia/issues/detail?id=1787
            expected_image_hashtype_and_digest = (
                expectations_per_test
                [gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS][0])
            expected_image_relative_url = (
                ExpectationComparisons._create_relative_url(
                    hashtype_and_digest=expected_image_hashtype_and_digest,
                    test_name=test))
            expectations_dict = {}
            for field in EXPECTATION_FIELDS_PASSED_THRU_VERBATIM:
              expectations_dict[field] = expectations_per_test.get(field)
          except (KeyError, TypeError):
            # There are several cases in which we would expect to find
            # no expectations for a given test:
            #
            # 1. result_type == NOCOMPARISON
            #   There are no expectations for this test yet!
            #
            # 2. alternate rendering mode failures (e.g. serialized)
            #   In cases like
            #   https://code.google.com/p/skia/issues/detail?id=1684
            #   ('tileimagefilter GM test failing in serialized render mode'),
            #   the gm-actuals will list a failure for the alternate
            #   rendering mode even though we don't have explicit expectations
            #   for the test (the implicit expectation is that it must
            #   render the same in all rendering modes).
            #
            # Don't log type 1, because it is common.
            # Log other types, because they are rare and we should know about
            # them, but don't throw an exception, because we need to keep our
            # tools working in the meanwhile!
            if result_type != results.KEY__RESULT_TYPE__NOCOMPARISON:
              logging.warning('No expectations found for test: %s' % {
                  results.KEY__EXTRACOLUMNS__BUILDER: builder,
                  results.KEY__EXTRACOLUMNS__RESULT_TYPE: result_type,
                  'image_name': image_name,
                  })

          # If this test was recently rebaselined, it will remain in
          # the 'failed' set of actuals until all the bots have
          # cycled (although the expectations have indeed been set
          # from the most recent actuals).  Treat these as successes
          # instead of failures.
          #
          # TODO(epoger): Do we need to do something similar in
          # other cases, such as when we have recently marked a test
          # as ignoreFailure but it still shows up in the 'failed'
          # category?  Maybe we should not rely on the result_type
          # categories recorded within the gm_actuals AT ALL, and
          # instead evaluate the result_type ourselves based on what
          # we see in expectations vs actual checksum?
          if expected_image_relative_url == actual_image_relative_url:
            updated_result_type = results.KEY__RESULT_TYPE__SUCCEEDED
          elif ((result_type == results.KEY__RESULT_TYPE__FAILED) and
                (test in self._ignore_failures_on_these_tests)):
            updated_result_type = results.KEY__RESULT_TYPE__FAILUREIGNORED
          else:
            updated_result_type = result_type
          extra_columns_dict = {
              results.KEY__EXTRACOLUMNS__RESULT_TYPE: updated_result_type,
              results.KEY__EXTRACOLUMNS__BUILDER: builder,
              results.KEY__EXTRACOLUMNS__TEST: test,
              results.KEY__EXTRACOLUMNS__CONFIG: config,
          }
          try:
            image_pair = imagepair.ImagePair(
                image_diff_db=self._image_diff_db,
                base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL,
                imageA_relative_url=expected_image_relative_url,
                imageB_relative_url=actual_image_relative_url,
                expectations=expectations_dict,
                extra_columns=extra_columns_dict)
            all_image_pairs.add_image_pair(image_pair)
            if updated_result_type != results.KEY__RESULT_TYPE__SUCCEEDED:
              failing_image_pairs.add_image_pair(image_pair)
          except Exception:
            logging.exception('got exception while creating new ImagePair')

    # pylint: disable=W0201
    self._results = {
      results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(
          column_ids_in_order=ORDERED_COLUMN_IDS),
      results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(
          column_ids_in_order=ORDERED_COLUMN_IDS),
    }


def main():
  logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
                      datefmt='%m/%d/%Y %H:%M:%S',
                      level=logging.INFO)
  parser = argparse.ArgumentParser()
  parser.add_argument(
      '--actuals', default=results.DEFAULT_ACTUALS_DIR,
      help='Directory containing all actual-result JSON files; defaults to '
      '\'%(default)s\' .')
  parser.add_argument(
      '--expectations', default=DEFAULT_EXPECTATIONS_DIR,
      help='Directory containing all expected-result JSON files; defaults to '
      '\'%(default)s\' .')
  parser.add_argument(
      '--ignore-failures-file', default=DEFAULT_IGNORE_FAILURES_FILE,
      help='If a file with this name is found within the EXPECTATIONS dir, '
      'ignore failures for any tests listed in the file; defaults to '
      '\'%(default)s\' .')
  parser.add_argument(
      '--outfile', required=True,
      help='File to write result summary into, in JSON format.')
  parser.add_argument(
      '--results', default=results.KEY__HEADER__RESULTS_FAILURES,
      help='Which result types to include. Defaults to \'%(default)s\'; '
      'must be one of ' +
      str([results.KEY__HEADER__RESULTS_FAILURES,
           results.KEY__HEADER__RESULTS_ALL]))
  parser.add_argument(
      '--workdir', default=results.DEFAULT_GENERATED_IMAGES_ROOT,
      help='Directory within which to download images and generate diffs; '
      'defaults to \'%(default)s\' .')
  args = parser.parse_args()
  image_diff_db = imagediffdb.ImageDiffDB(storage_root=args.workdir)
  results_obj = ExpectationComparisons(
      image_diff_db=image_diff_db,
      actuals_root=args.actuals,
      expected_root=args.expectations,
      ignore_failures_file=args.ignore_failures_file)
  gm_json.WriteToFile(
      results_obj.get_packaged_results_of_type(results_type=args.results),
      args.outfile)


if __name__ == '__main__':
  main()