path: root/gm/rebaseline_server/results.py

#!/usr/bin/python

"""
Copyright 2013 Google Inc.

Use of this source code is governed by a BSD-style license that can be
found in the LICENSE file.

Repackage expected/actual GM results as needed by our HTML rebaseline viewer.
"""

# System-level imports
import fnmatch
import os
import re
import sys

# Imports from within Skia
#
# We need to add the 'gm' directory, so that we can import gm_json.py within
# that directory.  That script allows us to parse the actual-results.json file
# written out by the GM tool.
# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end*
# so any dirs that are already in the PYTHONPATH will be preferred.
PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
GM_DIRECTORY = os.path.dirname(PARENT_DIRECTORY)
if GM_DIRECTORY not in sys.path:
  sys.path.append(GM_DIRECTORY)
import gm_json
import imagepairset

# Keys used to link an image to a particular GM test.
# NOTE: Keep these in sync with static/constants.js
VALUE__HEADER__SCHEMA_VERSION = 3
KEY__EXPECTATIONS__BUGS = gm_json.JSONKEY_EXPECTEDRESULTS_BUGS
KEY__EXPECTATIONS__IGNOREFAILURE = gm_json.JSONKEY_EXPECTEDRESULTS_IGNOREFAILURE
KEY__EXPECTATIONS__REVIEWED = gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED
KEY__EXTRACOLUMNS__BUILDER = 'builder'
KEY__EXTRACOLUMNS__CONFIG = 'config'
KEY__EXTRACOLUMNS__RESULT_TYPE = 'resultType'
KEY__EXTRACOLUMNS__TEST = 'test'
KEY__HEADER__DATAHASH = 'dataHash'
KEY__HEADER__IS_EDITABLE = 'isEditable'
KEY__HEADER__IS_EXPORTED = 'isExported'
KEY__HEADER__IS_STILL_LOADING = 'resultsStillLoading'
KEY__HEADER__RESULTS_ALL = 'all'
KEY__HEADER__RESULTS_FAILURES = 'failures'
KEY__HEADER__SCHEMA_VERSION = 'schemaVersion'
KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE = 'timeNextUpdateAvailable'
KEY__HEADER__TIME_UPDATED = 'timeUpdated'
KEY__HEADER__TYPE = 'type'
KEY__RESULT_TYPE__FAILED = gm_json.JSONKEY_ACTUALRESULTS_FAILED
KEY__RESULT_TYPE__FAILUREIGNORED = gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED
KEY__RESULT_TYPE__NOCOMPARISON = gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON
KEY__RESULT_TYPE__SUCCEEDED = gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED

IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
IMAGE_FILENAME_FORMATTER = '%s_%s.png'  # pass in (testname, config)

DEFAULT_ACTUALS_DIR = '.gm-actuals'
DEFAULT_GENERATED_IMAGES_ROOT = os.path.join(
    PARENT_DIRECTORY, '.generated-images')

# Define the default set of builders we will process expectations/actuals for.
# This allows us to ignore builders for which we don't maintain expectations
# (trybots, Valgrind, ASAN, TSAN), and avoid problems like
# https://code.google.com/p/skia/issues/detail?id=2036 ('rebaseline_server
# produces error when trying to add baselines for ASAN/TSAN builders')
DEFAULT_MATCH_BUILDERS_PATTERN_LIST = ['.*']
DEFAULT_SKIP_BUILDERS_PATTERN_LIST = [
    '.*-Trybot', '.*Valgrind.*', '.*TSAN.*', '.*ASAN.*']


class BaseComparisons(object):
  """Base class for generating summary of comparisons between two image sets.
  """

  def get_results_of_type(self, results_type):
    """Return results of some/all tests (depending on 'results_type' parameter).

    Args:
      results_type: string describing which types of results to include; must
          be one of the RESULTS_* constants

    Results are returned in a dictionary as output by ImagePairSet.as_dict().
    """
    return self._results[results_type]

  def get_packaged_results_of_type(self, results_type, reload_seconds=None,
                                   is_editable=False, is_exported=True):
    """Package the results of some/all tests as a complete response_dict.

    Args:
      results_type: string indicating which set of results to return;
          must be one of the RESULTS_* constants
      reload_seconds: if specified, note that new results may be available once
          these results are reload_seconds old
      is_editable: whether clients are allowed to submit new baselines
      is_exported: whether these results are being made available to other
          network hosts
    """
    response_dict = self._results[results_type]
    time_updated = self.get_timestamp()
    response_dict[imagepairset.KEY__ROOT__HEADER] = {
        KEY__HEADER__SCHEMA_VERSION: (
            VALUE__HEADER__SCHEMA_VERSION),

        # Timestamps:
        # 1. when this data was last updated
        # 2. when the caller should check back for new data (if ever)
        KEY__HEADER__TIME_UPDATED: time_updated,
        KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE: (
            (time_updated+reload_seconds) if reload_seconds else None),

        # The type we passed to get_results_of_type()
        KEY__HEADER__TYPE: results_type,

        # Hash of dataset, which the client must return with any edits--
        # this ensures that the edits were made to a particular dataset.
        KEY__HEADER__DATAHASH: str(hash(repr(
            response_dict[imagepairset.KEY__ROOT__IMAGEPAIRS]))),

        # Whether the server will accept edits back.
        KEY__HEADER__IS_EDITABLE: is_editable,

        # Whether the service is accessible from other hosts.
        KEY__HEADER__IS_EXPORTED: is_exported,
    }
    return response_dict

  def get_timestamp(self):
    """Return the time at which this object was created, in seconds past epoch
    (UTC).
    """
    return self._timestamp

  _match_builders_pattern_list = [
      re.compile(p) for p in DEFAULT_MATCH_BUILDERS_PATTERN_LIST]
  _skip_builders_pattern_list = [
      re.compile(p) for p in DEFAULT_SKIP_BUILDERS_PATTERN_LIST]

  def set_match_builders_pattern_list(self, pattern_list):
    """Override the default set of builders we should process.

    The default is DEFAULT_MATCH_BUILDERS_PATTERN_LIST .

    Note that skip_builders_pattern_list overrides this; regardless of whether a
    builder is in the "match" list, if it's in the "skip" list, we will skip it.

    Args:
      pattern_list: list of regex patterns; process builders that match any
          entry within this list
    """
    if pattern_list == None:
      pattern_list = []
    self._match_builders_pattern_list = [re.compile(p) for p in pattern_list]

  def set_skip_builders_pattern_list(self, pattern_list):
    """Override the default set of builders we should skip while processing.

    The default is DEFAULT_SKIP_BUILDERS_PATTERN_LIST .

    This overrides match_builders_pattern_list; regardless of whether a
    builder is in the "match" list, if it's in the "skip" list, we will skip it.

    Args:
      pattern_list: list of regex patterns; skip builders that match any
          entry within this list
    """
    if pattern_list == None:
      pattern_list = []
    self._skip_builders_pattern_list = [re.compile(p) for p in pattern_list]

  def _ignore_builder(self, builder):
    """Returns True if we should skip processing this builder.

    Args:
      builder: name of this builder, as a string

    Returns:
      True if we should ignore expectations and actuals for this builder.
    """
    for pattern in self._skip_builders_pattern_list:
      if pattern.match(builder):
        return True
    for pattern in self._match_builders_pattern_list:
      if pattern.match(builder):
        return False
    return True

  def _read_builder_dicts_from_root(self, root, pattern='*.json'):
    """Read all JSON dictionaries within a directory tree.

    Skips any dictionaries belonging to a builder we have chosen to ignore.

    Args:
      root: path to root of directory tree
      pattern: which files to read within root (fnmatch-style pattern)

    Returns:
      A meta-dictionary containing all the JSON dictionaries found within
      the directory tree, keyed by builder name (the basename of the directory
      where each JSON dictionary was found).

    Raises:
      IOError if root does not refer to an existing directory
    """
    # I considered making this call _read_dicts_from_root(), but I decided
    # it was better to prune out the ignored builders within the os.walk().
    if not os.path.isdir(root):
      raise IOError('no directory found at path %s' % root)
    meta_dict = {}
    for dirpath, dirnames, filenames in os.walk(root):
      for matching_filename in fnmatch.filter(filenames, pattern):
        builder = os.path.basename(dirpath)
        if self._ignore_builder(builder):
          continue
        full_path = os.path.join(dirpath, matching_filename)
        meta_dict[builder] = gm_json.LoadFromFile(full_path)
    return meta_dict

  def _read_dicts_from_root(self, root, pattern='*.json'):
    """Read all JSON dictionaries within a directory tree.

    Args:
      root: path to root of directory tree
      pattern: which files to read within root (fnmatch-style pattern)

    Returns:
      A meta-dictionary containing all the JSON dictionaries found within
      the directory tree, keyed by the pathname (relative to root) of each JSON
      dictionary.

    Raises:
      IOError if root does not refer to an existing directory
    """
    if not os.path.isdir(root):
      raise IOError('no directory found at path %s' % root)
    meta_dict = {}
    for abs_dirpath, dirnames, filenames in os.walk(root):
      rel_dirpath = os.path.relpath(abs_dirpath, root)
      for matching_filename in fnmatch.filter(filenames, pattern):
        abs_path = os.path.join(abs_dirpath, matching_filename)
        rel_path = os.path.join(rel_dirpath, matching_filename)
        meta_dict[rel_path] = gm_json.LoadFromFile(abs_path)
    return meta_dict

  @staticmethod
  def _read_noncomment_lines(path):
    """Return a list of all noncomment lines within a file.

    (A "noncomment" line is one that does not start with a '#'.)

    Args:
      path: path to file
    """
    lines = []
    with open(path, 'r') as fh:
      for line in fh:
        if not line.startswith('#'):
          lines.append(line.strip())
    return lines

  @staticmethod
  def _create_relative_url(hashtype_and_digest, test_name):
    """Returns the URL for this image, relative to GM_ACTUALS_ROOT_HTTP_URL.

    If we don't have a record of this image, returns None.

    Args:
      hashtype_and_digest: (hash_type, hash_digest) tuple, or None if we
          don't have a record of this image
      test_name: string; name of the GM test that created this image
    """
    if not hashtype_and_digest:
      return None
    return gm_json.CreateGmRelativeUrl(
        test_name=test_name,
        hash_type=hashtype_and_digest[0],
        hash_digest=hashtype_and_digest[1])

  @staticmethod
  def combine_subdicts(input_dict):
    """ Flatten out a dictionary structure by one level.

    Input:
      {
        KEY_A1 : {
          KEY_B1 : VALUE_B1,
        },
        KEY_A2 : {
          KEY_B2 : VALUE_B2,
        }
      }

    Output:
      {
        KEY_B1 : VALUE_B1,
        KEY_B2 : VALUE_B2,
      }

    If this would result in any repeated keys, it will raise an Exception.
    """
    output_dict = {}
    for key, subdict in input_dict.iteritems():
      for subdict_key, subdict_value in subdict.iteritems():
        if subdict_key in output_dict:
          raise Exception('duplicate key %s in combine_subdicts' % subdict_key)
        output_dict[subdict_key] = subdict_value
    return output_dict

  @staticmethod
  def get_multilevel(input_dict, *keys):
    """ Returns input_dict[key1][key2][...], or None if any key is not found.
    """
    for key in keys:
      if input_dict == None:
        return None
      input_dict = input_dict.get(key, None)
    return input_dict