diff options
author | epoger <epoger@google.com> | 2014-08-05 10:07:22 -0700 |
---|---|---|
committer | Commit bot <commit-bot@chromium.org> | 2014-08-05 10:07:22 -0700 |
commit | 0b7127635d8245de7ac704080d722d06e47621d0 (patch) | |
tree | a99796641d0825c38943398e0f33766274b045f7 /gm | |
parent | 8f7273399466c95c0c86b099de438d6ef1a15c88 (diff) |
teach rebaseline_server to generate diffs of rendered SKPs
Creates a new live-view.html page, served by the rebaseline_server, that will display diffs between two sets of rendered SKP images.
BUG=skia:1942
NOTRY=true
R=rmistry@google.com
Author: epoger@google.com
Review URL: https://codereview.chromium.org/424263005
Diffstat (limited to 'gm')
-rwxr-xr-x | gm/rebaseline_server/base_unittest.py | 9 | ||||
-rwxr-xr-x | gm/rebaseline_server/compare_rendered_pictures.py | 274 | ||||
-rwxr-xr-x | gm/rebaseline_server/compare_rendered_pictures_test.py | 17 | ||||
-rw-r--r-- | gm/rebaseline_server/imagediffdb.py | 220 | ||||
-rw-r--r-- | gm/rebaseline_server/imagepair.py | 37 | ||||
-rw-r--r-- | gm/rebaseline_server/imagepairset.py | 50 | ||||
-rwxr-xr-x | gm/rebaseline_server/results.py | 19 | ||||
-rwxr-xr-x | gm/rebaseline_server/server.py | 103 | ||||
-rw-r--r-- | gm/rebaseline_server/static/live-loader.js | 1031 | ||||
-rw-r--r-- | gm/rebaseline_server/static/live-view.html | 420 | ||||
-rw-r--r-- | gm/rebaseline_server/testdata/outputs/expected/compare_rendered_pictures_test.CompareRenderedPicturesTest.test_endToEnd/compare_rendered_pictures.json | 88 |
11 files changed, 2031 insertions, 237 deletions
diff --git a/gm/rebaseline_server/base_unittest.py b/gm/rebaseline_server/base_unittest.py index f8fdff19c9..b8a653866b 100755 --- a/gm/rebaseline_server/base_unittest.py +++ b/gm/rebaseline_server/base_unittest.py @@ -18,9 +18,14 @@ PARENT_DIR = os.path.abspath(os.path.dirname(__file__)) TRUNK_DIR = os.path.abspath(os.path.join(PARENT_DIR, os.pardir, os.pardir)) # Import the superclass base_unittest module from the tools dir. +# +# TODO(epoger): If I don't put this at the beginning of sys.path, the import of +# tests.base_unittest fails. That's bad. I need to come up with a cleaner way +# of doing this... I think this will involve changing how we import the "boto" +# library in gs_utils.py, within the common repo. TOOLS_DIR = os.path.join(TRUNK_DIR, 'tools') -if TOOLS_DIR not in sys.path: - sys.path.append(TOOLS_DIR) +if TOOLS_DIR != sys.path[0]: + sys.path.insert(0, TOOLS_DIR) import tests.base_unittest as superclass_module diff --git a/gm/rebaseline_server/compare_rendered_pictures.py b/gm/rebaseline_server/compare_rendered_pictures.py index a48d1c5763..907ea63136 100755 --- a/gm/rebaseline_server/compare_rendered_pictures.py +++ b/gm/rebaseline_server/compare_rendered_pictures.py @@ -7,18 +7,26 @@ Use of this source code is governed by a BSD-style license that can be found in the LICENSE file. Compare results of two render_pictures runs. + +TODO(epoger): Start using this module to compare ALL images (whether they +were generated from GMs or SKPs), and rename it accordingly. """ # System-level imports import logging import os +import shutil +import tempfile import time # Must fix up PYTHONPATH before importing from within Skia import fix_pythonpath # pylint: disable=W0611 # Imports from within Skia +from py.utils import gs_utils from py.utils import url_utils +import buildbot_globals +import column import gm_json import imagediffdb import imagepair @@ -27,118 +35,179 @@ import results # URL under which all render_pictures images can be found in Google Storage. # -# pylint: disable=C0301 -# TODO(epoger): Move this default value into -# https://skia.googlesource.com/buildbot/+/master/site_config/global_variables.json -# pylint: enable=C0301 -DEFAULT_IMAGE_BASE_URL = ( - 'http://chromium-skia-gm.commondatastorage.googleapis.com/' - 'render_pictures/images') +# TODO(epoger): In order to allow live-view of GMs and other images, read this +# from the input summary files, or allow the caller to set it within the +# GET_live_results call. +DEFAULT_IMAGE_BASE_GS_URL = 'gs://' + buildbot_globals.Get('skp_images_bucket') + +# Column descriptors, and display preferences for them. +COLUMN__RESULT_TYPE = results.KEY__EXTRACOLUMNS__RESULT_TYPE +COLUMN__SOURCE_SKP = 'sourceSkpFile' +COLUMN__TILED_OR_WHOLE = 'tiledOrWhole' +COLUMN__TILENUM = 'tilenum' +FREEFORM_COLUMN_IDS = [ + COLUMN__TILENUM, +] +ORDERED_COLUMN_IDS = [ + COLUMN__RESULT_TYPE, + COLUMN__SOURCE_SKP, + COLUMN__TILED_OR_WHOLE, + COLUMN__TILENUM, +] class RenderedPicturesComparisons(results.BaseComparisons): - """Loads results from two different render_pictures runs into an ImagePairSet. + """Loads results from multiple render_pictures runs into an ImagePairSet. """ - def __init__(self, subdirs, actuals_root, - generated_images_root=results.DEFAULT_GENERATED_IMAGES_ROOT, - image_base_url=DEFAULT_IMAGE_BASE_URL, - diff_base_url=None): + def __init__(self, setA_dirs, setB_dirs, image_diff_db, + image_base_gs_url=DEFAULT_IMAGE_BASE_GS_URL, + diff_base_url=None, setA_label='setA', + setB_label='setB', gs=None, + truncate_results=False): """ Args: - actuals_root: root directory containing all render_pictures-generated - JSON files - subdirs: (string, string) tuple; pair of subdirectories within - actuals_root to compare - generated_images_root: directory within which to create all pixel diffs; - if this directory does not yet exist, it will be created - image_base_url: URL under which all render_pictures result images can + setA_dirs: list of root directories to copy all JSON summaries from, + and to use as setA within the comparisons + setB_dirs: list of root directories to copy all JSON summaries from, + and to use as setB within the comparisons + image_diff_db: ImageDiffDB instance + image_base_gs_url: "gs://" URL pointing at the Google Storage bucket/dir + under which all render_pictures result images can be found; this will be used to read images for comparison within - this code, and included in the ImagePairSet so its consumers know - where to download the images from + this code, and included in the ImagePairSet (as an HTTP URL) so its + consumers know where to download the images from diff_base_url: base URL within which the client should look for diff images; if not specified, defaults to a "file:///" URL representation - of generated_images_root + of image_diff_db's storage_root + setA_label: description to use for results in setA + setB_label: description to use for results in setB + gs: instance of GSUtils object we can use to download summary files + truncate_results: FOR MANUAL TESTING: if True, truncate the set of images + we process, to speed up testing. """ - time_start = int(time.time()) - self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root) - self._image_base_url = image_base_url + super(RenderedPicturesComparisons, self).__init__() + self._image_diff_db = image_diff_db + self._image_base_gs_url = image_base_gs_url self._diff_base_url = ( diff_base_url or - url_utils.create_filepath_url(generated_images_root)) - self._load_result_pairs(actuals_root, subdirs) - self._timestamp = int(time.time()) - logging.info('Results complete; took %d seconds.' % - (self._timestamp - time_start)) + url_utils.create_filepath_url(image_diff_db.storage_root)) + self._setA_label = setA_label + self._setB_label = setB_label + self._gs = gs + self.truncate_results = truncate_results - def _load_result_pairs(self, actuals_root, subdirs): - """Loads all JSON files found within two subdirs in actuals_root, - compares across those two subdirs, and stores the summary in self._results. + tempdir = tempfile.mkdtemp() + try: + setA_root = os.path.join(tempdir, 'setA') + setB_root = os.path.join(tempdir, 'setB') + for source_dir in setA_dirs: + self._copy_dir_contents(source_dir=source_dir, dest_dir=setA_root) + for source_dir in setB_dirs: + self._copy_dir_contents(source_dir=source_dir, dest_dir=setB_root) + + time_start = int(time.time()) + # TODO(epoger): For now, this assumes that we are always comparing two + # sets of actual results, not actuals vs expectations. Allow the user + # to control this. + self._results = self._load_result_pairs( + setA_root=setA_root, setA_section=gm_json.JSONKEY_ACTUALRESULTS, + setB_root=setB_root, setB_section=gm_json.JSONKEY_ACTUALRESULTS) + self._timestamp = int(time.time()) + logging.info('Number of download file collisions: %s' % + imagediffdb.global_file_collisions) + logging.info('Results complete; took %d seconds.' % + (self._timestamp - time_start)) + finally: + shutil.rmtree(tempdir) + + def _load_result_pairs(self, setA_root, setA_section, setB_root, + setB_section): + """Loads all JSON image summaries from 2 directory trees and compares them. Args: - actuals_root: root directory containing all render_pictures-generated - JSON files - subdirs: (string, string) tuple; pair of subdirectories within - actuals_root to compare + setA_root: root directory containing JSON summaries of rendering results + setA_section: which section (gm_json.JSONKEY_ACTUALRESULTS or + gm_json.JSONKEY_EXPECTEDRESULTS) to load from the summaries in setA + setB_root: root directory containing JSON summaries of rendering results + setB_section: which section (gm_json.JSONKEY_ACTUALRESULTS or + gm_json.JSONKEY_EXPECTEDRESULTS) to load from the summaries in setB + + Returns the summary of all image diff results. """ - logging.info( - 'Reading actual-results JSON files from %s subdirs within %s...' % ( - subdirs, actuals_root)) - subdirA, subdirB = subdirs - subdirA_dicts = self._read_dicts_from_root( - os.path.join(actuals_root, subdirA)) - subdirB_dicts = self._read_dicts_from_root( - os.path.join(actuals_root, subdirB)) - logging.info('Comparing subdirs %s and %s...' % (subdirA, subdirB)) + logging.info('Reading JSON image summaries from dirs %s and %s...' % ( + setA_root, setB_root)) + setA_dicts = self._read_dicts_from_root(setA_root) + setB_dicts = self._read_dicts_from_root(setB_root) + logging.info('Comparing summary dicts...') all_image_pairs = imagepairset.ImagePairSet( - descriptions=subdirs, + descriptions=(self._setA_label, self._setB_label), diff_base_url=self._diff_base_url) failing_image_pairs = imagepairset.ImagePairSet( - descriptions=subdirs, + descriptions=(self._setA_label, self._setB_label), diff_base_url=self._diff_base_url) + # Override settings for columns that should be filtered using freeform text. + for column_id in FREEFORM_COLUMN_IDS: + factory = column.ColumnHeaderFactory( + header_text=column_id, use_freeform_filter=True) + all_image_pairs.set_column_header_factory( + column_id=column_id, column_header_factory=factory) + failing_image_pairs.set_column_header_factory( + column_id=column_id, column_header_factory=factory) + all_image_pairs.ensure_extra_column_values_in_summary( - column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[ + column_id=COLUMN__RESULT_TYPE, values=[ results.KEY__RESULT_TYPE__FAILED, results.KEY__RESULT_TYPE__NOCOMPARISON, results.KEY__RESULT_TYPE__SUCCEEDED, ]) failing_image_pairs.ensure_extra_column_values_in_summary( - column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[ + column_id=COLUMN__RESULT_TYPE, values=[ results.KEY__RESULT_TYPE__FAILED, results.KEY__RESULT_TYPE__NOCOMPARISON, ]) - common_dict_paths = sorted(set(subdirA_dicts.keys() + subdirB_dicts.keys())) - num_common_dict_paths = len(common_dict_paths) + union_dict_paths = sorted(set(setA_dicts.keys() + setB_dicts.keys())) + num_union_dict_paths = len(union_dict_paths) dict_num = 0 - for dict_path in common_dict_paths: + for dict_path in union_dict_paths: dict_num += 1 logging.info('Generating pixel diffs for dict #%d of %d, "%s"...' % - (dict_num, num_common_dict_paths, dict_path)) - dictA = subdirA_dicts[dict_path] - dictB = subdirB_dicts[dict_path] + (dict_num, num_union_dict_paths, dict_path)) + + dictA = self.get_default(setA_dicts, None, dict_path) self._validate_dict_version(dictA) + dictA_results = self.get_default(dictA, {}, setA_section) + + dictB = self.get_default(setB_dicts, None, dict_path) self._validate_dict_version(dictB) - dictA_results = dictA[gm_json.JSONKEY_ACTUALRESULTS] - dictB_results = dictB[gm_json.JSONKEY_ACTUALRESULTS] + dictB_results = self.get_default(dictB, {}, setB_section) + skp_names = sorted(set(dictA_results.keys() + dictB_results.keys())) + # Just for manual testing... truncate to an arbitrary subset. + if self.truncate_results: + skp_names = skp_names[1:3] for skp_name in skp_names: imagepairs_for_this_skp = [] - whole_image_A = RenderedPicturesComparisons.get_multilevel( - dictA_results, skp_name, gm_json.JSONKEY_SOURCE_WHOLEIMAGE) - whole_image_B = RenderedPicturesComparisons.get_multilevel( - dictB_results, skp_name, gm_json.JSONKEY_SOURCE_WHOLEIMAGE) + whole_image_A = self.get_default( + dictA_results, None, + skp_name, gm_json.JSONKEY_SOURCE_WHOLEIMAGE) + whole_image_B = self.get_default( + dictB_results, None, + skp_name, gm_json.JSONKEY_SOURCE_WHOLEIMAGE) imagepairs_for_this_skp.append(self._create_image_pair( - test=skp_name, config=gm_json.JSONKEY_SOURCE_WHOLEIMAGE, - image_dict_A=whole_image_A, image_dict_B=whole_image_B)) - - tiled_images_A = RenderedPicturesComparisons.get_multilevel( - dictA_results, skp_name, gm_json.JSONKEY_SOURCE_TILEDIMAGES) - tiled_images_B = RenderedPicturesComparisons.get_multilevel( - dictB_results, skp_name, gm_json.JSONKEY_SOURCE_TILEDIMAGES) + image_dict_A=whole_image_A, image_dict_B=whole_image_B, + source_skp_name=skp_name, tilenum=None)) + + tiled_images_A = self.get_default( + dictA_results, None, + skp_name, gm_json.JSONKEY_SOURCE_TILEDIMAGES) + tiled_images_B = self.get_default( + dictB_results, None, + skp_name, gm_json.JSONKEY_SOURCE_TILEDIMAGES) # TODO(epoger): Report an error if we find tiles for A but not B? if tiled_images_A and tiled_images_B: # TODO(epoger): Report an error if we find a different number of tiles @@ -146,34 +215,37 @@ class RenderedPicturesComparisons(results.BaseComparisons): num_tiles = len(tiled_images_A) for tile_num in range(num_tiles): imagepairs_for_this_skp.append(self._create_image_pair( - test=skp_name, - config='%s-%d' % (gm_json.JSONKEY_SOURCE_TILEDIMAGES, tile_num), image_dict_A=tiled_images_A[tile_num], - image_dict_B=tiled_images_B[tile_num])) + image_dict_B=tiled_images_B[tile_num], + source_skp_name=skp_name, tilenum=tile_num)) for one_imagepair in imagepairs_for_this_skp: if one_imagepair: all_image_pairs.add_image_pair(one_imagepair) result_type = one_imagepair.extra_columns_dict\ - [results.KEY__EXTRACOLUMNS__RESULT_TYPE] + [COLUMN__RESULT_TYPE] if result_type != results.KEY__RESULT_TYPE__SUCCEEDED: failing_image_pairs.add_image_pair(one_imagepair) - # pylint: disable=W0201 - self._results = { - results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(), - results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(), + return { + results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict( + column_ids_in_order=ORDERED_COLUMN_IDS), + results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict( + column_ids_in_order=ORDERED_COLUMN_IDS), } def _validate_dict_version(self, result_dict): """Raises Exception if the dict is not the type/version we know how to read. Args: - result_dict: dictionary holding output of render_pictures + result_dict: dictionary holding output of render_pictures; if None, + this method will return without raising an Exception """ expected_header_type = 'ChecksummedImages' expected_header_revision = 1 + if result_dict == None: + return header = result_dict[gm_json.JSONKEY_HEADER] header_type = header[gm_json.JSONKEY_HEADER_TYPE] if header_type != expected_header_type: @@ -184,14 +256,15 @@ class RenderedPicturesComparisons(results.BaseComparisons): raise Exception('expected header_revision %d, but got %d' % ( expected_header_revision, header_revision)) - def _create_image_pair(self, test, config, image_dict_A, image_dict_B): + def _create_image_pair(self, image_dict_A, image_dict_B, source_skp_name, + tilenum): """Creates an ImagePair object for this pair of images. Args: - test: string; name of the test - config: string; name of the config image_dict_A: dict with JSONKEY_IMAGE_* keys, or None if no image image_dict_B: dict with JSONKEY_IMAGE_* keys, or None if no image + source_skp_name: string; name of the source SKP file + tilenum: which tile, or None if a wholeimage Returns: An ImagePair object, or None if both image_dict_A and image_dict_B are @@ -223,28 +296,45 @@ class RenderedPicturesComparisons(results.BaseComparisons): result_type = results.KEY__RESULT_TYPE__FAILED extra_columns_dict = { - results.KEY__EXTRACOLUMNS__CONFIG: config, - results.KEY__EXTRACOLUMNS__RESULT_TYPE: result_type, - results.KEY__EXTRACOLUMNS__TEST: test, - # TODO(epoger): Right now, the client UI crashes if it receives - # results that do not include this column. - # Until we fix that, keep the client happy. - results.KEY__EXTRACOLUMNS__BUILDER: 'TODO', + COLUMN__RESULT_TYPE: result_type, + COLUMN__SOURCE_SKP: source_skp_name, } + if tilenum == None: + extra_columns_dict[COLUMN__TILED_OR_WHOLE] = 'whole' + extra_columns_dict[COLUMN__TILENUM] = 'N/A' + else: + extra_columns_dict[COLUMN__TILED_OR_WHOLE] = 'tiled' + extra_columns_dict[COLUMN__TILENUM] = str(tilenum) try: return imagepair.ImagePair( image_diff_db=self._image_diff_db, - base_url=self._image_base_url, + base_url=self._image_base_gs_url, imageA_relative_url=imageA_relative_url, imageB_relative_url=imageB_relative_url, extra_columns=extra_columns_dict) except (KeyError, TypeError): logging.exception( 'got exception while creating ImagePair for' - ' test="%s", config="%s", urlPair=("%s","%s")' % ( - test, config, imageA_relative_url, imageB_relative_url)) + ' urlPair=("%s","%s"), source_skp_name="%s", tilenum="%s"' % ( + imageA_relative_url, imageB_relative_url, source_skp_name, + tilenum)) return None + def _copy_dir_contents(self, source_dir, dest_dir): + """Copy all contents of source_dir into dest_dir, recursing into subdirs. + + Args: + source_dir: path to source dir (GS URL or local filepath) + dest_dir: path to destination dir (local filepath) -# TODO(epoger): Add main() so this can be called by vm_run_skia_try.sh + The copy operates as a "merge with overwrite": any files in source_dir will + be "overlaid" on top of the existing content in dest_dir. Existing files + with the same names will be overwritten. + """ + if gs_utils.GSUtils.is_gs_url(source_dir): + (bucket, path) = gs_utils.GSUtils.split_gs_url(source_dir) + self._gs.download_dir_contents(source_bucket=bucket, source_dir=path, + dest_dir=dest_dir) + else: + shutil.copytree(source_dir, dest_dir) diff --git a/gm/rebaseline_server/compare_rendered_pictures_test.py b/gm/rebaseline_server/compare_rendered_pictures_test.py index a8041ec802..5ddaf10017 100755 --- a/gm/rebaseline_server/compare_rendered_pictures_test.py +++ b/gm/rebaseline_server/compare_rendered_pictures_test.py @@ -30,6 +30,7 @@ import base_unittest import compare_rendered_pictures import find_run_binary import gm_json +import imagediffdb import results @@ -38,24 +39,28 @@ class CompareRenderedPicturesTest(base_unittest.TestCase): def test_endToEnd(self): """Generate two sets of SKPs, run render_pictures over both, and compare the results.""" + setA_label = 'before_patch' + setB_label = 'after_patch' self._generate_skps_and_run_render_pictures( - subdir='before_patch', skpdict={ + subdir=setA_label, skpdict={ 'changed.skp': 200, 'unchanged.skp': 100, 'only-in-before.skp': 128, }) self._generate_skps_and_run_render_pictures( - subdir='after_patch', skpdict={ + subdir=setB_label, skpdict={ 'changed.skp': 201, 'unchanged.skp': 100, 'only-in-after.skp': 128, }) results_obj = compare_rendered_pictures.RenderedPicturesComparisons( - actuals_root=self.temp_dir, - subdirs=('before_patch', 'after_patch'), - generated_images_root=self.temp_dir, - diff_base_url='/static/generated-images') + setA_dirs=[os.path.join(self.temp_dir, setA_label)], + setB_dirs=[os.path.join(self.temp_dir, setB_label)], + image_diff_db=imagediffdb.ImageDiffDB(self.temp_dir), + image_base_gs_url='gs://fakebucket/fake/path', + diff_base_url='/static/generated-images', + setA_label=setA_label, setB_label=setB_label) results_obj.get_timestamp = mock_get_timestamp gm_json.WriteToFile( diff --git a/gm/rebaseline_server/imagediffdb.py b/gm/rebaseline_server/imagediffdb.py index 89f9fef319..fbe7121140 100644 --- a/gm/rebaseline_server/imagediffdb.py +++ b/gm/rebaseline_server/imagediffdb.py @@ -11,12 +11,16 @@ Calulate differences between image pairs, and store them in a database. # System-level imports import contextlib +import errno import json import logging import os +import Queue import re import shutil import tempfile +import threading +import time import urllib # Must fix up PYTHONPATH before importing from within Skia @@ -24,11 +28,16 @@ import fix_pythonpath # pylint: disable=W0611 # Imports from within Skia import find_run_binary +from py.utils import gs_utils + SKPDIFF_BINARY = find_run_binary.find_path_to_program('skpdiff') DEFAULT_IMAGE_SUFFIX = '.png' DEFAULT_IMAGES_SUBDIR = 'images' +# TODO(epoger): Figure out a better default number of threads; for now, +# using a conservative default value. +DEFAULT_NUM_WORKER_THREADS = 1 DISALLOWED_FILEPATH_CHAR_REGEX = re.compile('[^\w\-]') @@ -42,11 +51,20 @@ KEY__DIFFERENCES__NUM_DIFF_PIXELS = 'numDifferingPixels' KEY__DIFFERENCES__PERCENT_DIFF_PIXELS = 'percentDifferingPixels' KEY__DIFFERENCES__PERCEPTUAL_DIFF = 'perceptualDifference' +# Special values within ImageDiffDB._diff_dict +_DIFFRECORD_FAILED = 'failed' +_DIFFRECORD_PENDING = 'pending' + +# Temporary variable to keep track of how many times we download +# the same file in multiple threads. +# TODO(epoger): Delete this, once we see that the number stays close to 0. +global_file_collisions = 0 + class DiffRecord(object): """ Record of differences between two images. """ - def __init__(self, storage_root, + def __init__(self, gs, storage_root, expected_image_url, expected_image_locator, actual_image_url, actual_image_locator, expected_images_subdir=DEFAULT_IMAGES_SUBDIR, @@ -55,18 +73,16 @@ class DiffRecord(object): """Download this pair of images (unless we already have them on local disk), and prepare a DiffRecord for them. - TODO(epoger): Make this asynchronously download images, rather than blocking - until the images have been downloaded and processed. - Args: + gs: instance of GSUtils object we can use to download images storage_root: root directory on local disk within which we store all images - expected_image_url: file or HTTP url from which we will download the + expected_image_url: file, GS, or HTTP url from which we will download the expected image expected_image_locator: a unique ID string under which we will store the expected image within storage_root (probably including a checksum to guarantee uniqueness) - actual_image_url: file or HTTP url from which we will download the + actual_image_url: file, GS, or HTTP url from which we will download the actual image actual_image_locator: a unique ID string under which we will store the actual image within storage_root (probably including a checksum to @@ -79,8 +95,6 @@ class DiffRecord(object): actual_image_locator = _sanitize_locator(actual_image_locator) # Download the expected/actual images, if we don't have them already. - # TODO(rmistry): Add a parameter that just tries to use already-present - # image files rather than downloading them. expected_image_file = os.path.join( storage_root, expected_images_subdir, str(expected_image_locator) + image_suffix) @@ -88,13 +102,13 @@ class DiffRecord(object): storage_root, actual_images_subdir, str(actual_image_locator) + image_suffix) try: - _download_file(expected_image_file, expected_image_url) + _download_file(gs, expected_image_file, expected_image_url) except Exception: logging.exception('unable to download expected_image_url %s to file %s' % (expected_image_url, expected_image_file)) raise try: - _download_file(actual_image_file, actual_image_url) + _download_file(gs, actual_image_file, actual_image_url) except Exception: logging.exception('unable to download actual_image_url %s to file %s' % (actual_image_url, actual_image_file)) @@ -112,8 +126,12 @@ class DiffRecord(object): actual_img = os.path.join(storage_root, actual_images_subdir, str(actual_image_locator) + image_suffix) - # TODO: Call skpdiff ONCE for all image pairs, instead of calling it - # repeatedly. This will allow us to parallelize a lot more work. + # TODO(epoger): Consider calling skpdiff ONCE for all image pairs, + # instead of calling it separately for each image pair. + # Pro: we'll incur less overhead from making repeated system calls, + # spinning up the skpdiff binary, etc. + # Con: we would have to wait until all image pairs were loaded before + # generating any of the diffs? find_run_binary.run_command( [SKPDIFF_BINARY, '-p', expected_img, actual_img, '--jsonp', 'false', @@ -211,16 +229,71 @@ class ImageDiffDB(object): """ Calculates differences between image pairs, maintaining a database of them for download.""" - def __init__(self, storage_root): + def __init__(self, storage_root, gs=None, + num_worker_threads=DEFAULT_NUM_WORKER_THREADS): """ Args: storage_root: string; root path within the DB will store all of its stuff + gs: instance of GSUtils object we can use to download images + num_worker_threads: how many threads that download images and + generate diffs simultaneously """ self._storage_root = storage_root + self._gs = gs # Dictionary of DiffRecords, keyed by (expected_image_locator, # actual_image_locator) tuples. + # Values can also be _DIFFRECORD_PENDING, _DIFFRECORD_FAILED. + # + # Any thread that modifies _diff_dict must first acquire + # _diff_dict_writelock! + # + # TODO(epoger): Disk is limitless, but RAM is not... so, we should probably + # remove items from self._diff_dict if they haven't been accessed for a + # long time. We can always regenerate them by diffing the images we + # previously downloaded to local disk. + # I guess we should figure out how expensive it is to download vs diff the + # image pairs... if diffing them is expensive too, we can write these + # _diff_dict objects out to disk if there's too many to hold in RAM. + # Or we could use virtual memory to handle that automatically. self._diff_dict = {} + self._diff_dict_writelock = threading.RLock() + + # Set up the queue for asynchronously loading DiffRecords, and start the + # worker threads reading from it. + self._tasks_queue = Queue.Queue(maxsize=2*num_worker_threads) + self._workers = [] + for i in range(num_worker_threads): + worker = threading.Thread(target=self.worker, args=(i,)) + worker.daemon = True + worker.start() + self._workers.append(worker) + + def worker(self, worker_num): + """Launch a worker thread that pulls tasks off self._tasks_queue. + + Args: + worker_num: (integer) which worker this is + """ + while True: + params = self._tasks_queue.get() + key, expected_image_url, actual_image_url = params + try: + diff_record = DiffRecord( + self._gs, self._storage_root, + expected_image_url=expected_image_url, + expected_image_locator=key[0], + actual_image_url=actual_image_url, + actual_image_locator=key[1]) + except Exception: + logging.exception( + 'exception while creating DiffRecord for key %s' % str(key)) + diff_record = _DIFFRECORD_FAILED + self._diff_dict_writelock.acquire() + try: + self._diff_dict[key] = diff_record + finally: + self._diff_dict_writelock.release() @property def storage_root(self): @@ -229,24 +302,21 @@ class ImageDiffDB(object): def add_image_pair(self, expected_image_url, expected_image_locator, actual_image_url, actual_image_locator): - """Download this pair of images (unless we already have them on local disk), - and prepare a DiffRecord for them. + """Asynchronously prepare a DiffRecord for a pair of images. + + This method will return quickly; calls to get_diff_record() will block + until the DiffRecord is available (or we have given up on creating it). - TODO(epoger): Make this asynchronously download images, rather than blocking - until the images have been downloaded and processed. - When we do that, we should probably add a new method that will block - until all of the images have been downloaded and processed. Otherwise, - we won't know when it's safe to start calling get_diff_record(). - jcgregorio notes: maybe just make ImageDiffDB thread-safe and create a - thread-pool/worker queue at a higher level that just uses ImageDiffDB? + If we already have a DiffRecord for this particular image pair, no work + will be done. Args: - expected_image_url: file or HTTP url from which we will download the + expected_image_url: file, GS, or HTTP url from which we will download the expected image expected_image_locator: a unique ID string under which we will store the expected image within storage_root (probably including a checksum to guarantee uniqueness) - actual_image_url: file or HTTP url from which we will download the + actual_image_url: file, GS, or HTTP url from which we will download the actual image actual_image_locator: a unique ID string under which we will store the actual image within storage_root (probably including a checksum to @@ -255,49 +325,96 @@ class ImageDiffDB(object): expected_image_locator = _sanitize_locator(expected_image_locator) actual_image_locator = _sanitize_locator(actual_image_locator) key = (expected_image_locator, actual_image_locator) - if not key in self._diff_dict: - try: - new_diff_record = DiffRecord( - self._storage_root, - expected_image_url=expected_image_url, - expected_image_locator=expected_image_locator, - actual_image_url=actual_image_url, - actual_image_locator=actual_image_locator) - except Exception: - # If we can't create a real DiffRecord for this (expected, actual) pair, - # store None and the UI will show whatever information we DO have. - # Fixes http://skbug.com/2368 . - logging.exception( - 'got exception while creating a DiffRecord for ' - 'expected_image_url=%s , actual_image_url=%s; returning None' % ( - expected_image_url, actual_image_url)) - new_diff_record = None - self._diff_dict[key] = new_diff_record + must_add_to_queue = False + + self._diff_dict_writelock.acquire() + try: + if not key in self._diff_dict: + # If we have already requested a diff between these two images, + # we don't need to request it again. + must_add_to_queue = True + self._diff_dict[key] = _DIFFRECORD_PENDING + finally: + self._diff_dict_writelock.release() + + if must_add_to_queue: + self._tasks_queue.put((key, expected_image_url, actual_image_url)) def get_diff_record(self, expected_image_locator, actual_image_locator): """Returns the DiffRecord for this image pair. - Raises a KeyError if we don't have a DiffRecord for this image pair. + This call will block until the diff record is available, or we were unable + to generate it. + + Args: + expected_image_locator: a unique ID string under which we will store the + expected image within storage_root (probably including a checksum to + guarantee uniqueness) + actual_image_locator: a unique ID string under which we will store the + actual image within storage_root (probably including a checksum to + guarantee uniqueness) + + Returns the DiffRecord for this image pair, or None if we were unable to + generate one. """ key = (_sanitize_locator(expected_image_locator), _sanitize_locator(actual_image_locator)) - return self._diff_dict[key] + diff_record = self._diff_dict[key] + + # If we have no results yet, block until we do. + while diff_record == _DIFFRECORD_PENDING: + time.sleep(1) + diff_record = self._diff_dict[key] + + # Once we have the result... + if diff_record == _DIFFRECORD_FAILED: + logging.error( + 'failed to create a DiffRecord for expected_image_locator=%s , ' + 'actual_image_locator=%s' % ( + expected_image_locator, actual_image_locator)) + return None + else: + return diff_record # Utility functions -def _download_file(local_filepath, url): +def _download_file(gs, local_filepath, url): """Download a file from url to local_filepath, unless it is already there. Args: + gs: instance of GSUtils object, in case the url points at Google Storage local_filepath: path on local disk where the image should be stored - url: URL from which we can download the image if we don't have it yet + url: HTTP or GS URL from which we can download the image if we don't have + it yet """ + global global_file_collisions if not os.path.exists(local_filepath): _mkdir_unless_exists(os.path.dirname(local_filepath)) - with contextlib.closing(urllib.urlopen(url)) as url_handle: - with open(local_filepath, 'wb') as file_handle: - shutil.copyfileobj(fsrc=url_handle, fdst=file_handle) + + # First download the file contents into a unique filename, and + # then rename that file. That way, if multiple threads are downloading + # the same filename at the same time, they won't interfere with each + # other (they will both download the file, and one will "win" in the end) + temp_filename = '%s-%d' % (local_filepath, + threading.current_thread().ident) + if gs_utils.GSUtils.is_gs_url(url): + (bucket, path) = gs_utils.GSUtils.split_gs_url(url) + gs.download_file(source_bucket=bucket, source_path=path, + dest_path=temp_filename) + else: + with contextlib.closing(urllib.urlopen(url)) as url_handle: + with open(temp_filename, 'wb') as file_handle: + shutil.copyfileobj(fsrc=url_handle, fdst=file_handle) + + # Rename the file to its real filename. + # Keep count of how many colliding downloads we encounter; + # if it's a large number, we may want to change our download strategy + # to minimize repeated downloads. + if os.path.exists(local_filepath): + global_file_collisions += 1 + else: + os.rename(temp_filename, local_filepath) def _mkdir_unless_exists(path): @@ -306,8 +423,11 @@ def _mkdir_unless_exists(path): Args: path: path on local disk """ - if not os.path.isdir(path): + try: os.makedirs(path) + except OSError as e: + if e.errno == errno.EEXIST: + pass def _sanitize_locator(locator): diff --git a/gm/rebaseline_server/imagepair.py b/gm/rebaseline_server/imagepair.py index 446858d3a5..12d6718857 100644 --- a/gm/rebaseline_server/imagepair.py +++ b/gm/rebaseline_server/imagepair.py @@ -21,6 +21,10 @@ KEY__IMAGEPAIRS__IMAGE_A_URL = 'imageAUrl' KEY__IMAGEPAIRS__IMAGE_B_URL = 'imageBUrl' KEY__IMAGEPAIRS__IS_DIFFERENT = 'isDifferent' +# If self._diff_record is set to this, we haven't asked ImageDiffDB for the +# image diff details yet. +_DIFF_RECORD_STILL_LOADING = 'still_loading' + class ImagePair(object): """Describes a pair of images, pixel difference info, and optional metadata. @@ -42,6 +46,7 @@ class ImagePair(object): extra_columns: optional dictionary containing more metadata (test name, builder name, etc.) """ + self._image_diff_db = image_diff_db self.base_url = base_url self.imageA_relative_url = imageA_relative_url self.imageB_relative_url = imageB_relative_url @@ -49,27 +54,20 @@ class ImagePair(object): self.extra_columns_dict = extra_columns if not imageA_relative_url or not imageB_relative_url: self._is_different = True - self.diff_record = None + self._diff_record = None elif imageA_relative_url == imageB_relative_url: self._is_different = False - self.diff_record = None + self._diff_record = None else: - # TODO(epoger): Rather than blocking until image_diff_db can read in - # the image pair and generate diffs, it would be better to do it - # asynchronously: tell image_diff_db to download a bunch of file pairs, - # and only block later if we're still waiting for diff_records to come - # back. + # Tell image_diff_db to add an entry for this diff asynchronously. + # Later on, we will call image_diff_db.get_diff_record() to find it. self._is_different = True + self._diff_record = _DIFF_RECORD_STILL_LOADING image_diff_db.add_image_pair( expected_image_locator=imageA_relative_url, expected_image_url=posixpath.join(base_url, imageA_relative_url), actual_image_locator=imageB_relative_url, actual_image_url=posixpath.join(base_url, imageB_relative_url)) - self.diff_record = image_diff_db.get_diff_record( - expected_image_locator=imageA_relative_url, - actual_image_locator=imageB_relative_url) - if self.diff_record and self.diff_record.get_num_pixels_differing() == 0: - self._is_different = False def as_dict(self): """Returns a dictionary describing this ImagePair. @@ -85,6 +83,17 @@ class ImagePair(object): asdict[KEY__IMAGEPAIRS__EXPECTATIONS] = self.expectations_dict if self.extra_columns_dict: asdict[KEY__IMAGEPAIRS__EXTRACOLUMNS] = self.extra_columns_dict - if self.diff_record and (self.diff_record.get_num_pixels_differing() > 0): - asdict[KEY__IMAGEPAIRS__DIFFERENCES] = self.diff_record.as_dict() + if self._diff_record is _DIFF_RECORD_STILL_LOADING: + # We have waited as long as we can to ask ImageDiffDB for details of + # this image diff. Now we must block until ImageDiffDB can provide + # those details. + # + # TODO(epoger): Is it wasteful for every imagepair to have its own + # reference to image_diff_db? If so, we could pass an image_diff_db + # reference into this method call instead... + self._diff_record = self._image_diff_db.get_diff_record( + expected_image_locator=self.imageA_relative_url, + actual_image_locator=self.imageB_relative_url) + if self._diff_record != None: + asdict[KEY__IMAGEPAIRS__DIFFERENCES] = self._diff_record.as_dict() return asdict diff --git a/gm/rebaseline_server/imagepairset.py b/gm/rebaseline_server/imagepairset.py index ef9acbc4bc..f6fe09fab7 100644 --- a/gm/rebaseline_server/imagepairset.py +++ b/gm/rebaseline_server/imagepairset.py @@ -12,9 +12,13 @@ ImagePairSet class; see its docstring below. # System-level imports import posixpath -# Local imports +# Must fix up PYTHONPATH before importing from within Skia +import fix_pythonpath # pylint: disable=W0611 + +# Imports from within Skia import column import imagediffdb +from py.utils import gs_utils # Keys used within dictionary representation of ImagePairSet. # NOTE: Keep these in sync with static/constants.js @@ -53,20 +57,25 @@ class ImagePairSet(object): self._descriptions = descriptions or DEFAULT_DESCRIPTIONS self._extra_column_tallies = {} # maps column_id -> values # -> instances_per_value - self._image_pair_dicts = [] self._image_base_url = None self._diff_base_url = diff_base_url + # We build self._image_pair_objects incrementally as calls come into + # add_image_pair(); self._image_pair_dicts is filled in lazily (so that + # we put off asking ImageDiffDB for results as long as possible). + self._image_pair_objects = [] + self._image_pair_dicts = None + def add_image_pair(self, image_pair): """Adds an ImagePair; this may be repeated any number of times.""" # Special handling when we add the first ImagePair... - if not self._image_pair_dicts: + if not self._image_pair_objects: self._image_base_url = image_pair.base_url if image_pair.base_url != self._image_base_url: raise Exception('added ImagePair with base_url "%s" instead of "%s"' % ( image_pair.base_url, self._image_base_url)) - self._image_pair_dicts.append(image_pair.as_dict()) + self._image_pair_objects.append(image_pair) extra_columns_dict = image_pair.extra_columns_dict if extra_columns_dict: for column_id, value in extra_columns_dict.iteritems(): @@ -141,7 +150,7 @@ class ImagePairSet(object): Uses the KEY__* constants as keys. - Params: + Args: column_ids_in_order: A list of all extracolumn IDs in the desired display order. If unspecified, they will be displayed in alphabetical order. If specified, this list must contain all the extracolumn IDs! @@ -162,6 +171,16 @@ class ImagePairSet(object): key_description = KEY__IMAGESETS__FIELD__DESCRIPTION key_base_url = KEY__IMAGESETS__FIELD__BASE_URL + if gs_utils.GSUtils.is_gs_url(self._image_base_url): + value_base_url = self._convert_gs_url_to_http_url(self._image_base_url) + else: + value_base_url = self._image_base_url + + # We've waited as long as we can to ask ImageDiffDB for details of the + # image diffs, so that it has time to compute them. + if self._image_pair_dicts == None: + self._image_pair_dicts = [ip.as_dict() for ip in self._image_pair_objects] + return { KEY__ROOT__EXTRACOLUMNHEADERS: self._column_headers_as_dict(), KEY__ROOT__EXTRACOLUMNORDER: column_ids_in_order, @@ -169,11 +188,11 @@ class ImagePairSet(object): KEY__ROOT__IMAGESETS: { KEY__IMAGESETS__SET__IMAGE_A: { key_description: self._descriptions[0], - key_base_url: self._image_base_url, + key_base_url: value_base_url, }, KEY__IMAGESETS__SET__IMAGE_B: { key_description: self._descriptions[1], - key_base_url: self._image_base_url, + key_base_url: value_base_url, }, KEY__IMAGESETS__SET__DIFFS: { key_description: 'color difference per channel', @@ -187,3 +206,20 @@ class ImagePairSet(object): }, }, } + + @staticmethod + def _convert_gs_url_to_http_url(gs_url): + """Returns HTTP URL that can be used to download this Google Storage file. + + TODO(epoger): Create functionality like this within gs_utils.py instead of + here? See https://codereview.chromium.org/428493005/ ('create + anyfile_utils.py for copying files between HTTP/GS/local filesystem') + + Args: + gs_url: "gs://bucket/path" format URL + """ + bucket, path = gs_utils.GSUtils.split_gs_url(gs_url) + http_url = 'http://storage.cloud.google.com/' + bucket + if path: + http_url += '/' + path + return http_url diff --git a/gm/rebaseline_server/results.py b/gm/rebaseline_server/results.py index da970af3c7..347666a3b5 100755 --- a/gm/rebaseline_server/results.py +++ b/gm/rebaseline_server/results.py @@ -303,11 +303,22 @@ class BaseComparisons(object): return output_dict @staticmethod - def get_multilevel(input_dict, *keys): - """ Returns input_dict[key1][key2][...], or None if any key is not found. + def get_default(input_dict, default_value, *keys): + """Returns input_dict[key1][key2][...], or default_value. + + If input_dict is None, or any key is missing along the way, this returns + default_value. + + Args: + input_dict: dictionary to look within + key: key indicating which value to return from input_dict + default_value: value to return if input_dict is None or any key cannot + be found along the way """ + if input_dict == None: + return default_value for key in keys: - if input_dict == None: - return None input_dict = input_dict.get(key, None) + if input_dict == None: + return default_value return input_dict diff --git a/gm/rebaseline_server/server.py b/gm/rebaseline_server/server.py index 50367f6f35..6271f97dd2 100755 --- a/gm/rebaseline_server/server.py +++ b/gm/rebaseline_server/server.py @@ -40,6 +40,7 @@ import gm_json # https://codereview.chromium.org/195943004/diff/1/gm/rebaseline_server/server.py#newcode44 # pylint: enable=C0301 import compare_configs +import compare_rendered_pictures import compare_to_expectations import download_actuals import imagediffdb @@ -73,8 +74,11 @@ DEFAULT_PORT = 8888 PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) TRUNK_DIRECTORY = os.path.dirname(os.path.dirname(PARENT_DIRECTORY)) # Directory, relative to PARENT_DIRECTORY, within which the server will serve -# out live results (not static files). -RESULTS_SUBDIR = 'results' +# out image diff data from within the precomputed _SERVER.results . +PRECOMPUTED_RESULTS_SUBDIR = 'results' +# Directory, relative to PARENT_DIRECTORY, within which the server will serve +# out live-generated image diff data. +LIVE_RESULTS_SUBDIR = 'live-results' # Directory, relative to PARENT_DIRECTORY, within which the server will serve # out static files. STATIC_CONTENTS_SUBDIR = 'static' @@ -83,6 +87,10 @@ GENERATED_HTML_SUBDIR = 'generated-html' GENERATED_IMAGES_SUBDIR = 'generated-images' GENERATED_JSON_SUBDIR = 'generated-json' +# Parameters we use within do_GET_live_results() +LIVE_PARAM__SET_A_DIR = 'setADir' +LIVE_PARAM__SET_B_DIR = 'setBDir' + # How often (in seconds) clients should reload while waiting for initial # results to load. RELOAD_INTERVAL_UNTIL_READY = 10 @@ -165,7 +173,7 @@ def _create_index(file_path, config_pairs): '<li><a href="/{static_subdir}/view.html#/view.html?' 'resultsToLoad=/{results_subdir}/{summary_type}">' '{summary_type}</a></li>'.format( - results_subdir=RESULTS_SUBDIR, + results_subdir=PRECOMPUTED_RESULTS_SUBDIR, static_subdir=STATIC_CONTENTS_SUBDIR, summary_type=summary_type)) file_handle.write('</ul>') @@ -193,7 +201,9 @@ class Server(object): json_filename=DEFAULT_JSON_FILENAME, gm_summaries_bucket=DEFAULT_GM_SUMMARIES_BUCKET, port=DEFAULT_PORT, export=False, editable=True, - reload_seconds=0, config_pairs=None, builder_regex_list=None): + reload_seconds=0, config_pairs=None, builder_regex_list=None, + boto_file_path=None, + imagediffdb_threads=imagediffdb.DEFAULT_NUM_WORKER_THREADS): """ Args: actuals_dir: directory under which we will check out the latest actual @@ -212,6 +222,10 @@ class Server(object): don't compare configs at all. builder_regex_list: List of regular expressions specifying which builders we will process. If None, process all builders. + boto_file_path: Path to .boto file giving us credentials to access + Google Storage buckets; if None, we will only be able to access + public GS buckets. + imagediffdb_threads: How many threads to spin up within imagediffdb. """ self._actuals_dir = actuals_dir self._json_filename = json_filename @@ -222,7 +236,13 @@ class Server(object): self._reload_seconds = reload_seconds self._config_pairs = config_pairs or [] self._builder_regex_list = builder_regex_list - self._gs = gs_utils.GSUtils() + self.truncate_results = False + + if boto_file_path: + self._gs = gs_utils.GSUtils(boto_file_path=boto_file_path) + else: + self._gs = gs_utils.GSUtils() + _create_index( file_path=os.path.join( PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR, GENERATED_HTML_SUBDIR, @@ -234,9 +254,16 @@ class Server(object): # 2. the expected or actual results on local disk self.results_rlock = threading.RLock() - # These will be filled in by calls to update_results() + # Create a single ImageDiffDB instance that is used by all our differs. + self._image_diff_db = imagediffdb.ImageDiffDB( + gs=self._gs, + storage_root=os.path.join( + PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR, + GENERATED_IMAGES_SUBDIR), + num_worker_threads=imagediffdb_threads) + + # This will be filled in by calls to update_results() self._results = None - self._image_diff_db = None @property def results(self): @@ -245,6 +272,16 @@ class Server(object): return self._results @property + def image_diff_db(self): + """ Returns reference to our ImageDiffDB object.""" + return self._image_diff_db + + @property + def gs(self): + """ Returns reference to our GSUtils object.""" + return self._gs + + @property def is_exported(self): """ Returns true iff HTTP clients on other hosts are allowed to access this server. """ @@ -343,12 +380,6 @@ class Server(object): compare_to_expectations.DEFAULT_EXPECTATIONS_DIR) _run_command(['gclient', 'sync'], TRUNK_DIRECTORY) - if not self._image_diff_db: - self._image_diff_db = imagediffdb.ImageDiffDB( - storage_root=os.path.join( - PARENT_DIRECTORY, STATIC_CONTENTS_SUBDIR, - GENERATED_IMAGES_SUBDIR)) - self._results = compare_to_expectations.ExpectationComparisons( image_diff_db=self._image_diff_db, actuals_root=self._actuals_dir, @@ -443,7 +474,8 @@ class HTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): normpath = posixpath.normpath(self.path) (dispatcher_name, remainder) = PATHSPLIT_RE.match(normpath).groups() dispatchers = { - RESULTS_SUBDIR: self.do_GET_results, + PRECOMPUTED_RESULTS_SUBDIR: self.do_GET_precomputed_results, + LIVE_RESULTS_SUBDIR: self.do_GET_live_results, STATIC_CONTENTS_SUBDIR: self.do_GET_static, } dispatcher = dispatchers[dispatcher_name] @@ -452,14 +484,15 @@ class HTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): self.send_error(404) raise - def do_GET_results(self, results_type): - """ Handle a GET request for GM results. + def do_GET_precomputed_results(self, results_type): + """ Handle a GET request for part of the precomputed _SERVER.results object. Args: results_type: string indicating which set of results to return; must be one of the results_mod.RESULTS_* constants """ - logging.debug('do_GET_results: sending results of type "%s"' % results_type) + logging.debug('do_GET_precomputed_results: sending results of type "%s"' % + results_type) # Since we must make multiple calls to the ExpectationComparisons object, # grab a reference to it in case it is updated to point at a new # ExpectationComparisons object within another thread. @@ -487,6 +520,23 @@ class HTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): } self.send_json_dict(response_dict) + def do_GET_live_results(self, url_remainder): + """ Handle a GET request for live-generated image diff data. + + Args: + url_remainder: string indicating which image diffs to generate + """ + logging.debug('do_GET_live_results: url_remainder="%s"' % url_remainder) + param_dict = urlparse.parse_qs(url_remainder) + results_obj = compare_rendered_pictures.RenderedPicturesComparisons( + setA_dirs=param_dict[LIVE_PARAM__SET_A_DIR], + setB_dirs=param_dict[LIVE_PARAM__SET_B_DIR], + image_diff_db=_SERVER.image_diff_db, + diff_base_url='/static/generated-images', + gs=_SERVER.gs, truncate_results=_SERVER.truncate_results) + self.send_json_dict(results_obj.get_packaged_results_of_type( + results_mod.KEY__HEADER__RESULTS_ALL)) + def do_GET_static(self, path): """ Handle a GET request for a file under STATIC_CONTENTS_SUBDIR . Only allow serving of files within STATIC_CONTENTS_SUBDIR that is a @@ -643,6 +693,12 @@ def main(): 'actual GM results. If this directory does not ' 'exist, it will be created. Defaults to %(default)s'), default=DEFAULT_ACTUALS_DIR) + parser.add_argument('--boto', + help=('Path to .boto file giving us credentials to access ' + 'Google Storage buckets. If not specified, we will ' + 'only be able to access public GS buckets (and thus ' + 'won\'t be able to download SKP images).'), + default='') # TODO(epoger): Before https://codereview.chromium.org/310093003 , # when this tool downloaded the JSON summaries from skia-autogen, # it had an --actuals-revision the caller could specify to download @@ -688,6 +744,14 @@ def main(): 'By default, we do not reload at all, and you ' 'must restart the server to pick up new data.'), default=0) + parser.add_argument('--threads', type=int, + help=('How many parallel threads we use to download ' + 'images and generate diffs; defaults to ' + '%(default)s'), + default=imagediffdb.DEFAULT_NUM_WORKER_THREADS) + parser.add_argument('--truncate', action='store_true', + help=('FOR TESTING ONLY: truncate the set of images we ' + 'process, to speed up testing.')) args = parser.parse_args() if args.compare_configs: config_pairs = CONFIG_PAIRS_TO_COMPARE @@ -700,7 +764,10 @@ def main(): gm_summaries_bucket=args.gm_summaries_bucket, port=args.port, export=args.export, editable=args.editable, reload_seconds=args.reload, config_pairs=config_pairs, - builder_regex_list=args.builders) + builder_regex_list=args.builders, boto_file_path=args.boto, + imagediffdb_threads=args.threads) + if args.truncate: + _SERVER.truncate_results = True _SERVER.run() diff --git a/gm/rebaseline_server/static/live-loader.js b/gm/rebaseline_server/static/live-loader.js new file mode 100644 index 0000000000..418e9cc708 --- /dev/null +++ b/gm/rebaseline_server/static/live-loader.js @@ -0,0 +1,1031 @@ +/* + * Loader: + * Reads GM result reports written out by results.py, and imports + * them into $scope.extraColumnHeaders and $scope.imagePairs . + */ +var Loader = angular.module( + 'Loader', + ['ConstantsModule'] +); + +Loader.directive( + 'resultsUpdatedCallbackDirective', + ['$timeout', + function($timeout) { + return function(scope, element, attrs) { + if (scope.$last) { + $timeout(function() { + scope.resultsUpdatedCallback(); + }); + } + }; + } + ] +); + +// TODO(epoger): Combine ALL of our filtering operations (including +// truncation) into this one filter, so that runs most efficiently? +// (We would have to make sure truncation still took place after +// sorting, though.) +Loader.filter( + 'removeHiddenImagePairs', + function(constants) { + return function(unfilteredImagePairs, filterableColumnNames, showingColumnValues, + viewingTab) { + var filteredImagePairs = []; + for (var i = 0; i < unfilteredImagePairs.length; i++) { + var imagePair = unfilteredImagePairs[i]; + var extraColumnValues = imagePair[constants.KEY__IMAGEPAIRS__EXTRACOLUMNS]; + var allColumnValuesAreVisible = true; + // Loop over all columns, and if any of them contain values not found in + // showingColumnValues[columnName], don't include this imagePair. + // + // We use this same filtering mechanism regardless of whether each column + // has USE_FREEFORM_FILTER set or not; if that flag is set, then we will + // have already used the freeform text entry block to populate + // showingColumnValues[columnName]. + for (var j = 0; j < filterableColumnNames.length; j++) { + var columnName = filterableColumnNames[j]; + var columnValue = extraColumnValues[columnName]; + if (!showingColumnValues[columnName][columnValue]) { + allColumnValuesAreVisible = false; + break; + } + } + if (allColumnValuesAreVisible && (viewingTab == imagePair.tab)) { + filteredImagePairs.push(imagePair); + } + } + return filteredImagePairs; + }; + } +); + +/** + * Limit the input imagePairs to some max number, and merge identical rows + * (adjacent rows which have the same (imageA, imageB) pair). + * + * @param unfilteredImagePairs imagePairs to filter + * @param maxPairs maximum number of pairs to output, or <0 for no limit + * @param mergeIdenticalRows if true, merge identical rows by setting + * ROWSPAN>1 on the first merged row, and ROWSPAN=0 for the rest + */ +Loader.filter( + 'mergeAndLimit', + function(constants) { + return function(unfilteredImagePairs, maxPairs, mergeIdenticalRows) { + var numPairs = unfilteredImagePairs.length; + if ((maxPairs > 0) && (maxPairs < numPairs)) { + numPairs = maxPairs; + } + var filteredImagePairs = []; + if (!mergeIdenticalRows || (numPairs == 1)) { + // Take a shortcut if we're not merging identical rows. + // We still need to set ROWSPAN to 1 for each row, for the HTML viewer. + for (var i = numPairs-1; i >= 0; i--) { + var imagePair = unfilteredImagePairs[i]; + imagePair[constants.KEY__IMAGEPAIRS__ROWSPAN] = 1; + filteredImagePairs[i] = imagePair; + } + } else if (numPairs > 1) { + // General case--there are at least 2 rows, so we may need to merge some. + // Work from the bottom up, so we can keep a running total of how many + // rows should be merged, and set ROWSPAN of the top row accordingly. + var imagePair = unfilteredImagePairs[numPairs-1]; + var nextRowImageAUrl = imagePair[constants.KEY__IMAGEPAIRS__IMAGE_A_URL]; + var nextRowImageBUrl = imagePair[constants.KEY__IMAGEPAIRS__IMAGE_B_URL]; + imagePair[constants.KEY__IMAGEPAIRS__ROWSPAN] = 1; + filteredImagePairs[numPairs-1] = imagePair; + for (var i = numPairs-2; i >= 0; i--) { + imagePair = unfilteredImagePairs[i]; + var thisRowImageAUrl = imagePair[constants.KEY__IMAGEPAIRS__IMAGE_A_URL]; + var thisRowImageBUrl = imagePair[constants.KEY__IMAGEPAIRS__IMAGE_B_URL]; + if ((thisRowImageAUrl == nextRowImageAUrl) && + (thisRowImageBUrl == nextRowImageBUrl)) { + imagePair[constants.KEY__IMAGEPAIRS__ROWSPAN] = + filteredImagePairs[i+1][constants.KEY__IMAGEPAIRS__ROWSPAN] + 1; + filteredImagePairs[i+1][constants.KEY__IMAGEPAIRS__ROWSPAN] = 0; + } else { + imagePair[constants.KEY__IMAGEPAIRS__ROWSPAN] = 1; + nextRowImageAUrl = thisRowImageAUrl; + nextRowImageBUrl = thisRowImageBUrl; + } + filteredImagePairs[i] = imagePair; + } + } else { + // No results. + } + return filteredImagePairs; + }; + } +); + + +Loader.controller( + 'Loader.Controller', + function($scope, $http, $filter, $location, $log, $timeout, constants) { + $scope.readyToDisplay = false; + $scope.constants = constants; + $scope.windowTitle = "Loading GM Results..."; + $scope.setADir = $location.search().setADir; + $scope.setBDir = $location.search().setBDir; + $scope.loadingMessage = "please wait..."; + + /** + * On initial page load, load a full dictionary of results. + * Once the dictionary is loaded, unhide the page elements so they can + * render the data. + */ + var liveQueryUrl = + "/live-results/setADir=" + encodeURIComponent($scope.setADir) + + "&setBDir=" + encodeURIComponent($scope.setBDir); + $http.get(liveQueryUrl).success( + function(data, status, header, config) { + var dataHeader = data[constants.KEY__ROOT__HEADER]; + if (dataHeader[constants.KEY__HEADER__SCHEMA_VERSION] != + constants.VALUE__HEADER__SCHEMA_VERSION) { + $scope.loadingMessage = "ERROR: Got JSON file with schema version " + + dataHeader[constants.KEY__HEADER__SCHEMA_VERSION] + + " but expected schema version " + + constants.VALUE__HEADER__SCHEMA_VERSION; + } else if (dataHeader[constants.KEY__HEADER__IS_STILL_LOADING]) { + // Apply the server's requested reload delay to local time, + // so we will wait the right number of seconds regardless of clock + // skew between client and server. + var reloadDelayInSeconds = + dataHeader[constants.KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE] - + dataHeader[constants.KEY__HEADER__TIME_UPDATED]; + var timeNow = new Date().getTime(); + var timeToReload = timeNow + reloadDelayInSeconds * 1000; + $scope.loadingMessage = + "server is still loading results; will retry at " + + $scope.localTimeString(timeToReload / 1000); + $timeout( + function(){location.reload();}, + timeToReload - timeNow); + } else { + $scope.loadingMessage = "processing data, please wait..."; + + $scope.header = dataHeader; + $scope.extraColumnHeaders = data[constants.KEY__ROOT__EXTRACOLUMNHEADERS]; + $scope.orderedColumnNames = data[constants.KEY__ROOT__EXTRACOLUMNORDER]; + $scope.imagePairs = data[constants.KEY__ROOT__IMAGEPAIRS]; + $scope.imageSets = data[constants.KEY__ROOT__IMAGESETS]; + $scope.sortColumnSubdict = constants.KEY__IMAGEPAIRS__DIFFERENCES; + $scope.sortColumnKey = constants.KEY__DIFFERENCES__PERCEPTUAL_DIFF; + + $scope.showSubmitAdvancedSettings = false; + $scope.submitAdvancedSettings = {}; + $scope.submitAdvancedSettings[ + constants.KEY__EXPECTATIONS__REVIEWED] = true; + $scope.submitAdvancedSettings[ + constants.KEY__EXPECTATIONS__IGNOREFAILURE] = false; + $scope.submitAdvancedSettings['bug'] = ''; + + // Create the list of tabs (lists into which the user can file each + // test). This may vary, depending on isEditable. + $scope.tabs = [ + 'Unfiled', 'Hidden' + ]; + if (dataHeader[constants.KEY__HEADER__IS_EDITABLE]) { + $scope.tabs = $scope.tabs.concat( + ['Pending Approval']); + } + $scope.defaultTab = $scope.tabs[0]; + $scope.viewingTab = $scope.defaultTab; + + // Track the number of results on each tab. + $scope.numResultsPerTab = {}; + for (var i = 0; i < $scope.tabs.length; i++) { + $scope.numResultsPerTab[$scope.tabs[i]] = 0; + } + $scope.numResultsPerTab[$scope.defaultTab] = $scope.imagePairs.length; + + // Add index and tab fields to all records. + for (var i = 0; i < $scope.imagePairs.length; i++) { + $scope.imagePairs[i].index = i; + $scope.imagePairs[i].tab = $scope.defaultTab; + } + + // Arrays within which the user can toggle individual elements. + $scope.selectedImagePairs = []; + + // Set up filters. + // + // filterableColumnNames is a list of all column names we can filter on. + // allColumnValues[columnName] is a list of all known values + // for a given column. + // showingColumnValues[columnName] is a set indicating which values + // in a given column would cause us to show a row, rather than hiding it. + // + // columnStringMatch[columnName] is a string used as a pattern to generate + // showingColumnValues[columnName] for columns we filter using free-form text. + // It is ignored for any columns with USE_FREEFORM_FILTER == false. + $scope.filterableColumnNames = []; + $scope.allColumnValues = {}; + $scope.showingColumnValues = {}; + $scope.columnStringMatch = {}; + + angular.forEach( + Object.keys($scope.extraColumnHeaders), + function(columnName) { + var columnHeader = $scope.extraColumnHeaders[columnName]; + if (columnHeader[constants.KEY__EXTRACOLUMNHEADERS__IS_FILTERABLE]) { + $scope.filterableColumnNames.push(columnName); + $scope.allColumnValues[columnName] = $scope.columnSliceOf2DArray( + columnHeader[constants.KEY__EXTRACOLUMNHEADERS__VALUES_AND_COUNTS], 0); + $scope.showingColumnValues[columnName] = {}; + $scope.toggleValuesInSet($scope.allColumnValues[columnName], + $scope.showingColumnValues[columnName]); + $scope.columnStringMatch[columnName] = ""; + } + } + ); + + // TODO(epoger): Special handling for RESULT_TYPE column: + // by default, show only KEY__RESULT_TYPE__FAILED results + $scope.showingColumnValues[constants.KEY__EXTRACOLUMNS__RESULT_TYPE] = {}; + $scope.showingColumnValues[constants.KEY__EXTRACOLUMNS__RESULT_TYPE][ + constants.KEY__RESULT_TYPE__FAILED] = true; + + // Set up mapping for URL parameters. + // parameter name -> copier object to load/save parameter value + $scope.queryParameters.map = { + 'setADir': $scope.queryParameters.copiers.simple, + 'setBDir': $scope.queryParameters.copiers.simple, + 'displayLimitPending': $scope.queryParameters.copiers.simple, + 'showThumbnailsPending': $scope.queryParameters.copiers.simple, + 'mergeIdenticalRowsPending': $scope.queryParameters.copiers.simple, + 'imageSizePending': $scope.queryParameters.copiers.simple, + 'sortColumnSubdict': $scope.queryParameters.copiers.simple, + 'sortColumnKey': $scope.queryParameters.copiers.simple, + }; + // Some parameters are handled differently based on whether they USE_FREEFORM_FILTER. + angular.forEach( + $scope.filterableColumnNames, + function(columnName) { + if ($scope.extraColumnHeaders[columnName] + [constants.KEY__EXTRACOLUMNHEADERS__USE_FREEFORM_FILTER]) { + $scope.queryParameters.map[columnName] = + $scope.queryParameters.copiers.columnStringMatch; + } else { + $scope.queryParameters.map[columnName] = + $scope.queryParameters.copiers.showingColumnValuesSet; + } + } + ); + + // If any defaults were overridden in the URL, get them now. + $scope.queryParameters.load(); + + // Any image URLs which are relative should be relative to the JSON + // file's source directory; absolute URLs should be left alone. + var baseUrlKey = constants.KEY__IMAGESETS__FIELD__BASE_URL; + angular.forEach( + $scope.imageSets, + function(imageSet) { + var baseUrl = imageSet[baseUrlKey]; + if ((baseUrl.substring(0, 1) != '/') && + (baseUrl.indexOf('://') == -1)) { + imageSet[baseUrlKey] = '/' + baseUrl; + } + } + ); + + $scope.readyToDisplay = true; + $scope.updateResults(); + $scope.loadingMessage = ""; + $scope.windowTitle = "Current GM Results"; + } + } + ).error( + function(data, status, header, config) { + $scope.loadingMessage = "FAILED to load."; + $scope.windowTitle = "Failed to Load GM Results"; + } + ); + + + // + // Select/Clear/Toggle all tests. + // + + /** + * Select all currently showing tests. + */ + $scope.selectAllImagePairs = function() { + var numImagePairsShowing = $scope.limitedImagePairs.length; + for (var i = 0; i < numImagePairsShowing; i++) { + var index = $scope.limitedImagePairs[i].index; + if (!$scope.isValueInArray(index, $scope.selectedImagePairs)) { + $scope.toggleValueInArray(index, $scope.selectedImagePairs); + } + } + } + + /** + * Deselect all currently showing tests. + */ + $scope.clearAllImagePairs = function() { + var numImagePairsShowing = $scope.limitedImagePairs.length; + for (var i = 0; i < numImagePairsShowing; i++) { + var index = $scope.limitedImagePairs[i].index; + if ($scope.isValueInArray(index, $scope.selectedImagePairs)) { + $scope.toggleValueInArray(index, $scope.selectedImagePairs); + } + } + } + + /** + * Toggle selection of all currently showing tests. + */ + $scope.toggleAllImagePairs = function() { + var numImagePairsShowing = $scope.limitedImagePairs.length; + for (var i = 0; i < numImagePairsShowing; i++) { + var index = $scope.limitedImagePairs[i].index; + $scope.toggleValueInArray(index, $scope.selectedImagePairs); + } + } + + /** + * Toggle selection state of a subset of the currently showing tests. + * + * @param startIndex index within $scope.limitedImagePairs of the first + * test to toggle selection state of + * @param num number of tests (in a contiguous block) to toggle + */ + $scope.toggleSomeImagePairs = function(startIndex, num) { + var numImagePairsShowing = $scope.limitedImagePairs.length; + for (var i = startIndex; i < startIndex + num; i++) { + var index = $scope.limitedImagePairs[i].index; + $scope.toggleValueInArray(index, $scope.selectedImagePairs); + } + } + + + // + // Tab operations. + // + + /** + * Change the selected tab. + * + * @param tab (string): name of the tab to select + */ + $scope.setViewingTab = function(tab) { + $scope.viewingTab = tab; + $scope.updateResults(); + } + + /** + * Move the imagePairs in $scope.selectedImagePairs to a different tab, + * and then clear $scope.selectedImagePairs. + * + * @param newTab (string): name of the tab to move the tests to + */ + $scope.moveSelectedImagePairsToTab = function(newTab) { + $scope.moveImagePairsToTab($scope.selectedImagePairs, newTab); + $scope.selectedImagePairs = []; + $scope.updateResults(); + } + + /** + * Move a subset of $scope.imagePairs to a different tab. + * + * @param imagePairIndices (array of ints): indices into $scope.imagePairs + * indicating which test results to move + * @param newTab (string): name of the tab to move the tests to + */ + $scope.moveImagePairsToTab = function(imagePairIndices, newTab) { + var imagePairIndex; + var numImagePairs = imagePairIndices.length; + for (var i = 0; i < numImagePairs; i++) { + imagePairIndex = imagePairIndices[i]; + $scope.numResultsPerTab[$scope.imagePairs[imagePairIndex].tab]--; + $scope.imagePairs[imagePairIndex].tab = newTab; + } + $scope.numResultsPerTab[newTab] += numImagePairs; + } + + + // + // $scope.queryParameters: + // Transfer parameter values between $scope and the URL query string. + // + $scope.queryParameters = {}; + + // load and save functions for parameters of each type + // (load a parameter value into $scope from nameValuePairs, + // save a parameter value from $scope into nameValuePairs) + $scope.queryParameters.copiers = { + 'simple': { + 'load': function(nameValuePairs, name) { + var value = nameValuePairs[name]; + if (value) { + $scope[name] = value; + } + }, + 'save': function(nameValuePairs, name) { + nameValuePairs[name] = $scope[name]; + } + }, + + 'columnStringMatch': { + 'load': function(nameValuePairs, name) { + var value = nameValuePairs[name]; + if (value) { + $scope.columnStringMatch[name] = value; + } + }, + 'save': function(nameValuePairs, name) { + nameValuePairs[name] = $scope.columnStringMatch[name]; + } + }, + + 'showingColumnValuesSet': { + 'load': function(nameValuePairs, name) { + var value = nameValuePairs[name]; + if (value) { + var valueArray = value.split(','); + $scope.showingColumnValues[name] = {}; + $scope.toggleValuesInSet(valueArray, $scope.showingColumnValues[name]); + } + }, + 'save': function(nameValuePairs, name) { + nameValuePairs[name] = Object.keys($scope.showingColumnValues[name]).join(','); + } + }, + + }; + + // Loads all parameters into $scope from the URL query string; + // any which are not found within the URL will keep their current value. + $scope.queryParameters.load = function() { + var nameValuePairs = $location.search(); + + // If urlSchemaVersion is not specified, we assume the current version. + var urlSchemaVersion = constants.URL_VALUE__SCHEMA_VERSION__CURRENT; + if (constants.URL_KEY__SCHEMA_VERSION in nameValuePairs) { + urlSchemaVersion = nameValuePairs[constants.URL_KEY__SCHEMA_VERSION]; + } else if ('hiddenResultTypes' in nameValuePairs) { + // The combination of: + // - absence of an explicit urlSchemaVersion, and + // - presence of the old 'hiddenResultTypes' field + // tells us that the URL is from the original urlSchemaVersion. + // See https://codereview.chromium.org/367173002/ + urlSchemaVersion = 0; + } + $scope.urlSchemaVersionLoaded = urlSchemaVersion; + + if (urlSchemaVersion != constants.URL_VALUE__SCHEMA_VERSION__CURRENT) { + nameValuePairs = $scope.upconvertUrlNameValuePairs(nameValuePairs, urlSchemaVersion); + } + angular.forEach($scope.queryParameters.map, + function(copier, paramName) { + copier.load(nameValuePairs, paramName); + } + ); + }; + + // Saves all parameters from $scope into the URL query string. + $scope.queryParameters.save = function() { + var nameValuePairs = {}; + nameValuePairs[constants.URL_KEY__SCHEMA_VERSION] = constants.URL_VALUE__SCHEMA_VERSION__CURRENT; + angular.forEach($scope.queryParameters.map, + function(copier, paramName) { + copier.save(nameValuePairs, paramName); + } + ); + $location.search(nameValuePairs); + }; + + /** + * Converts URL name/value pairs that were stored by a previous urlSchemaVersion + * to the currently needed format. + * + * @param oldNValuePairs name/value pairs found in the loaded URL + * @param oldUrlSchemaVersion which version of the schema was used to generate that URL + * + * @returns nameValuePairs as needed by the current URL parser + */ + $scope.upconvertUrlNameValuePairs = function(oldNameValuePairs, oldUrlSchemaVersion) { + var newNameValuePairs = {}; + angular.forEach(oldNameValuePairs, + function(value, name) { + if (oldUrlSchemaVersion < 1) { + if ('hiddenConfigs' == name) { + name = 'config'; + var valueSet = {}; + $scope.toggleValuesInSet(value.split(','), valueSet); + $scope.toggleValuesInSet( + $scope.allColumnValues[constants.KEY__EXTRACOLUMNS__CONFIG], + valueSet); + value = Object.keys(valueSet).join(','); + } else if ('hiddenResultTypes' == name) { + name = 'resultType'; + var valueSet = {}; + $scope.toggleValuesInSet(value.split(','), valueSet); + $scope.toggleValuesInSet( + $scope.allColumnValues[constants.KEY__EXTRACOLUMNS__RESULT_TYPE], + valueSet); + value = Object.keys(valueSet).join(','); + } + } + + newNameValuePairs[name] = value; + } + ); + return newNameValuePairs; + } + + + // + // updateResults() and friends. + // + + /** + * Set $scope.areUpdatesPending (to enable/disable the Update Results + * button). + * + * TODO(epoger): We could reduce the amount of code by just setting the + * variable directly (from, e.g., a button's ng-click handler). But when + * I tried that, the HTML elements depending on the variable did not get + * updated. + * It turns out that this is due to variable scoping within an ng-repeat + * element; see http://stackoverflow.com/questions/15388344/behavior-of-assignment-expression-invoked-by-ng-click-within-ng-repeat + * + * @param val boolean value to set $scope.areUpdatesPending to + */ + $scope.setUpdatesPending = function(val) { + $scope.areUpdatesPending = val; + } + + /** + * Update the displayed results, based on filters/settings, + * and call $scope.queryParameters.save() so that the new filter results + * can be bookmarked. + */ + $scope.updateResults = function() { + $scope.renderStartTime = window.performance.now(); + $log.debug("renderStartTime: " + $scope.renderStartTime); + $scope.displayLimit = $scope.displayLimitPending; + $scope.mergeIdenticalRows = $scope.mergeIdenticalRowsPending; + + // For each USE_FREEFORM_FILTER column, populate showingColumnValues. + // This is more efficient than applying the freeform filter within the + // tight loop in removeHiddenImagePairs. + angular.forEach( + $scope.filterableColumnNames, + function(columnName) { + var columnHeader = $scope.extraColumnHeaders[columnName]; + if (columnHeader[constants.KEY__EXTRACOLUMNHEADERS__USE_FREEFORM_FILTER]) { + var columnStringMatch = $scope.columnStringMatch[columnName]; + var showingColumnValues = {}; + angular.forEach( + $scope.allColumnValues[columnName], + function(columnValue) { + if (-1 != columnValue.indexOf(columnStringMatch)) { + showingColumnValues[columnValue] = true; + } + } + ); + $scope.showingColumnValues[columnName] = showingColumnValues; + } + } + ); + + // TODO(epoger): Every time we apply a filter, AngularJS creates + // another copy of the array. Is there a way we can filter out + // the imagePairs as they are displayed, rather than storing multiple + // array copies? (For better performance.) + + if ($scope.viewingTab == $scope.defaultTab) { + + // TODO(epoger): Until we allow the user to reverse sort order, + // there are certain columns we want to sort in a different order. + var doReverse = ( + ($scope.sortColumnKey == + constants.KEY__DIFFERENCES__PERCENT_DIFF_PIXELS) || + ($scope.sortColumnKey == + constants.KEY__DIFFERENCES__PERCEPTUAL_DIFF)); + + $scope.filteredImagePairs = + $filter("orderBy")( + $filter("removeHiddenImagePairs")( + $scope.imagePairs, + $scope.filterableColumnNames, + $scope.showingColumnValues, + $scope.viewingTab + ), + [$scope.getSortColumnValue, $scope.getSecondOrderSortValue], + doReverse); + $scope.limitedImagePairs = $filter("mergeAndLimit")( + $scope.filteredImagePairs, $scope.displayLimit, $scope.mergeIdenticalRows); + } else { + $scope.filteredImagePairs = + $filter("orderBy")( + $filter("filter")( + $scope.imagePairs, + {tab: $scope.viewingTab}, + true + ), + [$scope.getSortColumnValue, $scope.getSecondOrderSortValue]); + $scope.limitedImagePairs = $filter("mergeAndLimit")( + $scope.filteredImagePairs, -1, $scope.mergeIdenticalRows); + } + $scope.showThumbnails = $scope.showThumbnailsPending; + $scope.imageSize = $scope.imageSizePending; + $scope.setUpdatesPending(false); + $scope.queryParameters.save(); + } + + /** + * This function is called when the results have been completely rendered + * after updateResults(). + */ + $scope.resultsUpdatedCallback = function() { + $scope.renderEndTime = window.performance.now(); + $log.debug("renderEndTime: " + $scope.renderEndTime); + } + + /** + * Re-sort the displayed results. + * + * @param subdict (string): which KEY__IMAGEPAIRS__* subdictionary + * the sort column key is within, or 'none' if the sort column + * key is one of KEY__IMAGEPAIRS__* + * @param key (string): sort by value associated with this key in subdict + */ + $scope.sortResultsBy = function(subdict, key) { + $scope.sortColumnSubdict = subdict; + $scope.sortColumnKey = key; + $scope.updateResults(); + } + + /** + * For a particular ImagePair, return the value of the column we are + * sorting on (according to $scope.sortColumnSubdict and + * $scope.sortColumnKey). + * + * @param imagePair: imagePair to get a column value out of. + */ + $scope.getSortColumnValue = function(imagePair) { + if ($scope.sortColumnSubdict in imagePair) { + return imagePair[$scope.sortColumnSubdict][$scope.sortColumnKey]; + } else if ($scope.sortColumnKey in imagePair) { + return imagePair[$scope.sortColumnKey]; + } else { + return undefined; + } + } + + /** + * For a particular ImagePair, return the value we use for the + * second-order sort (tiebreaker when multiple rows have + * the same getSortColumnValue()). + * + * We join the imageA and imageB urls for this value, so that we merge + * adjacent rows as much as possible. + * + * @param imagePair: imagePair to get a column value out of. + */ + $scope.getSecondOrderSortValue = function(imagePair) { + return imagePair[constants.KEY__IMAGEPAIRS__IMAGE_A_URL] + "-vs-" + + imagePair[constants.KEY__IMAGEPAIRS__IMAGE_B_URL]; + } + + /** + * Set $scope.columnStringMatch[name] = value, and update results. + * + * @param name + * @param value + */ + $scope.setColumnStringMatch = function(name, value) { + $scope.columnStringMatch[name] = value; + $scope.updateResults(); + } + + /** + * Update $scope.showingColumnValues[columnName] and $scope.columnStringMatch[columnName] + * so that ONLY entries with this columnValue are showing, and update the visible results. + * (We update both of those, so we cover both freeform and checkbox filtered columns.) + * + * @param columnName + * @param columnValue + */ + $scope.showOnlyColumnValue = function(columnName, columnValue) { + $scope.columnStringMatch[columnName] = columnValue; + $scope.showingColumnValues[columnName] = {}; + $scope.toggleValueInSet(columnValue, $scope.showingColumnValues[columnName]); + $scope.updateResults(); + } + + /** + * Update $scope.showingColumnValues[columnName] and $scope.columnStringMatch[columnName] + * so that ALL entries are showing, and update the visible results. + * (We update both of those, so we cover both freeform and checkbox filtered columns.) + * + * @param columnName + */ + $scope.showAllColumnValues = function(columnName) { + $scope.columnStringMatch[columnName] = ""; + $scope.showingColumnValues[columnName] = {}; + $scope.toggleValuesInSet($scope.allColumnValues[columnName], + $scope.showingColumnValues[columnName]); + $scope.updateResults(); + } + + + // + // Operations for sending info back to the server. + // + + /** + * Tell the server that the actual results of these particular tests + * are acceptable. + * + * TODO(epoger): This assumes that the original expectations are in + * imageSetA, and the actuals are in imageSetB. + * + * @param imagePairsSubset an array of test results, most likely a subset of + * $scope.imagePairs (perhaps with some modifications) + */ + $scope.submitApprovals = function(imagePairsSubset) { + $scope.submitPending = true; + + // Convert bug text field to null or 1-item array. + var bugs = null; + var bugNumber = parseInt($scope.submitAdvancedSettings['bug']); + if (!isNaN(bugNumber)) { + bugs = [bugNumber]; + } + + // TODO(epoger): This is a suboptimal way to prevent users from + // rebaselining failures in alternative renderModes, but it does work. + // For a better solution, see + // https://code.google.com/p/skia/issues/detail?id=1748 ('gm: add new + // result type, RenderModeMismatch') + var encounteredComparisonConfig = false; + + var updatedExpectations = []; + for (var i = 0; i < imagePairsSubset.length; i++) { + var imagePair = imagePairsSubset[i]; + var updatedExpectation = {}; + updatedExpectation[constants.KEY__IMAGEPAIRS__EXPECTATIONS] = + imagePair[constants.KEY__IMAGEPAIRS__EXPECTATIONS]; + updatedExpectation[constants.KEY__IMAGEPAIRS__EXTRACOLUMNS] = + imagePair[constants.KEY__IMAGEPAIRS__EXTRACOLUMNS]; + // IMAGE_B_URL contains the actual image (which is now the expectation) + updatedExpectation[constants.KEY__IMAGEPAIRS__IMAGE_B_URL] = + imagePair[constants.KEY__IMAGEPAIRS__IMAGE_B_URL]; + if (0 == updatedExpectation[constants.KEY__IMAGEPAIRS__EXTRACOLUMNS] + [constants.KEY__EXTRACOLUMNS__CONFIG] + .indexOf('comparison-')) { + encounteredComparisonConfig = true; + } + + // Advanced settings... + if (null == updatedExpectation[constants.KEY__IMAGEPAIRS__EXPECTATIONS]) { + updatedExpectation[constants.KEY__IMAGEPAIRS__EXPECTATIONS] = {}; + } + updatedExpectation[constants.KEY__IMAGEPAIRS__EXPECTATIONS] + [constants.KEY__EXPECTATIONS__REVIEWED] = + $scope.submitAdvancedSettings[ + constants.KEY__EXPECTATIONS__REVIEWED]; + if (true == $scope.submitAdvancedSettings[ + constants.KEY__EXPECTATIONS__IGNOREFAILURE]) { + // if it's false, don't send it at all (just keep the default) + updatedExpectation[constants.KEY__IMAGEPAIRS__EXPECTATIONS] + [constants.KEY__EXPECTATIONS__IGNOREFAILURE] = true; + } + updatedExpectation[constants.KEY__IMAGEPAIRS__EXPECTATIONS] + [constants.KEY__EXPECTATIONS__BUGS] = bugs; + + updatedExpectations.push(updatedExpectation); + } + if (encounteredComparisonConfig) { + alert("Approval failed -- you cannot approve results with config " + + "type comparison-*"); + $scope.submitPending = false; + return; + } + var modificationData = {}; + modificationData[constants.KEY__EDITS__MODIFICATIONS] = + updatedExpectations; + modificationData[constants.KEY__EDITS__OLD_RESULTS_HASH] = + $scope.header[constants.KEY__HEADER__DATAHASH]; + modificationData[constants.KEY__EDITS__OLD_RESULTS_TYPE] = + $scope.header[constants.KEY__HEADER__TYPE]; + $http({ + method: "POST", + url: "/edits", + data: modificationData + }).success(function(data, status, headers, config) { + var imagePairIndicesToMove = []; + for (var i = 0; i < imagePairsSubset.length; i++) { + imagePairIndicesToMove.push(imagePairsSubset[i].index); + } + $scope.moveImagePairsToTab(imagePairIndicesToMove, + "HackToMakeSureThisImagePairDisappears"); + $scope.updateResults(); + alert("New baselines submitted successfully!\n\n" + + "You still need to commit the updated expectations files on " + + "the server side to the Skia repo.\n\n" + + "When you click OK, your web UI will reload; after that " + + "completes, you will see the updated data (once the server has " + + "finished loading the update results into memory!) and you can " + + "submit more baselines if you want."); + // I don't know why, but if I just call reload() here it doesn't work. + // Making a timer call it fixes the problem. + $timeout(function(){location.reload();}, 1); + }).error(function(data, status, headers, config) { + alert("There was an error submitting your baselines.\n\n" + + "Please see server-side log for details."); + $scope.submitPending = false; + }); + } + + + // + // Operations we use to mimic Set semantics, in such a way that + // checking for presence within the Set is as fast as possible. + // But getting a list of all values within the Set is not necessarily + // possible. + // TODO(epoger): move into a separate .js file? + // + + /** + * Returns the number of values present within set "set". + * + * @param set an Object which we use to mimic set semantics + */ + $scope.setSize = function(set) { + return Object.keys(set).length; + } + + /** + * Returns true if value "value" is present within set "set". + * + * @param value a value of any type + * @param set an Object which we use to mimic set semantics + * (this should make isValueInSet faster than if we used an Array) + */ + $scope.isValueInSet = function(value, set) { + return (true == set[value]); + } + + /** + * If value "value" is already in set "set", remove it; otherwise, add it. + * + * @param value a value of any type + * @param set an Object which we use to mimic set semantics + */ + $scope.toggleValueInSet = function(value, set) { + if (true == set[value]) { + delete set[value]; + } else { + set[value] = true; + } + } + + /** + * For each value in valueArray, call toggleValueInSet(value, set). + * + * @param valueArray + * @param set + */ + $scope.toggleValuesInSet = function(valueArray, set) { + var arrayLength = valueArray.length; + for (var i = 0; i < arrayLength; i++) { + $scope.toggleValueInSet(valueArray[i], set); + } + } + + + // + // Array operations; similar to our Set operations, but operate on a + // Javascript Array so we *can* easily get a list of all values in the Set. + // TODO(epoger): move into a separate .js file? + // + + /** + * Returns true if value "value" is present within array "array". + * + * @param value a value of any type + * @param array a Javascript Array + */ + $scope.isValueInArray = function(value, array) { + return (-1 != array.indexOf(value)); + } + + /** + * If value "value" is already in array "array", remove it; otherwise, + * add it. + * + * @param value a value of any type + * @param array a Javascript Array + */ + $scope.toggleValueInArray = function(value, array) { + var i = array.indexOf(value); + if (-1 == i) { + array.push(value); + } else { + array.splice(i, 1); + } + } + + + // + // Miscellaneous utility functions. + // TODO(epoger): move into a separate .js file? + // + + /** + * Returns a single "column slice" of a 2D array. + * + * For example, if array is: + * [[A0, A1], + * [B0, B1], + * [C0, C1]] + * and index is 0, this this will return: + * [A0, B0, C0] + * + * @param array a Javascript Array + * @param column (numeric): index within each row array + */ + $scope.columnSliceOf2DArray = function(array, column) { + var slice = []; + var numRows = array.length; + for (var row = 0; row < numRows; row++) { + slice.push(array[row][column]); + } + return slice; + } + + /** + * Returns a human-readable (in local time zone) time string for a + * particular moment in time. + * + * @param secondsPastEpoch (numeric): seconds past epoch in UTC + */ + $scope.localTimeString = function(secondsPastEpoch) { + var d = new Date(secondsPastEpoch * 1000); + return d.toString(); + } + + /** + * Returns a hex color string (such as "#aabbcc") for the given RGB values. + * + * @param r (numeric): red channel value, 0-255 + * @param g (numeric): green channel value, 0-255 + * @param b (numeric): blue channel value, 0-255 + */ + $scope.hexColorString = function(r, g, b) { + var rString = r.toString(16); + if (r < 16) { + rString = "0" + rString; + } + var gString = g.toString(16); + if (g < 16) { + gString = "0" + gString; + } + var bString = b.toString(16); + if (b < 16) { + bString = "0" + bString; + } + return '#' + rString + gString + bString; + } + + /** + * Returns a hex color string (such as "#aabbcc") for the given brightness. + * + * @param brightnessString (string): 0-255, 0 is completely black + * + * TODO(epoger): It might be nice to tint the color when it's not completely + * black or completely white. + */ + $scope.brightnessStringToHexColor = function(brightnessString) { + var v = parseInt(brightnessString); + return $scope.hexColorString(v, v, v); + } + + /** + * Returns the last path component of image diff URL for a given ImagePair. + * + * Depending on which diff this is (whitediffs, pixeldiffs, etc.) this + * will be relative to different base URLs. + * + * We must keep this function in sync with _get_difference_locator() in + * ../imagediffdb.py + * + * @param imagePair: ImagePair to generate image diff URL for + */ + $scope.getImageDiffRelativeUrl = function(imagePair) { + var before = + imagePair[constants.KEY__IMAGEPAIRS__IMAGE_A_URL] + "-vs-" + + imagePair[constants.KEY__IMAGEPAIRS__IMAGE_B_URL]; + return before.replace(/[^\w\-]/g, "_") + ".png"; + } + + } +); diff --git a/gm/rebaseline_server/static/live-view.html b/gm/rebaseline_server/static/live-view.html new file mode 100644 index 0000000000..9e22ed4998 --- /dev/null +++ b/gm/rebaseline_server/static/live-view.html @@ -0,0 +1,420 @@ +<!DOCTYPE html> + +<html ng-app="Loader" ng-controller="Loader.Controller"> + +<head> + <title ng-bind="windowTitle"></title> + <script src="https://ajax.googleapis.com/ajax/libs/angularjs/1.1.5/angular.js"></script> + <script src="constants.js"></script> + <script src="live-loader.js"></script> + <link rel="stylesheet" href="view.css"> +</head> + +<body> + <h2> + Instructions, roadmap, etc. are at + <a href="http://tinyurl.com/SkiaRebaselineServer"> + http://tinyurl.com/SkiaRebaselineServer + </a> + </h2> + + <em ng-show="!readyToDisplay"> + Loading results of query: + <ul> + <li>setA: {{setADir}}</li> + <li>setB: {{setBDir}}</li> + </ul> + <br> + {{loadingMessage}} + </em> + + <div ng-show="readyToDisplay"> + + <div class="warning-div" + ng-show="urlSchemaVersionLoaded != constants.URL_VALUE__SCHEMA_VERSION__CURRENT"> + WARNING! The URL you loaded used schema version {{urlSchemaVersionLoaded}}, rather than + the most recent version {{constants.URL_VALUE__SCHEMA_VERSION__CURRENT}}. It has been + converted to the most recent version on a best-effort basis; you may wish to double-check + which records are displayed. + </div> + + <div class="warning-div" + ng-show="header[constants.KEY__HEADER__IS_EDITABLE] && header[constants.KEY__HEADER__IS_EXPORTED]"> + WARNING! These results are editable and exported, so any user + who can connect to this server over the network can modify them. + </div> + + <div ng-show="header[constants.KEY__HEADER__TIME_UPDATED]"> + setA: {{setADir}}<br> + setB: {{setBDir}}<br> + These results current as of + {{localTimeString(header[constants.KEY__HEADER__TIME_UPDATED])}} + </div> + + <div class="tab-wrapper"><!-- tabs --> + <div class="tab-spacer" ng-repeat="tab in tabs"> + <div class="tab tab-{{tab == viewingTab}}" + ng-click="setViewingTab(tab)"> + {{tab}} ({{numResultsPerTab[tab]}}) + </div> + <div class="tab-spacer"> + + </div> + </div> + </div><!-- tabs --> + + <div class="tab-main"><!-- main display area of selected tab --> + + <br> + <!-- We only show the filters/settings table on the Unfiled tab. --> + <table ng-show="viewingTab == defaultTab" border="1"> + <tr> + <th colspan="4"> + Filters + </th> + <th> + Settings + </th> + </tr> + <tr valign="top"> + + <!-- filters --> + <td ng-repeat="columnName in orderedColumnNames"> + + <!-- Only display filterable columns here... --> + <div ng-if="extraColumnHeaders[columnName][constants.KEY__EXTRACOLUMNHEADERS__IS_FILTERABLE]"> + {{extraColumnHeaders[columnName][constants.KEY__EXTRACOLUMNHEADERS__HEADER_TEXT]}}<br> + + <!-- If we filter this column using free-form text match... --> + <div ng-if="extraColumnHeaders[columnName][constants.KEY__EXTRACOLUMNHEADERS__USE_FREEFORM_FILTER]"> + <input type="text" + ng-model="columnStringMatch[columnName]" + ng-change="setUpdatesPending(true)"/> + <br> + <button ng-click="setColumnStringMatch(columnName, '')" + ng-disabled="('' == columnStringMatch[columnName])"> + clear (show all) + </button> + </div> + + <!-- If we filter this column using checkboxes... --> + <div ng-if="!extraColumnHeaders[columnName][constants.KEY__EXTRACOLUMNHEADERS__USE_FREEFORM_FILTER]"> + <label ng-repeat="valueAndCount in extraColumnHeaders[columnName][constants.KEY__EXTRACOLUMNHEADERS__VALUES_AND_COUNTS]"> + <input type="checkbox" + name="resultTypes" + value="{{valueAndCount[0]}}" + ng-checked="isValueInSet(valueAndCount[0], showingColumnValues[columnName])" + ng-click="toggleValueInSet(valueAndCount[0], showingColumnValues[columnName]); setUpdatesPending(true)"> + {{valueAndCount[0]}} ({{valueAndCount[1]}})<br> + </label> + <button ng-click="showingColumnValues[columnName] = {}; toggleValuesInSet(allColumnValues[columnName], showingColumnValues[columnName]); updateResults()" + ng-disabled="!readyToDisplay || allColumnValues[columnName].length == setSize(showingColumnValues[columnName])"> + all + </button> + <button ng-click="showingColumnValues[columnName] = {}; updateResults()" + ng-disabled="!readyToDisplay || 0 == setSize(showingColumnValues[columnName])"> + none + </button> + <button ng-click="toggleValuesInSet(allColumnValues[columnName], showingColumnValues[columnName]); updateResults()"> + toggle + </button> + </div> + + </div> + </td> + + <!-- settings --> + <td><table> + <tr><td> + <input type="checkbox" ng-model="showThumbnailsPending" + ng-init="showThumbnailsPending = true" + ng-change="areUpdatesPending = true"/> + Show thumbnails + </td></tr> + <tr><td> + <input type="checkbox" ng-model="mergeIdenticalRowsPending" + ng-init="mergeIdenticalRowsPending = true" + ng-change="areUpdatesPending = true"/> + Merge identical rows + </td></tr> + <tr><td> + Image width + <input type="text" ng-model="imageSizePending" + ng-init="imageSizePending=100" + ng-change="areUpdatesPending = true" + maxlength="4"/> + </td></tr> + <tr><td> + Max records to display + <input type="text" ng-model="displayLimitPending" + ng-init="displayLimitPending=50" + ng-change="areUpdatesPending = true" + maxlength="4"/> + </td></tr> + <tr><td> + <button class="update-results-button" + ng-click="updateResults()" + ng-disabled="!areUpdatesPending"> + Update Results + </button> + </td></tr> + </tr></table></td> + </tr> + </table> + + <p> + + <!-- Submission UI that we only show in the Pending Approval tab. --> + <div ng-show="'Pending Approval' == viewingTab"> + <div style="display:inline-block"> + <button style="font-size:20px" + ng-click="submitApprovals(filteredImagePairs)" + ng-disabled="submitPending || (filteredImagePairs.length == 0)"> + Update these {{filteredImagePairs.length}} expectations on the server + </button> + </div> + <div style="display:inline-block"> + <div style="font-size:20px" + ng-show="submitPending"> + Submitting, please wait... + </div> + </div> + <div> + Advanced settings... + <input type="checkbox" ng-model="showSubmitAdvancedSettings"> + show + <ul ng-show="showSubmitAdvancedSettings"> + <li ng-repeat="setting in [constants.KEY__EXPECTATIONS__REVIEWED, constants.KEY__EXPECTATIONS__IGNOREFAILURE]"> + {{setting}} + <input type="checkbox" ng-model="submitAdvancedSettings[setting]"> + </li> + <li ng-repeat="setting in ['bug']"> + {{setting}} + <input type="text" ng-model="submitAdvancedSettings[setting]"> + </li> + </ul> + </div> + </div> + + <p> + + <table border="0"><tr><td> <!-- table holding results header + results table --> + <table border="0" width="100%"> <!-- results header --> + <tr> + <td> + Found {{filteredImagePairs.length}} matches; + <span ng-show="filteredImagePairs.length > limitedImagePairs.length"> + displaying the first {{limitedImagePairs.length}}. + </span> + <span ng-show="filteredImagePairs.length <= limitedImagePairs.length"> + displaying them all. + </span> + <span ng-show="renderEndTime > renderStartTime"> + Rendered in {{(renderEndTime - renderStartTime).toFixed(0)}} ms. + </span> + <br> + (click on the column header radio buttons to re-sort by that column) + </td> + <td align="right"> + <div> + all tests shown: + <button ng-click="selectAllImagePairs()"> + select + </button> + <button ng-click="clearAllImagePairs()"> + clear + </button> + <button ng-click="toggleAllImagePairs()"> + toggle + </button> + </div> + <div ng-repeat="otherTab in tabs"> + <button ng-click="moveSelectedImagePairsToTab(otherTab)" + ng-disabled="selectedImagePairs.length == 0" + ng-show="otherTab != viewingTab"> + move {{selectedImagePairs.length}} selected tests to {{otherTab}} tab + </button> + </div> + </td> + </tr> + </table> <!-- results header --> + </td></tr><tr><td> + <table border="1" ng-app="diff_viewer"> <!-- results --> + <tr> + <!-- Most column headers are displayed in a common fashion... --> + <th ng-repeat="columnName in orderedColumnNames"> + <input type="radio" + name="sortColumnRadio" + value="{{columnName}}" + ng-checked="(sortColumnKey == columnName)" + ng-click="sortResultsBy(constants.KEY__IMAGEPAIRS__EXTRACOLUMNS, columnName)"> + {{extraColumnHeaders[columnName][constants.KEY__EXTRACOLUMNHEADERS__HEADER_TEXT]}} + </th> + <!-- ... but there are a few columns where we display things differently. --> + <th> + <input type="radio" + name="sortColumnRadio" + value="bugs" + ng-checked="(sortColumnKey == constants.KEY__EXPECTATIONS__BUGS)" + ng-click="sortResultsBy(constants.KEY__IMAGEPAIRS__EXPECTATIONS, constants.KEY__EXPECTATIONS__BUGS)"> + bugs + </th> + <th width="{{imageSize}}"> + <input type="radio" + name="sortColumnRadio" + value="imageA" + ng-checked="(sortColumnKey == constants.KEY__IMAGEPAIRS__IMAGE_A_URL)" + ng-click="sortResultsBy('none', constants.KEY__IMAGEPAIRS__IMAGE_A_URL)"> + {{imageSets[constants.KEY__IMAGESETS__SET__IMAGE_A][constants.KEY__IMAGESETS__FIELD__DESCRIPTION]}} + </th> + <th width="{{imageSize}}"> + <input type="radio" + name="sortColumnRadio" + value="imageB" + ng-checked="(sortColumnKey == constants.KEY__IMAGEPAIRS__IMAGE_B_URL)" + ng-click="sortResultsBy('none', constants.KEY__IMAGEPAIRS__IMAGE_B_URL)"> + {{imageSets[constants.KEY__IMAGESETS__SET__IMAGE_B][constants.KEY__IMAGESETS__FIELD__DESCRIPTION]}} + </th> + <th width="{{imageSize}}"> + <input type="radio" + name="sortColumnRadio" + value="percentDifferingPixels" + ng-checked="(sortColumnKey == constants.KEY__DIFFERENCES__PERCENT_DIFF_PIXELS)" + ng-click="sortResultsBy(constants.KEY__IMAGEPAIRS__DIFFERENCES, constants.KEY__DIFFERENCES__PERCENT_DIFF_PIXELS)"> + differing pixels in white + </th> + <th width="{{imageSize}}"> + <input type="radio" + name="sortColumnRadio" + value="perceptualDiff" + ng-checked="(sortColumnKey == constants.KEY__DIFFERENCES__PERCEPTUAL_DIFF)" + ng-click="sortResultsBy(constants.KEY__IMAGEPAIRS__DIFFERENCES, constants.KEY__DIFFERENCES__PERCEPTUAL_DIFF)"> + perceptual difference + <br> + <input type="range" ng-model="pixelDiffBgColorBrightness" + ng-init="pixelDiffBgColorBrightness=64; pixelDiffBgColor=brightnessStringToHexColor(pixelDiffBgColorBrightness)" + ng-change="pixelDiffBgColor=brightnessStringToHexColor(pixelDiffBgColorBrightness)" + title="image background brightness" + min="0" max="255"/> + </th> + <th> + <!-- imagepair-selection checkbox column --> + </th> + </tr> + + <tr ng-repeat="imagePair in limitedImagePairs" valign="top" + ng-class-odd="'results-odd'" ng-class-even="'results-even'" + results-updated-callback-directive> + + <td ng-repeat="columnName in orderedColumnNames"> + {{imagePair[constants.KEY__IMAGEPAIRS__EXTRACOLUMNS][columnName]}} + <br> + <button class="show-only-button" + ng-show="viewingTab == defaultTab" + ng-disabled="1 == setSize(showingColumnValues[columnName])" + ng-click="showOnlyColumnValue(columnName, imagePair[constants.KEY__IMAGEPAIRS__EXTRACOLUMNS][columnName])" + title="show only results of {{extraColumnHeaders[columnName][constants.KEY__EXTRACOLUMNHEADERS__HEADER_TEXT]}} {{imagePair[constants.KEY__IMAGEPAIRS__EXTRACOLUMNS][columnName]}}"> + show only + </button> + <br> + <button class="show-all-button" + ng-show="viewingTab == defaultTab" + ng-disabled="allColumnValues[columnName].length == setSize(showingColumnValues[columnName])" + ng-click="showAllColumnValues(columnName)" + title="show results of all {{extraColumnHeaders[columnName][constants.KEY__EXTRACOLUMNHEADERS__HEADER_TEXT]}}s"> + show all + </button> + </td> + + <!-- bugs --> + <td> + <a ng-repeat="bug in imagePair[constants.KEY__IMAGEPAIRS__EXPECTATIONS][constants.KEY__EXPECTATIONS__BUGS]" + href="https://code.google.com/p/skia/issues/detail?id={{bug}}" + target="_blank"> + {{bug}} + </a> + </td> + + <!-- image A --> + <td width="{{imageSize}}" ng-if="imagePair[constants.KEY__IMAGEPAIRS__ROWSPAN] > 0" rowspan="{{imagePair[constants.KEY__IMAGEPAIRS__ROWSPAN]}}"> + <div ng-if="imagePair[constants.KEY__IMAGEPAIRS__IMAGE_A_URL] != null"> + <a href="{{imageSets[constants.KEY__IMAGESETS__SET__IMAGE_A][constants.KEY__IMAGESETS__FIELD__BASE_URL]}}/{{imagePair[constants.KEY__IMAGEPAIRS__IMAGE_A_URL]}}" target="_blank">View Image</a><br/> + <img ng-if="showThumbnails" + width="{{imageSize}}" + ng-src="{{imageSets[constants.KEY__IMAGESETS__SET__IMAGE_A][constants.KEY__IMAGESETS__FIELD__BASE_URL]}}/{{imagePair[constants.KEY__IMAGEPAIRS__IMAGE_A_URL]}}" /> + </div> + <div ng-show="imagePair[constants.KEY__IMAGEPAIRS__IMAGE_A_URL] == null" + style="text-align:center"> + –none– + </div> + </td> + + <!-- image B --> + <td width="{{imageSize}}" ng-if="imagePair[constants.KEY__IMAGEPAIRS__ROWSPAN] > 0" rowspan="{{imagePair[constants.KEY__IMAGEPAIRS__ROWSPAN]}}"> + <div ng-if="imagePair[constants.KEY__IMAGEPAIRS__IMAGE_B_URL] != null"> + <a href="{{imageSets[constants.KEY__IMAGESETS__SET__IMAGE_B][constants.KEY__IMAGESETS__FIELD__BASE_URL]}}/{{imagePair[constants.KEY__IMAGEPAIRS__IMAGE_B_URL]}}" target="_blank">View Image</a><br/> + <img ng-if="showThumbnails" + width="{{imageSize}}" + ng-src="{{imageSets[constants.KEY__IMAGESETS__SET__IMAGE_B][constants.KEY__IMAGESETS__FIELD__BASE_URL]}}/{{imagePair[constants.KEY__IMAGEPAIRS__IMAGE_B_URL]}}" /> + </div> + <div ng-show="imagePair[constants.KEY__IMAGEPAIRS__IMAGE_B_URL] == null" + style="text-align:center"> + –none– + </div> + </td> + + <!-- whitediffs: every differing pixel shown in white --> + <td width="{{imageSize}}" ng-if="imagePair[constants.KEY__IMAGEPAIRS__ROWSPAN] > 0" rowspan="{{imagePair[constants.KEY__IMAGEPAIRS__ROWSPAN]}}"> + <div ng-if="imagePair[constants.KEY__IMAGEPAIRS__IS_DIFFERENT]" + title="{{imagePair[constants.KEY__IMAGEPAIRS__DIFFERENCES][constants.KEY__DIFFERENCES__NUM_DIFF_PIXELS] | number:0}} of {{(100 * imagePair[constants.KEY__IMAGEPAIRS__DIFFERENCES][constants.KEY__DIFFERENCES__NUM_DIFF_PIXELS] / imagePair[constants.KEY__IMAGEPAIRS__DIFFERENCES][constants.KEY__DIFFERENCES__PERCENT_DIFF_PIXELS]) | number:0}} pixels ({{imagePair[constants.KEY__IMAGEPAIRS__DIFFERENCES][constants.KEY__DIFFERENCES__PERCENT_DIFF_PIXELS].toFixed(4)}}%) differ from expectation."> + + <a href="{{imageSets[constants.KEY__IMAGESETS__SET__WHITEDIFFS][constants.KEY__IMAGESETS__FIELD__BASE_URL]}}/{{getImageDiffRelativeUrl(imagePair)}}" target="_blank">View Image</a><br/> + <img ng-if="showThumbnails" + width="{{imageSize}}" + ng-src="{{imageSets[constants.KEY__IMAGESETS__SET__WHITEDIFFS][constants.KEY__IMAGESETS__FIELD__BASE_URL]}}/{{getImageDiffRelativeUrl(imagePair)}}" /> + <br/> + {{imagePair[constants.KEY__IMAGEPAIRS__DIFFERENCES][constants.KEY__DIFFERENCES__PERCENT_DIFF_PIXELS].toFixed(4)}}% + ({{imagePair[constants.KEY__IMAGEPAIRS__DIFFERENCES][constants.KEY__DIFFERENCES__NUM_DIFF_PIXELS]}}) + </div> + <div ng-show="!imagePair[constants.KEY__IMAGEPAIRS__IS_DIFFERENT]" + style="text-align:center"> + –none– + </div> + </td> + + <!-- diffs: per-channel RGB deltas --> + <td width="{{imageSize}}" ng-if="imagePair[constants.KEY__IMAGEPAIRS__ROWSPAN] > 0" rowspan="{{imagePair[constants.KEY__IMAGEPAIRS__ROWSPAN]}}"> + <div ng-if="imagePair[constants.KEY__IMAGEPAIRS__IS_DIFFERENT]" + title="Perceptual difference measure is {{imagePair[constants.KEY__IMAGEPAIRS__DIFFERENCES][constants.KEY__DIFFERENCES__PERCEPTUAL_DIFF].toFixed(4)}}%. Maximum difference per channel: R={{imagePair[constants.KEY__IMAGEPAIRS__DIFFERENCES][constants.KEY__DIFFERENCES__MAX_DIFF_PER_CHANNEL][0]}}, G={{imagePair[constants.KEY__IMAGEPAIRS__DIFFERENCES][constants.KEY__DIFFERENCES__MAX_DIFF_PER_CHANNEL][1]}}, B={{imagePair[constants.KEY__IMAGEPAIRS__DIFFERENCES][constants.KEY__DIFFERENCES__MAX_DIFF_PER_CHANNEL][2]}}"> + + <a href="{{imageSets[constants.KEY__IMAGESETS__SET__DIFFS][constants.KEY__IMAGESETS__FIELD__BASE_URL]}}/{{getImageDiffRelativeUrl(imagePair)}}" target="_blank">View Image</a><br/> + <img ng-if="showThumbnails" + ng-style="{backgroundColor: pixelDiffBgColor}" + width="{{imageSize}}" + ng-src="{{imageSets[constants.KEY__IMAGESETS__SET__DIFFS][constants.KEY__IMAGESETS__FIELD__BASE_URL]}}/{{getImageDiffRelativeUrl(imagePair)}}" /> + <br/> + {{imagePair[constants.KEY__IMAGEPAIRS__DIFFERENCES][constants.KEY__DIFFERENCES__PERCEPTUAL_DIFF].toFixed(4)}}% + {{imagePair[constants.KEY__IMAGEPAIRS__DIFFERENCES][constants.KEY__DIFFERENCES__MAX_DIFF_PER_CHANNEL]}} + </div> + <div ng-show="!imagePair[constants.KEY__IMAGEPAIRS__IS_DIFFERENT]" + style="text-align:center"> + –none– + </div> + </td> + + <td ng-if="imagePair[constants.KEY__IMAGEPAIRS__ROWSPAN] > 0" rowspan="{{imagePair[constants.KEY__IMAGEPAIRS__ROWSPAN]}}"> + <br/> + <input type="checkbox" + name="rowSelect" + value="{{imagePair.index}}" + ng-checked="isValueInArray(imagePair.index, selectedImagePairs)" + ng-click="toggleSomeImagePairs($index, imagePair[constants.KEY__IMAGEPAIRS__ROWSPAN])"> + </tr> + </table> <!-- imagePairs --> + </td></tr></table> <!-- table holding results header + imagePairs table --> + + </div><!-- main display area of selected tab --> + </div><!-- everything: hide until readyToDisplay --> + +</body> +</html> diff --git a/gm/rebaseline_server/testdata/outputs/expected/compare_rendered_pictures_test.CompareRenderedPicturesTest.test_endToEnd/compare_rendered_pictures.json b/gm/rebaseline_server/testdata/outputs/expected/compare_rendered_pictures_test.CompareRenderedPicturesTest.test_endToEnd/compare_rendered_pictures.json index e741e3ec99..03c5b2959a 100644 --- a/gm/rebaseline_server/testdata/outputs/expected/compare_rendered_pictures_test.CompareRenderedPicturesTest.test_endToEnd/compare_rendered_pictures.json +++ b/gm/rebaseline_server/testdata/outputs/expected/compare_rendered_pictures_test.CompareRenderedPicturesTest.test_endToEnd/compare_rendered_pictures.json @@ -1,29 +1,5 @@ { "extraColumnHeaders": { - "builder": { - "headerText": "builder", - "isFilterable": true, - "isSortable": true, - "useFreeformFilter": false, - "valuesAndCounts": [ - [ - "TODO", - 4 - ] - ] - }, - "config": { - "headerText": "config", - "isFilterable": true, - "isSortable": true, - "useFreeformFilter": false, - "valuesAndCounts": [ - [ - "whole-image", - 4 - ] - ] - }, "resultType": { "headerText": "resultType", "isFilterable": true, @@ -44,8 +20,8 @@ ] ] }, - "test": { - "headerText": "test", + "sourceSkpFile": { + "headerText": "sourceSkpFile", "isFilterable": true, "isSortable": true, "useFreeformFilter": false, @@ -67,16 +43,40 @@ 1 ] ] + }, + "tiledOrWhole": { + "headerText": "tiledOrWhole", + "isFilterable": true, + "isSortable": true, + "useFreeformFilter": false, + "valuesAndCounts": [ + [ + "whole", + 4 + ] + ] + }, + "tilenum": { + "headerText": "tilenum", + "isFilterable": true, + "isSortable": true, + "useFreeformFilter": true, + "valuesAndCounts": [ + [ + "N/A", + 4 + ] + ] } }, "extraColumnOrder": [ - "builder", - "config", "resultType", - "test" + "sourceSkpFile", + "tiledOrWhole", + "tilenum" ], "header": { - "dataHash": "-595743736412687673", + "dataHash": "-4754972663365911725", "isEditable": false, "isExported": true, "schemaVersion": 4, @@ -87,10 +87,10 @@ "imagePairs": [ { "extraColumns": { - "builder": "TODO", - "config": "whole-image", "resultType": "failed", - "test": "changed.skp" + "sourceSkpFile": "changed.skp", + "tiledOrWhole": "whole", + "tilenum": "N/A" }, "imageAUrl": "changed_skp/bitmap-64bitMD5_3101044995537104462.png", "imageBUrl": "changed_skp/bitmap-64bitMD5_13623922271964399662.png", @@ -98,10 +98,10 @@ }, { "extraColumns": { - "builder": "TODO", - "config": "whole-image", "resultType": "no-comparison", - "test": "only-in-after.skp" + "sourceSkpFile": "only-in-after.skp", + "tiledOrWhole": "whole", + "tilenum": "N/A" }, "imageAUrl": null, "imageBUrl": "only-in-after_skp/bitmap-64bitMD5_2320185040577047131.png", @@ -109,10 +109,10 @@ }, { "extraColumns": { - "builder": "TODO", - "config": "whole-image", "resultType": "no-comparison", - "test": "only-in-before.skp" + "sourceSkpFile": "only-in-before.skp", + "tiledOrWhole": "whole", + "tilenum": "N/A" }, "imageAUrl": "only-in-before_skp/bitmap-64bitMD5_2320185040577047131.png", "imageBUrl": null, @@ -120,10 +120,10 @@ }, { "extraColumns": { - "builder": "TODO", - "config": "whole-image", "resultType": "succeeded", - "test": "unchanged.skp" + "sourceSkpFile": "unchanged.skp", + "tiledOrWhole": "whole", + "tilenum": "N/A" }, "imageAUrl": "unchanged_skp/bitmap-64bitMD5_3322248763049618493.png", "imageBUrl": "unchanged_skp/bitmap-64bitMD5_3322248763049618493.png", @@ -136,11 +136,11 @@ "description": "color difference per channel" }, "imageA": { - "baseUrl": "http://chromium-skia-gm.commondatastorage.googleapis.com/render_pictures/images", + "baseUrl": "http://storage.cloud.google.com/fakebucket/fake/path", "description": "before_patch" }, "imageB": { - "baseUrl": "http://chromium-skia-gm.commondatastorage.googleapis.com/render_pictures/images", + "baseUrl": "http://storage.cloud.google.com/fakebucket/fake/path", "description": "after_patch" }, "whiteDiffs": { |