make compare_rendered_pictures process render_pictures's new JSON output format

BUG=skia:1942,skia:2230
NOTRY=True
R=borenet@google.com

Author: epoger@google.com

Review URL: https://codereview.chromium.org/265793013

git-svn-id: http://skia.googlecode.com/svn/trunk@14591 2bbb7eff-a529-9590-31e7-b0007b416f81

8 files changed, 278 insertions, 119 deletions

diff --git a/gm/gm_json.py b/gm/gm_json.py
index 6f4b324d46..3f43b340a5 100644
--- a/gm/gm_json.py
+++ b/gm/gm_json.py
@@ -23,7 +23,9 @@ import re
 # actual-results.json).
 #
 # NOTE: These constants must be kept in sync with the kJsonKey_ constants in
-# gm_expectations.cpp !
+# gm_expectations.cpp and tools/PictureRenderer.cpp !
+# Eric suggests: create gm/gm_expectations_constants.h containing ONLY variable
+# declarations so as to be readable by both gm/gm_expectations.cpp and Python.
 
 
 JSONKEY_ACTUALRESULTS = 'actual-results'
@@ -75,10 +77,20 @@ JSONKEY_EXPECTEDRESULTS_NOTES = 'notes'
 #   review of expectations.
 JSONKEY_EXPECTEDRESULTS_REVIEWED = 'reviewed-by-human'
 
-
 # Allowed hash types for test expectations.
 JSONKEY_HASHTYPE_BITMAP_64BITMD5 = 'bitmap-64bitMD5'
 
+JSONKEY_HEADER = 'header'
+JSONKEY_HEADER_TYPE = 'type'
+JSONKEY_HEADER_REVISION = 'revision'
+JSONKEY_IMAGE_CHECKSUMALGORITHM = 'checksumAlgorithm'
+JSONKEY_IMAGE_CHECKSUMVALUE = 'checksumValue'
+JSONKEY_IMAGE_COMPARISONRESULT = 'comparisonResult'
+JSONKEY_IMAGE_FILEPATH = 'filepath'
+JSONKEY_SOURCE_TILEDIMAGES = 'tiled-images'
+JSONKEY_SOURCE_WHOLEIMAGE = 'whole-image'
+
+
 # Root directory where the buildbots store their actually-generated images...
 #  as a publicly readable HTTP URL:
 GM_ACTUALS_ROOT_HTTP_URL = (
diff --git a/gm/rebaseline_server/compare_configs.py b/gm/rebaseline_server/compare_configs.py
index ba256cab60..e84de9ae15 100755
--- a/gm/rebaseline_server/compare_configs.py
+++ b/gm/rebaseline_server/compare_configs.py
@@ -88,7 +88,8 @@ class ConfigComparisons(results.BaseComparisons):
     """
     logging.info('Reading actual-results JSON files from %s...' %
                  self._actuals_root)
-    actual_builder_dicts = self._read_dicts_from_root(self._actuals_root)
+    actual_builder_dicts = self._read_builder_dicts_from_root(
+        self._actuals_root)
     configA, configB = configs
     logging.info('Comparing configs %s and %s...' % (configA, configB))
 
diff --git a/gm/rebaseline_server/compare_rendered_pictures.py b/gm/rebaseline_server/compare_rendered_pictures.py
index 80a42e51f9..ba621c3a35 100755
--- a/gm/rebaseline_server/compare_rendered_pictures.py
+++ b/gm/rebaseline_server/compare_rendered_pictures.py
@@ -39,9 +39,6 @@ import imagepair
 import imagepairset
 import results
 
-# Characters we don't want popping up just anywhere within filenames.
-DISALLOWED_FILEPATH_CHAR_REGEX = re.compile('[^\w\-]')
-
 # URL under which all render_pictures images can be found in Google Storage.
 # TODO(epoger): Move this default value into
 # https://skia.googlesource.com/buildbot/+/master/site_config/global_variables.json
@@ -97,9 +94,9 @@ class RenderedPicturesComparisons(results.BaseComparisons):
         'Reading actual-results JSON files from %s subdirs within %s...' % (
             subdirs, actuals_root))
     subdirA, subdirB = subdirs
-    subdirA_builder_dicts = self._read_dicts_from_root(
+    subdirA_dicts = self._read_dicts_from_root(
         os.path.join(actuals_root, subdirA))
-    subdirB_builder_dicts = self._read_dicts_from_root(
+    subdirB_dicts = self._read_dicts_from_root(
         os.path.join(actuals_root, subdirB))
     logging.info('Comparing subdirs %s and %s...' % (subdirA, subdirB))
 
@@ -122,87 +119,140 @@ class RenderedPicturesComparisons(results.BaseComparisons):
             results.KEY__RESULT_TYPE__NOCOMPARISON,
         ])
 
-    builders = sorted(set(subdirA_builder_dicts.keys() +
-                          subdirB_builder_dicts.keys()))
-    num_builders = len(builders)
-    builder_num = 0
-    for builder in builders:
-      builder_num += 1
-      logging.info('Generating pixel diffs for builder #%d of %d, "%s"...' %
-                   (builder_num, num_builders, builder))
-      # TODO(epoger): This will fail if we have results for this builder in
-      # subdirA but not subdirB (or vice versa).
-      subdirA_results = results.BaseComparisons.combine_subdicts(
-          subdirA_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
-      subdirB_results = results.BaseComparisons.combine_subdicts(
-          subdirB_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
-      image_names = sorted(set(subdirA_results.keys() +
-                               subdirB_results.keys()))
-      for image_name in image_names:
-        # The image name may contain funny characters or be ridiculously long
-        # (see https://code.google.com/p/skia/issues/detail?id=2344#c10 ),
-        # so make sure we sanitize it before using it in a URL path.
-        #
-        # TODO(epoger): Rather than sanitizing/truncating the image name here,
-        # do it in render_pictures instead.
-        # Reason: we will need to be consistent in applying this rule, so that
-        # the process which uploads the files to GS using these paths will
-        # match the paths created by downstream processes.
-        # So, we should make render_pictures write out images to paths that are
-        # "ready to upload" to Google Storage, like gm does.
-        sanitized_test_name = DISALLOWED_FILEPATH_CHAR_REGEX.sub(
-            '_', image_name)[:30]
-
-        subdirA_image_relative_url = (
-            results.BaseComparisons._create_relative_url(
-                hashtype_and_digest=subdirA_results.get(image_name),
-                test_name=sanitized_test_name))
-        subdirB_image_relative_url = (
-            results.BaseComparisons._create_relative_url(
-                hashtype_and_digest=subdirB_results.get(image_name),
-                test_name=sanitized_test_name))
-
-        # If we have images for at least one of these two subdirs,
-        # add them to our list.
-        if subdirA_image_relative_url or subdirB_image_relative_url:
-          if subdirA_image_relative_url == subdirB_image_relative_url:
-            result_type = results.KEY__RESULT_TYPE__SUCCEEDED
-          elif not subdirA_image_relative_url:
-            result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
-          elif not subdirB_image_relative_url:
-            result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
-          else:
-            result_type = results.KEY__RESULT_TYPE__FAILED
-
-        extra_columns_dict = {
-            results.KEY__EXTRACOLUMN__RESULT_TYPE: result_type,
-            results.KEY__EXTRACOLUMN__BUILDER: builder,
-            results.KEY__EXTRACOLUMN__TEST: image_name,
-            # TODO(epoger): Right now, the client UI crashes if it receives
-            # results that do not include a 'config' column.
-            # Until we fix that, keep the client happy.
-            results.KEY__EXTRACOLUMN__CONFIG: 'TODO',
-        }
-
-        try:
-          image_pair = imagepair.ImagePair(
-              image_diff_db=self._image_diff_db,
-              base_url=self._image_base_url,
-              imageA_relative_url=subdirA_image_relative_url,
-              imageB_relative_url=subdirB_image_relative_url,
-              extra_columns=extra_columns_dict)
-          all_image_pairs.add_image_pair(image_pair)
-          if result_type != results.KEY__RESULT_TYPE__SUCCEEDED:
-            failing_image_pairs.add_image_pair(image_pair)
-        except (KeyError, TypeError):
-          logging.exception(
-              'got exception while creating ImagePair for image_name '
-              '"%s", builder "%s"' % (image_name, builder))
+    common_dict_paths = sorted(set(subdirA_dicts.keys() + subdirB_dicts.keys()))
+    num_common_dict_paths = len(common_dict_paths)
+    dict_num = 0
+    for dict_path in common_dict_paths:
+      dict_num += 1
+      logging.info('Generating pixel diffs for dict #%d of %d, "%s"...' %
+                   (dict_num, num_common_dict_paths, dict_path))
+      dictA = subdirA_dicts[dict_path]
+      dictB = subdirB_dicts[dict_path]
+      self._validate_dict_version(dictA)
+      self._validate_dict_version(dictB)
+      dictA_results = dictA[gm_json.JSONKEY_ACTUALRESULTS]
+      dictB_results = dictB[gm_json.JSONKEY_ACTUALRESULTS]
+      skp_names = sorted(set(dictA_results.keys() + dictB_results.keys()))
+      for skp_name in skp_names:
+        imagepairs_for_this_skp = []
+
+        whole_image_A = RenderedPicturesComparisons.get_multilevel(
+            dictA_results, skp_name, gm_json.JSONKEY_SOURCE_WHOLEIMAGE)
+        whole_image_B = RenderedPicturesComparisons.get_multilevel(
+            dictB_results, skp_name, gm_json.JSONKEY_SOURCE_WHOLEIMAGE)
+        imagepairs_for_this_skp.append(self._create_image_pair(
+            test=skp_name, config=gm_json.JSONKEY_SOURCE_WHOLEIMAGE,
+            image_dict_A=whole_image_A, image_dict_B=whole_image_B))
+
+        tiled_images_A = RenderedPicturesComparisons.get_multilevel(
+            dictA_results, skp_name, gm_json.JSONKEY_SOURCE_TILEDIMAGES)
+        tiled_images_B = RenderedPicturesComparisons.get_multilevel(
+            dictB_results, skp_name, gm_json.JSONKEY_SOURCE_TILEDIMAGES)
+        # TODO(epoger): Report an error if we find tiles for A but not B?
+        if tiled_images_A and tiled_images_B:
+          # TODO(epoger): Report an error if we find a different number of tiles
+          # for A and B?
+          num_tiles = len(tiled_images_A)
+          for tile_num in range(num_tiles):
+            imagepairs_for_this_skp.append(self._create_image_pair(
+                test=skp_name,
+                config='%s-%d' % (gm_json.JSONKEY_SOURCE_TILEDIMAGES, tile_num),
+                image_dict_A=tiled_images_A[tile_num],
+                image_dict_B=tiled_images_B[tile_num]))
+
+        for imagepair in imagepairs_for_this_skp:
+          if imagepair:
+            all_image_pairs.add_image_pair(imagepair)
+            result_type = imagepair.extra_columns_dict\
+                [results.KEY__EXTRACOLUMN__RESULT_TYPE]
+            if result_type != results.KEY__RESULT_TYPE__SUCCEEDED:
+              failing_image_pairs.add_image_pair(imagepair)
 
     self._results = {
       results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(),
       results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(),
     }
 
+  def _validate_dict_version(self, result_dict):
+    """Raises Exception if the dict is not the type/version we know how to read.
+
+    Args:
+      result_dict: dictionary holding output of render_pictures
+    """
+    expected_header_type = 'ChecksummedImages'
+    expected_header_revision = 1
+
+    header = result_dict[gm_json.JSONKEY_HEADER]
+    header_type = header[gm_json.JSONKEY_HEADER_TYPE]
+    if header_type != expected_header_type:
+      raise Exception('expected header_type "%s", but got "%s"' % (
+          expected_header_type, header_type))
+    header_revision = header[gm_json.JSONKEY_HEADER_REVISION]
+    if header_revision != expected_header_revision:
+      raise Exception('expected header_revision %d, but got %d' % (
+          expected_header_revision, header_revision))
+
+  def _create_image_pair(self, test, config, image_dict_A, image_dict_B):
+    """Creates an ImagePair object for this pair of images.
+
+    Args:
+      test: string; name of the test
+      config: string; name of the config
+      image_dict_A: dict with JSONKEY_IMAGE_* keys, or None if no image
+      image_dict_B: dict with JSONKEY_IMAGE_* keys, or None if no image
+
+    Returns:
+      An ImagePair object, or None if both image_dict_A and image_dict_B are
+      None.
+    """
+    if (not image_dict_A) and (not image_dict_B):
+      return None
+
+    def _checksum_and_relative_url(dic):
+      if dic:
+        return ((dic[gm_json.JSONKEY_IMAGE_CHECKSUMALGORITHM],
+                 dic[gm_json.JSONKEY_IMAGE_CHECKSUMVALUE]),
+                dic[gm_json.JSONKEY_IMAGE_FILEPATH])
+      else:
+        return None, None
+
+    imageA_checksum, imageA_relative_url = _checksum_and_relative_url(
+        image_dict_A)
+    imageB_checksum, imageB_relative_url = _checksum_and_relative_url(
+        image_dict_B)
+
+    if not imageA_checksum:
+      result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
+    elif not imageB_checksum:
+      result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
+    elif imageA_checksum == imageB_checksum:
+      result_type = results.KEY__RESULT_TYPE__SUCCEEDED
+    else:
+      result_type = results.KEY__RESULT_TYPE__FAILED
+
+    extra_columns_dict = {
+        results.KEY__EXTRACOLUMN__CONFIG: config,
+        results.KEY__EXTRACOLUMN__RESULT_TYPE: result_type,
+        results.KEY__EXTRACOLUMN__TEST: test,
+        # TODO(epoger): Right now, the client UI crashes if it receives
+        # results that do not include this column.
+        # Until we fix that, keep the client happy.
+        results.KEY__EXTRACOLUMN__BUILDER: 'TODO',
+    }
+
+    try:
+      return imagepair.ImagePair(
+          image_diff_db=self._image_diff_db,
+          base_url=self._image_base_url,
+          imageA_relative_url=imageA_relative_url,
+          imageB_relative_url=imageB_relative_url,
+          extra_columns=extra_columns_dict)
+    except (KeyError, TypeError):
+      logging.exception(
+          'got exception while creating ImagePair for'
+          ' test="%s", config="%s", urlPair=("%s","%s")' % (
+              test, config, imageA_relative_url, imageB_relative_url))
+      return None
+
 
 # TODO(epoger): Add main() so this can be called by vm_run_skia_try.sh
diff --git a/gm/rebaseline_server/compare_to_expectations.py b/gm/rebaseline_server/compare_to_expectations.py
index 97286b90c4..ab05f6f51b 100755
--- a/gm/rebaseline_server/compare_to_expectations.py
+++ b/gm/rebaseline_server/compare_to_expectations.py
@@ -121,7 +121,8 @@ class ExpectationComparisons(results.BaseComparisons):
          ]
 
     """
-    expected_builder_dicts = self._read_dicts_from_root(self._expected_root)
+    expected_builder_dicts = self._read_builder_dicts_from_root(
+        self._expected_root)
     for mod in modifications:
       image_name = results.IMAGE_FILENAME_FORMATTER % (
           mod[imagepair.KEY__EXTRA_COLUMN_VALUES]
@@ -200,10 +201,12 @@ class ExpectationComparisons(results.BaseComparisons):
     """
     logging.info('Reading actual-results JSON files from %s...' %
                  self._actuals_root)
-    actual_builder_dicts = self._read_dicts_from_root(self._actuals_root)
+    actual_builder_dicts = self._read_builder_dicts_from_root(
+        self._actuals_root)
     logging.info('Reading expected-results JSON files from %s...' %
                  self._expected_root)
-    expected_builder_dicts = self._read_dicts_from_root(self._expected_root)
+    expected_builder_dicts = self._read_builder_dicts_from_root(
+        self._expected_root)
 
     all_image_pairs = imagepairset.ImagePairSet(
         descriptions=IMAGEPAIR_SET_DESCRIPTIONS,
diff --git a/gm/rebaseline_server/results.py b/gm/rebaseline_server/results.py
index 255dfa31e2..461e7463a6 100755
--- a/gm/rebaseline_server/results.py
+++ b/gm/rebaseline_server/results.py
@@ -190,20 +190,25 @@ class BaseComparisons(object):
         return False
     return True
 
-  def _read_dicts_from_root(self, root, pattern='*.json'):
+  def _read_builder_dicts_from_root(self, root, pattern='*.json'):
     """Read all JSON dictionaries within a directory tree.
 
+    Skips any dictionaries belonging to a builder we have chosen to ignore.
+
     Args:
       root: path to root of directory tree
       pattern: which files to read within root (fnmatch-style pattern)
 
     Returns:
       A meta-dictionary containing all the JSON dictionaries found within
-      the directory tree, keyed by the builder name of each dictionary.
+      the directory tree, keyed by builder name (the basename of the directory
+      where each JSON dictionary was found).
 
     Raises:
       IOError if root does not refer to an existing directory
     """
+    # I considered making this call _read_dicts_from_root(), but I decided
+    # it was better to prune out the ignored builders within the os.walk().
     if not os.path.isdir(root):
       raise IOError('no directory found at path %s' % root)
     meta_dict = {}
@@ -212,8 +217,34 @@ class BaseComparisons(object):
         builder = os.path.basename(dirpath)
         if self._ignore_builder(builder):
           continue
-        fullpath = os.path.join(dirpath, matching_filename)
-        meta_dict[builder] = gm_json.LoadFromFile(fullpath)
+        full_path = os.path.join(dirpath, matching_filename)
+        meta_dict[builder] = gm_json.LoadFromFile(full_path)
+    return meta_dict
+
+  def _read_dicts_from_root(self, root, pattern='*.json'):
+    """Read all JSON dictionaries within a directory tree.
+
+    Args:
+      root: path to root of directory tree
+      pattern: which files to read within root (fnmatch-style pattern)
+
+    Returns:
+      A meta-dictionary containing all the JSON dictionaries found within
+      the directory tree, keyed by the pathname (relative to root) of each JSON
+      dictionary.
+
+    Raises:
+      IOError if root does not refer to an existing directory
+    """
+    if not os.path.isdir(root):
+      raise IOError('no directory found at path %s' % root)
+    meta_dict = {}
+    for abs_dirpath, dirnames, filenames in os.walk(root):
+      rel_dirpath = os.path.relpath(abs_dirpath, root)
+      for matching_filename in fnmatch.filter(filenames, pattern):
+        abs_path = os.path.join(abs_dirpath, matching_filename)
+        rel_path = os.path.join(rel_dirpath, matching_filename)
+        meta_dict[rel_path] = gm_json.LoadFromFile(abs_path)
     return meta_dict
 
   @staticmethod
@@ -240,18 +271,18 @@ class BaseComparisons(object):
 
     Input:
       {
-        "failed" : {
-          "changed.png" : [ "bitmap-64bitMD5", 8891695120562235492 ],
+        KEY_A1 : {
+          KEY_B1 : VALUE_B1,
         },
-        "no-comparison" : {
-          "unchanged.png" : [ "bitmap-64bitMD5", 11092453015575919668 ],
+        KEY_A2 : {
+          KEY_B2 : VALUE_B2,
         }
       }
 
     Output:
       {
-        "changed.png" : [ "bitmap-64bitMD5", 8891695120562235492 ],
-        "unchanged.png" : [ "bitmap-64bitMD5", 11092453015575919668 ],
+        KEY_B1 : VALUE_B1,
+        KEY_B2 : VALUE_B2,
       }
 
     If this would result in any repeated keys, it will raise an Exception.
@@ -263,3 +294,13 @@ class BaseComparisons(object):
           raise Exception('duplicate key %s in combine_subdicts' % subdict_key)
         output_dict[subdict_key] = subdict_value
     return output_dict
+
+  @staticmethod
+  def get_multilevel(input_dict, *keys):
+    """ Returns input_dict[key1][key2][...], or None if any key is not found.
+    """
+    for key in keys:
+      if input_dict == None:
+        return None
+      input_dict = input_dict.get(key, None)
+    return input_dict
diff --git a/gm/rebaseline_server/testdata/inputs/render_pictures_output/after_patch/builder1/output.json b/gm/rebaseline_server/testdata/inputs/render_pictures_output/after_patch/builder1/output.json
index 6b9acc0450..9bd6607da3 100644
--- a/gm/rebaseline_server/testdata/inputs/render_pictures_output/after_patch/builder1/output.json
+++ b/gm/rebaseline_server/testdata/inputs/render_pictures_output/after_patch/builder1/output.json
@@ -1,8 +1,24 @@
 {
+   "header" : {
+      "type" : "ChecksummedImages",
+      "revision" : 1
+   },
    "actual-results" : {
-      "no-comparison" : {
-         "changed.png" : [ "bitmap-64bitMD5", 2520753504544298264 ],
-         "unchanged.png" : [ "bitmap-64bitMD5", 11092453015575919668 ]
+      "changed.skp" : {
+         "whole-image" : {
+           "checksumAlgorithm" : "bitmap-64bitMD5",
+           "checksumValue" : 2520753504544298264,
+           "comparisonResult" : "no-comparison",
+           "filepath" : "bitmap-64bitMD5_2520753504544298264.png"
+         }
+      },
+      "unchanged.skp" : {
+         "whole-image" : {
+           "checksumAlgorithm" : "bitmap-64bitMD5",
+           "checksumValue" : 11092453015575919668,
+           "comparisonResult" : "no-comparison",
+           "filepath" : "bitmap-64bitMD5_11092453015575919668.png"
+         }
       }
    }
 }
diff --git a/gm/rebaseline_server/testdata/inputs/render_pictures_output/before_patch/builder1/output.json b/gm/rebaseline_server/testdata/inputs/render_pictures_output/before_patch/builder1/output.json
index 16e1316bda..8bc81c85e0 100644
--- a/gm/rebaseline_server/testdata/inputs/render_pictures_output/before_patch/builder1/output.json
+++ b/gm/rebaseline_server/testdata/inputs/render_pictures_output/before_patch/builder1/output.json
@@ -1,8 +1,32 @@
 {
+   "header" : {
+      "type" : "ChecksummedImages",
+      "revision" : 1
+   },
    "actual-results" : {
-      "no-comparison" : {
-         "changed.png" : [ "bitmap-64bitMD5", 8891695120562235492 ],
-         "unchanged.png" : [ "bitmap-64bitMD5", 11092453015575919668 ]
+      "changed.skp" : {
+         "whole-image" : {
+           "checksumAlgorithm" : "bitmap-64bitMD5",
+           "checksumValue" : 8891695120562235492,
+           "comparisonResult" : "no-comparison",
+           "filepath" : "bitmap-64bitMD5_8891695120562235492.png"
+         }
+      },
+      "only-in-before.skp" : {
+         "whole-image" : {
+           "checksumAlgorithm" : "bitmap-64bitMD5",
+           "checksumValue" : 8891695120562235492,
+           "comparisonResult" : "no-comparison",
+           "filepath" : "bitmap-64bitMD5_8891695120562235492.png"
+         }
+      },
+      "unchanged.skp" : {
+         "whole-image" : {
+           "checksumAlgorithm" : "bitmap-64bitMD5",
+           "checksumValue" : 11092453015575919668,
+           "comparisonResult" : "no-comparison",
+           "filepath" : "bitmap-64bitMD5_11092453015575919668.png"
+         }
       }
    }
 }
diff --git a/gm/rebaseline_server/testdata/outputs/expected/compare_rendered_pictures_test.CompareRenderedPicturesTest.test_endToEnd/compare_rendered_pictures.json b/gm/rebaseline_server/testdata/outputs/expected/compare_rendered_pictures_test.CompareRenderedPicturesTest.test_endToEnd/compare_rendered_pictures.json
index c60c7873f9..7a0c912c43 100644
--- a/gm/rebaseline_server/testdata/outputs/expected/compare_rendered_pictures_test.CompareRenderedPicturesTest.test_endToEnd/compare_rendered_pictures.json
+++ b/gm/rebaseline_server/testdata/outputs/expected/compare_rendered_pictures_test.CompareRenderedPicturesTest.test_endToEnd/compare_rendered_pictures.json
@@ -5,7 +5,7 @@
       "isFilterable": true, 
       "isSortable": true, 
       "valuesAndCounts": {
-        "builder1": 2
+        "TODO": 3
       }
     }, 
     "config": {
@@ -13,7 +13,7 @@
       "isFilterable": true, 
       "isSortable": true, 
       "valuesAndCounts": {
-        "TODO": 2
+        "whole-image": 3
       }
     }, 
     "resultType": {
@@ -22,7 +22,7 @@
       "isSortable": true, 
       "valuesAndCounts": {
         "failed": 1, 
-        "no-comparison": 0, 
+        "no-comparison": 1, 
         "succeeded": 1
       }
     }, 
@@ -31,13 +31,14 @@
       "isFilterable": true, 
       "isSortable": true, 
       "valuesAndCounts": {
-        "changed.png": 1, 
-        "unchanged.png": 1
+        "changed.skp": 1, 
+        "only-in-before.skp": 1, 
+        "unchanged.skp": 1
       }
     }
   }, 
   "header": {
-    "dataHash": "3972200251153667246", 
+    "dataHash": "182723807383859624", 
     "isEditable": false, 
     "isExported": true, 
     "schemaVersion": 2, 
@@ -48,24 +49,35 @@
   "imagePairs": [
     {
       "extraColumns": {
-        "builder": "builder1", 
-        "config": "TODO", 
+        "builder": "TODO", 
+        "config": "whole-image", 
         "resultType": "failed", 
-        "test": "changed.png"
+        "test": "changed.skp"
       }, 
-      "imageAUrl": "bitmap-64bitMD5/changed_png/8891695120562235492.png", 
-      "imageBUrl": "bitmap-64bitMD5/changed_png/2520753504544298264.png", 
+      "imageAUrl": "bitmap-64bitMD5_8891695120562235492.png", 
+      "imageBUrl": "bitmap-64bitMD5_2520753504544298264.png", 
       "isDifferent": true
     }, 
     {
       "extraColumns": {
-        "builder": "builder1", 
-        "config": "TODO", 
+        "builder": "TODO", 
+        "config": "whole-image", 
+        "resultType": "no-comparison", 
+        "test": "only-in-before.skp"
+      }, 
+      "imageAUrl": "bitmap-64bitMD5_8891695120562235492.png", 
+      "imageBUrl": null, 
+      "isDifferent": true
+    }, 
+    {
+      "extraColumns": {
+        "builder": "TODO", 
+        "config": "whole-image", 
         "resultType": "succeeded", 
-        "test": "unchanged.png"
+        "test": "unchanged.skp"
       }, 
-      "imageAUrl": "bitmap-64bitMD5/unchanged_png/11092453015575919668.png", 
-      "imageBUrl": "bitmap-64bitMD5/unchanged_png/11092453015575919668.png", 
+      "imageAUrl": "bitmap-64bitMD5_11092453015575919668.png", 
+      "imageBUrl": "bitmap-64bitMD5_11092453015575919668.png", 
       "isDifferent": false
     }
   ],