add --readJsonSummaryPath to render_pictures

BUG=skia:1942
R=borenet@google.com

Author: epoger@google.com

Review URL: https://codereview.chromium.org/273783004

git-svn-id: http://skia.googlecode.com/svn/trunk@14695 2bbb7eff-a529-9590-31e7-b0007b416f81

6 files changed, 397 insertions, 112 deletions

diff --git a/tools/PictureRenderer.cpp b/tools/PictureRenderer.cpp
index b5dc0fc93d..74567001e6 100644
--- a/tools/PictureRenderer.cpp
+++ b/tools/PictureRenderer.cpp
@@ -275,7 +275,7 @@ uint32_t PictureRenderer::recordFlags() {
  * @return bool True if the operation completed successfully.
  */
 static bool write(SkCanvas* canvas, const SkString& outputDir, const SkString& inputFilename,
-                  ImageResultsSummary *jsonSummaryPtr, bool useChecksumBasedFilenames,
+                  ImageResultsAndExpectations *jsonSummaryPtr, bool useChecksumBasedFilenames,
                   const int* tileNumberPtr=NULL) {
     SkASSERT(canvas != NULL);
     if (NULL == canvas) {
@@ -284,14 +284,11 @@ static bool write(SkCanvas* canvas, const SkString& outputDir, const SkString& i
 
     SkBitmap bitmap;
     SkISize size = canvas->getDeviceSize();
-    sk_tools::setup_bitmap(&bitmap, size.width(), size.height());
-
-    // Make sure we only compute the bitmap hash once (at most).
-    uint64_t hash;
-    bool generatedHash = false;
+    setup_bitmap(&bitmap, size.width(), size.height());
 
     canvas->readPixels(&bitmap, 0, 0);
-    sk_tools::force_all_opaque(bitmap);
+    force_all_opaque(bitmap);
+    BitmapAndDigest bitmapAndDigest(bitmap);
 
     SkString escapedInputFilename(inputFilename);
     replace_char(&escapedInputFilename, '.', '_');
@@ -299,18 +296,13 @@ static bool write(SkCanvas* canvas, const SkString& outputDir, const SkString& i
     // TODO(epoger): what about including the config type within outputFilename?  That way,
     // we could combine results of different config types without conflicting filenames.
     SkString outputFilename;
+    const ImageDigest *imageDigestPtr = bitmapAndDigest.getImageDigestPtr();
     const char *outputSubdirPtr = NULL;
     if (useChecksumBasedFilenames) {
-        SkASSERT(!generatedHash);
-        SkAssertResult(SkBitmapHasher::ComputeDigest(bitmap, &hash));
-        generatedHash = true;
-
         outputSubdirPtr = escapedInputFilename.c_str();
-        // TODO(epoger): The string constant below will be removed when I land
-        // the second part of https://codereview.chromium.org/261313004/
-        // ('add --readJsonSummaryPath to render_pictures')
-        outputFilename.set("bitmap-64bitMD5_");
-        outputFilename.appendU64(hash);
+        outputFilename.set(imageDigestPtr->getHashType());
+        outputFilename.append("_");
+        outputFilename.appendU64(imageDigestPtr->getHashValue());
     } else {
         outputFilename.set(escapedInputFilename);
         if (NULL != tileNumberPtr) {
@@ -321,11 +313,7 @@ static bool write(SkCanvas* canvas, const SkString& outputDir, const SkString& i
     outputFilename.append(".png");
 
     if (NULL != jsonSummaryPtr) {
-        if (!generatedHash) {
-            SkAssertResult(SkBitmapHasher::ComputeDigest(bitmap, &hash));
-            generatedHash = true;
-        }
-
+        const ImageDigest *imageDigestPtr = bitmapAndDigest.getImageDigestPtr();
         SkString outputRelativePath;
         if (outputSubdirPtr) {
             outputRelativePath.set(outputSubdirPtr);
@@ -336,7 +324,7 @@ static bool write(SkCanvas* canvas, const SkString& outputDir, const SkString& i
         }
 
         jsonSummaryPtr->add(inputFilename.c_str(), outputRelativePath.c_str(),
-                            hash, tileNumberPtr);
+                            *imageDigestPtr, tileNumberPtr);
     }
 
     if (outputDir.isEmpty()) {
@@ -707,7 +695,8 @@ class CloneData : public SkRunnable {
 
 public:
     CloneData(SkPicture* clone, SkCanvas* canvas, SkTDArray<SkRect>& rects, int start, int end,
-              SkRunnable* done, ImageResultsSummary* jsonSummaryPtr, bool useChecksumBasedFilenames)
+              SkRunnable* done, ImageResultsAndExpectations* jsonSummaryPtr,
+              bool useChecksumBasedFilenames)
         : fClone(clone)
         , fCanvas(canvas)
         , fRects(rects)
@@ -778,7 +767,7 @@ private:
                                     // and only set to false upon failure to write to a PNG.
     SkRunnable*        fDone;
     SkBitmap*          fBitmap;
-    ImageResultsSummary* fJsonSummaryPtr;
+    ImageResultsAndExpectations* fJsonSummaryPtr;
     bool               fUseChecksumBasedFilenames;
 };
 
diff --git a/tools/PictureRenderer.h b/tools/PictureRenderer.h
index 445737e755..342df7872a 100644
--- a/tools/PictureRenderer.h
+++ b/tools/PictureRenderer.h
@@ -219,7 +219,7 @@ public:
         fGridInfo.fTileInterval.set(width, height);
     }
 
-    void setJsonSummaryPtr(ImageResultsSummary* jsonSummaryPtr) {
+    void setJsonSummaryPtr(ImageResultsAndExpectations* jsonSummaryPtr) {
         fJsonSummaryPtr = jsonSummaryPtr;
     }
 
@@ -365,7 +365,7 @@ protected:
     SkAutoTUnref<SkCanvas> fCanvas;
     SkAutoTUnref<SkPicture> fPicture;
     bool                   fUseChecksumBasedFilenames;
-    ImageResultsSummary*   fJsonSummaryPtr;
+    ImageResultsAndExpectations*   fJsonSummaryPtr;
     SkDeviceTypes          fDeviceType;
     BBoxHierarchyType      fBBoxHierarchyType;
     DrawFilterFlags        fDrawFilters[SkDrawFilter::kTypeCount];
diff --git a/tools/image_expectations.cpp b/tools/image_expectations.cpp
index 1ab53f24d3..9b180da5fe 100644
--- a/tools/image_expectations.cpp
+++ b/tools/image_expectations.cpp
@@ -7,6 +7,7 @@
 
 #include "SkBitmap.h"
 #include "SkBitmapHasher.h"
+#include "SkData.h"
 #include "SkJSONCPP.h"
 #include "SkOSFile.h"
 #include "SkStream.h"
@@ -27,45 +28,125 @@
  * output module.
  */
 const static char kJsonKey_ActualResults[] = "actual-results";
+const static char kJsonKey_ExpectedResults[] = "expected-results";
 const static char kJsonKey_Header[] = "header";
 const static char kJsonKey_Header_Type[] = "type";
-const static char kJsonKey_Header_Revision[] = "revision";  // unique within Type
+const static char kJsonKey_Header_Revision[] = "revision";
 const static char kJsonKey_Image_ChecksumAlgorithm[] = "checksumAlgorithm";
 const static char kJsonKey_Image_ChecksumValue[] = "checksumValue";
 const static char kJsonKey_Image_ComparisonResult[] = "comparisonResult";
 const static char kJsonKey_Image_Filepath[] = "filepath";
+const static char kJsonKey_Image_IgnoreFailure[] = "ignoreFailure";
 const static char kJsonKey_Source_TiledImages[] = "tiled-images";
 const static char kJsonKey_Source_WholeImage[] = "whole-image";
 // Values (not keys) that are written out by this JSON generator
 const static char kJsonValue_Header_Type[] = "ChecksummedImages";
 const static int kJsonValue_Header_Revision = 1;
 const static char kJsonValue_Image_ChecksumAlgorithm_Bitmap64bitMD5[] = "bitmap-64bitMD5";
+const static char kJsonValue_Image_ComparisonResult_Failed[] = "failed";
+const static char kJsonValue_Image_ComparisonResult_FailureIgnored[] = "failure-ignored";
 const static char kJsonValue_Image_ComparisonResult_NoComparison[] = "no-comparison";
+const static char kJsonValue_Image_ComparisonResult_Succeeded[] = "succeeded";
 
 namespace sk_tools {
 
-    void ImageResultsSummary::add(const char *sourceName, const char *fileName, uint64_t hash,
-                                  const int *tileNumber) {
-        Json::Value image;
-        image[kJsonKey_Image_ChecksumAlgorithm] = kJsonValue_Image_ChecksumAlgorithm_Bitmap64bitMD5;
-        image[kJsonKey_Image_ChecksumValue] = Json::UInt64(hash);
-        image[kJsonKey_Image_ComparisonResult] = kJsonValue_Image_ComparisonResult_NoComparison;
-        image[kJsonKey_Image_Filepath] = fileName;
-        if (NULL == tileNumber) {
-            fActualResults[sourceName][kJsonKey_Source_WholeImage] = image;
+    // ImageDigest class...
+
+    ImageDigest::ImageDigest(const SkBitmap &bitmap) {
+        if (!SkBitmapHasher::ComputeDigest(bitmap, &fHashValue)) {
+            SkFAIL("unable to compute image digest");
+        }
+    }
+
+    ImageDigest::ImageDigest(const SkString &hashType, uint64_t hashValue) {
+        if (!hashType.equals(kJsonValue_Image_ChecksumAlgorithm_Bitmap64bitMD5)) {
+            SkFAIL((SkString("unsupported hashType ")+=hashType).c_str());
+        } else {
+            fHashValue = hashValue;
+        }
+    }
+
+    SkString ImageDigest::getHashType() const {
+        // TODO(epoger): The current implementation assumes that the
+        // result digest is always of type kJsonValue_Image_ChecksumAlgorithm_Bitmap64bitMD5 .
+        return SkString(kJsonValue_Image_ChecksumAlgorithm_Bitmap64bitMD5);
+    }
+
+    uint64_t ImageDigest::getHashValue() const {
+        return fHashValue;
+    }
+
+    // BitmapAndDigest class...
+
+    BitmapAndDigest::BitmapAndDigest(const SkBitmap &bitmap) : fBitmap(bitmap) {
+    }
+
+    const ImageDigest *BitmapAndDigest::getImageDigestPtr() {
+        if (NULL == fImageDigestRef.get()) {
+            fImageDigestRef.reset(SkNEW_ARGS(ImageDigest, (fBitmap)));
+        }
+        return fImageDigestRef.get();
+    }
+
+    const SkBitmap *BitmapAndDigest::getBitmapPtr() const {
+        return &fBitmap;
+    }
+
+    // ImageResultsAndExpectations class...
+
+    bool ImageResultsAndExpectations::readExpectationsFile(const char *jsonPath) {
+        if (Parse(jsonPath, &fExpectedJsonRoot)) {
+            fExpectedResults = fExpectedJsonRoot[kJsonKey_ExpectedResults];
+            return true;
         } else {
-            fActualResults[sourceName][kJsonKey_Source_TiledImages][*tileNumber] = image;
+            return false;
         }
     }
 
-    void ImageResultsSummary::add(const char *sourceName, const char *fileName, const SkBitmap& bitmap,
-                                  const int *tileNumber) {
-        uint64_t hash;
-        SkAssertResult(SkBitmapHasher::ComputeDigest(bitmap, &hash));
-        this->add(sourceName, fileName, hash, tileNumber);
+    void ImageResultsAndExpectations::add(const char *sourceName, const char *fileName,
+                                  const ImageDigest &digest, const int *tileNumber) {
+        // Get expectation, if any.
+        Json::Value expectedImage;
+        if (!fExpectedResults.isNull()) {
+            if (NULL == tileNumber) {
+                expectedImage = fExpectedResults[sourceName][kJsonKey_Source_WholeImage];
+            } else {
+                expectedImage = fExpectedResults[sourceName][kJsonKey_Source_TiledImages]
+                                                [*tileNumber];
+            }
+        }
+
+        // Fill in info about the actual result itself.
+        Json::Value actualChecksumAlgorithm = digest.getHashType().c_str();
+        Json::Value actualChecksumValue = Json::UInt64(digest.getHashValue());
+        Json::Value actualImage;
+        actualImage[kJsonKey_Image_ChecksumAlgorithm] = actualChecksumAlgorithm;
+        actualImage[kJsonKey_Image_ChecksumValue] = actualChecksumValue;
+        actualImage[kJsonKey_Image_Filepath] = fileName;
+
+        // Compare against expectedImage to fill in comparisonResult.
+        Json::Value comparisonResult = kJsonValue_Image_ComparisonResult_NoComparison;
+        if (!expectedImage.isNull()) {
+            if ((actualChecksumAlgorithm == expectedImage[kJsonKey_Image_ChecksumAlgorithm]) &&
+                (actualChecksumValue == expectedImage[kJsonKey_Image_ChecksumValue])) {
+                comparisonResult = kJsonValue_Image_ComparisonResult_Succeeded;
+            } else if (expectedImage[kJsonKey_Image_IgnoreFailure] == true) {
+                comparisonResult = kJsonValue_Image_ComparisonResult_FailureIgnored;
+            } else {
+                comparisonResult = kJsonValue_Image_ComparisonResult_Failed;
+            }
+        }
+        actualImage[kJsonKey_Image_ComparisonResult] = comparisonResult;
+
+        // Add this actual result to our collection.
+        if (NULL == tileNumber) {
+            fActualResults[sourceName][kJsonKey_Source_WholeImage] = actualImage;
+        } else {
+            fActualResults[sourceName][kJsonKey_Source_TiledImages][*tileNumber] = actualImage;
+        }
     }
 
-    void ImageResultsSummary::writeToFile(const char *filename) {
+    void ImageResultsAndExpectations::writeToFile(const char *filename) const {
         Json::Value header;
         header[kJsonKey_Header_Type] = kJsonValue_Header_Type;
         header[kJsonKey_Header_Revision] = kJsonValue_Header_Revision;
@@ -77,4 +158,23 @@ namespace sk_tools {
         stream.write(jsonStdString.c_str(), jsonStdString.length());
     }
 
+    /*static*/ bool ImageResultsAndExpectations::Parse(const char *jsonPath,
+                                                       Json::Value *jsonRoot) {
+        SkAutoDataUnref dataRef(SkData::NewFromFileName(jsonPath));
+        if (NULL == dataRef.get()) {
+            SkDebugf("error reading JSON file %s\n", jsonPath);
+            return false;
+        }
+
+        const char *bytes = reinterpret_cast<const char *>(dataRef.get()->data());
+        size_t size = dataRef.get()->size();
+        Json::Reader reader;
+        if (!reader.parse(bytes, bytes+size, *jsonRoot)) {
+            SkDebugf("error parsing JSON file %s\n", jsonPath);
+            return false;
+        }
+
+        return true;
+    }
+
 } // namespace sk_tools
diff --git a/tools/image_expectations.h b/tools/image_expectations.h
index 432cf6d3a8..09a945a1b0 100644
--- a/tools/image_expectations.h
+++ b/tools/image_expectations.h
@@ -10,34 +10,89 @@
 
 #include "SkBitmap.h"
 #include "SkJSONCPP.h"
+#include "SkRefCnt.h"
 
 namespace sk_tools {
 
     /**
-     * Class for collecting image results (checksums) as we go.
+     * The digest of an image (either an image we have generated locally, or an image expectation).
+     *
+     * Currently, this is always a uint64_t hash digest of an SkBitmap.
      */
-    class ImageResultsSummary {
+    class ImageDigest : public SkRefCnt {
     public:
         /**
-         * Adds this image to the summary of results.
+         * Create an ImageDigest of a bitmap.
          *
-         * @param sourceName name of the source file that generated this result
-         * @param fileName relative path to the image output file on local disk
-         * @param hash hash to store
-         * @param tileNumber if not NULL, ptr to tile number
+         * Note that this is an expensive operation, because it has to examine all pixels in
+         * the bitmap.  You may wish to consider using the BitmapAndDigest class, which will
+         * compute the ImageDigest lazily.
+         *
+         * @param bitmap image to get the digest of
          */
-        void add(const char *sourceName, const char *fileName, uint64_t hash,
-                 const int *tileNumber=NULL);
+        explicit ImageDigest(const SkBitmap &bitmap);
+
+        /**
+         * Create an ImageDigest using a hashType/hashValue pair.
+         *
+         * @param hashType the algorithm used to generate the hash; for now, only
+         *     kJsonValue_Image_ChecksumAlgorithm_Bitmap64bitMD5 is allowed.
+         * @param hashValue the value generated by the hash algorithm for a particular image.
+         */
+        explicit ImageDigest(const SkString &hashType, uint64_t hashValue);
+
+        /**
+         * Returns the hash digest type as an SkString.
+         *
+         * For now, this always returns kJsonValue_Image_ChecksumAlgorithm_Bitmap64bitMD5 .
+         */
+        SkString getHashType() const;
+
+        /**
+         * Returns the hash digest value as a uint64_t.
+         */
+        uint64_t getHashValue() const;
+
+    private:
+        uint64_t fHashValue;
+    };
+
+    /**
+     * Container that holds a reference to an SkBitmap and computes its ImageDigest lazily.
+     *
+     * Computing the ImageDigest can be expensive, so this can help you postpone (or maybe even
+     * avoid) that work.
+     */
+    class BitmapAndDigest {
+    public:
+        explicit BitmapAndDigest(const SkBitmap &bitmap);
+
+        const ImageDigest *getImageDigestPtr();
+        const SkBitmap *getBitmapPtr() const;
+    private:
+        const SkBitmap fBitmap;
+        SkAutoTUnref<ImageDigest> fImageDigestRef;
+    };
+
+    /**
+     * Collects ImageDigests of actually rendered images, perhaps comparing to expectations.
+     */
+    class ImageResultsAndExpectations {
+    public:
+        /**
+         * Adds expectations from a JSON file, returning true if successful.
+         */
+        bool readExpectationsFile(const char *jsonPath);
 
         /**
          * Adds this image to the summary of results.
          *
          * @param sourceName name of the source file that generated this result
          * @param fileName relative path to the image output file on local disk
-         * @param bitmap bitmap to store the hash of
+         * @param digest description of the image's contents
          * @param tileNumber if not NULL, ptr to tile number
          */
-        void add(const char *sourceName, const char *fileName, const SkBitmap& bitmap,
+        void add(const char *sourceName, const char *fileName, const ImageDigest &digest,
                  const int *tileNumber=NULL);
 
         /**
@@ -45,10 +100,20 @@ namespace sk_tools {
          *
          * @param filename path to write the summary to
          */
-        void writeToFile(const char *filename);
+        void writeToFile(const char *filename) const;
 
     private:
+
+        /**
+         * Read the file contents from jsonPath and parse them into jsonRoot.
+         *
+         * Returns true if successful.
+         */
+        static bool Parse(const char *jsonPath, Json::Value *jsonRoot);
+
         Json::Value fActualResults;
+        Json::Value fExpectedJsonRoot;
+        Json::Value fExpectedResults;
     };
 
 } // namespace sk_tools
diff --git a/tools/render_pictures_main.cpp b/tools/render_pictures_main.cpp
index 1bc4d21e33..9f28bff4b9 100644
--- a/tools/render_pictures_main.cpp
+++ b/tools/render_pictures_main.cpp
@@ -31,6 +31,7 @@ DECLARE_bool(deferImageDecoding);
 DEFINE_int32(maxComponentDiff, 256, "Maximum diff on a component, 0 - 256. Components that differ "
              "by more than this amount are considered errors, though all diffs are reported. "
              "Requires --validate.");
+DEFINE_string(readJsonSummaryPath, "", "JSON file to read image expectations from.");
 DECLARE_string(readPath);
 DEFINE_bool(writeChecksumBasedFilenames, false,
             "When writing out images, use checksum-based filenames.");
@@ -254,7 +255,7 @@ private:
  */
 static bool render_picture(const SkString& inputPath, const SkString* outputDir,
                            sk_tools::PictureRenderer& renderer,
-                           sk_tools::ImageResultsSummary *jsonSummaryPtr) {
+                           sk_tools::ImageResultsAndExpectations *jsonSummaryPtr) {
     int diffs[256] = {0};
     SkBitmap* bitmap = NULL;
     renderer.setJsonSummaryPtr(jsonSummaryPtr);
@@ -348,9 +349,10 @@ static bool render_picture(const SkString& inputPath, const SkString* outputDir,
         outputPath.append(".png");
 
         if (NULL != jsonSummaryPtr) {
+            sk_tools::ImageDigest imageDigest(*bitmap);
             SkString outputFileBasename;
             sk_tools::get_basename(&outputFileBasename, outputPath);
-            jsonSummaryPtr->add(inputFilename.c_str(), outputFileBasename.c_str(), *bitmap);
+            jsonSummaryPtr->add(inputFilename.c_str(), outputFileBasename.c_str(), imageDigest);
         }
 
         if (NULL != outputDir) {
@@ -369,7 +371,7 @@ static bool render_picture(const SkString& inputPath, const SkString* outputDir,
 
 static int process_input(const char* input, const SkString* outputDir,
                          sk_tools::PictureRenderer& renderer,
-                         sk_tools::ImageResultsSummary *jsonSummaryPtr) {
+                         sk_tools::ImageResultsAndExpectations *jsonSummaryPtr) {
     SkOSFile::Iter iter(input, "skp");
     SkString inputFilename;
     int failures = 0;
@@ -449,10 +451,13 @@ int tool_main(int argc, char** argv) {
     if (FLAGS_writePath.count() == 1) {
         outputDir.set(FLAGS_writePath[0]);
     }
-    sk_tools::ImageResultsSummary jsonSummary;
-    sk_tools::ImageResultsSummary* jsonSummaryPtr = NULL;
+    sk_tools::ImageResultsAndExpectations jsonSummary;
+    sk_tools::ImageResultsAndExpectations* jsonSummaryPtr = NULL;
     if (FLAGS_writeJsonSummaryPath.count() == 1) {
         jsonSummaryPtr = &jsonSummary;
+        if (FLAGS_readJsonSummaryPath.count() == 1) {
+            SkASSERT(jsonSummary.readExpectationsFile(FLAGS_readJsonSummaryPath[0]));
+        }
     }
 
     int failures = 0;
diff --git a/tools/tests/render_pictures_test.py b/tools/tests/render_pictures_test.py
index d378a54688..4b11e56ae9 100755
--- a/tools/tests/render_pictures_test.py
+++ b/tools/tests/render_pictures_test.py
@@ -10,6 +10,7 @@ Test the render_pictures binary.
 """
 
 # System-level imports
+import copy
 import json
 import os
 import shutil
@@ -27,100 +28,138 @@ EXPECTED_HEADER_CONTENTS = {
 }
 
 # Manually verified: 640x400 red rectangle with black border
+# Standard expectations will be set up in such a way that this image fails
+# the comparison.
 RED_WHOLEIMAGE = {
     "checksumAlgorithm" : "bitmap-64bitMD5",
     "checksumValue" : 11092453015575919668,
-    "comparisonResult" : "no-comparison",
+    "comparisonResult" : "failed",
     "filepath" : "red_skp.png",
 }
 
 # Manually verified: 640x400 green rectangle with black border
+# Standard expectations will be set up in such a way that this image passes
+# the comparison.
 GREEN_WHOLEIMAGE = {
     "checksumAlgorithm" : "bitmap-64bitMD5",
     "checksumValue" : 8891695120562235492,
-    "comparisonResult" : "no-comparison",
+    "comparisonResult" : "succeeded",
     "filepath" : "green_skp.png",
 }
 
 # Manually verified these 6 images, all 256x256 tiles,
 # consistent with a tiled version of the 640x400 red rect
 # with black borders.
+# Standard expectations will be set up in such a way that these images fail
+# the comparison.
 RED_TILES = [{
     "checksumAlgorithm" : "bitmap-64bitMD5",
     "checksumValue" : 5815827069051002745,
-    "comparisonResult" : "no-comparison",
+    "comparisonResult" : "failed",
     "filepath" : "red_skp-tile0.png",
 },{
     "checksumAlgorithm" : "bitmap-64bitMD5",
     "checksumValue" : 9323613075234140270,
-    "comparisonResult" : "no-comparison",
+    "comparisonResult" : "failed",
     "filepath" : "red_skp-tile1.png",
 }, {
     "checksumAlgorithm" : "bitmap-64bitMD5",
     "checksumValue" : 16670399404877552232,
-    "comparisonResult" : "no-comparison",
+    "comparisonResult" : "failed",
     "filepath" : "red_skp-tile2.png",
 }, {
     "checksumAlgorithm" : "bitmap-64bitMD5",
     "checksumValue" : 2507897274083364964,
-    "comparisonResult" : "no-comparison",
+    "comparisonResult" : "failed",
     "filepath" : "red_skp-tile3.png",
 }, {
     "checksumAlgorithm" : "bitmap-64bitMD5",
     "checksumValue" : 7325267995523877959,
-    "comparisonResult" : "no-comparison",
+    "comparisonResult" : "failed",
     "filepath" : "red_skp-tile4.png",
 }, {
     "checksumAlgorithm" : "bitmap-64bitMD5",
     "checksumValue" : 2181381724594493116,
-    "comparisonResult" : "no-comparison",
+    "comparisonResult" : "failed",
     "filepath" : "red_skp-tile5.png",
 }]
 
 # Manually verified these 6 images, all 256x256 tiles,
 # consistent with a tiled version of the 640x400 green rect
 # with black borders.
+# Standard expectations will be set up in such a way that these images pass
+# the comparison.
 GREEN_TILES = [{
     "checksumAlgorithm" : "bitmap-64bitMD5",
     "checksumValue" : 12587324416545178013,
-    "comparisonResult" : "no-comparison",
+    "comparisonResult" : "succeeded",
     "filepath" : "green_skp-tile0.png",
 }, {
     "checksumAlgorithm" : "bitmap-64bitMD5",
     "checksumValue" : 7624374914829746293,
-    "comparisonResult" : "no-comparison",
+    "comparisonResult" : "succeeded",
     "filepath" : "green_skp-tile1.png",
 }, {
     "checksumAlgorithm" : "bitmap-64bitMD5",
     "checksumValue" : 5686489729535631913,
-    "comparisonResult" : "no-comparison",
+    "comparisonResult" : "succeeded",
     "filepath" : "green_skp-tile2.png",
 }, {
     "checksumAlgorithm" : "bitmap-64bitMD5",
     "checksumValue" : 7980646035555096146,
-    "comparisonResult" : "no-comparison",
+    "comparisonResult" : "succeeded",
     "filepath" : "green_skp-tile3.png",
 }, {
     "checksumAlgorithm" : "bitmap-64bitMD5",
     "checksumValue" : 17817086664365875131,
-    "comparisonResult" : "no-comparison",
+    "comparisonResult" : "succeeded",
     "filepath" : "green_skp-tile4.png",
 }, {
     "checksumAlgorithm" : "bitmap-64bitMD5",
     "checksumValue" : 10673669813016809363,
-    "comparisonResult" : "no-comparison",
+    "comparisonResult" : "succeeded",
     "filepath" : "green_skp-tile5.png",
 }]
 
 
+def modified_dict(input_dict, modification_dict):
+  """Returns a dict, with some modifications applied to it.
+
+  Args:
+    input_dict: a dictionary (which will be copied, not modified in place)
+    modification_dict: a set of key/value pairs to overwrite in the dict
+  """
+  output_dict = input_dict.copy()
+  output_dict.update(modification_dict)
+  return output_dict
+
+
+def modified_list_of_dicts(input_list, modification_dict):
+  """Returns a list of dicts, with some modifications applied to each dict.
+
+  Args:
+    input_list: a list of dictionaries; these dicts will be copied, not
+        modified in place
+    modification_dict: a set of key/value pairs to overwrite in each dict
+        within input_list
+  """
+  output_list = []
+  for input_dict in input_list:
+    output_dict = modified_dict(input_dict, modification_dict)
+    output_list.append(output_dict)
+  return output_list
+
+
 class RenderPicturesTest(base_unittest.TestCase):
 
   def setUp(self):
+    self.maxDiff = MAX_DIFF_LENGTH
+    self._expectations_dir = tempfile.mkdtemp()
     self._input_skp_dir = tempfile.mkdtemp()
     self._temp_dir = tempfile.mkdtemp()
-    self.maxDiff = MAX_DIFF_LENGTH
 
   def tearDown(self):
+    shutil.rmtree(self._expectations_dir)
     shutil.rmtree(self._input_skp_dir)
     shutil.rmtree(self._temp_dir)
 
@@ -137,14 +176,17 @@ class RenderPicturesTest(base_unittest.TestCase):
     probably shouldn't write out red_skp.png and green_skp.png at all!
     See http://skbug.com/2464
     """
-    output_json_path = os.path.join(self._temp_dir, 'output.json')
+    output_json_path = os.path.join(self._temp_dir, 'actuals.json')
     self._generate_skps()
-    self._run_render_pictures(['-r', self._input_skp_dir,
-                               '--bbh', 'grid', '256', '256',
-                               '--mode', 'tile', '256', '256',
-                               '--writeJsonSummaryPath', output_json_path,
-                               '--writePath', self._temp_dir,
-                               '--writeWholeImage'])
+    expectations_path = self._create_expectations()
+    self._run_render_pictures([
+        '-r', self._input_skp_dir,
+        '--bbh', 'grid', '256', '256',
+        '--mode', 'tile', '256', '256',
+        '--readJsonSummaryPath', expectations_path,
+        '--writeJsonSummaryPath', output_json_path,
+        '--writePath', self._temp_dir,
+        '--writeWholeImage'])
     expected_summary_dict = {
         "header" : EXPECTED_HEADER_CONTENTS,
         "actual-results" : {
@@ -160,15 +202,50 @@ class RenderPicturesTest(base_unittest.TestCase):
     }
     self._assert_json_contents(output_json_path, expected_summary_dict)
     self._assert_directory_contents(
-        self._temp_dir, ['red_skp.png', 'green_skp.png', 'output.json'])
+        self._temp_dir, ['red_skp.png', 'green_skp.png', 'actuals.json'])
+
+  def test_missing_tile_and_whole_image(self):
+    """test_tiled_whole_image, but missing expectations for some images.
+    """
+    output_json_path = os.path.join(self._temp_dir, 'actuals.json')
+    self._generate_skps()
+    expectations_path = self._create_expectations(missing_some_images=True)
+    self._run_render_pictures([
+        '-r', self._input_skp_dir,
+        '--bbh', 'grid', '256', '256',
+        '--mode', 'tile', '256', '256',
+        '--readJsonSummaryPath', expectations_path,
+        '--writeJsonSummaryPath', output_json_path,
+        '--writePath', self._temp_dir,
+        '--writeWholeImage'])
+    modified_red_tiles = copy.deepcopy(RED_TILES)
+    modified_red_tiles[5]['comparisonResult'] = 'no-comparison'
+    expected_summary_dict = {
+        "header" : EXPECTED_HEADER_CONTENTS,
+        "actual-results" : {
+            "red.skp": {
+                "tiled-images": modified_red_tiles,
+                "whole-image": modified_dict(
+                    RED_WHOLEIMAGE, {"comparisonResult" : "no-comparison"}),
+            },
+            "green.skp": {
+                "tiled-images": GREEN_TILES,
+                "whole-image": GREEN_WHOLEIMAGE,
+            }
+        }
+    }
+    self._assert_json_contents(output_json_path, expected_summary_dict)
 
   def test_untiled(self):
     """Run without tiles."""
-    output_json_path = os.path.join(self._temp_dir, 'output.json')
+    output_json_path = os.path.join(self._temp_dir, 'actuals.json')
     self._generate_skps()
-    self._run_render_pictures(['-r', self._input_skp_dir,
-                               '--writePath', self._temp_dir,
-                               '--writeJsonSummaryPath', output_json_path])
+    expectations_path = self._create_expectations()
+    self._run_render_pictures([
+        '-r', self._input_skp_dir,
+        '--readJsonSummaryPath', expectations_path,
+        '--writePath', self._temp_dir,
+        '--writeJsonSummaryPath', output_json_path])
     expected_summary_dict = {
         "header" : EXPECTED_HEADER_CONTENTS,
         "actual-results" : {
@@ -182,11 +259,11 @@ class RenderPicturesTest(base_unittest.TestCase):
     }
     self._assert_json_contents(output_json_path, expected_summary_dict)
     self._assert_directory_contents(
-        self._temp_dir, ['red_skp.png', 'green_skp.png', 'output.json'])
+        self._temp_dir, ['red_skp.png', 'green_skp.png', 'actuals.json'])
 
   def test_untiled_writeChecksumBasedFilenames(self):
     """Same as test_untiled, but with --writeChecksumBasedFilenames."""
-    output_json_path = os.path.join(self._temp_dir, 'output.json')
+    output_json_path = os.path.join(self._temp_dir, 'actuals.json')
     self._generate_skps()
     self._run_render_pictures(['-r', self._input_skp_dir,
                                '--writeChecksumBasedFilenames',
@@ -217,7 +294,7 @@ class RenderPicturesTest(base_unittest.TestCase):
     }
     self._assert_json_contents(output_json_path, expected_summary_dict)
     self._assert_directory_contents(self._temp_dir, [
-        'red_skp', 'green_skp', 'output.json'])
+        'red_skp', 'green_skp', 'actuals.json'])
     self._assert_directory_contents(
         os.path.join(self._temp_dir, 'red_skp'),
         ['bitmap-64bitMD5_11092453015575919668.png'])
@@ -227,12 +304,15 @@ class RenderPicturesTest(base_unittest.TestCase):
 
   def test_untiled_validate(self):
     """Same as test_untiled, but with --validate."""
-    output_json_path = os.path.join(self._temp_dir, 'output.json')
+    output_json_path = os.path.join(self._temp_dir, 'actuals.json')
     self._generate_skps()
-    self._run_render_pictures(['-r', self._input_skp_dir,
-                               '--validate',
-                               '--writePath', self._temp_dir,
-                               '--writeJsonSummaryPath', output_json_path])
+    expectations_path = self._create_expectations()
+    self._run_render_pictures([
+        '-r', self._input_skp_dir,
+        '--readJsonSummaryPath', expectations_path,
+        '--validate',
+        '--writePath', self._temp_dir,
+        '--writeJsonSummaryPath', output_json_path])
     expected_summary_dict = {
         "header" : EXPECTED_HEADER_CONTENTS,
         "actual-results" : {
@@ -246,14 +326,17 @@ class RenderPicturesTest(base_unittest.TestCase):
     }
     self._assert_json_contents(output_json_path, expected_summary_dict)
     self._assert_directory_contents(
-        self._temp_dir, ['red_skp.png', 'green_skp.png', 'output.json'])
+        self._temp_dir, ['red_skp.png', 'green_skp.png', 'actuals.json'])
 
   def test_untiled_without_writePath(self):
     """Same as test_untiled, but without --writePath."""
-    output_json_path = os.path.join(self._temp_dir, 'output.json')
+    output_json_path = os.path.join(self._temp_dir, 'actuals.json')
     self._generate_skps()
-    self._run_render_pictures(['-r', self._input_skp_dir,
-                               '--writeJsonSummaryPath', output_json_path])
+    expectations_path = self._create_expectations()
+    self._run_render_pictures([
+        '-r', self._input_skp_dir,
+        '--readJsonSummaryPath', expectations_path,
+        '--writeJsonSummaryPath', output_json_path])
     expected_summary_dict = {
         "header" : EXPECTED_HEADER_CONTENTS,
         "actual-results" : {
@@ -269,13 +352,16 @@ class RenderPicturesTest(base_unittest.TestCase):
 
   def test_tiled(self):
     """Generate individual tiles."""
-    output_json_path = os.path.join(self._temp_dir, 'output.json')
+    output_json_path = os.path.join(self._temp_dir, 'actuals.json')
     self._generate_skps()
-    self._run_render_pictures(['-r', self._input_skp_dir,
-                               '--bbh', 'grid', '256', '256',
-                               '--mode', 'tile', '256', '256',
-                               '--writePath', self._temp_dir,
-                               '--writeJsonSummaryPath', output_json_path])
+    expectations_path = self._create_expectations()
+    self._run_render_pictures([
+        '-r', self._input_skp_dir,
+        '--bbh', 'grid', '256', '256',
+        '--mode', 'tile', '256', '256',
+        '--readJsonSummaryPath', expectations_path,
+        '--writePath', self._temp_dir,
+        '--writeJsonSummaryPath', output_json_path])
     expected_summary_dict = {
         "header" : EXPECTED_HEADER_CONTENTS,
         "actual-results" : {
@@ -294,11 +380,11 @@ class RenderPicturesTest(base_unittest.TestCase):
          'red_skp-tile3.png', 'red_skp-tile4.png', 'red_skp-tile5.png',
          'green_skp-tile0.png', 'green_skp-tile1.png', 'green_skp-tile2.png',
          'green_skp-tile3.png', 'green_skp-tile4.png', 'green_skp-tile5.png',
-         'output.json'])
+         'actuals.json'])
 
   def test_tiled_writeChecksumBasedFilenames(self):
     """Same as test_tiled, but with --writeChecksumBasedFilenames."""
-    output_json_path = os.path.join(self._temp_dir, 'output.json')
+    output_json_path = os.path.join(self._temp_dir, 'actuals.json')
     self._generate_skps()
     self._run_render_pictures(['-r', self._input_skp_dir,
                                '--bbh', 'grid', '256', '256',
@@ -385,7 +471,7 @@ class RenderPicturesTest(base_unittest.TestCase):
     }
     self._assert_json_contents(output_json_path, expected_summary_dict)
     self._assert_directory_contents(self._temp_dir, [
-        'red_skp', 'green_skp', 'output.json'])
+        'red_skp', 'green_skp', 'actuals.json'])
     self._assert_directory_contents(
         os.path.join(self._temp_dir, 'red_skp'),
         ['bitmap-64bitMD5_5815827069051002745.png',
@@ -410,6 +496,43 @@ class RenderPicturesTest(base_unittest.TestCase):
                              '--config', '8888',
                              ] + args)
 
+  def _create_expectations(self, missing_some_images=False,
+                           rel_path='expectations.json'):
+    """Creates expectations JSON file within self._expectations_dir .
+
+    Args:
+      missing_some_images: (bool) whether to remove expectations for a subset
+          of the images
+      rel_path: (string) relative path within self._expectations_dir to write
+          the expectations into
+
+    Returns: full path to the expectations file created.
+    """
+    expectations_dict = {
+        "header" : EXPECTED_HEADER_CONTENTS,
+        "expected-results" : {
+            # red.skp: these should fail the comparison
+            "red.skp": {
+                "tiled-images": modified_list_of_dicts(
+                    RED_TILES, {'checksumValue': 11111}),
+                "whole-image": modified_dict(
+                    RED_WHOLEIMAGE, {'checksumValue': 22222}),
+            },
+            # green.skp: these should pass the comparison
+            "green.skp": {
+                "tiled-images": GREEN_TILES,
+                "whole-image": GREEN_WHOLEIMAGE,
+            }
+        }
+    }
+    if missing_some_images:
+      del expectations_dict['expected-results']['red.skp']['whole-image']
+      del expectations_dict['expected-results']['red.skp']['tiled-images'][-1]
+    path = os.path.join(self._expectations_dir, rel_path)
+    with open(path, 'w') as fh:
+      json.dump(expectations_dict, fh)
+    return path
+
   def _generate_skps(self):
     """Runs the skpmaker binary to generate files in self._input_skp_dir."""
     self._run_skpmaker(
@@ -452,7 +575,6 @@ class RenderPicturesTest(base_unittest.TestCase):
     """
     self.assertEqual(set(os.listdir(dir_path)), set(expected_filenames))
 
-
   def _assert_json_contents(self, json_path, expected_dict):
     """Asserts that contents of a JSON file are identical to expected_dict.
 
@@ -465,9 +587,13 @@ class RenderPicturesTest(base_unittest.TestCase):
       AssertionError: contents of the JSON file are not identical to
                       expected_dict.
     """
-    file_contents = open(json_path, 'r').read()
-    actual_dict = json.loads(file_contents)
-    self.assertEqual(actual_dict, expected_dict)
+    prettyprinted_expected_dict = json.dumps(expected_dict, sort_keys=True,
+                                             indent=2)
+    with open(json_path, 'r') as fh:
+      prettyprinted_json_dict = json.dumps(json.load(fh), sort_keys=True,
+                                           indent=2)
+    self.assertMultiLineEqual(prettyprinted_expected_dict,
+                              prettyprinted_json_dict)
 
 
 def main():