GM: run mode comparisons for all raster configs.

GM works like this today:
  - Test all enabled configs directly
  - Render into 8888
  - Test all enabled modes against that 8888.

This change makes it do this:
  - Test all enabled configs directly
    - for each config, if it's raster, test all enabled modes against its output.

Upshot is, we now check that picture draws identically to raster for 565 too, not just 8888.  optimizations-565 is correctly failing.

BUG=skia:1994
R=epoger@google.com, reed@google.com

Author: mtklein@google.com

Review URL: https://codereview.chromium.org/129203002

git-svn-id: http://skia.googlecode.com/svn/trunk@13050 2bbb7eff-a529-9590-31e7-b0007b416f81

21 files changed, 217 insertions, 158 deletions

diff --git a/gm/gmmain.cpp b/gm/gmmain.cpp
index e4c1c30110..daf0baed51 100644
--- a/gm/gmmain.cpp
+++ b/gm/gmmain.cpp
@@ -318,8 +318,8 @@ public:
             // and changing that now may affect other parts of our code.
             if (modeAsString.startsWith("-")) {
                 modeAsString.remove(0, 1);
-                modes.push_back(modeAsString);
             }
+            modes.push_back(modeAsString);
         }
     }
 
@@ -336,6 +336,27 @@ public:
     }
 
     /**
+     * Calls RecordTestResults to record that we skipped a test.
+     *
+     * Depending on the backend, this may mean that we skipped a single rendermode, or all
+     * rendermodes; see http://skbug.com/1994 and https://codereview.chromium.org/129203002/
+     */
+    void RecordSkippedTest(const SkString& shortNamePlusConfig,
+                           const char renderModeDescriptor [], Backend backend) {
+        if (kRaster_Backend == backend) {
+            // Skipping a test on kRaster_Backend means that we will skip ALL renderModes
+            // (as opposed to other backends, on which we only run the default renderMode).
+            //
+            // We cannot call RecordTestResults yet, because we won't know the full set of
+            // renderModes until we have run all tests.
+            fTestsSkippedOnAllRenderModes.push_back(shortNamePlusConfig);
+        } else {
+            this->RecordTestResults(kIntentionallySkipped_ErrorType, shortNamePlusConfig,
+                                    renderModeDescriptor);
+        }
+    }
+
+    /**
      * Records the results of this test in fTestsRun and fFailedTests.
      *
      * We even record successes, and errors that we regard as
@@ -1240,6 +1261,7 @@ public:
 
     // collection of tests that have failed with each ErrorType
     SkTArray<SkString> fFailedTests[kLast_ErrorType+1];
+    SkTArray<SkString> fTestsSkippedOnAllRenderModes;
     int fTestsRun;
     SkTDict<int> fRenderModesEncountered;
 
@@ -1515,127 +1537,6 @@ template <typename T> void appendUnique(SkTDArray<T>* array, const T& value) {
 }
 
 /**
- * Run this test in a number of different configs (8888, 565, PDF,
- * etc.), confirming that the resulting bitmaps match expectations
- * (which may be different for each config).
- *
- * Returns all errors encountered while doing so.
- */
-ErrorCombination run_multiple_configs(GMMain &gmmain, GM *gm,
-                                      const SkTDArray<size_t> &configs,
-                                      const SkTDArray<const PDFRasterizerData*> &pdfRasterizers,
-                                      GrContextFactory *grFactory);
-ErrorCombination run_multiple_configs(GMMain &gmmain, GM *gm,
-                                      const SkTDArray<size_t> &configs,
-                                      const SkTDArray<const PDFRasterizerData*> &pdfRasterizers,
-                                      GrContextFactory *grFactory) {
-    const char renderModeDescriptor[] = "";
-    ErrorCombination errorsForAllConfigs;
-    uint32_t gmFlags = gm->getFlags();
-
-    for (int i = 0; i < configs.count(); i++) {
-        ConfigData config = gRec[configs[i]];
-        const SkString shortNamePlusConfig = gmmain.make_shortname_plus_config(gm->shortName(),
-                                                                               config.fName);
-
-        // Skip any tests that we don't even need to try.
-        // If any of these were skipped on a per-GM basis, record them as
-        // kIntentionallySkipped.
-        if (kPDF_Backend == config.fBackend) {
-            if (gmFlags & GM::kSkipPDF_Flag) {
-                gmmain.RecordTestResults(kIntentionallySkipped_ErrorType, shortNamePlusConfig,
-                                         renderModeDescriptor);
-                errorsForAllConfigs.add(kIntentionallySkipped_ErrorType);
-                continue;
-            }
-        }
-        if ((gmFlags & GM::kSkip565_Flag) &&
-            (kRaster_Backend == config.fBackend) &&
-            (SkBitmap::kRGB_565_Config == config.fConfig)) {
-            gmmain.RecordTestResults(kIntentionallySkipped_ErrorType, shortNamePlusConfig,
-                                     renderModeDescriptor);
-            errorsForAllConfigs.add(kIntentionallySkipped_ErrorType);
-            continue;
-        }
-        if (((gmFlags & GM::kSkipGPU_Flag) && kGPU_Backend == config.fBackend) ||
-            ((gmFlags & GM::kGPUOnly_Flag) && kGPU_Backend != config.fBackend)) {
-            gmmain.RecordTestResults(kIntentionallySkipped_ErrorType, shortNamePlusConfig,
-                                     renderModeDescriptor);
-            errorsForAllConfigs.add(kIntentionallySkipped_ErrorType);
-            continue;
-        }
-
-        // Now we know that we want to run this test and record its
-        // success or failure.
-        ErrorCombination errorsForThisConfig;
-        GrSurface* gpuTarget = NULL;
-#if SK_SUPPORT_GPU
-        SkAutoTUnref<GrSurface> auGpuTarget;
-        if ((errorsForThisConfig.isEmpty()) && (kGPU_Backend == config.fBackend)) {
-            if (FLAGS_resetGpuContext) {
-                grFactory->destroyContexts();
-            }
-            GrContext* gr = grFactory->get(config.fGLContextType);
-            bool grSuccess = false;
-            if (gr) {
-                // create a render target to back the device
-                GrTextureDesc desc;
-                desc.fConfig = kSkia8888_GrPixelConfig;
-                desc.fFlags = kRenderTarget_GrTextureFlagBit;
-                desc.fWidth = gm->getISize().width();
-                desc.fHeight = gm->getISize().height();
-                desc.fSampleCnt = config.fSampleCnt;
-                auGpuTarget.reset(gr->createUncachedTexture(desc, NULL, 0));
-                if (NULL != auGpuTarget) {
-                    gpuTarget = auGpuTarget;
-                    grSuccess = true;
-                    // Set the user specified cache limits if non-default.
-                    size_t bytes;
-                    int count;
-                    gr->getTextureCacheLimits(&count, &bytes);
-                    if (DEFAULT_CACHE_VALUE != gGpuCacheSizeBytes) {
-                        bytes = static_cast<size_t>(gGpuCacheSizeBytes);
-                    }
-                    if (DEFAULT_CACHE_VALUE != gGpuCacheSizeCount) {
-                        count = gGpuCacheSizeCount;
-                    }
-                    gr->setTextureCacheLimits(count, bytes);
-                }
-            }
-            if (!grSuccess) {
-                errorsForThisConfig.add(kNoGpuContext_ErrorType);
-            }
-        }
-#endif
-
-        SkBitmap comparisonBitmap;
-
-        const char* writePath;
-        if (FLAGS_writePath.count() == 1) {
-            writePath = FLAGS_writePath[0];
-        } else {
-            writePath = NULL;
-        }
-
-        if (errorsForThisConfig.isEmpty()) {
-            errorsForThisConfig.add(gmmain.test_drawing(gm, config, pdfRasterizers,
-                                                        writePath, gpuTarget,
-                                                        &comparisonBitmap));
-            gmmain.RecordTestResults(errorsForThisConfig, shortNamePlusConfig, "");
-        }
-
-        if (FLAGS_deferred && errorsForThisConfig.isEmpty() &&
-            (kGPU_Backend == config.fBackend || kRaster_Backend == config.fBackend)) {
-            errorsForThisConfig.add(gmmain.test_deferred_drawing(gm, config, comparisonBitmap,
-                                                                 gpuTarget));
-        }
-
-        errorsForAllConfigs.add(errorsForThisConfig);
-    }
-    return errorsForAllConfigs;
-}
-
-/**
  * Run this test in a number of different drawing modes (pipe,
  * deferred, tiled, etc.), confirming that the resulting bitmaps all
  * *exactly* match comparisonBitmap.
@@ -1768,6 +1669,139 @@ ErrorCombination run_multiple_modes(GMMain &gmmain, GM *gm, const ConfigData &co
     return errorsForAllModes;
 }
 
+
+/**
+ * Run this test in a number of different configs (8888, 565, PDF,
+ * etc.), confirming that the resulting bitmaps match expectations
+ * (which may be different for each config).
+ *
+ * Returns all errors encountered while doing so.
+ */
+ErrorCombination run_multiple_configs(GMMain &gmmain, GM *gm,
+                                      const SkTDArray<size_t> &configs,
+                                      const SkTDArray<const PDFRasterizerData*> &pdfRasterizers,
+                                      const SkTDArray<SkScalar> &tileGridReplayScales,
+                                      GrContextFactory *grFactory);
+ErrorCombination run_multiple_configs(GMMain &gmmain, GM *gm,
+                                      const SkTDArray<size_t> &configs,
+                                      const SkTDArray<const PDFRasterizerData*> &pdfRasterizers,
+                                      const SkTDArray<SkScalar> &tileGridReplayScales,
+                                      GrContextFactory *grFactory) {
+    const char renderModeDescriptor[] = "";
+    ErrorCombination errorsForAllConfigs;
+    uint32_t gmFlags = gm->getFlags();
+
+    for (int i = 0; i < configs.count(); i++) {
+        ConfigData config = gRec[configs[i]];
+        const SkString shortNamePlusConfig = gmmain.make_shortname_plus_config(gm->shortName(),
+                                                                               config.fName);
+
+        // Skip any tests that we don't even need to try.
+        // If any of these were skipped on a per-GM basis, record them as
+        // kIntentionallySkipped.
+        if (kPDF_Backend == config.fBackend) {
+            if (gmFlags & GM::kSkipPDF_Flag) {
+                gmmain.RecordSkippedTest(shortNamePlusConfig,
+                                         renderModeDescriptor,
+                                         config.fBackend);
+                errorsForAllConfigs.add(kIntentionallySkipped_ErrorType);
+                continue;
+            }
+        }
+        if ((gmFlags & GM::kSkip565_Flag) &&
+            (kRaster_Backend == config.fBackend) &&
+            (SkBitmap::kRGB_565_Config == config.fConfig)) {
+            gmmain.RecordSkippedTest(shortNamePlusConfig,
+                                     renderModeDescriptor,
+                                     config.fBackend);
+            errorsForAllConfigs.add(kIntentionallySkipped_ErrorType);
+            continue;
+        }
+        if (((gmFlags & GM::kSkipGPU_Flag) && kGPU_Backend == config.fBackend) ||
+            ((gmFlags & GM::kGPUOnly_Flag) && kGPU_Backend != config.fBackend)) {
+            gmmain.RecordSkippedTest(shortNamePlusConfig,
+                                     renderModeDescriptor,
+                                     config.fBackend);
+            errorsForAllConfigs.add(kIntentionallySkipped_ErrorType);
+            continue;
+        }
+
+        // Now we know that we want to run this test and record its
+        // success or failure.
+        ErrorCombination errorsForThisConfig;
+        GrSurface* gpuTarget = NULL;
+#if SK_SUPPORT_GPU
+        SkAutoTUnref<GrSurface> auGpuTarget;
+        if ((errorsForThisConfig.isEmpty()) && (kGPU_Backend == config.fBackend)) {
+            if (FLAGS_resetGpuContext) {
+                grFactory->destroyContexts();
+            }
+            GrContext* gr = grFactory->get(config.fGLContextType);
+            bool grSuccess = false;
+            if (gr) {
+                // create a render target to back the device
+                GrTextureDesc desc;
+                desc.fConfig = kSkia8888_GrPixelConfig;
+                desc.fFlags = kRenderTarget_GrTextureFlagBit;
+                desc.fWidth = gm->getISize().width();
+                desc.fHeight = gm->getISize().height();
+                desc.fSampleCnt = config.fSampleCnt;
+                auGpuTarget.reset(gr->createUncachedTexture(desc, NULL, 0));
+                if (NULL != auGpuTarget) {
+                    gpuTarget = auGpuTarget;
+                    grSuccess = true;
+                    // Set the user specified cache limits if non-default.
+                    size_t bytes;
+                    int count;
+                    gr->getTextureCacheLimits(&count, &bytes);
+                    if (DEFAULT_CACHE_VALUE != gGpuCacheSizeBytes) {
+                        bytes = static_cast<size_t>(gGpuCacheSizeBytes);
+                    }
+                    if (DEFAULT_CACHE_VALUE != gGpuCacheSizeCount) {
+                        count = gGpuCacheSizeCount;
+                    }
+                    gr->setTextureCacheLimits(count, bytes);
+                }
+            }
+            if (!grSuccess) {
+                errorsForThisConfig.add(kNoGpuContext_ErrorType);
+            }
+        }
+#endif
+
+        SkBitmap comparisonBitmap;
+
+        const char* writePath;
+        if (FLAGS_writePath.count() == 1) {
+            writePath = FLAGS_writePath[0];
+        } else {
+            writePath = NULL;
+        }
+
+        if (errorsForThisConfig.isEmpty()) {
+            errorsForThisConfig.add(gmmain.test_drawing(gm, config, pdfRasterizers,
+                                                        writePath, gpuTarget,
+                                                        &comparisonBitmap));
+            gmmain.RecordTestResults(errorsForThisConfig, shortNamePlusConfig, "");
+        }
+
+        // TODO: run only if gmmain.test_drawing succeeded.
+        if (kRaster_Backend == config.fBackend) {
+            run_multiple_modes(gmmain, gm, config, comparisonBitmap, tileGridReplayScales);
+        }
+
+        if (FLAGS_deferred && errorsForThisConfig.isEmpty() &&
+            (kGPU_Backend == config.fBackend || kRaster_Backend == config.fBackend)) {
+            errorsForThisConfig.add(gmmain.test_deferred_drawing(gm, config, comparisonBitmap,
+                                                                 gpuTarget));
+        }
+
+        errorsForAllConfigs.add(errorsForThisConfig);
+    }
+    return errorsForAllConfigs;
+}
+
+
 /**
  * Read individual lines from a file, pushing them into the given array.
  *
@@ -2285,32 +2319,48 @@ int tool_main(int argc, char** argv) {
         gm_fprintf(stdout, "%sdrawing... %s [%d %d]\n", moduloStr.c_str(), shortName,
                    size.width(), size.height());
 
-        run_multiple_configs(gmmain, gm, configs, pdfRasterizers, grFactory);
-
-        SkBitmap comparisonBitmap;
-        const ConfigData compareConfig =
-            { SkBitmap::kARGB_8888_Config, kRaster_Backend, kDontCare_GLContextType, 0, kRW_ConfigFlag, "comparison", false };
-        gmmain.generate_image(gm, compareConfig, NULL, &comparisonBitmap, false);
-
-        // TODO(epoger): only run this if gmmain.generate_image() succeeded?
-        // Otherwise, what are we comparing against?
-        run_multiple_modes(gmmain, gm, compareConfig, comparisonBitmap, tileGridReplayScales);
+        run_multiple_configs(gmmain, gm, configs, pdfRasterizers, tileGridReplayScales, grFactory);
     }
 
     SkTArray<SkString> modes;
     gmmain.GetRenderModesEncountered(modes);
+    int modeCount = modes.count();
+
+    // Now that we have run all the tests and thus know the full set of renderModes that we
+    // tried to run, we can call RecordTestResults() to record the cases in which we skipped
+    // ALL renderModes.
+    // See http://skbug.com/1994 and https://codereview.chromium.org/129203002/
+    int testCount = gmmain.fTestsSkippedOnAllRenderModes.count();
+    for (int testNum = 0; testNum < testCount; ++testNum) {
+        const SkString &shortNamePlusConfig = gmmain.fTestsSkippedOnAllRenderModes[testNum];
+        for (int modeNum = 0; modeNum < modeCount; ++modeNum) {
+            gmmain.RecordTestResults(kIntentionallySkipped_ErrorType, shortNamePlusConfig,
+                                     modes[modeNum].c_str());
+        }
+    }
+
     bool reportError = false;
     if (gmmain.NumSignificantErrors() > 0) {
         reportError = true;
     }
-    int expectedNumberOfTests = gmsRun * (configs.count() + modes.count());
+
+    // We test every GM against every config, and for every raster config also test every mode.
+    int rasterConfigs = 0;
+    for (int i = 0; i < configs.count(); i++) {
+        if (gRec[configs[i]].fBackend == kRaster_Backend) {
+            rasterConfigs++;
+        }
+    }
+    // For raster configs, we run all renderModes; for non-raster configs, just default renderMode
+    const int expectedNumberOfTests = rasterConfigs * gmsRun * modeCount
+                                    + (configs.count() - rasterConfigs) * gmsRun;
 
     // Output summary to stdout.
     if (FLAGS_verbose) {
         gm_fprintf(stdout, "Ran %d GMs\n", gmsRun);
         gm_fprintf(stdout, "... over %2d configs [%s]\n", configs.count(),
                    list_all_config_names(configs).c_str());
-        gm_fprintf(stdout, "...  and %2d modes   [%s]\n", modes.count(), list_all(modes).c_str());
+        gm_fprintf(stdout, "...  and %2d modes   [%s]\n", modeCount, list_all(modes).c_str());
         gm_fprintf(stdout, "... so there should be a total of %d tests.\n", expectedNumberOfTests);
     }
     gmmain.ListErrors(FLAGS_verbose);
diff --git a/gm/tests/outputs/add-config-pdf/output-expected/stdout b/gm/tests/outputs/add-config-pdf/output-expected/stdout
index cbdb12a1a0..507fc7cec7 100644
--- a/gm/tests/outputs/add-config-pdf/output-expected/stdout
+++ b/gm/tests/outputs/add-config-pdf/output-expected/stdout
@@ -8,7 +8,7 @@ GM: reading resources from resources
 GM: drawing... selftest1 [300 200]
 GM: Ran 1 GMs
 GM: ... over  3 configs ["8888", "565", "pdf"]
-GM: ...  and  0 modes   []
+GM: ...  and  1 modes   [""]
 GM: ... so there should be a total of 3 tests.
 GM: Ran 3 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=0 MissingExpectations=1 WritingReferenceImage=0
 GM: [*] 0 NoGpuContext:
diff --git a/gm/tests/outputs/checksum-based-filenames/output-expected/stdout b/gm/tests/outputs/checksum-based-filenames/output-expected/stdout
index 6e33214898..8432287a0e 100644
--- a/gm/tests/outputs/checksum-based-filenames/output-expected/stdout
+++ b/gm/tests/outputs/checksum-based-filenames/output-expected/stdout
@@ -7,7 +7,7 @@ GM: reading resources from resources
 GM: drawing... selftest1 [300 200]
 GM: Ran 1 GMs
 GM: ... over  2 configs ["8888", "565"]
-GM: ...  and  0 modes   []
+GM: ...  and  1 modes   [""]
 GM: ... so there should be a total of 2 tests.
 GM: Ran 2 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=2 MissingExpectations=0 WritingReferenceImage=0
 GM: [*] 0 NoGpuContext:
diff --git a/gm/tests/outputs/compared-against-different-pixels-images/output-expected/stdout b/gm/tests/outputs/compared-against-different-pixels-images/output-expected/stdout
index eb634e525e..32f73cc729 100644
--- a/gm/tests/outputs/compared-against-different-pixels-images/output-expected/stdout
+++ b/gm/tests/outputs/compared-against-different-pixels-images/output-expected/stdout
@@ -7,7 +7,7 @@ GM: reading resources from resources
 GM: drawing... selftest1 [300 200]
 GM: Ran 1 GMs
 GM: ... over  2 configs ["8888", "565"]
-GM: ...  and  0 modes   []
+GM: ...  and  1 modes   [""]
 GM: ... so there should be a total of 2 tests.
 GM: Ran 2 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=2 MissingExpectations=0 WritingReferenceImage=0
 GM: [*] 0 NoGpuContext:
diff --git a/gm/tests/outputs/compared-against-different-pixels-json/output-expected/stdout b/gm/tests/outputs/compared-against-different-pixels-json/output-expected/stdout
index 0701282514..d68c495615 100644
--- a/gm/tests/outputs/compared-against-different-pixels-json/output-expected/stdout
+++ b/gm/tests/outputs/compared-against-different-pixels-json/output-expected/stdout
@@ -7,7 +7,7 @@ GM: reading resources from resources
 GM: drawing... selftest1 [300 200]
 GM: Ran 1 GMs
 GM: ... over  2 configs ["8888", "565"]
-GM: ...  and  0 modes   []
+GM: ...  and  1 modes   [""]
 GM: ... so there should be a total of 2 tests.
 GM: Ran 2 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=2 MissingExpectations=0 WritingReferenceImage=0
 GM: [*] 0 NoGpuContext:
diff --git a/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout b/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout
index 9cbcf265c4..e4f8ab8f7c 100644
--- a/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout
+++ b/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout
@@ -7,7 +7,7 @@ GM: reading resources from resources
 GM: drawing... selftest1 [300 200]
 GM: Ran 1 GMs
 GM: ... over  2 configs ["8888", "565"]
-GM: ...  and  0 modes   []
+GM: ...  and  1 modes   [""]
 GM: ... so there should be a total of 2 tests.
 GM: Ran 2 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=0 MissingExpectations=2 WritingReferenceImage=0
 GM: [*] 0 NoGpuContext:
diff --git a/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/stdout b/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/stdout
index 23aa058ad0..bb1cc8a1a1 100644
--- a/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/stdout
+++ b/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/stdout
@@ -7,7 +7,7 @@ GM: reading resources from resources
 GM: drawing... selftest1 [300 200]
 GM: Ran 1 GMs
 GM: ... over  2 configs ["8888", "565"]
-GM: ...  and  0 modes   []
+GM: ...  and  1 modes   [""]
 GM: ... so there should be a total of 2 tests.
 GM: Ran 2 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=0 MissingExpectations=0 WritingReferenceImage=0
 GM: [*] 0 NoGpuContext:
diff --git a/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/stdout b/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/stdout
index 82de4b72e6..dc08b87ebe 100644
--- a/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/stdout
+++ b/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/stdout
@@ -7,7 +7,7 @@ GM: reading resources from resources
 GM: drawing... selftest1 [300 200]
 GM: Ran 1 GMs
 GM: ... over  2 configs ["8888", "565"]
-GM: ...  and  0 modes   []
+GM: ...  and  1 modes   [""]
 GM: ... so there should be a total of 2 tests.
 GM: Ran 2 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=0 MissingExpectations=0 WritingReferenceImage=0
 GM: [*] 0 NoGpuContext:
diff --git a/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/stdout b/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/stdout
index e2b02fedf1..0b31ff6e78 100644
--- a/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/stdout
+++ b/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/stdout
@@ -7,7 +7,7 @@ GM: reading resources from resources
 GM: drawing... selftest1 [300 200]
 GM: Ran 1 GMs
 GM: ... over  2 configs ["8888", "565"]
-GM: ...  and  0 modes   []
+GM: ...  and  1 modes   [""]
 GM: ... so there should be a total of 2 tests.
 GM: Ran 2 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=0 MissingExpectations=0 WritingReferenceImage=0
 GM: [*] 0 NoGpuContext:
diff --git a/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/stdout b/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/stdout
index 7f474041df..969c24b632 100644
--- a/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/stdout
+++ b/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/stdout
@@ -7,7 +7,7 @@ GM: reading resources from resources
 GM: drawing... selftest1 [300 200]
 GM: Ran 1 GMs
 GM: ... over  2 configs ["8888", "565"]
-GM: ...  and  0 modes   []
+GM: ...  and  1 modes   [""]
 GM: ... so there should be a total of 2 tests.
 GM: Ran 2 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=0 MissingExpectations=0 WritingReferenceImage=0
 GM: [*] 0 NoGpuContext:
diff --git a/gm/tests/outputs/ignore-expectations-mismatch/output-expected/stdout b/gm/tests/outputs/ignore-expectations-mismatch/output-expected/stdout
index a7ce529bfb..09aff571c8 100644
--- a/gm/tests/outputs/ignore-expectations-mismatch/output-expected/stdout
+++ b/gm/tests/outputs/ignore-expectations-mismatch/output-expected/stdout
@@ -7,7 +7,7 @@ GM: reading resources from resources
 GM: drawing... selftest1 [300 200]
 GM: Ran 1 GMs
 GM: ... over  2 configs ["8888", "565"]
-GM: ...  and  0 modes   []
+GM: ...  and  1 modes   [""]
 GM: ... so there should be a total of 2 tests.
 GM: Ran 2 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=2 MissingExpectations=0 WritingReferenceImage=0
 GM: [ ] 0 NoGpuContext:
diff --git a/gm/tests/outputs/ignoring-one-test/output-expected/stdout b/gm/tests/outputs/ignoring-one-test/output-expected/stdout
index 07adb39933..af45a03554 100644
--- a/gm/tests/outputs/ignoring-one-test/output-expected/stdout
+++ b/gm/tests/outputs/ignoring-one-test/output-expected/stdout
@@ -7,7 +7,7 @@ GM: reading resources from resources
 GM: drawing... selftest1 [300 200]
 GM: Ran 1 GMs
 GM: ... over  2 configs ["8888", "565"]
-GM: ...  and  0 modes   []
+GM: ...  and  1 modes   [""]
 GM: ... so there should be a total of 2 tests.
 GM: Ran 2 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=2 MissingExpectations=0 WritingReferenceImage=0
 GM: [*] 0 NoGpuContext:
diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/stdout b/gm/tests/outputs/ignoring-some-failures/output-expected/stdout
index 29506202b2..ba05a575cd 100644
--- a/gm/tests/outputs/ignoring-some-failures/output-expected/stdout
+++ b/gm/tests/outputs/ignoring-some-failures/output-expected/stdout
@@ -7,7 +7,7 @@ GM: reading resources from resources
 GM: drawing... selftest1 [300 200]
 GM: Ran 1 GMs
 GM: ... over  2 configs ["8888", "565"]
-GM: ...  and  0 modes   []
+GM: ...  and  1 modes   [""]
 GM: ... so there should be a total of 2 tests.
 GM: Ran 2 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=2 MissingExpectations=0 WritingReferenceImage=0
 GM: [*] 0 NoGpuContext:
diff --git a/gm/tests/outputs/intentionally-skipped-tests/output-expected/stdout b/gm/tests/outputs/intentionally-skipped-tests/output-expected/stdout
index 963c652934..b25d8feaeb 100644
--- a/gm/tests/outputs/intentionally-skipped-tests/output-expected/stdout
+++ b/gm/tests/outputs/intentionally-skipped-tests/output-expected/stdout
@@ -7,7 +7,7 @@ GM: drawing... selftest2 [300 200]
 GM: drawing... selftest1 [300 200]
 GM: Ran 2 GMs
 GM: ... over  2 configs ["8888", "565"]
-GM: ...  and  0 modes   []
+GM: ...  and  1 modes   [""]
 GM: ... so there should be a total of 4 tests.
 GM: Ran 4 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=0 MissingExpectations=4 WritingReferenceImage=0
 GM: [*] 0 NoGpuContext:
diff --git a/gm/tests/outputs/no-hierarchy/output-expected/stdout b/gm/tests/outputs/no-hierarchy/output-expected/stdout
index 7248d26978..2ae755a8b1 100644
--- a/gm/tests/outputs/no-hierarchy/output-expected/stdout
+++ b/gm/tests/outputs/no-hierarchy/output-expected/stdout
@@ -7,7 +7,7 @@ GM: reading resources from resources
 GM: drawing... selftest1 [300 200]
 GM: Ran 1 GMs
 GM: ... over  2 configs ["8888", "565"]
-GM: ...  and  0 modes   []
+GM: ...  and  1 modes   [""]
 GM: ... so there should be a total of 2 tests.
 GM: Ran 2 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=2 MissingExpectations=0 WritingReferenceImage=0
 GM: [*] 0 NoGpuContext:
diff --git a/gm/tests/outputs/no-readpath/output-expected/stdout b/gm/tests/outputs/no-readpath/output-expected/stdout
index ffa60c053a..ec10ea9d1f 100644
--- a/gm/tests/outputs/no-readpath/output-expected/stdout
+++ b/gm/tests/outputs/no-readpath/output-expected/stdout
@@ -6,7 +6,7 @@ GM: reading resources from resources
 GM: drawing... selftest1 [300 200]
 GM: Ran 1 GMs
 GM: ... over  2 configs ["8888", "565"]
-GM: ...  and  0 modes   []
+GM: ...  and  1 modes   [""]
 GM: ... so there should be a total of 2 tests.
 GM: Ran 2 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=0 MissingExpectations=2 WritingReferenceImage=0
 GM: [*] 0 NoGpuContext:
diff --git a/gm/tests/outputs/pipe-playback-failure/output-expected/json-summary.txt b/gm/tests/outputs/pipe-playback-failure/output-expected/json-summary.txt
index c99a8e9651..ebe8b592b0 100644
--- a/gm/tests/outputs/pipe-playback-failure/output-expected/json-summary.txt
+++ b/gm/tests/outputs/pipe-playback-failure/output-expected/json-summary.txt
@@ -1,7 +1,8 @@
 {
    "actual-results" : {
       "failed" : {
-         "comparison/selftest1-pipe.png" : [ "bitmap-64bitMD5", 6140979239232854774 ]
+         "565/selftest1-pipe.png" : [ "bitmap-64bitMD5", 6140979239232854774 ],
+         "8888/selftest1-pipe.png" : [ "bitmap-64bitMD5", 6140979239232854774 ]
       },
       "failure-ignored" : null,
       "no-comparison" : null,
@@ -11,19 +12,25 @@
       }
    },
    "expected-results" : {
+      "565/selftest1-pipe.png" : {
+         "allowed-digests" : [
+            [ "bitmap-64bitMD5", 12927999507540085554 ]
+         ],
+         "ignore-failure" : false
+      },
       "565/selftest1.png" : {
          "allowed-digests" : [
             [ "bitmap-64bitMD5", 12927999507540085554 ]
          ],
          "ignore-failure" : false
       },
-      "8888/selftest1.png" : {
+      "8888/selftest1-pipe.png" : {
          "allowed-digests" : [
             [ "bitmap-64bitMD5", 1209453360120438698 ]
          ],
          "ignore-failure" : false
       },
-      "comparison/selftest1-pipe.png" : {
+      "8888/selftest1.png" : {
          "allowed-digests" : [
             [ "bitmap-64bitMD5", 1209453360120438698 ]
          ],
diff --git a/gm/tests/outputs/pipe-playback-failure/output-expected/mismatchPath/565/selftest1-pipe.png b/gm/tests/outputs/pipe-playback-failure/output-expected/mismatchPath/565/selftest1-pipe.png
new file mode 100644
index 0000000000..6e665c4e55
--- /dev/null
+++ b/gm/tests/outputs/pipe-playback-failure/output-expected/mismatchPath/565/selftest1-pipe.png
@@ -0,0 +1 @@
+[contents of gm/tests/outputs/pipe-playback-failure/output-actual/mismatchPath/565/selftest1-pipe.png]
diff --git a/gm/tests/outputs/pipe-playback-failure/output-expected/mismatchPath/8888/selftest1-pipe.png b/gm/tests/outputs/pipe-playback-failure/output-expected/mismatchPath/8888/selftest1-pipe.png
new file mode 100644
index 0000000000..0426b8ce9d
--- /dev/null
+++ b/gm/tests/outputs/pipe-playback-failure/output-expected/mismatchPath/8888/selftest1-pipe.png
@@ -0,0 +1 @@
+[contents of gm/tests/outputs/pipe-playback-failure/output-actual/mismatchPath/8888/selftest1-pipe.png]
diff --git a/gm/tests/outputs/pipe-playback-failure/output-expected/stderr b/gm/tests/outputs/pipe-playback-failure/output-expected/stderr
index 150a10b8cd..b40a74802d 100644
--- a/gm/tests/outputs/pipe-playback-failure/output-expected/stderr
+++ b/gm/tests/outputs/pipe-playback-failure/output-expected/stderr
@@ -1,2 +1,2 @@
-GM: FAILED to write bitmap: gm/tests/outputs/pipe-playback-failure/output-actual/mismatchPath/comparison/selftest1-pipe.png
-GM: ---- comparison/selftest1-pipe.png: 60000 (of 60000) differing pixels, max per-channel mismatch R=135 G=246 B=135 A=0
+GM: ---- 8888/selftest1-pipe.png: 60000 (of 60000) differing pixels, max per-channel mismatch R=135 G=246 B=135 A=0
+GM: ---- 565/selftest1-pipe.png: not computing max per-channel pixel mismatch because non-8888
diff --git a/gm/tests/outputs/pipe-playback-failure/output-expected/stdout b/gm/tests/outputs/pipe-playback-failure/output-expected/stdout
index fffe55ee67..dbb4dd8ad9 100644
--- a/gm/tests/outputs/pipe-playback-failure/output-expected/stdout
+++ b/gm/tests/outputs/pipe-playback-failure/output-expected/stdout
@@ -7,12 +7,12 @@ GM: reading resources from resources
 GM: drawing... selftest1 [300 200]
 GM: Ran 1 GMs
 GM: ... over  2 configs ["8888", "565"]
-GM: ...  and  1 modes   ["pipe"]
-GM: ... so there should be a total of 3 tests.
-GM: Ran 3 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=1 GeneratePdfFailed=0 ExpectationsMismatch=0 MissingExpectations=0 WritingReferenceImage=0
+GM: ...  and  2 modes   ["", "pipe"]
+GM: ... so there should be a total of 4 tests.
+GM: Ran 4 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=2 GeneratePdfFailed=0 ExpectationsMismatch=0 MissingExpectations=0 WritingReferenceImage=0
 GM: [*] 0 NoGpuContext:
 GM: [ ] 0 IntentionallySkipped:
-GM: [*] 1 RenderModeMismatch: comparison/selftest1-pipe
+GM: [*] 2 RenderModeMismatch: 8888/selftest1-pipe 565/selftest1-pipe
 GM: [*] 0 GeneratePdfFailed:
 GM: [*] 0 ExpectationsMismatch:
 GM: [ ] 0 MissingExpectations: