From 85195486939b19a88b25b3ba0a1ad6640980fb5a Mon Sep 17 00:00:00 2001 From: "commit-bot@chromium.org" Date: Mon, 13 Jan 2014 18:27:59 +0000 Subject: GM: run mode comparisons for all raster configs. GM works like this today: - Test all enabled configs directly - Render into 8888 - Test all enabled modes against that 8888. This change makes it do this: - Test all enabled configs directly - for each config, if it's raster, test all enabled modes against its output. Upshot is, we now check that picture draws identically to raster for 565 too, not just 8888. optimizations-565 is correctly failing. BUG=skia:1994 R=epoger@google.com, reed@google.com Author: mtklein@google.com Review URL: https://codereview.chromium.org/129203002 git-svn-id: http://skia.googlecode.com/svn/trunk@13050 2bbb7eff-a529-9590-31e7-b0007b416f81 --- gm/gmmain.cpp | 318 ++++++++++++--------- .../outputs/add-config-pdf/output-expected/stdout | 2 +- .../output-expected/stdout | 2 +- .../output-expected/stdout | 2 +- .../output-expected/stdout | 2 +- .../output-expected/stdout | 2 +- .../output-expected/stdout | 2 +- .../output-expected/stdout | 2 +- .../output-expected/stdout | 2 +- .../output-expected/stdout | 2 +- .../output-expected/stdout | 2 +- .../ignoring-one-test/output-expected/stdout | 2 +- .../ignoring-some-failures/output-expected/stdout | 2 +- .../output-expected/stdout | 2 +- .../outputs/no-hierarchy/output-expected/stdout | 2 +- .../outputs/no-readpath/output-expected/stdout | 2 +- .../output-expected/json-summary.txt | 13 +- .../mismatchPath/565/selftest1-pipe.png | 1 + .../mismatchPath/8888/selftest1-pipe.png | 1 + .../pipe-playback-failure/output-expected/stderr | 4 +- .../pipe-playback-failure/output-expected/stdout | 8 +- 21 files changed, 217 insertions(+), 158 deletions(-) create mode 100644 gm/tests/outputs/pipe-playback-failure/output-expected/mismatchPath/565/selftest1-pipe.png create mode 100644 gm/tests/outputs/pipe-playback-failure/output-expected/mismatchPath/8888/selftest1-pipe.png diff --git a/gm/gmmain.cpp b/gm/gmmain.cpp index e4c1c30110..daf0baed51 100644 --- a/gm/gmmain.cpp +++ b/gm/gmmain.cpp @@ -318,8 +318,8 @@ public: // and changing that now may affect other parts of our code. if (modeAsString.startsWith("-")) { modeAsString.remove(0, 1); - modes.push_back(modeAsString); } + modes.push_back(modeAsString); } } @@ -335,6 +335,27 @@ public: return false; } + /** + * Calls RecordTestResults to record that we skipped a test. + * + * Depending on the backend, this may mean that we skipped a single rendermode, or all + * rendermodes; see http://skbug.com/1994 and https://codereview.chromium.org/129203002/ + */ + void RecordSkippedTest(const SkString& shortNamePlusConfig, + const char renderModeDescriptor [], Backend backend) { + if (kRaster_Backend == backend) { + // Skipping a test on kRaster_Backend means that we will skip ALL renderModes + // (as opposed to other backends, on which we only run the default renderMode). + // + // We cannot call RecordTestResults yet, because we won't know the full set of + // renderModes until we have run all tests. + fTestsSkippedOnAllRenderModes.push_back(shortNamePlusConfig); + } else { + this->RecordTestResults(kIntentionallySkipped_ErrorType, shortNamePlusConfig, + renderModeDescriptor); + } + } + /** * Records the results of this test in fTestsRun and fFailedTests. * @@ -1240,6 +1261,7 @@ public: // collection of tests that have failed with each ErrorType SkTArray fFailedTests[kLast_ErrorType+1]; + SkTArray fTestsSkippedOnAllRenderModes; int fTestsRun; SkTDict fRenderModesEncountered; @@ -1514,127 +1536,6 @@ template void appendUnique(SkTDArray* array, const T& value) { } } -/** - * Run this test in a number of different configs (8888, 565, PDF, - * etc.), confirming that the resulting bitmaps match expectations - * (which may be different for each config). - * - * Returns all errors encountered while doing so. - */ -ErrorCombination run_multiple_configs(GMMain &gmmain, GM *gm, - const SkTDArray &configs, - const SkTDArray &pdfRasterizers, - GrContextFactory *grFactory); -ErrorCombination run_multiple_configs(GMMain &gmmain, GM *gm, - const SkTDArray &configs, - const SkTDArray &pdfRasterizers, - GrContextFactory *grFactory) { - const char renderModeDescriptor[] = ""; - ErrorCombination errorsForAllConfigs; - uint32_t gmFlags = gm->getFlags(); - - for (int i = 0; i < configs.count(); i++) { - ConfigData config = gRec[configs[i]]; - const SkString shortNamePlusConfig = gmmain.make_shortname_plus_config(gm->shortName(), - config.fName); - - // Skip any tests that we don't even need to try. - // If any of these were skipped on a per-GM basis, record them as - // kIntentionallySkipped. - if (kPDF_Backend == config.fBackend) { - if (gmFlags & GM::kSkipPDF_Flag) { - gmmain.RecordTestResults(kIntentionallySkipped_ErrorType, shortNamePlusConfig, - renderModeDescriptor); - errorsForAllConfigs.add(kIntentionallySkipped_ErrorType); - continue; - } - } - if ((gmFlags & GM::kSkip565_Flag) && - (kRaster_Backend == config.fBackend) && - (SkBitmap::kRGB_565_Config == config.fConfig)) { - gmmain.RecordTestResults(kIntentionallySkipped_ErrorType, shortNamePlusConfig, - renderModeDescriptor); - errorsForAllConfigs.add(kIntentionallySkipped_ErrorType); - continue; - } - if (((gmFlags & GM::kSkipGPU_Flag) && kGPU_Backend == config.fBackend) || - ((gmFlags & GM::kGPUOnly_Flag) && kGPU_Backend != config.fBackend)) { - gmmain.RecordTestResults(kIntentionallySkipped_ErrorType, shortNamePlusConfig, - renderModeDescriptor); - errorsForAllConfigs.add(kIntentionallySkipped_ErrorType); - continue; - } - - // Now we know that we want to run this test and record its - // success or failure. - ErrorCombination errorsForThisConfig; - GrSurface* gpuTarget = NULL; -#if SK_SUPPORT_GPU - SkAutoTUnref auGpuTarget; - if ((errorsForThisConfig.isEmpty()) && (kGPU_Backend == config.fBackend)) { - if (FLAGS_resetGpuContext) { - grFactory->destroyContexts(); - } - GrContext* gr = grFactory->get(config.fGLContextType); - bool grSuccess = false; - if (gr) { - // create a render target to back the device - GrTextureDesc desc; - desc.fConfig = kSkia8888_GrPixelConfig; - desc.fFlags = kRenderTarget_GrTextureFlagBit; - desc.fWidth = gm->getISize().width(); - desc.fHeight = gm->getISize().height(); - desc.fSampleCnt = config.fSampleCnt; - auGpuTarget.reset(gr->createUncachedTexture(desc, NULL, 0)); - if (NULL != auGpuTarget) { - gpuTarget = auGpuTarget; - grSuccess = true; - // Set the user specified cache limits if non-default. - size_t bytes; - int count; - gr->getTextureCacheLimits(&count, &bytes); - if (DEFAULT_CACHE_VALUE != gGpuCacheSizeBytes) { - bytes = static_cast(gGpuCacheSizeBytes); - } - if (DEFAULT_CACHE_VALUE != gGpuCacheSizeCount) { - count = gGpuCacheSizeCount; - } - gr->setTextureCacheLimits(count, bytes); - } - } - if (!grSuccess) { - errorsForThisConfig.add(kNoGpuContext_ErrorType); - } - } -#endif - - SkBitmap comparisonBitmap; - - const char* writePath; - if (FLAGS_writePath.count() == 1) { - writePath = FLAGS_writePath[0]; - } else { - writePath = NULL; - } - - if (errorsForThisConfig.isEmpty()) { - errorsForThisConfig.add(gmmain.test_drawing(gm, config, pdfRasterizers, - writePath, gpuTarget, - &comparisonBitmap)); - gmmain.RecordTestResults(errorsForThisConfig, shortNamePlusConfig, ""); - } - - if (FLAGS_deferred && errorsForThisConfig.isEmpty() && - (kGPU_Backend == config.fBackend || kRaster_Backend == config.fBackend)) { - errorsForThisConfig.add(gmmain.test_deferred_drawing(gm, config, comparisonBitmap, - gpuTarget)); - } - - errorsForAllConfigs.add(errorsForThisConfig); - } - return errorsForAllConfigs; -} - /** * Run this test in a number of different drawing modes (pipe, * deferred, tiled, etc.), confirming that the resulting bitmaps all @@ -1768,6 +1669,139 @@ ErrorCombination run_multiple_modes(GMMain &gmmain, GM *gm, const ConfigData &co return errorsForAllModes; } + +/** + * Run this test in a number of different configs (8888, 565, PDF, + * etc.), confirming that the resulting bitmaps match expectations + * (which may be different for each config). + * + * Returns all errors encountered while doing so. + */ +ErrorCombination run_multiple_configs(GMMain &gmmain, GM *gm, + const SkTDArray &configs, + const SkTDArray &pdfRasterizers, + const SkTDArray &tileGridReplayScales, + GrContextFactory *grFactory); +ErrorCombination run_multiple_configs(GMMain &gmmain, GM *gm, + const SkTDArray &configs, + const SkTDArray &pdfRasterizers, + const SkTDArray &tileGridReplayScales, + GrContextFactory *grFactory) { + const char renderModeDescriptor[] = ""; + ErrorCombination errorsForAllConfigs; + uint32_t gmFlags = gm->getFlags(); + + for (int i = 0; i < configs.count(); i++) { + ConfigData config = gRec[configs[i]]; + const SkString shortNamePlusConfig = gmmain.make_shortname_plus_config(gm->shortName(), + config.fName); + + // Skip any tests that we don't even need to try. + // If any of these were skipped on a per-GM basis, record them as + // kIntentionallySkipped. + if (kPDF_Backend == config.fBackend) { + if (gmFlags & GM::kSkipPDF_Flag) { + gmmain.RecordSkippedTest(shortNamePlusConfig, + renderModeDescriptor, + config.fBackend); + errorsForAllConfigs.add(kIntentionallySkipped_ErrorType); + continue; + } + } + if ((gmFlags & GM::kSkip565_Flag) && + (kRaster_Backend == config.fBackend) && + (SkBitmap::kRGB_565_Config == config.fConfig)) { + gmmain.RecordSkippedTest(shortNamePlusConfig, + renderModeDescriptor, + config.fBackend); + errorsForAllConfigs.add(kIntentionallySkipped_ErrorType); + continue; + } + if (((gmFlags & GM::kSkipGPU_Flag) && kGPU_Backend == config.fBackend) || + ((gmFlags & GM::kGPUOnly_Flag) && kGPU_Backend != config.fBackend)) { + gmmain.RecordSkippedTest(shortNamePlusConfig, + renderModeDescriptor, + config.fBackend); + errorsForAllConfigs.add(kIntentionallySkipped_ErrorType); + continue; + } + + // Now we know that we want to run this test and record its + // success or failure. + ErrorCombination errorsForThisConfig; + GrSurface* gpuTarget = NULL; +#if SK_SUPPORT_GPU + SkAutoTUnref auGpuTarget; + if ((errorsForThisConfig.isEmpty()) && (kGPU_Backend == config.fBackend)) { + if (FLAGS_resetGpuContext) { + grFactory->destroyContexts(); + } + GrContext* gr = grFactory->get(config.fGLContextType); + bool grSuccess = false; + if (gr) { + // create a render target to back the device + GrTextureDesc desc; + desc.fConfig = kSkia8888_GrPixelConfig; + desc.fFlags = kRenderTarget_GrTextureFlagBit; + desc.fWidth = gm->getISize().width(); + desc.fHeight = gm->getISize().height(); + desc.fSampleCnt = config.fSampleCnt; + auGpuTarget.reset(gr->createUncachedTexture(desc, NULL, 0)); + if (NULL != auGpuTarget) { + gpuTarget = auGpuTarget; + grSuccess = true; + // Set the user specified cache limits if non-default. + size_t bytes; + int count; + gr->getTextureCacheLimits(&count, &bytes); + if (DEFAULT_CACHE_VALUE != gGpuCacheSizeBytes) { + bytes = static_cast(gGpuCacheSizeBytes); + } + if (DEFAULT_CACHE_VALUE != gGpuCacheSizeCount) { + count = gGpuCacheSizeCount; + } + gr->setTextureCacheLimits(count, bytes); + } + } + if (!grSuccess) { + errorsForThisConfig.add(kNoGpuContext_ErrorType); + } + } +#endif + + SkBitmap comparisonBitmap; + + const char* writePath; + if (FLAGS_writePath.count() == 1) { + writePath = FLAGS_writePath[0]; + } else { + writePath = NULL; + } + + if (errorsForThisConfig.isEmpty()) { + errorsForThisConfig.add(gmmain.test_drawing(gm, config, pdfRasterizers, + writePath, gpuTarget, + &comparisonBitmap)); + gmmain.RecordTestResults(errorsForThisConfig, shortNamePlusConfig, ""); + } + + // TODO: run only if gmmain.test_drawing succeeded. + if (kRaster_Backend == config.fBackend) { + run_multiple_modes(gmmain, gm, config, comparisonBitmap, tileGridReplayScales); + } + + if (FLAGS_deferred && errorsForThisConfig.isEmpty() && + (kGPU_Backend == config.fBackend || kRaster_Backend == config.fBackend)) { + errorsForThisConfig.add(gmmain.test_deferred_drawing(gm, config, comparisonBitmap, + gpuTarget)); + } + + errorsForAllConfigs.add(errorsForThisConfig); + } + return errorsForAllConfigs; +} + + /** * Read individual lines from a file, pushing them into the given array. * @@ -2285,32 +2319,48 @@ int tool_main(int argc, char** argv) { gm_fprintf(stdout, "%sdrawing... %s [%d %d]\n", moduloStr.c_str(), shortName, size.width(), size.height()); - run_multiple_configs(gmmain, gm, configs, pdfRasterizers, grFactory); - - SkBitmap comparisonBitmap; - const ConfigData compareConfig = - { SkBitmap::kARGB_8888_Config, kRaster_Backend, kDontCare_GLContextType, 0, kRW_ConfigFlag, "comparison", false }; - gmmain.generate_image(gm, compareConfig, NULL, &comparisonBitmap, false); - - // TODO(epoger): only run this if gmmain.generate_image() succeeded? - // Otherwise, what are we comparing against? - run_multiple_modes(gmmain, gm, compareConfig, comparisonBitmap, tileGridReplayScales); + run_multiple_configs(gmmain, gm, configs, pdfRasterizers, tileGridReplayScales, grFactory); } SkTArray modes; gmmain.GetRenderModesEncountered(modes); + int modeCount = modes.count(); + + // Now that we have run all the tests and thus know the full set of renderModes that we + // tried to run, we can call RecordTestResults() to record the cases in which we skipped + // ALL renderModes. + // See http://skbug.com/1994 and https://codereview.chromium.org/129203002/ + int testCount = gmmain.fTestsSkippedOnAllRenderModes.count(); + for (int testNum = 0; testNum < testCount; ++testNum) { + const SkString &shortNamePlusConfig = gmmain.fTestsSkippedOnAllRenderModes[testNum]; + for (int modeNum = 0; modeNum < modeCount; ++modeNum) { + gmmain.RecordTestResults(kIntentionallySkipped_ErrorType, shortNamePlusConfig, + modes[modeNum].c_str()); + } + } + bool reportError = false; if (gmmain.NumSignificantErrors() > 0) { reportError = true; } - int expectedNumberOfTests = gmsRun * (configs.count() + modes.count()); + + // We test every GM against every config, and for every raster config also test every mode. + int rasterConfigs = 0; + for (int i = 0; i < configs.count(); i++) { + if (gRec[configs[i]].fBackend == kRaster_Backend) { + rasterConfigs++; + } + } + // For raster configs, we run all renderModes; for non-raster configs, just default renderMode + const int expectedNumberOfTests = rasterConfigs * gmsRun * modeCount + + (configs.count() - rasterConfigs) * gmsRun; // Output summary to stdout. if (FLAGS_verbose) { gm_fprintf(stdout, "Ran %d GMs\n", gmsRun); gm_fprintf(stdout, "... over %2d configs [%s]\n", configs.count(), list_all_config_names(configs).c_str()); - gm_fprintf(stdout, "... and %2d modes [%s]\n", modes.count(), list_all(modes).c_str()); + gm_fprintf(stdout, "... and %2d modes [%s]\n", modeCount, list_all(modes).c_str()); gm_fprintf(stdout, "... so there should be a total of %d tests.\n", expectedNumberOfTests); } gmmain.ListErrors(FLAGS_verbose); diff --git a/gm/tests/outputs/add-config-pdf/output-expected/stdout b/gm/tests/outputs/add-config-pdf/output-expected/stdout index cbdb12a1a0..507fc7cec7 100644 --- a/gm/tests/outputs/add-config-pdf/output-expected/stdout +++ b/gm/tests/outputs/add-config-pdf/output-expected/stdout @@ -8,7 +8,7 @@ GM: reading resources from resources GM: drawing... selftest1 [300 200] GM: Ran 1 GMs GM: ... over 3 configs ["8888", "565", "pdf"] -GM: ... and 0 modes [] +GM: ... and 1 modes [""] GM: ... so there should be a total of 3 tests. GM: Ran 3 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=0 MissingExpectations=1 WritingReferenceImage=0 GM: [*] 0 NoGpuContext: diff --git a/gm/tests/outputs/checksum-based-filenames/output-expected/stdout b/gm/tests/outputs/checksum-based-filenames/output-expected/stdout index 6e33214898..8432287a0e 100644 --- a/gm/tests/outputs/checksum-based-filenames/output-expected/stdout +++ b/gm/tests/outputs/checksum-based-filenames/output-expected/stdout @@ -7,7 +7,7 @@ GM: reading resources from resources GM: drawing... selftest1 [300 200] GM: Ran 1 GMs GM: ... over 2 configs ["8888", "565"] -GM: ... and 0 modes [] +GM: ... and 1 modes [""] GM: ... so there should be a total of 2 tests. GM: Ran 2 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=2 MissingExpectations=0 WritingReferenceImage=0 GM: [*] 0 NoGpuContext: diff --git a/gm/tests/outputs/compared-against-different-pixels-images/output-expected/stdout b/gm/tests/outputs/compared-against-different-pixels-images/output-expected/stdout index eb634e525e..32f73cc729 100644 --- a/gm/tests/outputs/compared-against-different-pixels-images/output-expected/stdout +++ b/gm/tests/outputs/compared-against-different-pixels-images/output-expected/stdout @@ -7,7 +7,7 @@ GM: reading resources from resources GM: drawing... selftest1 [300 200] GM: Ran 1 GMs GM: ... over 2 configs ["8888", "565"] -GM: ... and 0 modes [] +GM: ... and 1 modes [""] GM: ... so there should be a total of 2 tests. GM: Ran 2 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=2 MissingExpectations=0 WritingReferenceImage=0 GM: [*] 0 NoGpuContext: diff --git a/gm/tests/outputs/compared-against-different-pixels-json/output-expected/stdout b/gm/tests/outputs/compared-against-different-pixels-json/output-expected/stdout index 0701282514..d68c495615 100644 --- a/gm/tests/outputs/compared-against-different-pixels-json/output-expected/stdout +++ b/gm/tests/outputs/compared-against-different-pixels-json/output-expected/stdout @@ -7,7 +7,7 @@ GM: reading resources from resources GM: drawing... selftest1 [300 200] GM: Ran 1 GMs GM: ... over 2 configs ["8888", "565"] -GM: ... and 0 modes [] +GM: ... and 1 modes [""] GM: ... so there should be a total of 2 tests. GM: Ran 2 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=2 MissingExpectations=0 WritingReferenceImage=0 GM: [*] 0 NoGpuContext: diff --git a/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout b/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout index 9cbcf265c4..e4f8ab8f7c 100644 --- a/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout +++ b/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout @@ -7,7 +7,7 @@ GM: reading resources from resources GM: drawing... selftest1 [300 200] GM: Ran 1 GMs GM: ... over 2 configs ["8888", "565"] -GM: ... and 0 modes [] +GM: ... and 1 modes [""] GM: ... so there should be a total of 2 tests. GM: Ran 2 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=0 MissingExpectations=2 WritingReferenceImage=0 GM: [*] 0 NoGpuContext: diff --git a/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/stdout b/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/stdout index 23aa058ad0..bb1cc8a1a1 100644 --- a/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/stdout +++ b/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/stdout @@ -7,7 +7,7 @@ GM: reading resources from resources GM: drawing... selftest1 [300 200] GM: Ran 1 GMs GM: ... over 2 configs ["8888", "565"] -GM: ... and 0 modes [] +GM: ... and 1 modes [""] GM: ... so there should be a total of 2 tests. GM: Ran 2 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=0 MissingExpectations=0 WritingReferenceImage=0 GM: [*] 0 NoGpuContext: diff --git a/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/stdout b/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/stdout index 82de4b72e6..dc08b87ebe 100644 --- a/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/stdout +++ b/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/stdout @@ -7,7 +7,7 @@ GM: reading resources from resources GM: drawing... selftest1 [300 200] GM: Ran 1 GMs GM: ... over 2 configs ["8888", "565"] -GM: ... and 0 modes [] +GM: ... and 1 modes [""] GM: ... so there should be a total of 2 tests. GM: Ran 2 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=0 MissingExpectations=0 WritingReferenceImage=0 GM: [*] 0 NoGpuContext: diff --git a/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/stdout b/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/stdout index e2b02fedf1..0b31ff6e78 100644 --- a/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/stdout +++ b/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/stdout @@ -7,7 +7,7 @@ GM: reading resources from resources GM: drawing... selftest1 [300 200] GM: Ran 1 GMs GM: ... over 2 configs ["8888", "565"] -GM: ... and 0 modes [] +GM: ... and 1 modes [""] GM: ... so there should be a total of 2 tests. GM: Ran 2 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=0 MissingExpectations=0 WritingReferenceImage=0 GM: [*] 0 NoGpuContext: diff --git a/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/stdout b/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/stdout index 7f474041df..969c24b632 100644 --- a/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/stdout +++ b/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/stdout @@ -7,7 +7,7 @@ GM: reading resources from resources GM: drawing... selftest1 [300 200] GM: Ran 1 GMs GM: ... over 2 configs ["8888", "565"] -GM: ... and 0 modes [] +GM: ... and 1 modes [""] GM: ... so there should be a total of 2 tests. GM: Ran 2 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=0 MissingExpectations=0 WritingReferenceImage=0 GM: [*] 0 NoGpuContext: diff --git a/gm/tests/outputs/ignore-expectations-mismatch/output-expected/stdout b/gm/tests/outputs/ignore-expectations-mismatch/output-expected/stdout index a7ce529bfb..09aff571c8 100644 --- a/gm/tests/outputs/ignore-expectations-mismatch/output-expected/stdout +++ b/gm/tests/outputs/ignore-expectations-mismatch/output-expected/stdout @@ -7,7 +7,7 @@ GM: reading resources from resources GM: drawing... selftest1 [300 200] GM: Ran 1 GMs GM: ... over 2 configs ["8888", "565"] -GM: ... and 0 modes [] +GM: ... and 1 modes [""] GM: ... so there should be a total of 2 tests. GM: Ran 2 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=2 MissingExpectations=0 WritingReferenceImage=0 GM: [ ] 0 NoGpuContext: diff --git a/gm/tests/outputs/ignoring-one-test/output-expected/stdout b/gm/tests/outputs/ignoring-one-test/output-expected/stdout index 07adb39933..af45a03554 100644 --- a/gm/tests/outputs/ignoring-one-test/output-expected/stdout +++ b/gm/tests/outputs/ignoring-one-test/output-expected/stdout @@ -7,7 +7,7 @@ GM: reading resources from resources GM: drawing... selftest1 [300 200] GM: Ran 1 GMs GM: ... over 2 configs ["8888", "565"] -GM: ... and 0 modes [] +GM: ... and 1 modes [""] GM: ... so there should be a total of 2 tests. GM: Ran 2 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=2 MissingExpectations=0 WritingReferenceImage=0 GM: [*] 0 NoGpuContext: diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/stdout b/gm/tests/outputs/ignoring-some-failures/output-expected/stdout index 29506202b2..ba05a575cd 100644 --- a/gm/tests/outputs/ignoring-some-failures/output-expected/stdout +++ b/gm/tests/outputs/ignoring-some-failures/output-expected/stdout @@ -7,7 +7,7 @@ GM: reading resources from resources GM: drawing... selftest1 [300 200] GM: Ran 1 GMs GM: ... over 2 configs ["8888", "565"] -GM: ... and 0 modes [] +GM: ... and 1 modes [""] GM: ... so there should be a total of 2 tests. GM: Ran 2 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=2 MissingExpectations=0 WritingReferenceImage=0 GM: [*] 0 NoGpuContext: diff --git a/gm/tests/outputs/intentionally-skipped-tests/output-expected/stdout b/gm/tests/outputs/intentionally-skipped-tests/output-expected/stdout index 963c652934..b25d8feaeb 100644 --- a/gm/tests/outputs/intentionally-skipped-tests/output-expected/stdout +++ b/gm/tests/outputs/intentionally-skipped-tests/output-expected/stdout @@ -7,7 +7,7 @@ GM: drawing... selftest2 [300 200] GM: drawing... selftest1 [300 200] GM: Ran 2 GMs GM: ... over 2 configs ["8888", "565"] -GM: ... and 0 modes [] +GM: ... and 1 modes [""] GM: ... so there should be a total of 4 tests. GM: Ran 4 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=0 MissingExpectations=4 WritingReferenceImage=0 GM: [*] 0 NoGpuContext: diff --git a/gm/tests/outputs/no-hierarchy/output-expected/stdout b/gm/tests/outputs/no-hierarchy/output-expected/stdout index 7248d26978..2ae755a8b1 100644 --- a/gm/tests/outputs/no-hierarchy/output-expected/stdout +++ b/gm/tests/outputs/no-hierarchy/output-expected/stdout @@ -7,7 +7,7 @@ GM: reading resources from resources GM: drawing... selftest1 [300 200] GM: Ran 1 GMs GM: ... over 2 configs ["8888", "565"] -GM: ... and 0 modes [] +GM: ... and 1 modes [""] GM: ... so there should be a total of 2 tests. GM: Ran 2 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=2 MissingExpectations=0 WritingReferenceImage=0 GM: [*] 0 NoGpuContext: diff --git a/gm/tests/outputs/no-readpath/output-expected/stdout b/gm/tests/outputs/no-readpath/output-expected/stdout index ffa60c053a..ec10ea9d1f 100644 --- a/gm/tests/outputs/no-readpath/output-expected/stdout +++ b/gm/tests/outputs/no-readpath/output-expected/stdout @@ -6,7 +6,7 @@ GM: reading resources from resources GM: drawing... selftest1 [300 200] GM: Ran 1 GMs GM: ... over 2 configs ["8888", "565"] -GM: ... and 0 modes [] +GM: ... and 1 modes [""] GM: ... so there should be a total of 2 tests. GM: Ran 2 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 GeneratePdfFailed=0 ExpectationsMismatch=0 MissingExpectations=2 WritingReferenceImage=0 GM: [*] 0 NoGpuContext: diff --git a/gm/tests/outputs/pipe-playback-failure/output-expected/json-summary.txt b/gm/tests/outputs/pipe-playback-failure/output-expected/json-summary.txt index c99a8e9651..ebe8b592b0 100644 --- a/gm/tests/outputs/pipe-playback-failure/output-expected/json-summary.txt +++ b/gm/tests/outputs/pipe-playback-failure/output-expected/json-summary.txt @@ -1,7 +1,8 @@ { "actual-results" : { "failed" : { - "comparison/selftest1-pipe.png" : [ "bitmap-64bitMD5", 6140979239232854774 ] + "565/selftest1-pipe.png" : [ "bitmap-64bitMD5", 6140979239232854774 ], + "8888/selftest1-pipe.png" : [ "bitmap-64bitMD5", 6140979239232854774 ] }, "failure-ignored" : null, "no-comparison" : null, @@ -11,19 +12,25 @@ } }, "expected-results" : { + "565/selftest1-pipe.png" : { + "allowed-digests" : [ + [ "bitmap-64bitMD5", 12927999507540085554 ] + ], + "ignore-failure" : false + }, "565/selftest1.png" : { "allowed-digests" : [ [ "bitmap-64bitMD5", 12927999507540085554 ] ], "ignore-failure" : false }, - "8888/selftest1.png" : { + "8888/selftest1-pipe.png" : { "allowed-digests" : [ [ "bitmap-64bitMD5", 1209453360120438698 ] ], "ignore-failure" : false }, - "comparison/selftest1-pipe.png" : { + "8888/selftest1.png" : { "allowed-digests" : [ [ "bitmap-64bitMD5", 1209453360120438698 ] ], diff --git a/gm/tests/outputs/pipe-playback-failure/output-expected/mismatchPath/565/selftest1-pipe.png b/gm/tests/outputs/pipe-playback-failure/output-expected/mismatchPath/565/selftest1-pipe.png new file mode 100644 index 0000000000..6e665c4e55 --- /dev/null +++ b/gm/tests/outputs/pipe-playback-failure/output-expected/mismatchPath/565/selftest1-pipe.png @@ -0,0 +1 @@ +[contents of gm/tests/outputs/pipe-playback-failure/output-actual/mismatchPath/565/selftest1-pipe.png] diff --git a/gm/tests/outputs/pipe-playback-failure/output-expected/mismatchPath/8888/selftest1-pipe.png b/gm/tests/outputs/pipe-playback-failure/output-expected/mismatchPath/8888/selftest1-pipe.png new file mode 100644 index 0000000000..0426b8ce9d --- /dev/null +++ b/gm/tests/outputs/pipe-playback-failure/output-expected/mismatchPath/8888/selftest1-pipe.png @@ -0,0 +1 @@ +[contents of gm/tests/outputs/pipe-playback-failure/output-actual/mismatchPath/8888/selftest1-pipe.png] diff --git a/gm/tests/outputs/pipe-playback-failure/output-expected/stderr b/gm/tests/outputs/pipe-playback-failure/output-expected/stderr index 150a10b8cd..b40a74802d 100644 --- a/gm/tests/outputs/pipe-playback-failure/output-expected/stderr +++ b/gm/tests/outputs/pipe-playback-failure/output-expected/stderr @@ -1,2 +1,2 @@ -GM: FAILED to write bitmap: gm/tests/outputs/pipe-playback-failure/output-actual/mismatchPath/comparison/selftest1-pipe.png -GM: ---- comparison/selftest1-pipe.png: 60000 (of 60000) differing pixels, max per-channel mismatch R=135 G=246 B=135 A=0 +GM: ---- 8888/selftest1-pipe.png: 60000 (of 60000) differing pixels, max per-channel mismatch R=135 G=246 B=135 A=0 +GM: ---- 565/selftest1-pipe.png: not computing max per-channel pixel mismatch because non-8888 diff --git a/gm/tests/outputs/pipe-playback-failure/output-expected/stdout b/gm/tests/outputs/pipe-playback-failure/output-expected/stdout index fffe55ee67..dbb4dd8ad9 100644 --- a/gm/tests/outputs/pipe-playback-failure/output-expected/stdout +++ b/gm/tests/outputs/pipe-playback-failure/output-expected/stdout @@ -7,12 +7,12 @@ GM: reading resources from resources GM: drawing... selftest1 [300 200] GM: Ran 1 GMs GM: ... over 2 configs ["8888", "565"] -GM: ... and 1 modes ["pipe"] -GM: ... so there should be a total of 3 tests. -GM: Ran 3 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=1 GeneratePdfFailed=0 ExpectationsMismatch=0 MissingExpectations=0 WritingReferenceImage=0 +GM: ... and 2 modes ["", "pipe"] +GM: ... so there should be a total of 4 tests. +GM: Ran 4 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=2 GeneratePdfFailed=0 ExpectationsMismatch=0 MissingExpectations=0 WritingReferenceImage=0 GM: [*] 0 NoGpuContext: GM: [ ] 0 IntentionallySkipped: -GM: [*] 1 RenderModeMismatch: comparison/selftest1-pipe +GM: [*] 2 RenderModeMismatch: 8888/selftest1-pipe 565/selftest1-pipe GM: [*] 0 GeneratePdfFailed: GM: [*] 0 ExpectationsMismatch: GM: [ ] 0 MissingExpectations: -- cgit v1.2.3