From 5852c8b06df9ed961526efb8df02bc00c87b9d4c Mon Sep 17 00:00:00 2001 From: "epoger@google.com" Date: Fri, 30 Aug 2013 20:19:25 +0000 Subject: gm: add self-test to confirm that ignore-failure field is respected BUG=skia:1578 R=scroggo@google.com Review URL: https://codereview.chromium.org/23799004 git-svn-id: http://skia.googlecode.com/svn/trunk@11047 2bbb7eff-a529-9590-31e7-b0007b416f81 --- gm/gmmain.cpp | 6 ----- .../output-expected/command_line | 1 + .../output-expected/json-summary.txt | 26 ++++++++++++++++++++++ .../output-expected/mismatchPath/565/bogusfile | 1 + .../output-expected/mismatchPath/565/selftest1.png | 1 + .../output-expected/mismatchPath/8888/bogusfile | 1 + .../mismatchPath/8888/selftest1.png | 1 + .../output-expected/mismatchPath/bogusfile | 1 + .../missingExpectationsPath/565/bogusfile | 1 + .../missingExpectationsPath/8888/bogusfile | 1 + .../missingExpectationsPath/bogusfile | 1 + .../output-expected/return_value | 1 + .../ignoring-some-failures/output-expected/stderr | 0 .../ignoring-some-failures/output-expected/stdout | 18 +++++++++++++++ .../output-expected/writePath/565/bogusfile | 1 + .../output-expected/writePath/565/selftest1.png | 1 + .../output-expected/writePath/8888/bogusfile | 1 + .../output-expected/writePath/8888/selftest1.png | 1 + .../output-expected/writePath/bogusfile | 1 + gm/tests/run.sh | 17 +++++++++++--- 20 files changed, 73 insertions(+), 9 deletions(-) create mode 100644 gm/tests/outputs/ignoring-some-failures/output-expected/command_line create mode 100644 gm/tests/outputs/ignoring-some-failures/output-expected/json-summary.txt create mode 100644 gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/565/bogusfile create mode 100644 gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/565/selftest1.png create mode 100644 gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/8888/bogusfile create mode 100644 gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/8888/selftest1.png create mode 100644 gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/bogusfile create mode 100644 gm/tests/outputs/ignoring-some-failures/output-expected/missingExpectationsPath/565/bogusfile create mode 100644 gm/tests/outputs/ignoring-some-failures/output-expected/missingExpectationsPath/8888/bogusfile create mode 100644 gm/tests/outputs/ignoring-some-failures/output-expected/missingExpectationsPath/bogusfile create mode 100644 gm/tests/outputs/ignoring-some-failures/output-expected/return_value create mode 100644 gm/tests/outputs/ignoring-some-failures/output-expected/stderr create mode 100644 gm/tests/outputs/ignoring-some-failures/output-expected/stdout create mode 100644 gm/tests/outputs/ignoring-some-failures/output-expected/writePath/565/bogusfile create mode 100644 gm/tests/outputs/ignoring-some-failures/output-expected/writePath/565/selftest1.png create mode 100644 gm/tests/outputs/ignoring-some-failures/output-expected/writePath/8888/bogusfile create mode 100644 gm/tests/outputs/ignoring-some-failures/output-expected/writePath/8888/selftest1.png create mode 100644 gm/tests/outputs/ignoring-some-failures/output-expected/writePath/bogusfile (limited to 'gm') diff --git a/gm/gmmain.cpp b/gm/gmmain.cpp index c24d9c70e4..5ebccb24e7 100644 --- a/gm/gmmain.cpp +++ b/gm/gmmain.cpp @@ -821,12 +821,6 @@ public: this->fJsonActualResults_Succeeded[testName] = jsonActualResults; } else { if (ignoreFailure) { - // TODO: Once we have added the ability to compare - // actual results against expectations in a JSON file - // (where we can set ignore-failure to either true or - // false), add test cases that exercise ignored - // failures (both for kMissingExpectations_ErrorType - // and kExpectationsMismatch_ErrorType). this->fJsonActualResults_FailureIgnored[testName] = jsonActualResults; } else { diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/command_line b/gm/tests/outputs/ignoring-some-failures/output-expected/command_line new file mode 100644 index 0000000000..37be27526c --- /dev/null +++ b/gm/tests/outputs/ignoring-some-failures/output-expected/command_line @@ -0,0 +1 @@ +out/Debug/gm --verbose --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/json/different-pixels-ignore-some-failures.json --writeJsonSummaryPath gm/tests/outputs/ignoring-some-failures/output-actual/json-summary.txt --writePath gm/tests/outputs/ignoring-some-failures/output-actual/writePath --mismatchPath gm/tests/outputs/ignoring-some-failures/output-actual/mismatchPath --missingExpectationsPath gm/tests/outputs/ignoring-some-failures/output-actual/missingExpectationsPath diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/json-summary.txt b/gm/tests/outputs/ignoring-some-failures/output-expected/json-summary.txt new file mode 100644 index 0000000000..b152d6d035 --- /dev/null +++ b/gm/tests/outputs/ignoring-some-failures/output-expected/json-summary.txt @@ -0,0 +1,26 @@ +{ + "actual-results" : { + "failed" : { + "8888/selftest1.png" : [ "bitmap-64bitMD5", 1209453360120438698 ] + }, + "failure-ignored" : { + "565/selftest1.png" : [ "bitmap-64bitMD5", 12927999507540085554 ] + }, + "no-comparison" : null, + "succeeded" : null + }, + "expected-results" : { + "565/selftest1.png" : { + "allowed-digests" : [ + [ "bitmap-64bitMD5", 8863920166200910451 ] + ], + "ignore-failure" : true + }, + "8888/selftest1.png" : { + "allowed-digests" : [ + [ "bitmap-64bitMD5", 13451349865803053525 ] + ], + "ignore-failure" : false + } + } +} diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/565/bogusfile b/gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/565/bogusfile new file mode 100644 index 0000000000..d86cd5bdd8 --- /dev/null +++ b/gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/565/bogusfile @@ -0,0 +1 @@ +Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories. diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/565/selftest1.png b/gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/565/selftest1.png new file mode 100644 index 0000000000..5a30830f5a --- /dev/null +++ b/gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/565/selftest1.png @@ -0,0 +1 @@ +[contents of gm/tests/outputs/ignoring-some-failures/output-actual/mismatchPath/565/selftest1.png] diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/8888/bogusfile b/gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/8888/bogusfile new file mode 100644 index 0000000000..d86cd5bdd8 --- /dev/null +++ b/gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/8888/bogusfile @@ -0,0 +1 @@ +Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories. diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/8888/selftest1.png b/gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/8888/selftest1.png new file mode 100644 index 0000000000..734abd0f49 --- /dev/null +++ b/gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/8888/selftest1.png @@ -0,0 +1 @@ +[contents of gm/tests/outputs/ignoring-some-failures/output-actual/mismatchPath/8888/selftest1.png] diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/bogusfile b/gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/bogusfile new file mode 100644 index 0000000000..d86cd5bdd8 --- /dev/null +++ b/gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/bogusfile @@ -0,0 +1 @@ +Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories. diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/missingExpectationsPath/565/bogusfile b/gm/tests/outputs/ignoring-some-failures/output-expected/missingExpectationsPath/565/bogusfile new file mode 100644 index 0000000000..d86cd5bdd8 --- /dev/null +++ b/gm/tests/outputs/ignoring-some-failures/output-expected/missingExpectationsPath/565/bogusfile @@ -0,0 +1 @@ +Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories. diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/missingExpectationsPath/8888/bogusfile b/gm/tests/outputs/ignoring-some-failures/output-expected/missingExpectationsPath/8888/bogusfile new file mode 100644 index 0000000000..d86cd5bdd8 --- /dev/null +++ b/gm/tests/outputs/ignoring-some-failures/output-expected/missingExpectationsPath/8888/bogusfile @@ -0,0 +1 @@ +Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories. diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/missingExpectationsPath/bogusfile b/gm/tests/outputs/ignoring-some-failures/output-expected/missingExpectationsPath/bogusfile new file mode 100644 index 0000000000..d86cd5bdd8 --- /dev/null +++ b/gm/tests/outputs/ignoring-some-failures/output-expected/missingExpectationsPath/bogusfile @@ -0,0 +1 @@ +Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories. diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/return_value b/gm/tests/outputs/ignoring-some-failures/output-expected/return_value new file mode 100644 index 0000000000..ace9d03621 --- /dev/null +++ b/gm/tests/outputs/ignoring-some-failures/output-expected/return_value @@ -0,0 +1 @@ +255 diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/stderr b/gm/tests/outputs/ignoring-some-failures/output-expected/stderr new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/stdout b/gm/tests/outputs/ignoring-some-failures/output-expected/stdout new file mode 100644 index 0000000000..02859bb2d1 --- /dev/null +++ b/gm/tests/outputs/ignoring-some-failures/output-expected/stdout @@ -0,0 +1,18 @@ +GM: These configs will be run: 8888 565 +GM: reading expectations from JSON summary file gm/tests/inputs/json/different-pixels-ignore-some-failures.json +GM: writing to gm/tests/outputs/ignoring-some-failures/output-actual/writePath +GM: writing mismatches to gm/tests/outputs/ignoring-some-failures/output-actual/mismatchPath +GM: writing images without expectations to gm/tests/outputs/ignoring-some-failures/output-actual/missingExpectationsPath +GM: drawing... selftest1 [300 200] +GM: Ran 1 GMs +GM: ... over 2 configs ["8888", "565"] +GM: ... and 7 modes ["pipe", "pipe cross-process", "pipe cross-process, shared address", "replay", "rtree", "serialize", "tilegrid"] +GM: ... so there should be a total of 9 tests. +GM: Ran 9 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 ExpectationsMismatch=2 MissingExpectations=0 WritingReferenceImage=0 +GM: [*] 0 NoGpuContext: +GM: [ ] 0 IntentionallySkipped: +GM: [*] 0 RenderModeMismatch: +GM: [*] 2 ExpectationsMismatch: 8888/selftest1 565/selftest1 +GM: [ ] 0 MissingExpectations: +GM: [*] 0 WritingReferenceImage: +GM: (results marked with [*] will cause nonzero return value) diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/writePath/565/bogusfile b/gm/tests/outputs/ignoring-some-failures/output-expected/writePath/565/bogusfile new file mode 100644 index 0000000000..d86cd5bdd8 --- /dev/null +++ b/gm/tests/outputs/ignoring-some-failures/output-expected/writePath/565/bogusfile @@ -0,0 +1 @@ +Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories. diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/writePath/565/selftest1.png b/gm/tests/outputs/ignoring-some-failures/output-expected/writePath/565/selftest1.png new file mode 100644 index 0000000000..4bc8ce56b4 --- /dev/null +++ b/gm/tests/outputs/ignoring-some-failures/output-expected/writePath/565/selftest1.png @@ -0,0 +1 @@ +[contents of gm/tests/outputs/ignoring-some-failures/output-actual/writePath/565/selftest1.png] diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/writePath/8888/bogusfile b/gm/tests/outputs/ignoring-some-failures/output-expected/writePath/8888/bogusfile new file mode 100644 index 0000000000..d86cd5bdd8 --- /dev/null +++ b/gm/tests/outputs/ignoring-some-failures/output-expected/writePath/8888/bogusfile @@ -0,0 +1 @@ +Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories. diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/writePath/8888/selftest1.png b/gm/tests/outputs/ignoring-some-failures/output-expected/writePath/8888/selftest1.png new file mode 100644 index 0000000000..f7f6539d50 --- /dev/null +++ b/gm/tests/outputs/ignoring-some-failures/output-expected/writePath/8888/selftest1.png @@ -0,0 +1 @@ +[contents of gm/tests/outputs/ignoring-some-failures/output-actual/writePath/8888/selftest1.png] diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/writePath/bogusfile b/gm/tests/outputs/ignoring-some-failures/output-expected/writePath/bogusfile new file mode 100644 index 0000000000..d86cd5bdd8 --- /dev/null +++ b/gm/tests/outputs/ignoring-some-failures/output-expected/writePath/bogusfile @@ -0,0 +1 @@ +Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories. diff --git a/gm/tests/run.sh b/gm/tests/run.sh index cf1b814583..adc0d27ba9 100755 --- a/gm/tests/run.sh +++ b/gm/tests/run.sh @@ -163,6 +163,14 @@ function create_inputs_dir { $GM_BINARY --hierarchy --match selftest1 $CONFIGS -r $THIS_IMAGE_DIR \ --writeJsonSummaryPath $JSON_DIR/different-pixels.json + # Create another JSON expectations file which is identical to + # different-pixels.json, but in which the *first* ignore-failure is changed + # from false to true. + OLD='"ignore-failure" : false' + NEW='"ignore-failure" : true' + sed -e "0,/$OLD/{s/$OLD/$NEW/}" $JSON_DIR/different-pixels.json \ + >$JSON_DIR/different-pixels-ignore-some-failures.json + THIS_IMAGE_DIR=$IMAGES_DIR/different-pixels-no-hierarchy mkdir -p $THIS_IMAGE_DIR $GM_BINARY --match selftest2 $CONFIGS -w $THIS_IMAGE_DIR @@ -193,6 +201,10 @@ gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/ide gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/different-pixels" "$GM_OUTPUTS/compared-against-different-pixels-images" gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OUTPUTS/compared-against-different-pixels-json" +# Compare different pixels, but with a SUBSET of the expectations marked as +# ignore-failure. +gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels-ignore-some-failures.json" "$GM_OUTPUTS/ignoring-some-failures" + # Compare generated image against an empty "expected image" dir. gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/empty-dir" "$GM_OUTPUTS/compared-against-empty-dir" @@ -205,9 +217,8 @@ gm_test "--hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/empty-dir" # Add pdf to the list of configs. gm_test "--verbose --hierarchy --match selftest1 $CONFIGS pdf -r $GM_INPUTS/json/identical-bytes.json" "$GM_OUTPUTS/add-config-pdf" -# If run without "-r", the JSON's "actual-results" section should contain -# actual checksums marked as "failure-ignored", but the "expected-results" -# section should be empty. +# Test what happens if run without -r (no expected-results.json to compare +# against). gm_test "--verbose --hierarchy --match selftest1 $CONFIGS" "$GM_OUTPUTS/no-readpath" # Test what happens if a subset of the renderModes fail (e.g. pipe) -- cgit v1.2.3