gm: add self-test to confirm that ignore-failure field is respected

BUG=skia:1578
R=scroggo@google.com

Review URL: https://codereview.chromium.org/23799004

git-svn-id: http://skia.googlecode.com/svn/trunk@11047 2bbb7eff-a529-9590-31e7-b0007b416f81

20 files changed, 73 insertions, 9 deletions

diff --git a/gm/gmmain.cpp b/gm/gmmain.cpp
index c24d9c70e4..5ebccb24e7 100644
--- a/gm/gmmain.cpp
+++ b/gm/gmmain.cpp
@@ -821,12 +821,6 @@ public:
             this->fJsonActualResults_Succeeded[testName] = jsonActualResults;
         } else {
             if (ignoreFailure) {
-                // TODO: Once we have added the ability to compare
-                // actual results against expectations in a JSON file
-                // (where we can set ignore-failure to either true or
-                // false), add test cases that exercise ignored
-                // failures (both for kMissingExpectations_ErrorType
-                // and kExpectationsMismatch_ErrorType).
                 this->fJsonActualResults_FailureIgnored[testName] =
                     jsonActualResults;
             } else {
diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/command_line b/gm/tests/outputs/ignoring-some-failures/output-expected/command_line
new file mode 100644
index 0000000000..37be27526c
--- /dev/null
+++ b/gm/tests/outputs/ignoring-some-failures/output-expected/command_line
@@ -0,0 +1 @@
+out/Debug/gm --verbose --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/json/different-pixels-ignore-some-failures.json --writeJsonSummaryPath gm/tests/outputs/ignoring-some-failures/output-actual/json-summary.txt --writePath gm/tests/outputs/ignoring-some-failures/output-actual/writePath --mismatchPath gm/tests/outputs/ignoring-some-failures/output-actual/mismatchPath --missingExpectationsPath gm/tests/outputs/ignoring-some-failures/output-actual/missingExpectationsPath
diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/json-summary.txt b/gm/tests/outputs/ignoring-some-failures/output-expected/json-summary.txt
new file mode 100644
index 0000000000..b152d6d035
--- /dev/null
+++ b/gm/tests/outputs/ignoring-some-failures/output-expected/json-summary.txt
@@ -0,0 +1,26 @@
+{
+   "actual-results" : {
+      "failed" : {
+         "8888/selftest1.png" : [ "bitmap-64bitMD5", 1209453360120438698 ]
+      },
+      "failure-ignored" : {
+         "565/selftest1.png" : [ "bitmap-64bitMD5", 12927999507540085554 ]
+      },
+      "no-comparison" : null,
+      "succeeded" : null
+   },
+   "expected-results" : {
+      "565/selftest1.png" : {
+         "allowed-digests" : [
+            [ "bitmap-64bitMD5", 8863920166200910451 ]
+         ],
+         "ignore-failure" : true
+      },
+      "8888/selftest1.png" : {
+         "allowed-digests" : [
+            [ "bitmap-64bitMD5", 13451349865803053525 ]
+         ],
+         "ignore-failure" : false
+      }
+   }
+}
diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/565/bogusfile b/gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/565/bogusfile
new file mode 100644
index 0000000000..d86cd5bdd8
--- /dev/null
+++ b/gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/565/bogusfile
@@ -0,0 +1 @@
+Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories.
diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/565/selftest1.png b/gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/565/selftest1.png
new file mode 100644
index 0000000000..5a30830f5a
--- /dev/null
+++ b/gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/565/selftest1.png
@@ -0,0 +1 @@
+[contents of gm/tests/outputs/ignoring-some-failures/output-actual/mismatchPath/565/selftest1.png]
diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/8888/bogusfile b/gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/8888/bogusfile
new file mode 100644
index 0000000000..d86cd5bdd8
--- /dev/null
+++ b/gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/8888/bogusfile
@@ -0,0 +1 @@
+Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories.
diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/8888/selftest1.png b/gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/8888/selftest1.png
new file mode 100644
index 0000000000..734abd0f49
--- /dev/null
+++ b/gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/8888/selftest1.png
@@ -0,0 +1 @@
+[contents of gm/tests/outputs/ignoring-some-failures/output-actual/mismatchPath/8888/selftest1.png]
diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/bogusfile b/gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/bogusfile
new file mode 100644
index 0000000000..d86cd5bdd8
--- /dev/null
+++ b/gm/tests/outputs/ignoring-some-failures/output-expected/mismatchPath/bogusfile
@@ -0,0 +1 @@
+Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories.
diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/missingExpectationsPath/565/bogusfile b/gm/tests/outputs/ignoring-some-failures/output-expected/missingExpectationsPath/565/bogusfile
new file mode 100644
index 0000000000..d86cd5bdd8
--- /dev/null
+++ b/gm/tests/outputs/ignoring-some-failures/output-expected/missingExpectationsPath/565/bogusfile
@@ -0,0 +1 @@
+Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories.
diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/missingExpectationsPath/8888/bogusfile b/gm/tests/outputs/ignoring-some-failures/output-expected/missingExpectationsPath/8888/bogusfile
new file mode 100644
index 0000000000..d86cd5bdd8
--- /dev/null
+++ b/gm/tests/outputs/ignoring-some-failures/output-expected/missingExpectationsPath/8888/bogusfile
@@ -0,0 +1 @@
+Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories.
diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/missingExpectationsPath/bogusfile b/gm/tests/outputs/ignoring-some-failures/output-expected/missingExpectationsPath/bogusfile
new file mode 100644
index 0000000000..d86cd5bdd8
--- /dev/null
+++ b/gm/tests/outputs/ignoring-some-failures/output-expected/missingExpectationsPath/bogusfile
@@ -0,0 +1 @@
+Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories.
diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/return_value b/gm/tests/outputs/ignoring-some-failures/output-expected/return_value
new file mode 100644
index 0000000000..ace9d03621
--- /dev/null
+++ b/gm/tests/outputs/ignoring-some-failures/output-expected/return_value
@@ -0,0 +1 @@
+255
diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/stderr b/gm/tests/outputs/ignoring-some-failures/output-expected/stderr
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/gm/tests/outputs/ignoring-some-failures/output-expected/stderr
diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/stdout b/gm/tests/outputs/ignoring-some-failures/output-expected/stdout
new file mode 100644
index 0000000000..02859bb2d1
--- /dev/null
+++ b/gm/tests/outputs/ignoring-some-failures/output-expected/stdout
@@ -0,0 +1,18 @@
+GM: These configs will be run: 8888 565
+GM: reading expectations from JSON summary file gm/tests/inputs/json/different-pixels-ignore-some-failures.json
+GM: writing to gm/tests/outputs/ignoring-some-failures/output-actual/writePath
+GM: writing mismatches to gm/tests/outputs/ignoring-some-failures/output-actual/mismatchPath
+GM: writing images without expectations to gm/tests/outputs/ignoring-some-failures/output-actual/missingExpectationsPath
+GM: drawing... selftest1 [300 200]
+GM: Ran 1 GMs
+GM: ... over  2 configs ["8888", "565"]
+GM: ...  and  7 modes   ["pipe", "pipe cross-process", "pipe cross-process, shared address", "replay", "rtree", "serialize", "tilegrid"]
+GM: ... so there should be a total of 9 tests.
+GM: Ran 9 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 ExpectationsMismatch=2 MissingExpectations=0 WritingReferenceImage=0
+GM: [*] 0 NoGpuContext:
+GM: [ ] 0 IntentionallySkipped:
+GM: [*] 0 RenderModeMismatch:
+GM: [*] 2 ExpectationsMismatch: 8888/selftest1 565/selftest1
+GM: [ ] 0 MissingExpectations:
+GM: [*] 0 WritingReferenceImage:
+GM: (results marked with [*] will cause nonzero return value)
diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/writePath/565/bogusfile b/gm/tests/outputs/ignoring-some-failures/output-expected/writePath/565/bogusfile
new file mode 100644
index 0000000000..d86cd5bdd8
--- /dev/null
+++ b/gm/tests/outputs/ignoring-some-failures/output-expected/writePath/565/bogusfile
@@ -0,0 +1 @@
+Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories.
diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/writePath/565/selftest1.png b/gm/tests/outputs/ignoring-some-failures/output-expected/writePath/565/selftest1.png
new file mode 100644
index 0000000000..4bc8ce56b4
--- /dev/null
+++ b/gm/tests/outputs/ignoring-some-failures/output-expected/writePath/565/selftest1.png
@@ -0,0 +1 @@
+[contents of gm/tests/outputs/ignoring-some-failures/output-actual/writePath/565/selftest1.png]
diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/writePath/8888/bogusfile b/gm/tests/outputs/ignoring-some-failures/output-expected/writePath/8888/bogusfile
new file mode 100644
index 0000000000..d86cd5bdd8
--- /dev/null
+++ b/gm/tests/outputs/ignoring-some-failures/output-expected/writePath/8888/bogusfile
@@ -0,0 +1 @@
+Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories.
diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/writePath/8888/selftest1.png b/gm/tests/outputs/ignoring-some-failures/output-expected/writePath/8888/selftest1.png
new file mode 100644
index 0000000000..f7f6539d50
--- /dev/null
+++ b/gm/tests/outputs/ignoring-some-failures/output-expected/writePath/8888/selftest1.png
@@ -0,0 +1 @@
+[contents of gm/tests/outputs/ignoring-some-failures/output-actual/writePath/8888/selftest1.png]
diff --git a/gm/tests/outputs/ignoring-some-failures/output-expected/writePath/bogusfile b/gm/tests/outputs/ignoring-some-failures/output-expected/writePath/bogusfile
new file mode 100644
index 0000000000..d86cd5bdd8
--- /dev/null
+++ b/gm/tests/outputs/ignoring-some-failures/output-expected/writePath/bogusfile
@@ -0,0 +1 @@
+Created additional file to make sure directory isn't empty, because self-test cannot handle empty directories.
diff --git a/gm/tests/run.sh b/gm/tests/run.sh
index cf1b814583..adc0d27ba9 100755
--- a/gm/tests/run.sh
+++ b/gm/tests/run.sh
@@ -163,6 +163,14 @@ function create_inputs_dir {
   $GM_BINARY --hierarchy --match selftest1 $CONFIGS -r $THIS_IMAGE_DIR \
     --writeJsonSummaryPath $JSON_DIR/different-pixels.json
 
+  # Create another JSON expectations file which is identical to
+  # different-pixels.json, but in which the *first* ignore-failure is changed
+  # from false to true.
+  OLD='"ignore-failure" : false'
+  NEW='"ignore-failure" : true'
+  sed -e "0,/$OLD/{s/$OLD/$NEW/}" $JSON_DIR/different-pixels.json \
+      >$JSON_DIR/different-pixels-ignore-some-failures.json
+
   THIS_IMAGE_DIR=$IMAGES_DIR/different-pixels-no-hierarchy
   mkdir -p $THIS_IMAGE_DIR
   $GM_BINARY --match selftest2 $CONFIGS -w $THIS_IMAGE_DIR
@@ -193,6 +201,10 @@ gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/ide
 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/different-pixels" "$GM_OUTPUTS/compared-against-different-pixels-images"
 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OUTPUTS/compared-against-different-pixels-json"
 
+# Compare different pixels, but with a SUBSET of the expectations marked as
+# ignore-failure.
+gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels-ignore-some-failures.json" "$GM_OUTPUTS/ignoring-some-failures"
+
 # Compare generated image against an empty "expected image" dir.
 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/empty-dir" "$GM_OUTPUTS/compared-against-empty-dir"
 
@@ -205,9 +217,8 @@ gm_test "--hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/empty-dir"
 # Add pdf to the list of configs.
 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS pdf -r $GM_INPUTS/json/identical-bytes.json" "$GM_OUTPUTS/add-config-pdf"
 
-# If run without "-r", the JSON's "actual-results" section should contain
-# actual checksums marked as "failure-ignored", but the "expected-results"
-# section should be empty.
+# Test what happens if run without -r (no expected-results.json to compare
+# against).
 gm_test "--verbose --hierarchy --match selftest1 $CONFIGS" "$GM_OUTPUTS/no-readpath"
 
 # Test what happens if a subset of the renderModes fail (e.g. pipe)