gm: record IntentionallySkipped tests

Review URL: https://codereview.chromium.org/13663003

git-svn-id: http://skia.googlecode.com/svn/trunk@8586 2bbb7eff-a529-9590-31e7-b0007b416f81

14 files changed, 56 insertions, 9 deletions

diff --git a/gm/tests/outputs/compared-against-different-pixels-images/output-expected/stdout b/gm/tests/outputs/compared-against-different-pixels-images/output-expected/stdout
index 8b8e61ddc3..02ccb62a68 100644
--- a/gm/tests/outputs/compared-against-different-pixels-images/output-expected/stdout
+++ b/gm/tests/outputs/compared-against-different-pixels-images/output-expected/stdout
@@ -4,8 +4,9 @@ GM: Ran 1 GMs
 GM: ... over  2 configs ["8888", "565"]
 GM: ...  and  7 modes   ["pipe", "pipe cross-process", "pipe cross-process, shared address", "replay", "rtree", "serialize", "tilegrid"]
 GM: ... so there should be a total of 9 tests.
-GM: Ran 9 tests: NoGpuContext=0 RenderModeMismatch=0 ExpectationsMismatch=2 MissingExpectations=0 WritingReferenceImage=0
+GM: Ran 9 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 ExpectationsMismatch=2 MissingExpectations=0 WritingReferenceImage=0
 GM: [*] 0 NoGpuContext:
+GM: [ ] 0 IntentionallySkipped:
 GM: [*] 0 RenderModeMismatch:
 GM: [*] 2 ExpectationsMismatch: 8888/selftest1 565/selftest1
 GM: [ ] 0 MissingExpectations:
diff --git a/gm/tests/outputs/compared-against-different-pixels-json/output-expected/stdout b/gm/tests/outputs/compared-against-different-pixels-json/output-expected/stdout
index 2fd0269ea3..5e34f19c24 100644
--- a/gm/tests/outputs/compared-against-different-pixels-json/output-expected/stdout
+++ b/gm/tests/outputs/compared-against-different-pixels-json/output-expected/stdout
@@ -4,8 +4,9 @@ GM: Ran 1 GMs
 GM: ... over  2 configs ["8888", "565"]
 GM: ...  and  7 modes   ["pipe", "pipe cross-process", "pipe cross-process, shared address", "replay", "rtree", "serialize", "tilegrid"]
 GM: ... so there should be a total of 9 tests.
-GM: Ran 9 tests: NoGpuContext=0 RenderModeMismatch=0 ExpectationsMismatch=2 MissingExpectations=0 WritingReferenceImage=0
+GM: Ran 9 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 ExpectationsMismatch=2 MissingExpectations=0 WritingReferenceImage=0
 GM: [*] 0 NoGpuContext:
+GM: [ ] 0 IntentionallySkipped:
 GM: [*] 0 RenderModeMismatch:
 GM: [*] 2 ExpectationsMismatch: 8888/selftest1 565/selftest1
 GM: [ ] 0 MissingExpectations:
diff --git a/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout b/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout
index 829c05e6df..9482216fa4 100644
--- a/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout
+++ b/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout
@@ -4,8 +4,9 @@ GM: Ran 1 GMs
 GM: ... over  2 configs ["8888", "565"]
 GM: ...  and  7 modes   ["pipe", "pipe cross-process", "pipe cross-process, shared address", "replay", "rtree", "serialize", "tilegrid"]
 GM: ... so there should be a total of 9 tests.
-GM: Ran 9 tests: NoGpuContext=0 RenderModeMismatch=0 ExpectationsMismatch=0 MissingExpectations=2 WritingReferenceImage=0
+GM: Ran 9 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 ExpectationsMismatch=0 MissingExpectations=2 WritingReferenceImage=0
 GM: [*] 0 NoGpuContext:
+GM: [ ] 0 IntentionallySkipped:
 GM: [*] 0 RenderModeMismatch:
 GM: [*] 0 ExpectationsMismatch:
 GM: [ ] 2 MissingExpectations: 8888/selftest1 565/selftest1
diff --git a/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/stdout b/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/stdout
index 5788d69cca..ee43b3ed72 100644
--- a/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/stdout
+++ b/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/stdout
@@ -4,8 +4,9 @@ GM: Ran 1 GMs
 GM: ... over  2 configs ["8888", "565"]
 GM: ...  and  7 modes   ["pipe", "pipe cross-process", "pipe cross-process, shared address", "replay", "rtree", "serialize", "tilegrid"]
 GM: ... so there should be a total of 9 tests.
-GM: Ran 9 tests: NoGpuContext=0 RenderModeMismatch=0 ExpectationsMismatch=0 MissingExpectations=0 WritingReferenceImage=0
+GM: Ran 9 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 ExpectationsMismatch=0 MissingExpectations=0 WritingReferenceImage=0
 GM: [*] 0 NoGpuContext:
+GM: [ ] 0 IntentionallySkipped:
 GM: [*] 0 RenderModeMismatch:
 GM: [*] 0 ExpectationsMismatch:
 GM: [ ] 0 MissingExpectations:
diff --git a/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/stdout b/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/stdout
index 303a23fc35..92c1b9fd90 100644
--- a/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/stdout
+++ b/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/stdout
@@ -4,8 +4,9 @@ GM: Ran 1 GMs
 GM: ... over  2 configs ["8888", "565"]
 GM: ...  and  7 modes   ["pipe", "pipe cross-process", "pipe cross-process, shared address", "replay", "rtree", "serialize", "tilegrid"]
 GM: ... so there should be a total of 9 tests.
-GM: Ran 9 tests: NoGpuContext=0 RenderModeMismatch=0 ExpectationsMismatch=0 MissingExpectations=0 WritingReferenceImage=0
+GM: Ran 9 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 ExpectationsMismatch=0 MissingExpectations=0 WritingReferenceImage=0
 GM: [*] 0 NoGpuContext:
+GM: [ ] 0 IntentionallySkipped:
 GM: [*] 0 RenderModeMismatch:
 GM: [*] 0 ExpectationsMismatch:
 GM: [ ] 0 MissingExpectations:
diff --git a/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/stdout b/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/stdout
index 4c4edba270..8a3dbaf216 100644
--- a/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/stdout
+++ b/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/stdout
@@ -4,8 +4,9 @@ GM: Ran 1 GMs
 GM: ... over  2 configs ["8888", "565"]
 GM: ...  and  7 modes   ["pipe", "pipe cross-process", "pipe cross-process, shared address", "replay", "rtree", "serialize", "tilegrid"]
 GM: ... so there should be a total of 9 tests.
-GM: Ran 9 tests: NoGpuContext=0 RenderModeMismatch=0 ExpectationsMismatch=0 MissingExpectations=0 WritingReferenceImage=0
+GM: Ran 9 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 ExpectationsMismatch=0 MissingExpectations=0 WritingReferenceImage=0
 GM: [*] 0 NoGpuContext:
+GM: [ ] 0 IntentionallySkipped:
 GM: [*] 0 RenderModeMismatch:
 GM: [*] 0 ExpectationsMismatch:
 GM: [ ] 0 MissingExpectations:
diff --git a/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/stdout b/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/stdout
index 9a9d91b25f..0bc0fc4a4e 100644
--- a/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/stdout
+++ b/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/stdout
@@ -4,8 +4,9 @@ GM: Ran 1 GMs
 GM: ... over  2 configs ["8888", "565"]
 GM: ...  and  7 modes   ["pipe", "pipe cross-process", "pipe cross-process, shared address", "replay", "rtree", "serialize", "tilegrid"]
 GM: ... so there should be a total of 9 tests.
-GM: Ran 9 tests: NoGpuContext=0 RenderModeMismatch=0 ExpectationsMismatch=0 MissingExpectations=0 WritingReferenceImage=0
+GM: Ran 9 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 ExpectationsMismatch=0 MissingExpectations=0 WritingReferenceImage=0
 GM: [*] 0 NoGpuContext:
+GM: [ ] 0 IntentionallySkipped:
 GM: [*] 0 RenderModeMismatch:
 GM: [*] 0 ExpectationsMismatch:
 GM: [ ] 0 MissingExpectations:
diff --git a/gm/tests/outputs/intentionally-skipped-tests/output-expected/command_line b/gm/tests/outputs/intentionally-skipped-tests/output-expected/command_line
new file mode 100644
index 0000000000..c80a9a20ce
--- /dev/null
+++ b/gm/tests/outputs/intentionally-skipped-tests/output-expected/command_line
@@ -0,0 +1 @@
+out/Debug/gm --hierarchy --match selftest1 selftest2 --config 8888 565 --writeJsonSummaryPath gm/tests/outputs/intentionally-skipped-tests/output-actual/json-summary.txt
diff --git a/gm/tests/outputs/intentionally-skipped-tests/output-expected/json-summary.txt b/gm/tests/outputs/intentionally-skipped-tests/output-expected/json-summary.txt
new file mode 100644
index 0000000000..a3ac39fb59
--- /dev/null
+++ b/gm/tests/outputs/intentionally-skipped-tests/output-expected/json-summary.txt
@@ -0,0 +1,22 @@
+{
+   "actual-results" : {
+      "failed" : null,
+      "failure-ignored" : null,
+      "no-comparison" : {
+         "565/selftest1" : {
+            "checksum" : 9512553915271796906
+         },
+         "565/selftest2" : {
+            "checksum" : 11071285354315388429
+         },
+         "8888/selftest1" : {
+            "checksum" : 14022967492765711532
+         },
+         "8888/selftest2" : {
+            "checksum" : 16527650414256125612
+         }
+      },
+      "succeeded" : null
+   },
+   "expected-results" : null
+}
diff --git a/gm/tests/outputs/intentionally-skipped-tests/output-expected/return_value b/gm/tests/outputs/intentionally-skipped-tests/output-expected/return_value
new file mode 100644
index 0000000000..573541ac97
--- /dev/null
+++ b/gm/tests/outputs/intentionally-skipped-tests/output-expected/return_value
@@ -0,0 +1 @@
+0
diff --git a/gm/tests/outputs/intentionally-skipped-tests/output-expected/stderr b/gm/tests/outputs/intentionally-skipped-tests/output-expected/stderr
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/gm/tests/outputs/intentionally-skipped-tests/output-expected/stderr
diff --git a/gm/tests/outputs/intentionally-skipped-tests/output-expected/stdout b/gm/tests/outputs/intentionally-skipped-tests/output-expected/stdout
new file mode 100644
index 0000000000..3165dd4246
--- /dev/null
+++ b/gm/tests/outputs/intentionally-skipped-tests/output-expected/stdout
@@ -0,0 +1,14 @@
+GM: drawing... selftest2 [300 200]
+GM: drawing... selftest1 [300 200]
+GM: Ran 2 GMs
+GM: ... over  2 configs ["8888", "565"]
+GM: ...  and  7 modes   ["pipe", "pipe cross-process", "pipe cross-process, shared address", "replay", "rtree", "serialize", "tilegrid"]
+GM: ... so there should be a total of 18 tests.
+GM: Ran 18 tests: NoGpuContext=0 IntentionallySkipped=3 RenderModeMismatch=0 ExpectationsMismatch=0 MissingExpectations=4 WritingReferenceImage=0
+GM: [*] 0 NoGpuContext:
+GM: [ ] 3 IntentionallySkipped: comparison/selftest2-pipe comparison/selftest2-pipe cross-process comparison/selftest2-pipe cross-process, shared address
+GM: [*] 0 RenderModeMismatch:
+GM: [*] 0 ExpectationsMismatch:
+GM: [ ] 4 MissingExpectations: 8888/selftest2 565/selftest2 8888/selftest1 565/selftest1
+GM: [*] 0 WritingReferenceImage:
+GM: (results marked with [*] will cause nonzero return value)
diff --git a/gm/tests/outputs/no-readpath/output-expected/stdout b/gm/tests/outputs/no-readpath/output-expected/stdout
index 1bdc86daf7..a892db02f9 100644
--- a/gm/tests/outputs/no-readpath/output-expected/stdout
+++ b/gm/tests/outputs/no-readpath/output-expected/stdout
@@ -3,8 +3,9 @@ GM: Ran 1 GMs
 GM: ... over  2 configs ["8888", "565"]
 GM: ...  and  7 modes   ["pipe", "pipe cross-process", "pipe cross-process, shared address", "replay", "rtree", "serialize", "tilegrid"]
 GM: ... so there should be a total of 9 tests.
-GM: Ran 9 tests: NoGpuContext=0 RenderModeMismatch=0 ExpectationsMismatch=0 MissingExpectations=2 WritingReferenceImage=0
+GM: Ran 9 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 ExpectationsMismatch=0 MissingExpectations=2 WritingReferenceImage=0
 GM: [*] 0 NoGpuContext:
+GM: [ ] 0 IntentionallySkipped:
 GM: [*] 0 RenderModeMismatch:
 GM: [*] 0 ExpectationsMismatch:
 GM: [ ] 2 MissingExpectations: 8888/selftest1 565/selftest1
diff --git a/gm/tests/outputs/pipe-playback-failure/output-expected/stdout b/gm/tests/outputs/pipe-playback-failure/output-expected/stdout
index 52fc54f2a1..64127be93d 100644
--- a/gm/tests/outputs/pipe-playback-failure/output-expected/stdout
+++ b/gm/tests/outputs/pipe-playback-failure/output-expected/stdout
@@ -4,8 +4,9 @@ GM: Ran 1 GMs
 GM: ... over  2 configs ["8888", "565"]
 GM: ...  and  5 modes   ["pipe", "replay", "rtree", "serialize", "tilegrid"]
 GM: ... so there should be a total of 7 tests.
-GM: Ran 7 tests: NoGpuContext=0 RenderModeMismatch=1 ExpectationsMismatch=0 MissingExpectations=0 WritingReferenceImage=0
+GM: Ran 7 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=1 ExpectationsMismatch=0 MissingExpectations=0 WritingReferenceImage=0
 GM: [*] 0 NoGpuContext:
+GM: [ ] 0 IntentionallySkipped:
 GM: [*] 1 RenderModeMismatch: comparison/selftest1-pipe
 GM: [*] 0 ExpectationsMismatch:
 GM: [ ] 0 MissingExpectations: