gm: hide list of tests with ignored errors, by default

Review URL: https://codereview.chromium.org/13811031

git-svn-id: http://skia.googlecode.com/svn/trunk@8591 2bbb7eff-a529-9590-31e7-b0007b416f81

17 files changed, 104 insertions, 53 deletions

diff --git a/gm/gmmain.cpp b/gm/gmmain.cpp
index cf0ab41b66..5b0f662649 100644
--- a/gm/gmmain.cpp
+++ b/gm/gmmain.cpp
@@ -299,9 +299,40 @@ public:
     }
 
     /**
+     * Display the summary of results with this ErrorType.
+     *
+     * @param type which ErrorType
+     * @param verbose whether to be all verbose about it
+     */
+    void DisplayResultTypeSummary(ErrorType type, bool verbose) {
+        bool isIgnorableType = fIgnorableErrorCombination.includes(type);
+
+        SkString line;
+        if (isIgnorableType) {
+            line.append("[ ] ");
+        } else {
+            line.append("[*] ");
+        }
+
+        SkTArray<SkString> *failedTestsOfThisType = &fFailedTests[type];
+        int count = failedTestsOfThisType->count();
+        line.appendf("%d %s", count, getErrorTypeName(type));
+        if (!isIgnorableType || verbose) {
+            line.append(":");
+            for (int i = 0; i < count; ++i) {
+                line.append(" ");
+                line.append((*failedTestsOfThisType)[i]);
+            }
+        }
+        gm_fprintf(stdout, "%s\n", line.c_str());
+    }
+
+    /**
      * List contents of fFailedTests to stdout.
+     *
+     * @param verbose whether to be all verbose about it
      */
-    void ListErrors() {
+    void ListErrors(bool verbose) {
         // First, print a single summary line.
         SkString summary;
         summary.appendf("Ran %d tests:", fTestsRun);
@@ -313,22 +344,7 @@ public:
 
         // Now, for each failure type, list the tests that failed that way.
         for (int typeInt = 0; typeInt <= kLast_ErrorType; typeInt++) {
-            SkString line;
-            ErrorType type = static_cast<ErrorType>(typeInt);
-            if (fIgnorableErrorCombination.includes(type)) {
-                line.append("[ ] ");
-            } else {
-                line.append("[*] ");
-            }
-
-            SkTArray<SkString> *failedTestsOfThisType = &fFailedTests[type];
-            int count = failedTestsOfThisType->count();
-            line.appendf("%d %s:", count, getErrorTypeName(type));
-            for (int i = 0; i < count; ++i) {
-                line.append(" ");
-                line.append((*failedTestsOfThisType)[i]);
-            }
-            gm_fprintf(stdout, "%s\n", line.c_str());
+            this->DisplayResultTypeSummary(static_cast<ErrorType>(typeInt), verbose);
         }
         gm_fprintf(stdout, "(results marked with [*] will cause nonzero return value)\n");
     }
@@ -1686,15 +1702,6 @@ int tool_main(int argc, char** argv) {
     GrContextFactory* grFactory = NULL;
 #endif
 
-    if (FLAGS_verbose) {
-        SkString str;
-        str.printf("%d configs:", configs.count());
-        for (int i = 0; i < configs.count(); ++i) {
-            str.appendf(" %s", gRec[configs[i]].fName);
-        }
-        gm_fprintf(stderr, "%s\n", str.c_str());
-    }
-
     if (FLAGS_resourcePath.count() == 1) {
         GM::SetResourcePath(FLAGS_resourcePath[0]);
     }
@@ -1798,15 +1805,17 @@ int tool_main(int argc, char** argv) {
     if (gmmain.NumSignificantErrors() > 0) {
         reportError = true;
     }
+    int expectedNumberOfTests = gmsRun * (configs.count() + modes.count());
 
     // Output summary to stdout.
-    gm_fprintf(stdout, "Ran %d GMs\n", gmsRun);
-    gm_fprintf(stdout, "... over %2d configs [%s]\n", configs.count(),
-               list_all_config_names(configs).c_str());
-    gm_fprintf(stdout, "...  and %2d modes   [%s]\n", modes.count(), list_all(modes).c_str());
-    int expectedNumberOfTests = gmsRun * (configs.count() + modes.count());
-    gm_fprintf(stdout, "... so there should be a total of %d tests.\n", expectedNumberOfTests);
-    gmmain.ListErrors();
+    if (FLAGS_verbose) {
+        gm_fprintf(stdout, "Ran %d GMs\n", gmsRun);
+        gm_fprintf(stdout, "... over %2d configs [%s]\n", configs.count(),
+                   list_all_config_names(configs).c_str());
+        gm_fprintf(stdout, "...  and %2d modes   [%s]\n", modes.count(), list_all(modes).c_str());
+        gm_fprintf(stdout, "... so there should be a total of %d tests.\n", expectedNumberOfTests);
+    }
+    gmmain.ListErrors(FLAGS_verbose);
 
     // TODO(epoger): in a standalone CL, enable this new check.
 #if 0
diff --git a/gm/tests/outputs/compared-against-different-pixels-images/output-expected/command_line b/gm/tests/outputs/compared-against-different-pixels-images/output-expected/command_line
index fcf9744b15..8b4bf99d74 100644
--- a/gm/tests/outputs/compared-against-different-pixels-images/output-expected/command_line
+++ b/gm/tests/outputs/compared-against-different-pixels-images/output-expected/command_line
@@ -1 +1 @@
-out/Debug/gm --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/images/different-pixels --writeJsonSummaryPath gm/tests/outputs/compared-against-different-pixels-images/output-actual/json-summary.txt
+out/Debug/gm --verbose --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/images/different-pixels --writeJsonSummaryPath gm/tests/outputs/compared-against-different-pixels-images/output-actual/json-summary.txt
diff --git a/gm/tests/outputs/compared-against-different-pixels-json/output-expected/command_line b/gm/tests/outputs/compared-against-different-pixels-json/output-expected/command_line
index be8405209c..73227a9510 100644
--- a/gm/tests/outputs/compared-against-different-pixels-json/output-expected/command_line
+++ b/gm/tests/outputs/compared-against-different-pixels-json/output-expected/command_line
@@ -1 +1 @@
-out/Debug/gm --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/json/different-pixels.json --writeJsonSummaryPath gm/tests/outputs/compared-against-different-pixels-json/output-actual/json-summary.txt
+out/Debug/gm --verbose --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/json/different-pixels.json --writeJsonSummaryPath gm/tests/outputs/compared-against-different-pixels-json/output-actual/json-summary.txt
diff --git a/gm/tests/outputs/compared-against-empty-dir/output-expected/command_line b/gm/tests/outputs/compared-against-empty-dir/output-expected/command_line
index ad626bc6af..4151af3b8f 100644
--- a/gm/tests/outputs/compared-against-empty-dir/output-expected/command_line
+++ b/gm/tests/outputs/compared-against-empty-dir/output-expected/command_line
@@ -1 +1 @@
-out/Debug/gm --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/images/empty-dir --writeJsonSummaryPath gm/tests/outputs/compared-against-empty-dir/output-actual/json-summary.txt
+out/Debug/gm --verbose --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/images/empty-dir --writeJsonSummaryPath gm/tests/outputs/compared-against-empty-dir/output-actual/json-summary.txt
diff --git a/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/command_line b/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/command_line
index 614cb2b943..ab8bac67fb 100644
--- a/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/command_line
+++ b/gm/tests/outputs/compared-against-identical-bytes-images/output-expected/command_line
@@ -1 +1 @@
-out/Debug/gm --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/images/identical-bytes --writeJsonSummaryPath gm/tests/outputs/compared-against-identical-bytes-images/output-actual/json-summary.txt
+out/Debug/gm --verbose --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/images/identical-bytes --writeJsonSummaryPath gm/tests/outputs/compared-against-identical-bytes-images/output-actual/json-summary.txt
diff --git a/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/command_line b/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/command_line
index aae0b60615..8f34c11de5 100644
--- a/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/command_line
+++ b/gm/tests/outputs/compared-against-identical-bytes-json/output-expected/command_line
@@ -1 +1 @@
-out/Debug/gm --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/json/identical-bytes.json --writeJsonSummaryPath gm/tests/outputs/compared-against-identical-bytes-json/output-actual/json-summary.txt
+out/Debug/gm --verbose --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/json/identical-bytes.json --writeJsonSummaryPath gm/tests/outputs/compared-against-identical-bytes-json/output-actual/json-summary.txt
diff --git a/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/command_line b/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/command_line
index 34fcc9800f..23f498ee93 100644
--- a/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/command_line
+++ b/gm/tests/outputs/compared-against-identical-pixels-images/output-expected/command_line
@@ -1 +1 @@
-out/Debug/gm --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/images/identical-pixels --writeJsonSummaryPath gm/tests/outputs/compared-against-identical-pixels-images/output-actual/json-summary.txt
+out/Debug/gm --verbose --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/images/identical-pixels --writeJsonSummaryPath gm/tests/outputs/compared-against-identical-pixels-images/output-actual/json-summary.txt
diff --git a/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/command_line b/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/command_line
index a7a9c82620..9fe0158b51 100644
--- a/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/command_line
+++ b/gm/tests/outputs/compared-against-identical-pixels-json/output-expected/command_line
@@ -1 +1 @@
-out/Debug/gm --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/json/identical-pixels.json --writeJsonSummaryPath gm/tests/outputs/compared-against-identical-pixels-json/output-actual/json-summary.txt
+out/Debug/gm --verbose --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/json/identical-pixels.json --writeJsonSummaryPath gm/tests/outputs/compared-against-identical-pixels-json/output-actual/json-summary.txt
diff --git a/gm/tests/outputs/intentionally-skipped-tests/output-expected/command_line b/gm/tests/outputs/intentionally-skipped-tests/output-expected/command_line
index c80a9a20ce..361cb5213e 100644
--- a/gm/tests/outputs/intentionally-skipped-tests/output-expected/command_line
+++ b/gm/tests/outputs/intentionally-skipped-tests/output-expected/command_line
@@ -1 +1 @@
-out/Debug/gm --hierarchy --match selftest1 selftest2 --config 8888 565 --writeJsonSummaryPath gm/tests/outputs/intentionally-skipped-tests/output-actual/json-summary.txt
+out/Debug/gm --verbose --hierarchy --match selftest1 selftest2 --config 8888 565 --writeJsonSummaryPath gm/tests/outputs/intentionally-skipped-tests/output-actual/json-summary.txt
diff --git a/gm/tests/outputs/no-readpath/output-expected/command_line b/gm/tests/outputs/no-readpath/output-expected/command_line
index a75c735ccc..3192ddbb3e 100644
--- a/gm/tests/outputs/no-readpath/output-expected/command_line
+++ b/gm/tests/outputs/no-readpath/output-expected/command_line
@@ -1 +1 @@
-out/Debug/gm --hierarchy --match selftest1 --config 8888 565 --writeJsonSummaryPath gm/tests/outputs/no-readpath/output-actual/json-summary.txt
+out/Debug/gm --verbose --hierarchy --match selftest1 --config 8888 565 --writeJsonSummaryPath gm/tests/outputs/no-readpath/output-actual/json-summary.txt
diff --git a/gm/tests/outputs/nonverbose/output-expected/command_line b/gm/tests/outputs/nonverbose/output-expected/command_line
new file mode 100644
index 0000000000..7c71a72a41
--- /dev/null
+++ b/gm/tests/outputs/nonverbose/output-expected/command_line
@@ -0,0 +1 @@
+out/Debug/gm --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/images/empty-dir --writeJsonSummaryPath gm/tests/outputs/nonverbose/output-actual/json-summary.txt
diff --git a/gm/tests/outputs/nonverbose/output-expected/json-summary.txt b/gm/tests/outputs/nonverbose/output-expected/json-summary.txt
new file mode 100644
index 0000000000..69ef127716
--- /dev/null
+++ b/gm/tests/outputs/nonverbose/output-expected/json-summary.txt
@@ -0,0 +1,25 @@
+{
+   "actual-results" : {
+      "failed" : null,
+      "failure-ignored" : null,
+      "no-comparison" : {
+         "565/selftest1" : {
+            "checksum" : 9512553915271796906
+         },
+         "8888/selftest1" : {
+            "checksum" : 14022967492765711532
+         }
+      },
+      "succeeded" : null
+   },
+   "expected-results" : {
+      "565/selftest1" : {
+         "checksums" : null,
+         "ignore-failure" : false
+      },
+      "8888/selftest1" : {
+         "checksums" : null,
+         "ignore-failure" : false
+      }
+   }
+}
diff --git a/gm/tests/outputs/nonverbose/output-expected/return_value b/gm/tests/outputs/nonverbose/output-expected/return_value
new file mode 100644
index 0000000000..573541ac97
--- /dev/null
+++ b/gm/tests/outputs/nonverbose/output-expected/return_value
@@ -0,0 +1 @@
+0
diff --git a/gm/tests/outputs/nonverbose/output-expected/stderr b/gm/tests/outputs/nonverbose/output-expected/stderr
new file mode 100644
index 0000000000..2e951cfede
--- /dev/null
+++ b/gm/tests/outputs/nonverbose/output-expected/stderr
@@ -0,0 +1,2 @@
+GM: FAILED to read gm/tests/inputs/images/empty-dir/8888/selftest1.png
+GM: FAILED to read gm/tests/inputs/images/empty-dir/565/selftest1.png
diff --git a/gm/tests/outputs/nonverbose/output-expected/stdout b/gm/tests/outputs/nonverbose/output-expected/stdout
new file mode 100644
index 0000000000..6139fdbdc4
--- /dev/null
+++ b/gm/tests/outputs/nonverbose/output-expected/stdout
@@ -0,0 +1,10 @@
+GM: reading from gm/tests/inputs/images/empty-dir
+GM: drawing... selftest1 [300 200]
+GM: Ran 9 tests: NoGpuContext=0 IntentionallySkipped=0 RenderModeMismatch=0 ExpectationsMismatch=0 MissingExpectations=2 WritingReferenceImage=0
+GM: [*] 0 NoGpuContext:
+GM: [ ] 0 IntentionallySkipped
+GM: [*] 0 RenderModeMismatch:
+GM: [*] 0 ExpectationsMismatch:
+GM: [ ] 2 MissingExpectations
+GM: [*] 0 WritingReferenceImage:
+GM: (results marked with [*] will cause nonzero return value)
diff --git a/gm/tests/outputs/pipe-playback-failure/output-expected/command_line b/gm/tests/outputs/pipe-playback-failure/output-expected/command_line
index 950339da65..28c5a67da0 100644
--- a/gm/tests/outputs/pipe-playback-failure/output-expected/command_line
+++ b/gm/tests/outputs/pipe-playback-failure/output-expected/command_line
@@ -1 +1 @@
-out/Debug/gm --simulatePipePlaybackFailure --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/json/identical-pixels.json --writeJsonSummaryPath gm/tests/outputs/pipe-playback-failure/output-actual/json-summary.txt
+out/Debug/gm --simulatePipePlaybackFailure --verbose --hierarchy --match selftest1 --config 8888 565 -r gm/tests/inputs/json/identical-pixels.json --writeJsonSummaryPath gm/tests/outputs/pipe-playback-failure/output-actual/json-summary.txt
diff --git a/gm/tests/run.sh b/gm/tests/run.sh
index 8348cce0c0..844294a0a2 100755
--- a/gm/tests/run.sh
+++ b/gm/tests/run.sh
@@ -137,29 +137,32 @@ GM_TEMPFILES=$GM_TESTDIR/tempfiles
 create_inputs_dir $GM_INPUTS
 
 # Compare generated image against an input image file with identical bytes.
-gm_test "--hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/identical-bytes" "$GM_OUTPUTS/compared-against-identical-bytes-images"
-gm_test "--hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/identical-bytes.json" "$GM_OUTPUTS/compared-against-identical-bytes-json"
+gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/identical-bytes" "$GM_OUTPUTS/compared-against-identical-bytes-images"
+gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/identical-bytes.json" "$GM_OUTPUTS/compared-against-identical-bytes-json"
 
 # Compare generated image against an input image file with identical pixels but different PNG encoding.
-gm_test "--hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/identical-pixels" "$GM_OUTPUTS/compared-against-identical-pixels-images"
-gm_test "--hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/identical-pixels.json" "$GM_OUTPUTS/compared-against-identical-pixels-json"
+gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/identical-pixels" "$GM_OUTPUTS/compared-against-identical-pixels-images"
+gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/identical-pixels.json" "$GM_OUTPUTS/compared-against-identical-pixels-json"
 
 # Compare generated image against an input image file with different pixels.
-gm_test "--hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/different-pixels" "$GM_OUTPUTS/compared-against-different-pixels-images"
-gm_test "--hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OUTPUTS/compared-against-different-pixels-json"
+gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/different-pixels" "$GM_OUTPUTS/compared-against-different-pixels-images"
+gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels.json" "$GM_OUTPUTS/compared-against-different-pixels-json"
 
 # Compare generated image against an empty "expected image" dir.
-gm_test "--hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/empty-dir" "$GM_OUTPUTS/compared-against-empty-dir"
+gm_test "--verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/empty-dir" "$GM_OUTPUTS/compared-against-empty-dir"
+
+# Compare generated image against an empty "expected image" dir, but NOT in verbose mode.
+gm_test "--hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/images/empty-dir" "$GM_OUTPUTS/nonverbose"
 
 # If run without "-r", the JSON's "actual-results" section should contain
 # actual checksums marked as "failure-ignored", but the "expected-results"
 # section should be empty.
-gm_test "--hierarchy --match selftest1 $CONFIGS" "$GM_OUTPUTS/no-readpath"
+gm_test "--verbose --hierarchy --match selftest1 $CONFIGS" "$GM_OUTPUTS/no-readpath"
 
 # Test what happens if a subset of the renderModes fail (e.g. pipe)
-gm_test "--simulatePipePlaybackFailure --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/identical-pixels.json" "$GM_OUTPUTS/pipe-playback-failure"
+gm_test "--simulatePipePlaybackFailure --verbose --hierarchy --match selftest1 $CONFIGS -r $GM_INPUTS/json/identical-pixels.json" "$GM_OUTPUTS/pipe-playback-failure"
 
 # Confirm that IntentionallySkipped tests are recorded as such.
-gm_test "--hierarchy --match selftest1 selftest2 $CONFIGS" "$GM_OUTPUTS/intentionally-skipped-tests"
+gm_test "--verbose --hierarchy --match selftest1 selftest2 $CONFIGS" "$GM_OUTPUTS/intentionally-skipped-tests"
 
 echo "All tests passed."