skdiff: add --failonmismatches and --listfilename options, plus cleanup

These changes are needed in order to switch the buildbots from using "gm -r" to "skdiff" to compare gm results, and should be generally good for humans too.
Review URL: https://codereview.appspot.com/6392054

git-svn-id: http://skia.googlecode.com/svn/trunk@4579 2bbb7eff-a529-9590-31e7-b0007b416f81

10 files changed, 34 insertions, 28 deletions

diff --git a/tools/tests/run.sh b/tools/tests/run.sh
index f6dbe27ca8..25baf1987a 100755
--- a/tools/tests/run.sh
+++ b/tools/tests/run.sh
@@ -42,19 +42,20 @@ SKDIFF_TESTDIR=tools/tests/skdiff
 
 # Run skdiff over a variety of file pair types: identical bits, identical
 # pixels, missing from baseDir, etc.
-# TODO: In the near future, skdiff will return a nonzero exit code in this case.
 skdiff_test "$SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/test1"
 
-# Same as above but without generating HTML output files.
-# TODO: In the near future, skdiff will return a nonzero exit code in this case.
-skdiff_test "--nodiffs $SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/test2"
+# Same as above, except:
+# - return the number of mismatching file pairs
+# - list filenames with each result type to stdout
+# - don't generate HTML output files
+skdiff_test "--failonmismatches --listfilenames --nodiffs $SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/test2"
 
 # Run skdiff over just the files that have identical bits, to validate any
 # behavior/return value differences in this case.
-skdiff_test "--nodiffs --match identical-bits $SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/identical-bits"
+skdiff_test "--failonmismatches --nodiffs --match identical-bits $SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/identical-bits"
 
 # Run skdiff over just the files that have identical bits or identical pixels,
 # to validate any behavior/return value differences in this case.
-skdiff_test "--nodiffs --match identical-bits --match identical-pixels $SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/identical-bits-or-pixels"
+skdiff_test "--failonmismatches --nodiffs --match identical-bits --match identical-pixels $SKDIFF_TESTDIR/baseDir $SKDIFF_TESTDIR/comparisonDir" "$SKDIFF_TESTDIR/identical-bits-or-pixels"
 
 echo "All tests passed."
diff --git a/tools/tests/skdiff/identical-bits-or-pixels/output-expected/command_line b/tools/tests/skdiff/identical-bits-or-pixels/output-expected/command_line
index 8a20667901..eed0759ac5 100644
--- a/tools/tests/skdiff/identical-bits-or-pixels/output-expected/command_line
+++ b/tools/tests/skdiff/identical-bits-or-pixels/output-expected/command_line
@@ -1 +1 @@
-out/Debug/skdiff --nodiffs --match identical-bits --match identical-pixels tools/tests/skdiff/baseDir tools/tests/skdiff/comparisonDir tools/tests/skdiff/identical-bits-or-pixels/output-actual
+out/Debug/skdiff --failonmismatches --nodiffs --match identical-bits --match identical-pixels tools/tests/skdiff/baseDir tools/tests/skdiff/comparisonDir tools/tests/skdiff/identical-bits-or-pixels/output-actual
diff --git a/tools/tests/skdiff/identical-bits-or-pixels/output-expected/stdout b/tools/tests/skdiff/identical-bits-or-pixels/output-expected/stdout
index 937f8c77c4..c5d12ec3da 100644
--- a/tools/tests/skdiff/identical-bits-or-pixels/output-expected/stdout
+++ b/tools/tests/skdiff/identical-bits-or-pixels/output-expected/stdout
@@ -1,4 +1,6 @@
 baseDir is [tools/tests/skdiff/baseDir/]
 comparisonDir is [tools/tests/skdiff/comparisonDir/]
 not writing any diffs to outputDir [tools/tests/skdiff/identical-bits-or-pixels/output-actual/]
-3 of 3 images matched.
+compared 3 file pairs:
+ 2 file pairs contain exactly the same bits
+ 1 file pairs contain the same pixel values, but not the same bits
diff --git a/tools/tests/skdiff/identical-bits/output-expected/command_line b/tools/tests/skdiff/identical-bits/output-expected/command_line
index 49f9ad5e74..5c8c27bbb6 100644
--- a/tools/tests/skdiff/identical-bits/output-expected/command_line
+++ b/tools/tests/skdiff/identical-bits/output-expected/command_line
@@ -1 +1 @@
-out/Debug/skdiff --nodiffs --match identical-bits tools/tests/skdiff/baseDir tools/tests/skdiff/comparisonDir tools/tests/skdiff/identical-bits/output-actual
+out/Debug/skdiff --failonmismatches --nodiffs --match identical-bits tools/tests/skdiff/baseDir tools/tests/skdiff/comparisonDir tools/tests/skdiff/identical-bits/output-actual
diff --git a/tools/tests/skdiff/identical-bits/output-expected/stdout b/tools/tests/skdiff/identical-bits/output-expected/stdout
index ccb24f8be4..c40ae334c9 100644
--- a/tools/tests/skdiff/identical-bits/output-expected/stdout
+++ b/tools/tests/skdiff/identical-bits/output-expected/stdout
@@ -1,4 +1,5 @@
 baseDir is [tools/tests/skdiff/baseDir/]
 comparisonDir is [tools/tests/skdiff/comparisonDir/]
 not writing any diffs to outputDir [tools/tests/skdiff/identical-bits/output-actual/]
-2 of 2 images matched.
+compared 2 file pairs:
+ 2 file pairs contain exactly the same bits
diff --git a/tools/tests/skdiff/test1/output-expected/index.html b/tools/tests/skdiff/test1/output-expected/index.html
index 9f3ce45bae..88b64edc17 100644
--- a/tools/tests/skdiff/test1/output-expected/index.html
+++ b/tools/tests/skdiff/test1/output-expected/index.html
@@ -4,8 +4,8 @@
 <tr><th>3 of 12 images matched exactly.<br></th>
 <th>every different pixel shown in white</th>
 <th>color difference at each pixel</th>
-<th>tools/tests/skdiff/baseDir/</th>
-<th>tools/tests/skdiff/comparisonDir/</th>
+<th>baseDir: tools/tests/skdiff/baseDir/</th>
+<th>comparisonDir: tools/tests/skdiff/comparisonDir/</th>
 </tr>
 <tr>
 <td><b>missing-from-baseDir.png</b><br>Missing from baseDir</td><td>N/A</td><td>N/A</td><td>N/A</td><td><a href="../../../../../tools/tests/skdiff/comparisonDir/missing-from-baseDir.png"><img src="../../../../../tools/tests/skdiff/comparisonDir/missing-from-baseDir.png" height="240px"></a></td></tr>
diff --git a/tools/tests/skdiff/test1/output-expected/return_value b/tools/tests/skdiff/test1/output-expected/return_value
index ec635144f6..573541ac97 100644
--- a/tools/tests/skdiff/test1/output-expected/return_value
+++ b/tools/tests/skdiff/test1/output-expected/return_value
@@ -1 +1 @@
-9
+0
diff --git a/tools/tests/skdiff/test1/output-expected/stdout b/tools/tests/skdiff/test1/output-expected/stdout
index 3d337d3376..142093f1dd 100644
--- a/tools/tests/skdiff/test1/output-expected/stdout
+++ b/tools/tests/skdiff/test1/output-expected/stdout
@@ -6,12 +6,13 @@ ERROR: no codec found for <tools/tests/skdiff/comparisonDir/different-bits-unkno
 baseDir is [tools/tests/skdiff/baseDir/]
 comparisonDir is [tools/tests/skdiff/comparisonDir/]
 writing diffs to outputDir is [tools/tests/skdiff/test1/output-actual/]
-Missing in baseDir:
-	missing-from-baseDir.png
-	missing-from-baseDir.xyz
-Missing in comparisonDir:
-	missing-from-comparisonDir.png
-	missing-from-comparisonDir.xyz
-3 of 12 images matched.
+compared 12 file pairs:
+ 2 file pairs contain exactly the same bits
+ 1 file pairs contain the same pixel values, but not the same bits
+ 2 file pairs have identical dimensions but some differing pixels
+ 2 file pairs have differing dimensions
+ 1 file pairs contain different bits and are not parsable images
+ 2 file pairs missing from comparisonDir
+ 2 file pairs missing from baseDir
 Maximum pixel intensity mismatch 239
 Largest area mismatch was 97.99% of pixels
diff --git a/tools/tests/skdiff/test2/output-expected/command_line b/tools/tests/skdiff/test2/output-expected/command_line
index b0bdd6291e..22dcf74a76 100644
--- a/tools/tests/skdiff/test2/output-expected/command_line
+++ b/tools/tests/skdiff/test2/output-expected/command_line
@@ -1 +1 @@
-out/Debug/skdiff --nodiffs tools/tests/skdiff/baseDir tools/tests/skdiff/comparisonDir tools/tests/skdiff/test2/output-actual
+out/Debug/skdiff --failonmismatches --listfilenames --nodiffs tools/tests/skdiff/baseDir tools/tests/skdiff/comparisonDir tools/tests/skdiff/test2/output-actual
diff --git a/tools/tests/skdiff/test2/output-expected/stdout b/tools/tests/skdiff/test2/output-expected/stdout
index eae392fc54..855deaee6e 100644
--- a/tools/tests/skdiff/test2/output-expected/stdout
+++ b/tools/tests/skdiff/test2/output-expected/stdout
@@ -2,12 +2,13 @@ ERROR: no codec found for basePath <tools/tests/skdiff/baseDir/different-bits-un
 baseDir is [tools/tests/skdiff/baseDir/]
 comparisonDir is [tools/tests/skdiff/comparisonDir/]
 not writing any diffs to outputDir [tools/tests/skdiff/test2/output-actual/]
-Missing in baseDir:
-	missing-from-baseDir.png
-	missing-from-baseDir.xyz
-Missing in comparisonDir:
-	missing-from-comparisonDir.png
-	missing-from-comparisonDir.xyz
-3 of 12 images matched.
+compared 12 file pairs:
+ 2 file pairs contain exactly the same bits: identical-bits-unknown-format.xyz identical-bits.png 
+ 1 file pairs contain the same pixel values, but not the same bits: different-bits-identical-pixels.png 
+ 2 file pairs have identical dimensions but some differing pixels: slightly-different-pixels-same-size.png very-different-pixels-same-size.png 
+ 2 file pairs have differing dimensions: slightly-different-sizes.png very-different-sizes.png 
+ 1 file pairs contain different bits and are not parsable images: different-bits-unknown-format.xyz 
+ 2 file pairs missing from comparisonDir: missing-from-comparisonDir.png missing-from-comparisonDir.xyz 
+ 2 file pairs missing from baseDir: missing-from-baseDir.png missing-from-baseDir.xyz 
 Maximum pixel intensity mismatch 239
 Largest area mismatch was 97.99% of pixels