Add --writeJsonSummary option to gm

This writes out a JSON file containing expected and actual checksum values for every test.
Next step: allow the same checksum file to be compared against, instead of PNG files.
Review URL: https://codereview.appspot.com/6940053

git-svn-id: http://skia.googlecode.com/svn/trunk@6843 2bbb7eff-a529-9590-31e7-b0007b416f81

13 files changed, 84 insertions, 4 deletions

diff --git a/gm/tests/inputs/empty-dir/README b/gm/tests/inputs/empty-dir/README
new file mode 100644
index 0000000000..4d39134f2b
--- /dev/null
+++ b/gm/tests/inputs/empty-dir/README
@@ -0,0 +1 @@
+This directory intentionally left empty. Except for this file.
diff --git a/gm/tests/outputs/compared-against-different-pixels/output-expected/command_line b/gm/tests/outputs/compared-against-different-pixels/output-expected/command_line
index ef8c5afda4..4253e730ef 100644
--- a/gm/tests/outputs/compared-against-different-pixels/output-expected/command_line
+++ b/gm/tests/outputs/compared-against-different-pixels/output-expected/command_line
@@ -1 +1 @@
-out/Debug/gm --hierarchy --match dashing2 --config 8888 -r gm/tests/inputs/different-pixels -w gm/tests/outputs/compared-against-different-pixels/output-actual/images
+out/Debug/gm --hierarchy --match dashing2 --config 8888 -r gm/tests/inputs/different-pixels --writeJsonSummary gm/tests/outputs/compared-against-different-pixels/output-actual/json-summary.txt -w gm/tests/outputs/compared-against-different-pixels/output-actual/images
diff --git a/gm/tests/outputs/compared-against-different-pixels/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-different-pixels/output-expected/json-summary.txt
new file mode 100644
index 0000000000..179c60c162
--- /dev/null
+++ b/gm/tests/outputs/compared-against-different-pixels/output-expected/json-summary.txt
@@ -0,0 +1,17 @@
+{
+   "actual-results" : {
+      "failed" : {
+         "8888/dashing2" : {
+            "checksum" : 2675870163990933333
+         }
+      },
+      "failure-ignored" : null,
+      "succeeded" : null
+   },
+   "expected-results" : {
+      "8888/dashing2" : {
+         "checksums" : [ 15161495552186645995 ],
+         "ignore-failure" : false
+      }
+   }
+}
diff --git a/gm/tests/outputs/compared-against-empty-dir/output-expected/command_line b/gm/tests/outputs/compared-against-empty-dir/output-expected/command_line
new file mode 100644
index 0000000000..0ee96beddd
--- /dev/null
+++ b/gm/tests/outputs/compared-against-empty-dir/output-expected/command_line
@@ -0,0 +1 @@
+out/Debug/gm --hierarchy --match dashing2 --config 8888 -r gm/tests/inputs/empty-dir --writeJsonSummary gm/tests/outputs/compared-against-empty-dir/output-actual/json-summary.txt -w gm/tests/outputs/compared-against-empty-dir/output-actual/images
diff --git a/gm/tests/outputs/compared-against-empty-dir/output-expected/images/8888/dashing2.png b/gm/tests/outputs/compared-against-empty-dir/output-expected/images/8888/dashing2.png
new file mode 100644
index 0000000000..465c0199df
--- /dev/null
+++ b/gm/tests/outputs/compared-against-empty-dir/output-expected/images/8888/dashing2.png
diff --git a/gm/tests/outputs/compared-against-empty-dir/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-empty-dir/output-expected/json-summary.txt
new file mode 100644
index 0000000000..935ba5037d
--- /dev/null
+++ b/gm/tests/outputs/compared-against-empty-dir/output-expected/json-summary.txt
@@ -0,0 +1,17 @@
+{
+   "actual-results" : {
+      "failed" : null,
+      "failure-ignored" : {
+         "8888/dashing2" : {
+            "checksum" : 2675870163990933333
+         }
+      },
+      "succeeded" : null
+   },
+   "expected-results" : {
+      "8888/dashing2" : {
+         "checksums" : null,
+         "ignore-failure" : true
+      }
+   }
+}
diff --git a/gm/tests/outputs/compared-against-empty-dir/output-expected/return_value b/gm/tests/outputs/compared-against-empty-dir/output-expected/return_value
new file mode 100644
index 0000000000..573541ac97
--- /dev/null
+++ b/gm/tests/outputs/compared-against-empty-dir/output-expected/return_value
@@ -0,0 +1 @@
+0
diff --git a/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout b/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout
new file mode 100644
index 0000000000..436554d29e
--- /dev/null
+++ b/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout
@@ -0,0 +1,5 @@
+reading from gm/tests/inputs/empty-dir
+writing to gm/tests/outputs/compared-against-empty-dir/output-actual/images
+drawing... dashing2 [640 480]
+FAILED to read gm/tests/inputs/empty-dir/8888/dashing2.png
+Ran 1 tests: 0 passed, 0 failed, 1 missing reference images
diff --git a/gm/tests/outputs/compared-against-identical-bytes/output-expected/command_line b/gm/tests/outputs/compared-against-identical-bytes/output-expected/command_line
index 6e89999781..556498ab12 100644
--- a/gm/tests/outputs/compared-against-identical-bytes/output-expected/command_line
+++ b/gm/tests/outputs/compared-against-identical-bytes/output-expected/command_line
@@ -1 +1 @@
-out/Debug/gm --hierarchy --match dashing2 --config 8888 -r gm/tests/inputs/identical-bytes -w gm/tests/outputs/compared-against-identical-bytes/output-actual/images
+out/Debug/gm --hierarchy --match dashing2 --config 8888 -r gm/tests/inputs/identical-bytes --writeJsonSummary gm/tests/outputs/compared-against-identical-bytes/output-actual/json-summary.txt -w gm/tests/outputs/compared-against-identical-bytes/output-actual/images
diff --git a/gm/tests/outputs/compared-against-identical-bytes/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-identical-bytes/output-expected/json-summary.txt
new file mode 100644
index 0000000000..cbb28e1172
--- /dev/null
+++ b/gm/tests/outputs/compared-against-identical-bytes/output-expected/json-summary.txt
@@ -0,0 +1,17 @@
+{
+   "actual-results" : {
+      "failed" : null,
+      "failure-ignored" : null,
+      "succeeded" : {
+         "8888/dashing2" : {
+            "checksum" : 2675870163990933333
+         }
+      }
+   },
+   "expected-results" : {
+      "8888/dashing2" : {
+         "checksums" : [ 2675870163990933333 ],
+         "ignore-failure" : false
+      }
+   }
+}
diff --git a/gm/tests/outputs/compared-against-identical-pixels/output-expected/command_line b/gm/tests/outputs/compared-against-identical-pixels/output-expected/command_line
index 285cef2125..ecdef271da 100644
--- a/gm/tests/outputs/compared-against-identical-pixels/output-expected/command_line
+++ b/gm/tests/outputs/compared-against-identical-pixels/output-expected/command_line
@@ -1 +1 @@
-out/Debug/gm --hierarchy --match dashing2 --config 8888 -r gm/tests/inputs/identical-pixels -w gm/tests/outputs/compared-against-identical-pixels/output-actual/images
+out/Debug/gm --hierarchy --match dashing2 --config 8888 -r gm/tests/inputs/identical-pixels --writeJsonSummary gm/tests/outputs/compared-against-identical-pixels/output-actual/json-summary.txt -w gm/tests/outputs/compared-against-identical-pixels/output-actual/images
diff --git a/gm/tests/outputs/compared-against-identical-pixels/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-identical-pixels/output-expected/json-summary.txt
new file mode 100644
index 0000000000..cbb28e1172
--- /dev/null
+++ b/gm/tests/outputs/compared-against-identical-pixels/output-expected/json-summary.txt
@@ -0,0 +1,17 @@
+{
+   "actual-results" : {
+      "failed" : null,
+      "failure-ignored" : null,
+      "succeeded" : {
+         "8888/dashing2" : {
+            "checksum" : 2675870163990933333
+         }
+      }
+   },
+   "expected-results" : {
+      "8888/dashing2" : {
+         "checksums" : [ 2675870163990933333 ],
+         "ignore-failure" : false
+      }
+   }
+}
diff --git a/gm/tests/run.sh b/gm/tests/run.sh
index b97b28904b..2bb9441d46 100755
--- a/gm/tests/run.sh
+++ b/gm/tests/run.sh
@@ -38,6 +38,7 @@ function compare_directories {
 # - with the arguments in $1
 # - writing resulting images into $2/output-actual/images
 # - writing stdout into $2/output-actual/stdout
+# - writing json summary into $2/output-actual/json-summary.txt
 # - writing return value into $2/output-actual/return_value
 # Then compare all of those against $2/output-expected .
 function gm_test {
@@ -51,7 +52,7 @@ function gm_test {
 
   rm -rf $ACTUAL_OUTPUT_DIR
   mkdir -p $ACTUAL_OUTPUT_DIR
-  COMMAND="$GM_BINARY $GM_ARGS -w $ACTUAL_OUTPUT_DIR/images"
+  COMMAND="$GM_BINARY $GM_ARGS --writeJsonSummary $ACTUAL_OUTPUT_DIR/json-summary.txt -w $ACTUAL_OUTPUT_DIR/images"
   echo "$COMMAND" >$ACTUAL_OUTPUT_DIR/command_line
   $COMMAND &>$ACTUAL_OUTPUT_DIR/stdout
   echo $? >$ACTUAL_OUTPUT_DIR/return_value
@@ -72,4 +73,7 @@ gm_test "--hierarchy --match dashing2 --config 8888 -r $GM_INPUTS/identical-pixe
 # Compare generated image against an input image file with different pixels.
 gm_test "--hierarchy --match dashing2 --config 8888 -r $GM_INPUTS/different-pixels" "$GM_OUTPUTS/compared-against-different-pixels"
 
+# Compare generated image against an empty "expected image" dir.
+gm_test "--hierarchy --match dashing2 --config 8888 -r $GM_INPUTS/empty-dir" "$GM_OUTPUTS/compared-against-empty-dir"
+
 echo "All tests passed."