Make gm's json results include checksums even if run without --readPath

Review URL: https://codereview.appspot.com/6979043

git-svn-id: http://skia.googlecode.com/svn/trunk@6917 2bbb7eff-a529-9590-31e7-b0007b416f81

12 files changed, 99 insertions, 27 deletions

diff --git a/gm/gmmain.cpp b/gm/gmmain.cpp
index 690ad0d845..314df8571c 100644
--- a/gm/gmmain.cpp
+++ b/gm/gmmain.cpp
@@ -80,6 +80,7 @@ const static ErrorBitfield ERROR_WRITING_REFERENCE_IMAGE = 0x10;
 const static char kJsonKey_ActualResults[]   = "actual-results";
 const static char kJsonKey_ActualResults_Failed[]        = "failed";
 const static char kJsonKey_ActualResults_FailureIgnored[]= "failure-ignored";
+const static char kJsonKey_ActualResults_NoComparison[]  = "no-comparison";
 const static char kJsonKey_ActualResults_Succeeded[]     = "succeeded";
 const static char kJsonKey_ActualResults_AnyStatus_Checksum[]    = "checksum";
 
@@ -623,7 +624,7 @@ public:
                 fJsonActualResults_Failed[name.c_str()] = actualResults;
             }
         } else {
-            fJsonActualResults_FailureIgnored[name.c_str()] = actualResults;
+            fJsonActualResults_NoComparison[name.c_str()] = actualResults;
         }
 
         // Add this test to the JSON collection of expected results.
@@ -659,6 +660,17 @@ public:
             retval |= compare_to_reference_image_on_disk(readPath, name, bitmap,
                                                          diffPath,
                                                          renderModeDescriptor);
+        } else if (NULL == referenceBitmap) {
+            // If we are running without "--readPath", we still want to
+            // record the actual results.
+            //
+            // For now, though, we don't record results of comparisons against
+            // different in-memory representations (hence the referenceBitmap
+            // NULL check).
+            Json::Value actualResults;
+            actualResults[kJsonKey_ActualResults_AnyStatus_Checksum] =
+                Json::UInt64(SkBitmapChecksummer::Compute64(bitmap));
+            fJsonActualResults_NoComparison[name.c_str()] = actualResults;
         }
         if (writePath && (gRec.fFlags & kWrite_ConfigFlag)) {
             retval |= write_reference_image(gRec, writePath,
@@ -845,6 +857,7 @@ public:
     Json::Value fJsonExpectedResults;
     Json::Value fJsonActualResults_Failed;
     Json::Value fJsonActualResults_FailureIgnored;
+    Json::Value fJsonActualResults_NoComparison;
     Json::Value fJsonActualResults_Succeeded;
 
 }; // end of GMMain class definition
@@ -1480,6 +1493,8 @@ int tool_main(int argc, char** argv) {
             gmmain.fJsonActualResults_Failed;
         actualResults[kJsonKey_ActualResults_FailureIgnored] =
             gmmain.fJsonActualResults_FailureIgnored;
+        actualResults[kJsonKey_ActualResults_NoComparison] =
+            gmmain.fJsonActualResults_NoComparison;
         actualResults[kJsonKey_ActualResults_Succeeded] =
             gmmain.fJsonActualResults_Succeeded;
         Json::Value root;
diff --git a/gm/tests/outputs/compared-against-different-pixels/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-different-pixels/output-expected/json-summary.txt
index 179c60c162..180b9efaa2 100644
--- a/gm/tests/outputs/compared-against-different-pixels/output-expected/json-summary.txt
+++ b/gm/tests/outputs/compared-against-different-pixels/output-expected/json-summary.txt
@@ -6,6 +6,7 @@
          }
       },
       "failure-ignored" : null,
+      "no-comparison" : null,
       "succeeded" : null
    },
    "expected-results" : {
diff --git a/gm/tests/outputs/compared-against-empty-dir/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-empty-dir/output-expected/json-summary.txt
index 935ba5037d..bdc8f8ab54 100644
--- a/gm/tests/outputs/compared-against-empty-dir/output-expected/json-summary.txt
+++ b/gm/tests/outputs/compared-against-empty-dir/output-expected/json-summary.txt
@@ -1,7 +1,8 @@
 {
    "actual-results" : {
       "failed" : null,
-      "failure-ignored" : {
+      "failure-ignored" : null,
+      "no-comparison" : {
          "8888/dashing2" : {
             "checksum" : 2675870163990933333
          }
diff --git a/gm/tests/outputs/compared-against-identical-bytes/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-identical-bytes/output-expected/json-summary.txt
index cbb28e1172..dcfa9f6d67 100644
--- a/gm/tests/outputs/compared-against-identical-bytes/output-expected/json-summary.txt
+++ b/gm/tests/outputs/compared-against-identical-bytes/output-expected/json-summary.txt
@@ -2,6 +2,7 @@
    "actual-results" : {
       "failed" : null,
       "failure-ignored" : null,
+      "no-comparison" : null,
       "succeeded" : {
          "8888/dashing2" : {
             "checksum" : 2675870163990933333
diff --git a/gm/tests/outputs/compared-against-identical-pixels/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-identical-pixels/output-expected/json-summary.txt
index cbb28e1172..dcfa9f6d67 100644
--- a/gm/tests/outputs/compared-against-identical-pixels/output-expected/json-summary.txt
+++ b/gm/tests/outputs/compared-against-identical-pixels/output-expected/json-summary.txt
@@ -2,6 +2,7 @@
    "actual-results" : {
       "failed" : null,
       "failure-ignored" : null,
+      "no-comparison" : null,
       "succeeded" : {
          "8888/dashing2" : {
             "checksum" : 2675870163990933333
diff --git a/gm/tests/outputs/no-readpath/output-expected/command_line b/gm/tests/outputs/no-readpath/output-expected/command_line
new file mode 100644
index 0000000000..8eab263135
--- /dev/null
+++ b/gm/tests/outputs/no-readpath/output-expected/command_line
@@ -0,0 +1 @@
+out/Debug/gm --hierarchy --match dashing2 --config 8888 --writeJsonSummary gm/tests/outputs/no-readpath/output-actual/json-summary.txt -w gm/tests/outputs/no-readpath/output-actual/images
diff --git a/gm/tests/outputs/no-readpath/output-expected/images/8888/dashing2.png b/gm/tests/outputs/no-readpath/output-expected/images/8888/dashing2.png
new file mode 100644
index 0000000000..465c0199df
--- /dev/null
+++ b/gm/tests/outputs/no-readpath/output-expected/images/8888/dashing2.png
diff --git a/gm/tests/outputs/no-readpath/output-expected/json-summary.txt b/gm/tests/outputs/no-readpath/output-expected/json-summary.txt
new file mode 100644
index 0000000000..09f095cc26
--- /dev/null
+++ b/gm/tests/outputs/no-readpath/output-expected/json-summary.txt
@@ -0,0 +1,13 @@
+{
+   "actual-results" : {
+      "failed" : null,
+      "failure-ignored" : null,
+      "no-comparison" : {
+         "8888/dashing2" : {
+            "checksum" : 2675870163990933333
+         }
+      },
+      "succeeded" : null
+   },
+   "expected-results" : null
+}
diff --git a/gm/tests/outputs/no-readpath/output-expected/return_value b/gm/tests/outputs/no-readpath/output-expected/return_value
new file mode 100644
index 0000000000..573541ac97
--- /dev/null
+++ b/gm/tests/outputs/no-readpath/output-expected/return_value
@@ -0,0 +1 @@
+0
diff --git a/gm/tests/outputs/no-readpath/output-expected/stdout b/gm/tests/outputs/no-readpath/output-expected/stdout
new file mode 100644
index 0000000000..ca556457b1
--- /dev/null
+++ b/gm/tests/outputs/no-readpath/output-expected/stdout
@@ -0,0 +1,3 @@
+writing to gm/tests/outputs/no-readpath/output-actual/images
+drawing... dashing2 [640 480]
+Ran 1 tests: 1 passed, 0 failed, 0 missing reference images
diff --git a/gm/tests/rebaseline.sh b/gm/tests/rebaseline.sh
index 77f8711e18..be8ccf291b 100755
--- a/gm/tests/rebaseline.sh
+++ b/gm/tests/rebaseline.sh
@@ -5,34 +5,64 @@
 # Use with caution: are you sure the new results are actually correct?
 #
 # YOU MUST RE-RUN THIS UNTIL THE SELF-TESTS SUCCEED!
-# (It takes one run for each call to gm_test in run.sh)
+# (It takes one run for each failing call to gm_test in run.sh)
+
+function replace_expected_with_actual {
+  # Delete all the expected output files
+  EXPECTED_FILES=$(find outputs/*/output-expected -type f | grep -v /\.svn/)
+  for EXPECTED_FILE in $EXPECTED_FILES; do
+    rm $EXPECTED_FILE
+  done
+
+  # Copy all the actual output files into the "expected" directories,
+  # creating new subdirs as we go.
+  ACTUAL_FILES=$(find outputs/*/output-actual -type f | grep -v /\.svn/)
+  for ACTUAL_FILE in $ACTUAL_FILES; do
+    EXPECTED_FILE=${ACTUAL_FILE//actual/expected}
+    mkdir -p $(dirname $EXPECTED_FILE)
+    cp $ACTUAL_FILE $EXPECTED_FILE
+  done
+}
+
+function svn_add_new_files {
+  # Delete all the "actual" directories, so we can svn-add any new "expected"
+  # directories without adding the "actual" ones.
+  rm -rf outputs/*/output-actual
+  FILES=$(svn stat outputs/* | grep ^\? | awk '{print $2}')
+  for FILE in $FILES; do
+    svn add $FILE
+  done
+  FILES=$(svn stat outputs/*/output-expected | grep ^\? | awk '{print $2}')
+  for FILE in $FILES; do
+    svn add $FILE
+  done
+}
+
+function svn_delete_old_files {
+  FILES=$(svn stat outputs/*/output-expected | grep ^\! | awk '{print $2}')
+  for FILE in $FILES; do
+    svn rm $FILE
+  done
+  FILES=$(svn stat outputs/* | grep ^\! | awk '{print $2}')
+  for FILE in $FILES; do
+    svn rm $FILE
+  done
+}
+
 
 # cd into the gm self-test dir
 cd $(dirname $0)
 
 ./run.sh
+SELFTEST_RESULT=$?
+echo
+if [ "$SELFTEST_RESULT" != "0" ]; then
+  replace_expected_with_actual
+  echo "Self-tests still failing, you should probably run this again..."
+else
+  svn_add_new_files
+  svn_delete_old_files
+  echo "Self-tests succeeded this time, you should be done!"
+fi
+exit $SELFTEST_RESULT
 
-# Delete all the expected output files
-EXPECTED_FILES=$(find outputs/*/output-expected -type f | grep -v /\.svn/)
-for EXPECTED_FILE in $EXPECTED_FILES; do
-  rm $EXPECTED_FILE
-done
-
-# Copy all the actual output files into the "expected" directories,
-# creating new subdirs as we go.
-ACTUAL_FILES=$(find outputs/*/output-actual -type f | grep -v /\.svn/)
-for ACTUAL_FILE in $ACTUAL_FILES; do
-  EXPECTED_FILE=${ACTUAL_FILE//actual/expected}
-  mkdir -p $(dirname $EXPECTED_FILE)
-  cp $ACTUAL_FILE $EXPECTED_FILE
-done
-
-# "svn add" any newly expected files/dirs, and "svn rm" any that are gone now
-FILES=$(svn stat outputs/*/output-expected | grep ^\? | awk '{print $2}')
-for FILE in $FILES; do
-  svn add $FILE
-done
-FILES=$(svn stat outputs/*/output-expected | grep ^\! | awk '{print $2}')
-for FILE in $FILES; do
-  svn rm $FILE
-done
diff --git a/gm/tests/run.sh b/gm/tests/run.sh
index 2bb9441d46..46c8929114 100755
--- a/gm/tests/run.sh
+++ b/gm/tests/run.sh
@@ -76,4 +76,9 @@ gm_test "--hierarchy --match dashing2 --config 8888 -r $GM_INPUTS/different-pixe
 # Compare generated image against an empty "expected image" dir.
 gm_test "--hierarchy --match dashing2 --config 8888 -r $GM_INPUTS/empty-dir" "$GM_OUTPUTS/compared-against-empty-dir"
 
+# If run without "-r", the JSON's "actual-results" section should contain
+# actual checksums marked as "failure-ignored", but the "expected-results"
+# section should be empty.
+gm_test "--hierarchy --match dashing2 --config 8888" "$GM_OUTPUTS/no-readpath"
+
 echo "All tests passed."