make gm self-test more robust

BUG=http://code.google.com/p/skia/issues/detail?id=1068
Review URL: https://codereview.appspot.com/7164043

git-svn-id: http://skia.googlecode.com/svn/trunk@7284 2bbb7eff-a529-9590-31e7-b0007b416f81

15 files changed, 72 insertions, 27 deletions

diff --git a/gm/tests/inputs/different-pixels/8888/dashing2.png b/gm/tests/inputs/different-pixels/8888/dashing2.png
deleted file mode 100644
index 3a0bc2e902..0000000000
--- a/gm/tests/inputs/different-pixels/8888/dashing2.png
+++ /dev/null
diff --git a/gm/tests/inputs/empty-dir/README b/gm/tests/inputs/empty-dir/README
deleted file mode 100644
index 4d39134f2b..0000000000
--- a/gm/tests/inputs/empty-dir/README
+++ /dev/null
@@ -1 +0,0 @@
-This directory intentionally left empty. Except for this file.
diff --git a/gm/tests/inputs/identical-bytes/8888/dashing2.png b/gm/tests/inputs/identical-bytes/8888/dashing2.png
deleted file mode 100644
index cb858d7575..0000000000
--- a/gm/tests/inputs/identical-bytes/8888/dashing2.png
+++ /dev/null
diff --git a/gm/tests/inputs/identical-pixels/8888/dashing2.png b/gm/tests/inputs/identical-pixels/8888/dashing2.png
deleted file mode 100644
index 86d5bce6d3..0000000000
--- a/gm/tests/inputs/identical-pixels/8888/dashing2.png
+++ /dev/null
diff --git a/gm/tests/outputs/compared-against-different-pixels/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-different-pixels/output-expected/json-summary.txt
index 182dd0ed5a..d6c59c0650 100644
--- a/gm/tests/outputs/compared-against-different-pixels/output-expected/json-summary.txt
+++ b/gm/tests/outputs/compared-against-different-pixels/output-expected/json-summary.txt
@@ -2,7 +2,7 @@
    "actual-results" : {
       "failed" : {
          "8888/dashing2" : {
-            "checksum" : 16595425910463109922
+            "checksum" : FAKE
          }
       },
       "failure-ignored" : null,
@@ -11,7 +11,7 @@
    },
    "expected-results" : {
       "8888/dashing2" : {
-         "checksums" : [ 15161495552186645995 ],
+         "checksums" : [ FAKE ],
          "ignore-failure" : false
       }
    }
diff --git a/gm/tests/outputs/compared-against-different-pixels/output-expected/stdout b/gm/tests/outputs/compared-against-different-pixels/output-expected/stdout
index e4dbd8cb4d..4286eb5488 100644
--- a/gm/tests/outputs/compared-against-different-pixels/output-expected/stdout
+++ b/gm/tests/outputs/compared-against-different-pixels/output-expected/stdout
@@ -1,6 +1,4 @@
-Non-default runtime configuration options:
 reading from gm/tests/inputs/different-pixels
 writing to gm/tests/outputs/compared-against-different-pixels/output-actual/images
 drawing... dashing2 [640 480]
 Ran 1 tests: 0 passed, 1 failed, 0 missing reference images
-		8888/dashing2 pixel_error
diff --git a/gm/tests/outputs/compared-against-empty-dir/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-empty-dir/output-expected/json-summary.txt
index 226b087a6c..73ff348f66 100644
--- a/gm/tests/outputs/compared-against-empty-dir/output-expected/json-summary.txt
+++ b/gm/tests/outputs/compared-against-empty-dir/output-expected/json-summary.txt
@@ -4,7 +4,7 @@
       "failure-ignored" : null,
       "no-comparison" : {
          "8888/dashing2" : {
-            "checksum" : 16595425910463109922
+            "checksum" : FAKE
          }
       },
       "succeeded" : null
diff --git a/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout b/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout
index 86216686aa..436554d29e 100644
--- a/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout
+++ b/gm/tests/outputs/compared-against-empty-dir/output-expected/stdout
@@ -1,4 +1,3 @@
-Non-default runtime configuration options:
 reading from gm/tests/inputs/empty-dir
 writing to gm/tests/outputs/compared-against-empty-dir/output-actual/images
 drawing... dashing2 [640 480]
diff --git a/gm/tests/outputs/compared-against-identical-bytes/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-identical-bytes/output-expected/json-summary.txt
index 6f812e05c8..9156fb4147 100644
--- a/gm/tests/outputs/compared-against-identical-bytes/output-expected/json-summary.txt
+++ b/gm/tests/outputs/compared-against-identical-bytes/output-expected/json-summary.txt
@@ -5,13 +5,13 @@
       "no-comparison" : null,
       "succeeded" : {
          "8888/dashing2" : {
-            "checksum" : 16595425910463109922
+            "checksum" : FAKE
          }
       }
    },
    "expected-results" : {
       "8888/dashing2" : {
-         "checksums" : [ 16595425910463109922 ],
+         "checksums" : [ FAKE ],
          "ignore-failure" : false
       }
    }
diff --git a/gm/tests/outputs/compared-against-identical-bytes/output-expected/stdout b/gm/tests/outputs/compared-against-identical-bytes/output-expected/stdout
index 8e67783ef7..301f17f5e7 100644
--- a/gm/tests/outputs/compared-against-identical-bytes/output-expected/stdout
+++ b/gm/tests/outputs/compared-against-identical-bytes/output-expected/stdout
@@ -1,4 +1,3 @@
-Non-default runtime configuration options:
 reading from gm/tests/inputs/identical-bytes
 writing to gm/tests/outputs/compared-against-identical-bytes/output-actual/images
 drawing... dashing2 [640 480]
diff --git a/gm/tests/outputs/compared-against-identical-pixels/output-expected/json-summary.txt b/gm/tests/outputs/compared-against-identical-pixels/output-expected/json-summary.txt
index 6f812e05c8..9156fb4147 100644
--- a/gm/tests/outputs/compared-against-identical-pixels/output-expected/json-summary.txt
+++ b/gm/tests/outputs/compared-against-identical-pixels/output-expected/json-summary.txt
@@ -5,13 +5,13 @@
       "no-comparison" : null,
       "succeeded" : {
          "8888/dashing2" : {
-            "checksum" : 16595425910463109922
+            "checksum" : FAKE
          }
       }
    },
    "expected-results" : {
       "8888/dashing2" : {
-         "checksums" : [ 16595425910463109922 ],
+         "checksums" : [ FAKE ],
          "ignore-failure" : false
       }
    }
diff --git a/gm/tests/outputs/compared-against-identical-pixels/output-expected/stdout b/gm/tests/outputs/compared-against-identical-pixels/output-expected/stdout
index 8829ef564f..00868ac5b1 100644
--- a/gm/tests/outputs/compared-against-identical-pixels/output-expected/stdout
+++ b/gm/tests/outputs/compared-against-identical-pixels/output-expected/stdout
@@ -1,4 +1,3 @@
-Non-default runtime configuration options:
 reading from gm/tests/inputs/identical-pixels
 writing to gm/tests/outputs/compared-against-identical-pixels/output-actual/images
 drawing... dashing2 [640 480]
diff --git a/gm/tests/outputs/no-readpath/output-expected/json-summary.txt b/gm/tests/outputs/no-readpath/output-expected/json-summary.txt
index 3b93856f37..04a042ca7b 100644
--- a/gm/tests/outputs/no-readpath/output-expected/json-summary.txt
+++ b/gm/tests/outputs/no-readpath/output-expected/json-summary.txt
@@ -4,7 +4,7 @@
       "failure-ignored" : null,
       "no-comparison" : {
          "8888/dashing2" : {
-            "checksum" : 16595425910463109922
+            "checksum" : FAKE
          }
       },
       "succeeded" : null
diff --git a/gm/tests/outputs/no-readpath/output-expected/stdout b/gm/tests/outputs/no-readpath/output-expected/stdout
index 61b3131c6c..4781221981 100644
--- a/gm/tests/outputs/no-readpath/output-expected/stdout
+++ b/gm/tests/outputs/no-readpath/output-expected/stdout
@@ -1,4 +1,3 @@
-Non-default runtime configuration options:
 writing to gm/tests/outputs/no-readpath/output-actual/images
 drawing... dashing2 [640 480]
 Ran 1 tests: 0 passed, 0 failed, 1 missing reference images
diff --git a/gm/tests/run.sh b/gm/tests/run.sh
index 2c9b719050..2bcff4e1ef 100755
--- a/gm/tests/run.sh
+++ b/gm/tests/run.sh
@@ -20,6 +20,9 @@ cd $(dirname $0)/../..
 # TODO(epoger): make it look in Release and/or Debug
 GM_BINARY=out/Debug/gm
 
+OUTPUT_ACTUAL_SUBDIR=output-actual
+OUTPUT_EXPECTED_SUBDIR=output-expected
+
 # Compare contents of all files within directories $1 and $2,
 # EXCEPT for any dotfiles.
 # If there are any differences, a description is written to stdout and
@@ -39,40 +42,89 @@ function compare_directories {
 
 # Run gm...
 # - with the arguments in $1
-# - writing resulting images into $2/output-actual/images
-# - writing stdout into $2/output-actual/stdout
-# - writing json summary into $2/output-actual/json-summary.txt
-# - writing return value into $2/output-actual/return_value
-# Then compare all of those against $2/output-expected .
+# - writing resulting images into $2/$OUTPUT_ACTUAL_SUBDIR/images
+# - writing stdout into $2/$OUTPUT_ACTUAL_SUBDIR/stdout
+# - writing json summary into $2/$OUTPUT_ACTUAL_SUBDIR/json-summary.txt
+# - writing return value into $2/$OUTPUT_ACTUAL_SUBDIR/return_value
+# Then compare all of those against $2/$OUTPUT_EXPECTED_SUBDIR .
 function gm_test {
   if [ $# != 2 ]; then
     echo "gm_test requires exactly 2 parameters, got $#"
     exit 1
   fi
   GM_ARGS="$1"
-  ACTUAL_OUTPUT_DIR="$2/output-actual"
-  EXPECTED_OUTPUT_DIR="$2/output-expected"
+  ACTUAL_OUTPUT_DIR="$2/$OUTPUT_ACTUAL_SUBDIR"
+  EXPECTED_OUTPUT_DIR="$2/$OUTPUT_EXPECTED_SUBDIR"
+  JSON_SUMMARY_FILE="$ACTUAL_OUTPUT_DIR/json-summary.txt"
 
   rm -rf $ACTUAL_OUTPUT_DIR
   mkdir -p $ACTUAL_OUTPUT_DIR
-  COMMAND="$GM_BINARY $GM_ARGS --writeJsonSummary $ACTUAL_OUTPUT_DIR/json-summary.txt -w $ACTUAL_OUTPUT_DIR/images"
+  COMMAND="$GM_BINARY $GM_ARGS --writeJsonSummary $JSON_SUMMARY_FILE -w $ACTUAL_OUTPUT_DIR/images"
   echo "$COMMAND" >$ACTUAL_OUTPUT_DIR/command_line
   $COMMAND &>$ACTUAL_OUTPUT_DIR/stdout
   echo $? >$ACTUAL_OUTPUT_DIR/return_value
 
-  # Temporary hack to alleviate
-  # http://code.google.com/p/skia/issues/detail?id=1068 ('GM self-test failures')
-  # (Remove instance-counting lines from the Debug output)
-  grep -v Leaked $ACTUAL_OUTPUT_DIR/stdout >$ACTUAL_OUTPUT_DIR/stdout-tmp
+  # Only compare selected lines in the output, to ignore any spurious lines
+  # as noted in http://code.google.com/p/skia/issues/detail?id=1068 .
+  #
+  # TODO(epoger): This is still hacky... we need to rewrite this script in
+  # Python soon, and make stuff like this more maintainable.
+  grep --regexp=^reading --regexp=^writing --regexp=^drawing \
+    --regexp=^FAILED --regexp=^Ran $ACTUAL_OUTPUT_DIR/stdout \
+    >$ACTUAL_OUTPUT_DIR/stdout-tmp
   mv $ACTUAL_OUTPUT_DIR/stdout-tmp $ACTUAL_OUTPUT_DIR/stdout
 
+  # Replace particular checksums in json output with a placeholder, so
+  # we don't need to rebaseline these json files when our drawing routines
+  # change.
+  sed -e 's/"checksum" : [0-9]*/"checksum" : FAKE/g' \
+    --in-place $JSON_SUMMARY_FILE
+  sed -e 's/"checksums" : \[ [0-9]* \]/"checksums" : [ FAKE ]/g' \
+    --in-place $JSON_SUMMARY_FILE
+
   compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR
 }
 
+# Create input dir (at path $1) with images that match or mismatch
+# as appropriate.
+#
+# We used to check these files into SVN, but then we needed to rebasline them
+# when our drawing changed at all... so, as proposed in
+# http://code.google.com/p/skia/issues/detail?id=1068 , we generate them
+# new each time.
+function create_inputs_dir {
+  if [ $# != 1 ]; then
+    echo "create_inputs_dir requires exactly 1 parameter, got $#"
+    exit 1
+  fi
+  INPUTS_DIR="$1"
+  mkdir -p $INPUTS_DIR
+
+  mkdir -p $INPUTS_DIR/identical-bytes
+  $GM_BINARY --hierarchy --match dashing2 --config 8888 \
+    -w $INPUTS_DIR/identical-bytes
+
+  mkdir -p $INPUTS_DIR/identical-pixels
+  $GM_BINARY --hierarchy --match dashing2 --config 8888 \
+    -w $INPUTS_DIR/identical-pixels
+  echo "more bytes that do not change the image pixels" \
+    >> $INPUTS_DIR/identical-pixels/8888/dashing2.png
+
+  mkdir -p $INPUTS_DIR/different-pixels
+  $GM_BINARY --hierarchy --match dashing3 --config 8888 \
+    -w $INPUTS_DIR/different-pixels
+  mv $INPUTS_DIR/different-pixels/8888/dashing3.png \
+    $INPUTS_DIR/different-pixels/8888/dashing2.png
+
+  mkdir -p $INPUTS_DIR/empty-dir
+}
+
 GM_TESTDIR=gm/tests
 GM_INPUTS=$GM_TESTDIR/inputs
 GM_OUTPUTS=$GM_TESTDIR/outputs
 
+create_inputs_dir $GM_INPUTS
+
 # Compare generated image against an input image file with identical bytes.
 gm_test "--hierarchy --match dashing2 --config 8888 -r $GM_INPUTS/identical-bytes" "$GM_OUTPUTS/compared-against-identical-bytes"