More work to integrate skimage with rebaseline tools.

tools/skimage_main.cpp:
Add the ability to write the results to checksum based filenames,
much like GM uses. This will allow using the skpdiff server to
rebaseline images.
Write the keys in the JSON file as <original image>_<pref config>.png,
so it matches gm_json.IMAGE_FILENAME_PATTERN. Also replace '_' with
'-' in the original file name, to avoid confusing the pattern matcher.
The '_' to '-' replacement also happens on the output filename.
Read the keys in a similar manner.
In make_outname, no longer remove a suffix. This fixes a bug where
subset decoding writes multiple subsets to the same file.

tools/rebaseline.py:
Since the filenames written to json files now match
gm_json.IMAGE_FILENAME_PATTERN, enable the option to match based
on configs/tests when rebaselining skimage.

test json files:
Update to match the new format of output.

gm/gm_expectations:
Add a constructor that takes a BitmapAndDigest as input.

tools/tests/skimage_self_test.py:
Test that reading the expectations file just created by skimage with
the same file actually compares to the original file (rather than just
succeeding because expectations were missing).

Change the expectations files to match the new format.

Will require a buildbot change to use the new flag: https://codereview.chromium.org/27389002/

BUG=1466
R=epoger@google.com

Review URL: https://codereview.chromium.org/26297004

git-svn-id: http://skia.googlecode.com/svn/trunk@11902 2bbb7eff-a529-9590-31e7-b0007b416f81

7 files changed, 51 insertions, 13 deletions

diff --git a/tools/tests/skimage/input/bad-images/empty-results.json b/tools/tests/skimage/input/bad-images/empty-results.json
index 8ae56100de..72a486430e 100644
--- a/tools/tests/skimage/input/bad-images/empty-results.json
+++ b/tools/tests/skimage/input/bad-images/empty-results.json
@@ -6,7 +6,7 @@
       "succeeded" : null
    },
    "expected-results" : {
-      "invalid.png" : {
+      "invalid-png_None.png" : {
       }
    }
 }
diff --git a/tools/tests/skimage/input/bad-images/ignore-results.json b/tools/tests/skimage/input/bad-images/ignore-results.json
index 4a0326e108..06a1979507 100644
--- a/tools/tests/skimage/input/bad-images/ignore-results.json
+++ b/tools/tests/skimage/input/bad-images/ignore-results.json
@@ -6,7 +6,7 @@
       "succeeded" : null
    },
    "expected-results" : {
-      "invalid.png" : {
+      "invalid-png_None.png" : {
          "allowed-digests" : [
             [ "bitmap-64bitMD5", 8011846386646810712 ]
          ],
diff --git a/tools/tests/skimage/input/bad-images/incorrect-results.json b/tools/tests/skimage/input/bad-images/incorrect-results.json
index 67b8bdb0c0..6ba68e34ec 100644
--- a/tools/tests/skimage/input/bad-images/incorrect-results.json
+++ b/tools/tests/skimage/input/bad-images/incorrect-results.json
@@ -6,7 +6,7 @@
       "succeeded" : null
    },
    "expected-results" : {
-      "invalid.png" : {
+      "invalid-png_None.png" : {
          "allowed-digests" : [
             [ "bitmap-64bitMD5", 8011846386646810712 ]
          ],
diff --git a/tools/tests/skimage/input/images-with-known-hashes/ignore-failures.json b/tools/tests/skimage/input/images-with-known-hashes/ignore-failures.json
index a99cd3e321..f82ac17c3c 100644
--- a/tools/tests/skimage/input/images-with-known-hashes/ignore-failures.json
+++ b/tools/tests/skimage/input/images-with-known-hashes/ignore-failures.json
@@ -6,7 +6,7 @@
       "succeeded" : null
    },
    "expected-results" : {
-      "1209453360120438698.png" : {
+      "1209453360120438698-png_None.png" : {
          "allowed-digests" : [
             [ "bitmap-64bitMD5", 1111111111111111111 ]
          ],
diff --git a/tools/tests/skimage/input/images-with-known-hashes/incorrect-results.json b/tools/tests/skimage/input/images-with-known-hashes/incorrect-results.json
index 9cf72c3cd3..9cb7cbdae7 100644
--- a/tools/tests/skimage/input/images-with-known-hashes/incorrect-results.json
+++ b/tools/tests/skimage/input/images-with-known-hashes/incorrect-results.json
@@ -6,7 +6,7 @@
       "succeeded" : null
    },
    "expected-results" : {
-      "1209453360120438698.png" : {
+      "1209453360120438698-png_None.png" : {
          "allowed-digests" : [
             [ "bitmap-64bitMD5", 1111111111111111111 ]
          ],
diff --git a/tools/tests/skimage/output-expected/create-expectations/expectations.json b/tools/tests/skimage/output-expected/create-expectations/expectations.json
index c143c1bfa2..0669624f0f 100644
--- a/tools/tests/skimage/output-expected/create-expectations/expectations.json
+++ b/tools/tests/skimage/output-expected/create-expectations/expectations.json
@@ -6,13 +6,13 @@
       "succeeded" : null
    },
    "expected-results" : {
-      "1209453360120438698.png" : {
+      "1209453360120438698-png_None.png" : {
          "allowed-digests" : [
             [ "bitmap-64bitMD5", 1209453360120438698 ]
          ],
          "ignore-failure" : false
       },
-      "12927999507540085554.png" : {
+      "12927999507540085554-png_None.png" : {
          "allowed-digests" : [
             [ "bitmap-64bitMD5", 12927999507540085554 ]
          ],
diff --git a/tools/tests/skimage_self_test.py b/tools/tests/skimage_self_test.py
index 6229d6a9ff..8191a9d323 100755
--- a/tools/tests/skimage_self_test.py
+++ b/tools/tests/skimage_self_test.py
@@ -69,13 +69,31 @@ def test_invalid_file(file_dir, skimage_binary):
 
     # Empty expectations:
     empty_expectations = os.path.join(expectations_dir, "empty-results.json")
-    subprocess.check_call([skimage_binary, "--readPath", invalid_file,
-                           "--readExpectationsPath", empty_expectations])
+    output = subprocess.check_output([skimage_binary, "--readPath", invalid_file,
+                                      "--readExpectationsPath",
+                                      empty_expectations],
+                                     stderr=subprocess.STDOUT)
+    if not "Missing" in output:
+      # Another test (in main()) tests to ensure that "Missing" does not appear
+      # in the output. That test could be passed if the output changed so
+      # "Missing" never appears. This ensures that an error is not missed if
+      # that happens.
+      print "skimage output changed! This may cause other self tests to fail!"
+      exit(1)
 
     # Ignore failure:
     ignore_expectations = os.path.join(expectations_dir, "ignore-results.json")
-    subprocess.check_call([skimage_binary, "--readPath", invalid_file,
-                           "--readExpectationsPath", ignore_expectations])
+    output = subprocess.check_output([skimage_binary, "--readPath", invalid_file,
+                                      "--readExpectationsPath",
+                                      ignore_expectations],
+                                     stderr=subprocess.STDOUT)
+    if not "failures" in output:
+      # Another test (in main()) tests to ensure that "failures" does not
+      # appear in the output. That test could be passed if the output changed
+      # so "failures" never appears. This ensures that an error is not missed
+      # if that happens.
+      print "skimage output changed! This may cause other self tests to fail!"
+      exit(1)
 
 def test_incorrect_expectations(file_dir, skimage_binary):
     """ Test that comparing to incorrect expectations fails, unless
@@ -125,8 +143,28 @@ def main():
 
     # Tell skimage to read back the expectations file it just wrote, and
     # confirm that the images in images_dir match it.
-    subprocess.check_call([skimage_binary, "--readPath", images_dir,
-                           "--readExpectationsPath", expectations_path])
+    output = subprocess.check_output([skimage_binary, "--readPath", images_dir,
+                                      "--readExpectationsPath",
+                                      expectations_path],
+                                     stderr=subprocess.STDOUT)
+
+    # Although skimage succeeded, it would have reported success if the file
+    # was missing from the expectations file. Consider this a failure, since
+    # the expectations file was created from this same image. (It will print
+    # "Missing" in this case before listing the missing expectations).
+    if "Missing" in output:
+      print "Expectations file was missing expectations!"
+      print output
+      exit(1)
+
+    # Again, skimage would succeed if there were known failures (and print
+    # "failures"), but there should be no failures, since the file just
+    # created did not include failures to ignore.
+    if "failures" in output:
+      print "Image failed!"
+      print output
+      exit(1)
+
 
     test_incorrect_expectations(file_dir=file_dir,
                                 skimage_binary=skimage_binary)