diff options
| -rw-r--r-- | gm/gm_json.py | 6 | ||||
| -rwxr-xr-x | tools/jsondiff.py | 175 | ||||
| -rw-r--r-- | tools/tests/jsondiff/input/new.json | 22 | ||||
| -rw-r--r-- | tools/tests/jsondiff/input/old.json | 22 | ||||
| -rw-r--r-- | tools/tests/jsondiff/output/old-vs-new/output-expected/command_line | 1 | ||||
| -rw-r--r-- | tools/tests/jsondiff/output/old-vs-new/output-expected/return_value | 1 | ||||
| -rw-r--r-- | tools/tests/jsondiff/output/old-vs-new/output-expected/stdout | 14 | ||||
| -rwxr-xr-x | tools/tests/rebaseline.sh | 2 | ||||
| -rwxr-xr-x | tools/tests/run.sh | 30 |
9 files changed, 272 insertions, 1 deletions
diff --git a/gm/gm_json.py b/gm/gm_json.py index 6ba3a261a5..cd4198415c 100644 --- a/gm/gm_json.py +++ b/gm/gm_json.py @@ -23,6 +23,12 @@ JSONKEY_ACTUALRESULTS_FAILUREIGNORED = 'failure-ignored' JSONKEY_ACTUALRESULTS_NOCOMPARISON = 'no-comparison' JSONKEY_ACTUALRESULTS_SUCCEEDED = 'succeeded' +JSONKEY_EXPECTEDRESULTS = 'expected-results' +JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS = 'allowed-digests' +JSONKEY_EXPECTEDRESULTS_IGNOREFAILURE = 'ignore-failure' + +JSONKEY_HASHTYPE_BITMAP_64BITMD5 = 'bitmap-64bitMD5' + def LoadFromString(file_contents): """Loads the JSON summary written out by the GM tool. Returns a dictionary keyed by the values listed as JSONKEY_ constants diff --git a/tools/jsondiff.py b/tools/jsondiff.py new file mode 100755 index 0000000000..a1ca257572 --- /dev/null +++ b/tools/jsondiff.py @@ -0,0 +1,175 @@ +#!/usr/bin/python + +''' +Copyright 2013 Google Inc. + +Use of this source code is governed by a BSD-style license that can be +found in the LICENSE file. +''' + +''' +Gathers diffs between 2 JSON expectations files, or between actual and +expected results within a single JSON actual-results file, +and generates an old-vs-new diff dictionary. +''' + +# System-level imports +import argparse +import json +import os +import sys +import urllib2 + +# Imports from within Skia +# +# We need to add the 'gm' directory, so that we can import gm_json.py within +# that directory. That script allows us to parse the actual-results.json file +# written out by the GM tool. +# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end* +# so any dirs that are already in the PYTHONPATH will be preferred. +# +# This assumes that the 'gm' directory has been checked out as a sibling of +# the 'tools' directory containing this script, which will be the case if +# 'trunk' was checked out as a single unit. +GM_DIRECTORY = os.path.realpath( + os.path.join(os.path.dirname(os.path.dirname(__file__)), 'gm')) +if GM_DIRECTORY not in sys.path: + sys.path.append(GM_DIRECTORY) +import gm_json + + +# Object that generates diffs between two JSON gm result files. +class GMDiffer(object): + + def __init__(self): + pass + + def _GetFileContentsAsString(self, filepath): + """Returns the full contents of a file, as a single string. + If the filename looks like a URL, download its contents...""" + if filepath.startswith('http:') or filepath.startswith('https:'): + return urllib2.urlopen(filepath).read() + else: + return open(filepath, 'r').read() + + def _GetExpectedResults(self, filepath): + """Returns the dictionary of expected results from a JSON file, + in this form: + + { + 'test1' : 14760033689012826769, + 'test2' : 9151974350149210736, + ... + } + + We make these simplifying assumptions: + 1. Each test has either 0 or 1 allowed results. + 2. All expectations are of type JSONKEY_HASHTYPE_BITMAP_64BITMD5. + + Any tests which violate those assumptions will cause an exception to + be raised. + + Any tests for which we have no expectations will be left out of the + returned dictionary. + """ + result_dict = {} + contents = self._GetFileContentsAsString(filepath) + json_dict = gm_json.LoadFromString(contents) + all_expectations = json_dict[gm_json.JSONKEY_EXPECTEDRESULTS] + for test_name in all_expectations.keys(): + test_expectations = all_expectations[test_name] + allowed_digests = test_expectations[ + gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS] + if allowed_digests: + num_allowed_digests = len(allowed_digests) + if num_allowed_digests > 1: + raise ValueError( + 'test %s in file %s has %d allowed digests' % ( + test_name, filepath, num_allowed_digests)) + digest_pair = allowed_digests[0] + if digest_pair[0] != gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5: + raise ValueError( + 'test %s in file %s has unsupported hashtype %s' % ( + test_name, filepath, digest_pair[0])) + result_dict[test_name] = digest_pair[1] + return result_dict + + def _GetActualResults(self, filepath): + """Returns the dictionary of actual results from a JSON file, + in this form: + + { + 'test1' : 14760033689012826769, + 'test2' : 9151974350149210736, + ... + } + + We make these simplifying assumptions: + 1. All results are of type JSONKEY_HASHTYPE_BITMAP_64BITMD5. + + Any tests which violate those assumptions will cause an exception to + be raised. + + Any tests for which we have no actual results will be left out of the + returned dictionary. + """ + result_dict = {} + contents = self._GetFileContentsAsString(filepath) + json_dict = gm_json.LoadFromString(contents) + all_result_types = json_dict[gm_json.JSONKEY_ACTUALRESULTS] + for result_type in all_result_types.keys(): + results_of_this_type = all_result_types[result_type] + if results_of_this_type: + for test_name in results_of_this_type.keys(): + digest_pair = results_of_this_type[test_name] + if digest_pair[0] != gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5: + raise ValueError( + 'test %s in file %s has unsupported hashtype %s' % ( + test_name, filepath, digest_pair[0])) + result_dict[test_name] = digest_pair[1] + return result_dict + + def _DictionaryDiff(self, old_dict, new_dict): + """Generate a dictionary showing the diffs between old_dict and new_dict. + Any entries which are identical across them will be left out.""" + diff_dict = {} + all_keys = set(old_dict.keys() + new_dict.keys()) + for key in all_keys: + if old_dict.get(key) != new_dict.get(key): + new_entry = {} + new_entry['old'] = old_dict.get(key) + new_entry['new'] = new_dict.get(key) + diff_dict[key] = new_entry + return diff_dict + + def GenerateDiffDict(self, oldfile, newfile=None): + """Generate a dictionary showing the diffs: + old = expectations within oldfile + new = expectations within newfile + + If newfile is not specified, then 'new' is the actual results within + oldfile. + """ + old_results = self._GetExpectedResults(oldfile) + if newfile: + new_results = self._GetExpectedResults(newfile) + else: + new_results = self._GetActualResults(oldfile) + return self._DictionaryDiff(old_results, new_results) + + +# main... +parser = argparse.ArgumentParser() +parser.add_argument('old', + help='Path to JSON file whose expectations to display on ' + + 'the "old" side of the diff. This can be a filepath on ' + + 'local storage, or a URL.') +parser.add_argument('new', nargs='?', + help='Path to JSON file whose expectations to display on ' + + 'the "new" side of the diff; if not specified, uses the ' + + 'ACTUAL results from the "old" JSON file. This can be a ' + + 'filepath on local storage, or a URL.') +args = parser.parse_args() +differ = GMDiffer() +diffs = differ.GenerateDiffDict(oldfile=args.old, newfile=args.new) +json.dump(diffs, sys.stdout, sort_keys=True, indent=2) diff --git a/tools/tests/jsondiff/input/new.json b/tools/tests/jsondiff/input/new.json new file mode 100644 index 0000000000..0bf352f5f7 --- /dev/null +++ b/tools/tests/jsondiff/input/new.json @@ -0,0 +1,22 @@ +{ + "expected-results" : { + "identical.png" : { + "allowed-digests" : [ + [ "bitmap-64bitMD5", 1111111 ] + ], + "ignore-failure" : false + }, + "differing.png" : { + "allowed-digests" : [ + [ "bitmap-64bitMD5", 777777777 ] + ], + "ignore-failure" : false + }, + "missing-from-old.png" : { + "allowed-digests" : [ + [ "bitmap-64bitMD5", 3333333 ] + ], + "ignore-failure" : false + } + } +} diff --git a/tools/tests/jsondiff/input/old.json b/tools/tests/jsondiff/input/old.json new file mode 100644 index 0000000000..b599b99fec --- /dev/null +++ b/tools/tests/jsondiff/input/old.json @@ -0,0 +1,22 @@ +{ + "expected-results" : { + "identical.png" : { + "allowed-digests" : [ + [ "bitmap-64bitMD5", 1111111 ] + ], + "ignore-failure" : false + }, + "differing.png" : { + "allowed-digests" : [ + [ "bitmap-64bitMD5", 888888888 ] + ], + "ignore-failure" : false + }, + "missing-from-new.png" : { + "allowed-digests" : [ + [ "bitmap-64bitMD5", 44444444 ] + ], + "ignore-failure" : false + } + } +} diff --git a/tools/tests/jsondiff/output/old-vs-new/output-expected/command_line b/tools/tests/jsondiff/output/old-vs-new/output-expected/command_line new file mode 100644 index 0000000000..4d5c36bde7 --- /dev/null +++ b/tools/tests/jsondiff/output/old-vs-new/output-expected/command_line @@ -0,0 +1 @@ +python tools/jsondiff.py tools/tests/jsondiff/input/old.json tools/tests/jsondiff/input/new.json diff --git a/tools/tests/jsondiff/output/old-vs-new/output-expected/return_value b/tools/tests/jsondiff/output/old-vs-new/output-expected/return_value new file mode 100644 index 0000000000..573541ac97 --- /dev/null +++ b/tools/tests/jsondiff/output/old-vs-new/output-expected/return_value @@ -0,0 +1 @@ +0 diff --git a/tools/tests/jsondiff/output/old-vs-new/output-expected/stdout b/tools/tests/jsondiff/output/old-vs-new/output-expected/stdout new file mode 100644 index 0000000000..212c305bf1 --- /dev/null +++ b/tools/tests/jsondiff/output/old-vs-new/output-expected/stdout @@ -0,0 +1,14 @@ +{ + "differing.png": { + "new": 777777777, + "old": 888888888 + }, + "missing-from-new.png": { + "new": null, + "old": 44444444 + }, + "missing-from-old.png": { + "new": 3333333, + "old": null + } +}
\ No newline at end of file diff --git a/tools/tests/rebaseline.sh b/tools/tests/rebaseline.sh index 38f19499ed..4fe044dbbe 100755 --- a/tools/tests/rebaseline.sh +++ b/tools/tests/rebaseline.sh @@ -80,7 +80,7 @@ cd $(dirname $0) ./run.sh SELFTEST_RESULT=$? -SUBDIRS="skdiff benchgraphs rebaseline/output" +SUBDIRS="skdiff benchgraphs rebaseline/output jsondiff/output" echo if [ "$SELFTEST_RESULT" != "0" ]; then for SUBDIR in $SUBDIRS; do diff --git a/tools/tests/run.sh b/tools/tests/run.sh index 2b919cb884..3f76de1b58 100755 --- a/tools/tests/run.sh +++ b/tools/tests/run.sh @@ -150,6 +150,27 @@ function rebaseline_test { compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR } +# Run jsondiff.py with arguments in $1, recording its output. +# Then compare that output to the content of $2/output-expected. +function jsondiff_test { + if [ $# != 2 ]; then + echo "jsondiff_test requires exactly 2 parameters, got $#" + exit 1 + fi + ARGS="$1" + ACTUAL_OUTPUT_DIR="$2/output-actual" + EXPECTED_OUTPUT_DIR="$2/output-expected" + + rm -rf $ACTUAL_OUTPUT_DIR + mkdir -p $ACTUAL_OUTPUT_DIR + COMMAND="python tools/jsondiff.py $ARGS" + echo "$COMMAND" >$ACTUAL_OUTPUT_DIR/command_line + $COMMAND &>$ACTUAL_OUTPUT_DIR/stdout + echo $? >$ACTUAL_OUTPUT_DIR/return_value + + compare_directories $EXPECTED_OUTPUT_DIR $ACTUAL_OUTPUT_DIR +} + # @@ -215,4 +236,13 @@ rebaseline_test "--json-base-url file:$REBASELINE_INPUT/json1 --subdirs base-and rebaseline_test "--json-base-url file:$REBASELINE_INPUT/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float --add-new" "$REBASELINE_OUTPUT/using-json1-add-new" rebaseline_test "--json-base-url file:$REBASELINE_INPUT/json1 --subdirs base-android-galaxy-nexus base-shuttle-win7-intel-float --expectations-root $REBASELINE_INPUT/json1" "$REBASELINE_OUTPUT/using-json1-expectations" +# +# Test jsondiff.py ... +# + +JSONDIFF_INPUT=tools/tests/jsondiff/input +JSONDIFF_OUTPUT=tools/tests/jsondiff/output +jsondiff_test "$JSONDIFF_INPUT/old.json $JSONDIFF_INPUT/new.json" "$JSONDIFF_OUTPUT/old-vs-new" + + echo "All tests passed." |