diff options
author | 2013-05-21 16:06:40 +0000 | |
---|---|---|
committer | 2013-05-21 16:06:40 +0000 | |
commit | a55e48d6ef67daa90d17731b49d45b0ac94dcc34 (patch) | |
tree | fbc43d19474a1606fff678e35bc76a95a8f794cc | |
parent | 2d51677b1916ee5d08f040855c8c2002ff47a38b (diff) |
Improve tool that analyzes gm JSON summary
BUG=https://code.google.com/p/skia/issues/detail?id=1300
R=borenet@google.com
Review URL: https://codereview.chromium.org/15298010
git-svn-id: http://skia.googlecode.com/svn/trunk@9217 2bbb7eff-a529-9590-31e7-b0007b416f81
-rw-r--r-- | gm/confirm_no_failures_in_json.py | 47 | ||||
-rw-r--r-- | gm/display_json_results.py | 120 | ||||
-rw-r--r-- | gm/gm_expectations.cpp | 2 | ||||
-rwxr-xr-x | gm/tests/run.sh | 6 |
4 files changed, 124 insertions, 51 deletions
diff --git a/gm/confirm_no_failures_in_json.py b/gm/confirm_no_failures_in_json.py deleted file mode 100644 index c84992670b..0000000000 --- a/gm/confirm_no_failures_in_json.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2013 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Utility to confirm that a JSON summary written by GM contains no failures. - -Usage: - python confirm_no_failures_in_json.py <filename> -""" - -__author__ = 'Elliot Poger' - - -import json -import sys - - -# These constants must be kept in sync with the kJsonKey_ constants in -# gm_expectations.cpp ! -JSONKEY_ACTUALRESULTS = 'actual-results' -JSONKEY_ACTUALRESULTS_FAILED = 'failed' - -# This is the same indent level as used by jsoncpp, just for consistency. -JSON_INDENTLEVEL = 3 - - -def Assert(filepath): - """Raises an exception if the JSON summary at filepath contains any failed - tests, or if we were unable to read the JSON summary.""" - failed_tests = GetFailedTests(filepath) - if failed_tests: - raise Exception('JSON file %s contained these test failures...\n%s' % ( - filepath, json.dumps(failed_tests, indent=JSON_INDENTLEVEL))) - - -def GetFailedTests(filepath): - """Returns the dictionary of failed tests from the JSON file at filepath.""" - json_dict = json.load(open(filepath)) - actual_results = json_dict[JSONKEY_ACTUALRESULTS] - return actual_results[JSONKEY_ACTUALRESULTS_FAILED] - - -if '__main__' == __name__: - if len(sys.argv) != 2: - raise Exception('usage: %s <input-json-filepath>' % sys.argv[0]) - Assert(sys.argv[1]) diff --git a/gm/display_json_results.py b/gm/display_json_results.py new file mode 100644 index 0000000000..00c0c0e8d1 --- /dev/null +++ b/gm/display_json_results.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python +# Copyright (c) 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Utility to display a summary of JSON-format GM results, and exit with +a nonzero errorcode if there were non-ignored failures in the GM results. + +Usage: + python display_json_results.py <filename> + +TODO(epoger): We may want to add flags to set the following: +- which error types cause a nonzero return code +- maximum number of tests to list for any one ResultAccumulator + (to keep the output reasonably short) +""" + +__author__ = 'Elliot Poger' + + +import json +import sys + + +# These constants must be kept in sync with the kJsonKey_ constants in +# gm_expectations.cpp ! +JSONKEY_ACTUALRESULTS = 'actual-results' +JSONKEY_ACTUALRESULTS_FAILED = 'failed' +JSONKEY_ACTUALRESULTS_FAILUREIGNORED = 'failure-ignored' +JSONKEY_ACTUALRESULTS_NOCOMPARISON = 'no-comparison' +JSONKEY_ACTUALRESULTS_SUCCEEDED = 'succeeded' + + +class ResultAccumulator(object): + """Object that accumulates results of a given type, and can generate a + summary upon request.""" + + def __init__(self, name, do_list, do_fail): + """name: name of the category this result type falls into + do_list: whether to list all of the tests with this results type + do_fail: whether to return with nonzero exit code if there are any + results of this type + """ + self._name = name + self._do_list = do_list + self._do_fail = do_fail + self._testnames = [] + + def AddResult(self, testname): + """Adds a result of this particular type. + testname: (string) name of the test""" + self._testnames.append(testname) + + def ShouldSignalFailure(self): + """Returns true if this result type is serious (self._do_fail is True) + and there were any results of this type.""" + if self._do_fail and self._testnames: + return True + else: + return False + + def GetSummaryLine(self): + """Returns a single-line string summary of all results added to this + accumulator so far.""" + summary = '' + if self._do_fail: + summary += '[*] ' + else: + summary += '[ ] ' + summary += str(len(self._testnames)) + summary += ' ' + summary += self._name + if self._do_list: + summary += ': ' + for testname in self._testnames: + summary += testname + summary += ' ' + return summary + + +def Display(filepath): + """Displays a summary of the results in a JSON file. + Returns True if the results are free of any significant failures. + filepath: (string) path to JSON file""" + + # Map labels within the JSON file to the ResultAccumulator for each label. + results_map = { + JSONKEY_ACTUALRESULTS_FAILED: + ResultAccumulator(name='ExpectationsMismatch', + do_list=True, do_fail=True), + JSONKEY_ACTUALRESULTS_FAILUREIGNORED: + ResultAccumulator(name='IgnoredExpectationsMismatch', + do_list=True, do_fail=False), + JSONKEY_ACTUALRESULTS_NOCOMPARISON: + ResultAccumulator(name='MissingExpectations', + do_list=False, do_fail=False), + JSONKEY_ACTUALRESULTS_SUCCEEDED: + ResultAccumulator(name='Passed', + do_list=False, do_fail=False), + } + + success = True + json_dict = json.load(open(filepath)) + actual_results = json_dict[JSONKEY_ACTUALRESULTS] + for label, accumulator in results_map.iteritems(): + results = actual_results[label] + if results: + for result in results: + accumulator.AddResult(result) + print accumulator.GetSummaryLine() + if accumulator.ShouldSignalFailure(): + success = False + print '(results marked with [*] will cause nonzero return value)' + return success + + +if '__main__' == __name__: + if len(sys.argv) != 2: + raise Exception('usage: %s <input-json-filepath>' % sys.argv[0]) + sys.exit(0 if Display(sys.argv[1]) else 1) diff --git a/gm/gm_expectations.cpp b/gm/gm_expectations.cpp index 19e5de6347..71311d5e11 100644 --- a/gm/gm_expectations.cpp +++ b/gm/gm_expectations.cpp @@ -12,7 +12,7 @@ #define DEBUGFAIL_SEE_STDERR SkDEBUGFAIL("see stderr for message") // These constants must be kept in sync with the JSONKEY_ constants in -// confirm_no_failures_in_json.py ! +// display_json_results.py ! const static char kJsonKey_ActualResults[] = "actual-results"; const static char kJsonKey_ActualResults_Failed[] = "failed"; const static char kJsonKey_ActualResults_FailureIgnored[]= "failure-ignored"; diff --git a/gm/tests/run.sh b/gm/tests/run.sh index 2038c49cc0..809c7d038d 100755 --- a/gm/tests/run.sh +++ b/gm/tests/run.sh @@ -216,14 +216,14 @@ gm_test "--ignoreErrorTypes ExpectationsMismatch NoGpuContext --verbose --hierar # Test non-hierarchical mode. gm_test "--verbose --match selftest1 $CONFIGS -r $GM_INPUTS/json/different-pixels-no-hierarchy.json" "$GM_OUTPUTS/no-hierarchy" -# Exercise confirm_no_failures_in_json.py +# Exercise display_json_results.py PASSING_CASES="compared-against-identical-bytes-json compared-against-identical-pixels-json" FAILING_CASES="compared-against-different-pixels-json" for CASE in $PASSING_CASES; do - assert_passes "python gm/confirm_no_failures_in_json.py $GM_OUTPUTS/$CASE/$OUTPUT_EXPECTED_SUBDIR/json-summary.txt" + assert_passes "python gm/display_json_results.py $GM_OUTPUTS/$CASE/$OUTPUT_EXPECTED_SUBDIR/json-summary.txt" done for CASE in $FAILING_CASES; do - assert_fails "python gm/confirm_no_failures_in_json.py $GM_OUTPUTS/$CASE/$OUTPUT_EXPECTED_SUBDIR/json-summary.txt" + assert_fails "python gm/display_json_results.py $GM_OUTPUTS/$CASE/$OUTPUT_EXPECTED_SUBDIR/json-summary.txt" done if [ $ENCOUNTERED_ANY_ERRORS == 0 ]; then |