1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
|
#!/usr/bin/python
'''
Copyright 2013 Google Inc.
Use of this source code is governed by a BSD-style license that can be
found in the LICENSE file.
'''
'''
Repackage expected/actual GM results as needed by our HTML rebaseline viewer.
'''
# System-level imports
import fnmatch
import json
import os
import re
import sys
# Imports from within Skia
#
# We need to add the 'gm' directory, so that we can import gm_json.py within
# that directory. That script allows us to parse the actual-results.json file
# written out by the GM tool.
# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end*
# so any dirs that are already in the PYTHONPATH will be preferred.
GM_DIRECTORY = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
if GM_DIRECTORY not in sys.path:
sys.path.append(GM_DIRECTORY)
import gm_json
IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
class Results(object):
""" Loads actual and expected results from all builders, supplying combined
reports as requested. """
def __init__(self, actuals_root, expected_root):
"""
params:
actuals_root: root directory containing all actual-results.json files
expected_root: root directory containing all expected-results.json files
"""
self._actual_builder_dicts = Results._GetDictsFromRoot(actuals_root)
self._expected_builder_dicts = Results._GetDictsFromRoot(expected_root)
self._all_results = self._Combine()
def GetAll(self):
"""Return results of all tests, as a list in this form:
[
{
"builder": "Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug",
"test": "bigmatrix",
"config": "8888",
"resultType": "failed",
"expectedHashType": "bitmap-64bitMD5",
"expectedHashDigest": "10894408024079689926",
"actualHashType": "bitmap-64bitMD5",
"actualHashDigest": "2409857384569",
},
...
]
"""
return self._all_results
@staticmethod
def _GetDictsFromRoot(root, pattern='*.json'):
"""Read all JSON dictionaries within a directory tree, returning them within
a meta-dictionary (keyed by the builder name for each dictionary).
params:
root: path to root of directory tree
pattern: which files to read within root (fnmatch-style pattern)
"""
meta_dict = {}
for dirpath, dirnames, filenames in os.walk(root):
for matching_filename in fnmatch.filter(filenames, pattern):
builder = os.path.basename(dirpath)
if builder.endswith('-Trybot'):
continue
fullpath = os.path.join(dirpath, matching_filename)
meta_dict[builder] = gm_json.LoadFromFile(fullpath)
return meta_dict
def _Combine(self):
"""Returns a list of all tests, across all builders, based on the
contents of self._actual_builder_dicts and self._expected_builder_dicts .
Returns the list in the same form needed for GetAllResults().
"""
all_tests = []
for builder in sorted(self._actual_builder_dicts.keys()):
actual_results_for_this_builder = (
self._actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
for result_type in sorted(actual_results_for_this_builder.keys()):
results_of_this_type = actual_results_for_this_builder[result_type]
if not results_of_this_type:
continue
for image_name in sorted(results_of_this_type.keys()):
actual_image = results_of_this_type[image_name]
try:
# TODO(epoger): assumes a single allowed digest per test
expected_image = (
self._expected_builder_dicts
[builder][gm_json.JSONKEY_EXPECTEDRESULTS]
[image_name][gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS]
[0])
except (KeyError, TypeError):
# There are several cases in which we would expect to find
# no expectations for a given test:
#
# 1. result_type == NOCOMPARISON
# There are no expectations for this test yet!
#
# 2. ignore-tests.txt
# If a test has been listed in ignore-tests.txt, then its status
# may show as FAILUREIGNORED even if it doesn't have any
# expectations yet.
#
# 3. alternate rendering mode failures (e.g. serialized)
# In cases like
# https://code.google.com/p/skia/issues/detail?id=1684
# ('tileimagefilter GM test failing in serialized render mode'),
# the gm-actuals will list a failure for the alternate
# rendering mode even though we don't have explicit expectations
# for the test (the implicit expectation is that it must
# render the same in all rendering modes).
#
# Don't log types 1 or 2, because they are common.
# Log other types, because they are rare and we should know about
# them, but don't throw an exception, because we need to keep our
# tools working in the meanwhile!
if result_type not in [
gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON,
gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED] :
print 'WARNING: No expectations found for test: %s' % {
'builder': builder,
'image_name': image_name,
'result_type': result_type,
}
expected_image = [None, None]
# If this test was recently rebaselined, it will remain in
# the "failed" set of actuals until all the bots have
# cycled (although the expectations have indeed been set
# from the most recent actuals). Treat these as successes
# instead of failures.
#
# TODO(epoger): Do we need to do something similar in
# other cases, such as when we have recently marked a test
# as ignoreFailure but it still shows up in the "failed"
# category? Maybe we should not rely on the result_type
# categories recorded within the gm_actuals AT ALL, and
# instead evaluate the result_type ourselves based on what
# we see in expectations vs actual checksum?
if expected_image == actual_image:
updated_result_type = gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED
else:
updated_result_type = result_type
# TODO(epoger): For now, don't include succeeded results.
# There are so many of them that they make the client too slow.
if updated_result_type == gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED:
continue
(test, config) = IMAGE_FILENAME_RE.match(image_name).groups()
all_tests.append({
"builder": builder,
"test": test,
"config": config,
"resultType": updated_result_type,
"actualHashType": actual_image[0],
"actualHashDigest": str(actual_image[1]),
"expectedHashType": expected_image[0],
"expectedHashDigest": str(expected_image[1]),
})
return all_tests
|
