1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
|
#!/usr/bin/python
"""
Copyright 2014 Google Inc.
Use of this source code is governed by a BSD-style license that can be
found in the LICENSE file.
Compare GM results for two configs, across all builders.
"""
# System-level imports
import argparse
import fnmatch
import json
import logging
import os
import re
import sys
import time
# Imports from within Skia
#
# TODO(epoger): Once we move the create_filepath_url() function out of
# download_actuals into a shared utility module, we won't need to import
# download_actuals anymore.
#
# We need to add the 'gm' directory, so that we can import gm_json.py within
# that directory. That script allows us to parse the actual-results.json file
# written out by the GM tool.
# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end*
# so any dirs that are already in the PYTHONPATH will be preferred.
PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
GM_DIRECTORY = os.path.dirname(PARENT_DIRECTORY)
TRUNK_DIRECTORY = os.path.dirname(GM_DIRECTORY)
if GM_DIRECTORY not in sys.path:
sys.path.append(GM_DIRECTORY)
import download_actuals
import gm_json
import imagediffdb
import imagepair
import imagepairset
import results
class ConfigComparisons(results.BaseComparisons):
"""Loads results from two different configurations into an ImagePairSet.
Loads actual and expected results from all builders, except for those skipped
by _ignore_builder().
"""
def __init__(self, configs, actuals_root=results.DEFAULT_ACTUALS_DIR,
generated_images_root=results.DEFAULT_GENERATED_IMAGES_ROOT,
diff_base_url=None, builder_regex_list=None):
"""
Args:
configs: (string, string) tuple; pair of configs to compare
actuals_root: root directory containing all actual-results.json files
generated_images_root: directory within which to create all pixel diffs;
if this directory does not yet exist, it will be created
diff_base_url: base URL within which the client should look for diff
images; if not specified, defaults to a "file:///" URL representation
of generated_images_root
builder_regex_list: List of regular expressions specifying which builders
we will process. If None, process all builders.
"""
time_start = int(time.time())
if builder_regex_list != None:
self.set_match_builders_pattern_list(builder_regex_list)
self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root)
self._diff_base_url = (
diff_base_url or
download_actuals.create_filepath_url(generated_images_root))
self._actuals_root = actuals_root
self._load_config_pairs(configs)
self._timestamp = int(time.time())
logging.info('Results complete; took %d seconds.' %
(self._timestamp - time_start))
def _load_config_pairs(self, configs):
"""Loads the results of all tests, across all builders (based on the
files within self._actuals_root), compares them across two configs,
and stores the summary in self._results.
Args:
configs: tuple of strings; pair of configs to compare
"""
logging.info('Reading actual-results JSON files from %s...' %
self._actuals_root)
actual_builder_dicts = self._read_dicts_from_root(self._actuals_root)
configA, configB = configs
logging.info('Comparing configs %s and %s...' % (configA, configB))
all_image_pairs = imagepairset.ImagePairSet(
descriptions=configs,
diff_base_url=self._diff_base_url)
failing_image_pairs = imagepairset.ImagePairSet(
descriptions=configs,
diff_base_url=self._diff_base_url)
all_image_pairs.ensure_extra_column_values_in_summary(
column_id=results.KEY__EXTRACOLUMN__RESULT_TYPE, values=[
results.KEY__RESULT_TYPE__FAILED,
results.KEY__RESULT_TYPE__NOCOMPARISON,
results.KEY__RESULT_TYPE__SUCCEEDED,
])
failing_image_pairs.ensure_extra_column_values_in_summary(
column_id=results.KEY__EXTRACOLUMN__RESULT_TYPE, values=[
results.KEY__RESULT_TYPE__FAILED,
results.KEY__RESULT_TYPE__NOCOMPARISON,
])
builders = sorted(actual_builder_dicts.keys())
num_builders = len(builders)
builder_num = 0
for builder in builders:
builder_num += 1
logging.info('Generating pixel diffs for builder #%d of %d, "%s"...' %
(builder_num, num_builders, builder))
actual_results_for_this_builder = (
actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
for result_type in sorted(actual_results_for_this_builder.keys()):
results_of_this_type = actual_results_for_this_builder[result_type]
if not results_of_this_type:
continue
tests_found = set()
for image_name in sorted(results_of_this_type.keys()):
(test, config) = results.IMAGE_FILENAME_RE.match(image_name).groups()
tests_found.add(test)
for test in tests_found:
# Get image_relative_url (or None) for each of configA, configB
image_name_A = results.IMAGE_FILENAME_FORMATTER % (test, configA)
configA_image_relative_url = ConfigComparisons._create_relative_url(
hashtype_and_digest=results_of_this_type.get(image_name_A),
test_name=test)
image_name_B = results.IMAGE_FILENAME_FORMATTER % (test, configB)
configB_image_relative_url = ConfigComparisons._create_relative_url(
hashtype_and_digest=results_of_this_type.get(image_name_B),
test_name=test)
# If we have images for at least one of these two configs,
# add them to our list.
if configA_image_relative_url or configB_image_relative_url:
if configA_image_relative_url == configB_image_relative_url:
result_type = results.KEY__RESULT_TYPE__SUCCEEDED
elif not configA_image_relative_url:
result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
elif not configB_image_relative_url:
result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
else:
result_type = results.KEY__RESULT_TYPE__FAILED
extra_columns_dict = {
results.KEY__EXTRACOLUMN__RESULT_TYPE: result_type,
results.KEY__EXTRACOLUMN__BUILDER: builder,
results.KEY__EXTRACOLUMN__TEST: test,
# TODO(epoger): Right now, the client UI crashes if it receives
# results that do not include a 'config' column.
# Until we fix that, keep the client happy.
results.KEY__EXTRACOLUMN__CONFIG: 'TODO',
}
try:
image_pair = imagepair.ImagePair(
image_diff_db=self._image_diff_db,
base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL,
imageA_relative_url=configA_image_relative_url,
imageB_relative_url=configB_image_relative_url,
extra_columns=extra_columns_dict)
all_image_pairs.add_image_pair(image_pair)
if result_type != results.KEY__RESULT_TYPE__SUCCEEDED:
failing_image_pairs.add_image_pair(image_pair)
except (KeyError, TypeError):
logging.exception(
'got exception while creating ImagePair for image_name '
'"%s", builder "%s"' % (image_name, builder))
self._results = {
results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(),
results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(),
}
def main():
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
datefmt='%m/%d/%Y %H:%M:%S',
level=logging.INFO)
parser = argparse.ArgumentParser()
parser.add_argument(
'--actuals', default=results.DEFAULT_ACTUALS_DIR,
help='Directory containing all actual-result JSON files; defaults to '
'\'%(default)s\' .')
parser.add_argument(
'config', nargs=2,
help='Two configurations to compare (8888, gpu, etc.).')
parser.add_argument(
'--outfile', required=True,
help='File to write result summary into, in JSON format.')
parser.add_argument(
'--results', default=results.KEY__HEADER__RESULTS_FAILURES,
help='Which result types to include. Defaults to \'%(default)s\'; '
'must be one of ' +
str([results.KEY__HEADER__RESULTS_FAILURES,
results.KEY__HEADER__RESULTS_ALL]))
parser.add_argument(
'--workdir', default=results.DEFAULT_GENERATED_IMAGES_ROOT,
help='Directory within which to download images and generate diffs; '
'defaults to \'%(default)s\' .')
args = parser.parse_args()
results_obj = ConfigComparisons(configs=args.config,
actuals_root=args.actuals,
generated_images_root=args.workdir)
gm_json.WriteToFile(
results_obj.get_packaged_results_of_type(results_type=args.results),
args.outfile)
if __name__ == '__main__':
main()
|
