1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
|
#!/usr/bin/python
"""
Copyright 2014 Google Inc.
Use of this source code is governed by a BSD-style license that can be
found in the LICENSE file.
Compare results of two render_pictures runs.
"""
# System-level imports
import logging
import os
import re
import time
# Imports from within Skia
import fix_pythonpath # must do this first
from pyutils import url_utils
import gm_json
import imagediffdb
import imagepair
import imagepairset
import results
# URL under which all render_pictures images can be found in Google Storage.
# TODO(epoger): Move this default value into
# https://skia.googlesource.com/buildbot/+/master/site_config/global_variables.json
DEFAULT_IMAGE_BASE_URL = 'http://chromium-skia-gm.commondatastorage.googleapis.com/render_pictures/images'
class RenderedPicturesComparisons(results.BaseComparisons):
"""Loads results from two different render_pictures runs into an ImagePairSet.
"""
def __init__(self, subdirs, actuals_root,
generated_images_root=results.DEFAULT_GENERATED_IMAGES_ROOT,
image_base_url=DEFAULT_IMAGE_BASE_URL,
diff_base_url=None):
"""
Args:
actuals_root: root directory containing all render_pictures-generated
JSON files
subdirs: (string, string) tuple; pair of subdirectories within
actuals_root to compare
generated_images_root: directory within which to create all pixel diffs;
if this directory does not yet exist, it will be created
image_base_url: URL under which all render_pictures result images can
be found; this will be used to read images for comparison within
this code, and included in the ImagePairSet so its consumers know
where to download the images from
diff_base_url: base URL within which the client should look for diff
images; if not specified, defaults to a "file:///" URL representation
of generated_images_root
"""
time_start = int(time.time())
self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root)
self._image_base_url = image_base_url
self._diff_base_url = (
diff_base_url or
url_utils.create_filepath_url(generated_images_root))
self._load_result_pairs(actuals_root, subdirs)
self._timestamp = int(time.time())
logging.info('Results complete; took %d seconds.' %
(self._timestamp - time_start))
def _load_result_pairs(self, actuals_root, subdirs):
"""Loads all JSON files found within two subdirs in actuals_root,
compares across those two subdirs, and stores the summary in self._results.
Args:
actuals_root: root directory containing all render_pictures-generated
JSON files
subdirs: (string, string) tuple; pair of subdirectories within
actuals_root to compare
"""
logging.info(
'Reading actual-results JSON files from %s subdirs within %s...' % (
subdirs, actuals_root))
subdirA, subdirB = subdirs
subdirA_dicts = self._read_dicts_from_root(
os.path.join(actuals_root, subdirA))
subdirB_dicts = self._read_dicts_from_root(
os.path.join(actuals_root, subdirB))
logging.info('Comparing subdirs %s and %s...' % (subdirA, subdirB))
all_image_pairs = imagepairset.ImagePairSet(
descriptions=subdirs,
diff_base_url=self._diff_base_url)
failing_image_pairs = imagepairset.ImagePairSet(
descriptions=subdirs,
diff_base_url=self._diff_base_url)
all_image_pairs.ensure_extra_column_values_in_summary(
column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[
results.KEY__RESULT_TYPE__FAILED,
results.KEY__RESULT_TYPE__NOCOMPARISON,
results.KEY__RESULT_TYPE__SUCCEEDED,
])
failing_image_pairs.ensure_extra_column_values_in_summary(
column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[
results.KEY__RESULT_TYPE__FAILED,
results.KEY__RESULT_TYPE__NOCOMPARISON,
])
common_dict_paths = sorted(set(subdirA_dicts.keys() + subdirB_dicts.keys()))
num_common_dict_paths = len(common_dict_paths)
dict_num = 0
for dict_path in common_dict_paths:
dict_num += 1
logging.info('Generating pixel diffs for dict #%d of %d, "%s"...' %
(dict_num, num_common_dict_paths, dict_path))
dictA = subdirA_dicts[dict_path]
dictB = subdirB_dicts[dict_path]
self._validate_dict_version(dictA)
self._validate_dict_version(dictB)
dictA_results = dictA[gm_json.JSONKEY_ACTUALRESULTS]
dictB_results = dictB[gm_json.JSONKEY_ACTUALRESULTS]
skp_names = sorted(set(dictA_results.keys() + dictB_results.keys()))
for skp_name in skp_names:
imagepairs_for_this_skp = []
whole_image_A = RenderedPicturesComparisons.get_multilevel(
dictA_results, skp_name, gm_json.JSONKEY_SOURCE_WHOLEIMAGE)
whole_image_B = RenderedPicturesComparisons.get_multilevel(
dictB_results, skp_name, gm_json.JSONKEY_SOURCE_WHOLEIMAGE)
imagepairs_for_this_skp.append(self._create_image_pair(
test=skp_name, config=gm_json.JSONKEY_SOURCE_WHOLEIMAGE,
image_dict_A=whole_image_A, image_dict_B=whole_image_B))
tiled_images_A = RenderedPicturesComparisons.get_multilevel(
dictA_results, skp_name, gm_json.JSONKEY_SOURCE_TILEDIMAGES)
tiled_images_B = RenderedPicturesComparisons.get_multilevel(
dictB_results, skp_name, gm_json.JSONKEY_SOURCE_TILEDIMAGES)
# TODO(epoger): Report an error if we find tiles for A but not B?
if tiled_images_A and tiled_images_B:
# TODO(epoger): Report an error if we find a different number of tiles
# for A and B?
num_tiles = len(tiled_images_A)
for tile_num in range(num_tiles):
imagepairs_for_this_skp.append(self._create_image_pair(
test=skp_name,
config='%s-%d' % (gm_json.JSONKEY_SOURCE_TILEDIMAGES, tile_num),
image_dict_A=tiled_images_A[tile_num],
image_dict_B=tiled_images_B[tile_num]))
for imagepair in imagepairs_for_this_skp:
if imagepair:
all_image_pairs.add_image_pair(imagepair)
result_type = imagepair.extra_columns_dict\
[results.KEY__EXTRACOLUMNS__RESULT_TYPE]
if result_type != results.KEY__RESULT_TYPE__SUCCEEDED:
failing_image_pairs.add_image_pair(imagepair)
self._results = {
results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(),
results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(),
}
def _validate_dict_version(self, result_dict):
"""Raises Exception if the dict is not the type/version we know how to read.
Args:
result_dict: dictionary holding output of render_pictures
"""
expected_header_type = 'ChecksummedImages'
expected_header_revision = 1
header = result_dict[gm_json.JSONKEY_HEADER]
header_type = header[gm_json.JSONKEY_HEADER_TYPE]
if header_type != expected_header_type:
raise Exception('expected header_type "%s", but got "%s"' % (
expected_header_type, header_type))
header_revision = header[gm_json.JSONKEY_HEADER_REVISION]
if header_revision != expected_header_revision:
raise Exception('expected header_revision %d, but got %d' % (
expected_header_revision, header_revision))
def _create_image_pair(self, test, config, image_dict_A, image_dict_B):
"""Creates an ImagePair object for this pair of images.
Args:
test: string; name of the test
config: string; name of the config
image_dict_A: dict with JSONKEY_IMAGE_* keys, or None if no image
image_dict_B: dict with JSONKEY_IMAGE_* keys, or None if no image
Returns:
An ImagePair object, or None if both image_dict_A and image_dict_B are
None.
"""
if (not image_dict_A) and (not image_dict_B):
return None
def _checksum_and_relative_url(dic):
if dic:
return ((dic[gm_json.JSONKEY_IMAGE_CHECKSUMALGORITHM],
dic[gm_json.JSONKEY_IMAGE_CHECKSUMVALUE]),
dic[gm_json.JSONKEY_IMAGE_FILEPATH])
else:
return None, None
imageA_checksum, imageA_relative_url = _checksum_and_relative_url(
image_dict_A)
imageB_checksum, imageB_relative_url = _checksum_and_relative_url(
image_dict_B)
if not imageA_checksum:
result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
elif not imageB_checksum:
result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
elif imageA_checksum == imageB_checksum:
result_type = results.KEY__RESULT_TYPE__SUCCEEDED
else:
result_type = results.KEY__RESULT_TYPE__FAILED
extra_columns_dict = {
results.KEY__EXTRACOLUMNS__CONFIG: config,
results.KEY__EXTRACOLUMNS__RESULT_TYPE: result_type,
results.KEY__EXTRACOLUMNS__TEST: test,
# TODO(epoger): Right now, the client UI crashes if it receives
# results that do not include this column.
# Until we fix that, keep the client happy.
results.KEY__EXTRACOLUMNS__BUILDER: 'TODO',
}
try:
return imagepair.ImagePair(
image_diff_db=self._image_diff_db,
base_url=self._image_base_url,
imageA_relative_url=imageA_relative_url,
imageB_relative_url=imageB_relative_url,
extra_columns=extra_columns_dict)
except (KeyError, TypeError):
logging.exception(
'got exception while creating ImagePair for'
' test="%s", config="%s", urlPair=("%s","%s")' % (
test, config, imageA_relative_url, imageB_relative_url))
return None
# TODO(epoger): Add main() so this can be called by vm_run_skia_try.sh
|
