1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
|
#!/usr/bin/python
"""
Copyright 2014 Google Inc.
Use of this source code is governed by a BSD-style license that can be
found in the LICENSE file.
Compare results of two render_pictures runs.
"""
# System-level imports
import logging
import os
import re
import sys
import time
# Imports from within Skia
#
# TODO(epoger): Once we move the create_filepath_url() function out of
# download_actuals into a shared utility module, we won't need to import
# download_actuals anymore.
#
# We need to add the 'gm' directory, so that we can import gm_json.py within
# that directory. That script allows us to parse the actual-results.json file
# written out by the GM tool.
# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end*
# so any dirs that are already in the PYTHONPATH will be preferred.
PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
GM_DIRECTORY = os.path.dirname(PARENT_DIRECTORY)
TRUNK_DIRECTORY = os.path.dirname(GM_DIRECTORY)
if GM_DIRECTORY not in sys.path:
sys.path.append(GM_DIRECTORY)
import download_actuals
import gm_json
import imagediffdb
import imagepair
import imagepairset
import results
# Characters we don't want popping up just anywhere within filenames.
DISALLOWED_FILEPATH_CHAR_REGEX = re.compile('[^\w\-]')
# URL under which all render_pictures images can be found in Google Storage.
# TODO(epoger): Move this default value into
# https://skia.googlesource.com/buildbot/+/master/site_config/global_variables.json
DEFAULT_IMAGE_BASE_URL = 'http://chromium-skia-gm.commondatastorage.googleapis.com/render_pictures/images'
class RenderedPicturesComparisons(results.BaseComparisons):
"""Loads results from two different render_pictures runs into an ImagePairSet.
"""
def __init__(self, subdirs, actuals_root,
generated_images_root=results.DEFAULT_GENERATED_IMAGES_ROOT,
image_base_url=DEFAULT_IMAGE_BASE_URL,
diff_base_url=None):
"""
Args:
actuals_root: root directory containing all render_pictures-generated
JSON files
subdirs: (string, string) tuple; pair of subdirectories within
actuals_root to compare
generated_images_root: directory within which to create all pixel diffs;
if this directory does not yet exist, it will be created
image_base_url: URL under which all render_pictures result images can
be found; this will be used to read images for comparison within
this code, and included in the ImagePairSet so its consumers know
where to download the images from
diff_base_url: base URL within which the client should look for diff
images; if not specified, defaults to a "file:///" URL representation
of generated_images_root
"""
time_start = int(time.time())
self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root)
self._image_base_url = image_base_url
self._diff_base_url = (
diff_base_url or
download_actuals.create_filepath_url(generated_images_root))
self._load_result_pairs(actuals_root, subdirs)
self._timestamp = int(time.time())
logging.info('Results complete; took %d seconds.' %
(self._timestamp - time_start))
def _load_result_pairs(self, actuals_root, subdirs):
"""Loads all JSON files found within two subdirs in actuals_root,
compares across those two subdirs, and stores the summary in self._results.
Args:
actuals_root: root directory containing all render_pictures-generated
JSON files
subdirs: (string, string) tuple; pair of subdirectories within
actuals_root to compare
"""
logging.info(
'Reading actual-results JSON files from %s subdirs within %s...' % (
subdirs, actuals_root))
subdirA, subdirB = subdirs
subdirA_builder_dicts = results.BaseComparisons._read_dicts_from_root(
os.path.join(actuals_root, subdirA))
subdirB_builder_dicts = results.BaseComparisons._read_dicts_from_root(
os.path.join(actuals_root, subdirB))
logging.info('Comparing subdirs %s and %s...' % (subdirA, subdirB))
all_image_pairs = imagepairset.ImagePairSet(
descriptions=subdirs,
diff_base_url=self._diff_base_url)
failing_image_pairs = imagepairset.ImagePairSet(
descriptions=subdirs,
diff_base_url=self._diff_base_url)
all_image_pairs.ensure_extra_column_values_in_summary(
column_id=results.KEY__EXTRACOLUMN__RESULT_TYPE, values=[
results.KEY__RESULT_TYPE__FAILED,
results.KEY__RESULT_TYPE__NOCOMPARISON,
results.KEY__RESULT_TYPE__SUCCEEDED,
])
failing_image_pairs.ensure_extra_column_values_in_summary(
column_id=results.KEY__EXTRACOLUMN__RESULT_TYPE, values=[
results.KEY__RESULT_TYPE__FAILED,
results.KEY__RESULT_TYPE__NOCOMPARISON,
])
builders = sorted(set(subdirA_builder_dicts.keys() +
subdirB_builder_dicts.keys()))
num_builders = len(builders)
builder_num = 0
for builder in builders:
builder_num += 1
logging.info('Generating pixel diffs for builder #%d of %d, "%s"...' %
(builder_num, num_builders, builder))
# TODO(epoger): This will fail if we have results for this builder in
# subdirA but not subdirB (or vice versa).
subdirA_results = results.BaseComparisons.combine_subdicts(
subdirA_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
subdirB_results = results.BaseComparisons.combine_subdicts(
subdirB_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
image_names = sorted(set(subdirA_results.keys() +
subdirB_results.keys()))
for image_name in image_names:
# The image name may contain funny characters or be ridiculously long
# (see https://code.google.com/p/skia/issues/detail?id=2344#c10 ),
# so make sure we sanitize it before using it in a URL path.
#
# TODO(epoger): Rather than sanitizing/truncating the image name here,
# do it in render_pictures instead.
# Reason: we will need to be consistent in applying this rule, so that
# the process which uploads the files to GS using these paths will
# match the paths created by downstream processes.
# So, we should make render_pictures write out images to paths that are
# "ready to upload" to Google Storage, like gm does.
sanitized_test_name = DISALLOWED_FILEPATH_CHAR_REGEX.sub(
'_', image_name)[:30]
subdirA_image_relative_url = (
results.BaseComparisons._create_relative_url(
hashtype_and_digest=subdirA_results.get(image_name),
test_name=sanitized_test_name))
subdirB_image_relative_url = (
results.BaseComparisons._create_relative_url(
hashtype_and_digest=subdirB_results.get(image_name),
test_name=sanitized_test_name))
# If we have images for at least one of these two subdirs,
# add them to our list.
if subdirA_image_relative_url or subdirB_image_relative_url:
if subdirA_image_relative_url == subdirB_image_relative_url:
result_type = results.KEY__RESULT_TYPE__SUCCEEDED
elif not subdirA_image_relative_url:
result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
elif not subdirB_image_relative_url:
result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
else:
result_type = results.KEY__RESULT_TYPE__FAILED
extra_columns_dict = {
results.KEY__EXTRACOLUMN__RESULT_TYPE: result_type,
results.KEY__EXTRACOLUMN__BUILDER: builder,
results.KEY__EXTRACOLUMN__TEST: image_name,
# TODO(epoger): Right now, the client UI crashes if it receives
# results that do not include a 'config' column.
# Until we fix that, keep the client happy.
results.KEY__EXTRACOLUMN__CONFIG: 'TODO',
}
try:
image_pair = imagepair.ImagePair(
image_diff_db=self._image_diff_db,
base_url=self._image_base_url,
imageA_relative_url=subdirA_image_relative_url,
imageB_relative_url=subdirB_image_relative_url,
extra_columns=extra_columns_dict)
all_image_pairs.add_image_pair(image_pair)
if result_type != results.KEY__RESULT_TYPE__SUCCEEDED:
failing_image_pairs.add_image_pair(image_pair)
except (KeyError, TypeError):
logging.exception(
'got exception while creating ImagePair for image_name '
'"%s", builder "%s"' % (image_name, builder))
self._results = {
results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(),
results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(),
}
# TODO(epoger): Add main() so this can be called by vm_run_skia_try.sh
|
