1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
|
'''
Downloads the actual gm results most recently generated by the Skia buildbots,
and adds any new ones to SVN control.
Launch with --help to see more information.
Copyright 2011 Google Inc.
Use of this source code is governed by a BSD-style license that can be
found in the LICENSE file.
'''
# common Python modules
import fnmatch
import optparse
import os
import re
import shutil
import sys
import tempfile
# modules declared within this same directory
import compare_baselines
import svn
USAGE_STRING = 'Usage: %s [options] [baseline_subdir]...'
HELP_STRING = '''
Downloads the actual gm results most recently generated by the Skia buildbots,
and adds any new ones to SVN control.
If no baseline_subdir is given, then this tool will download the most-recently
generated actual gm results for ALL platforms.
''' + compare_baselines.HOWTO_STRING
# Base URL of SVN repository where buildbots store actual gm image results.
GM_ACTUAL_URL = 'http://skia-autogen.googlecode.com/svn/gm-actual'
# GM baseline image URL in regular Skia SVN repository
GM_BASELINE_URL = 'https://skia.googlecode.com/svn/gm-expected'
GM_EXPECTED_DIR = 'gm-expected'
OPTION_ADD_NEW_FILES = '--add-new-files'
OPTION_BUILDER_SUFFIX = '--builder-suffix'
DEFAULT_BUILDER_SUFFIX = '32'
OPTION_IGNORE_LOCAL_MODS = '--ignore-local-mods'
def GetLatestResultsSvnUrl(svn, baseline_subdir, builder_suffix):
"""Return SVN URL from which we can check out the MOST RECENTLY generated images for this
baseline type.
@param svn an Svn object we can use to call ListSubdirs()
@param baseline_subdir indicates which platform we want images for
@param builder_suffix if multiple builders uploaded actual GM images for this baseline type,
choose the one whose builder_name matches this suffix
"""
root_url = '%s/%s' % (GM_ACTUAL_URL, baseline_subdir)
subdirs = sorted(svn.ListSubdirs(root_url))
num_subdirs = len(subdirs)
print('Within actual-results root URL %s, found these %d subdirs (presumably builder_names): %s'
% (root_url, num_subdirs, subdirs))
selected_subdir = None
if num_subdirs == 0:
print 'Found no builder_name subdirs, so reading actual images from the root_url itself.'
return root_url
elif num_subdirs == 1:
selected_subdir = subdirs[0]
print 'Found exactly one subdir in actual-results root_url: %s' % selected_subdir
else:
for possible_subdir in subdirs:
if possible_subdir.endswith(builder_suffix):
selected_subdir = possible_subdir
print 'Selected the first subdir ending in "%s": %s' % (
builder_suffix, selected_subdir)
break
if selected_subdir:
return '%s/%s/%s' % (root_url, selected_subdir, baseline_subdir)
else:
raise Exception('none of these subdirs of %s ended in "%s": %s' % (
root_url, builder_suffix, subdirs))
def GetBaselineSvnUrl(baseline_subdir):
"""Return SVN URL from which we can check out the baseline images for this
baseline type.
@param baseline_subdir indicates which platform we want baselines for
"""
return '%s/%s' % (GM_BASELINE_URL, baseline_subdir)
def CopyMatchingFiles(source_dir, dest_dir, filename_pattern, only_copy_updates=False):
"""Copy all files from source_dir that match filename_pattern, and save them (with their
original filenames) in dest_dir.
@param source_dir
@param dest_dir where to save the copied files
@param filename_pattern only copy files that match this Unix-style filename
pattern (e.g., '*.jpg')
@param only_copy_updates if True, only copy files that are already present in dest_dir
"""
all_filenames = os.listdir(source_dir)
matching_filenames = fnmatch.filter(all_filenames, filename_pattern)
for filename in matching_filenames:
source_path = os.path.join(source_dir, filename)
dest_path = os.path.join(dest_dir, filename)
if only_copy_updates and not os.path.isfile(dest_path):
continue
shutil.copyfile(source_path, dest_path)
def DownloadBaselinesForOnePlatform(baseline_subdir):
"""Download most recently generated baseline images for a single platform,
and add any new ones to SVN control.
@param baseline_subdir
"""
# Create repo_to_modify to handle the SVN repository we will add files to.
gm_dir = os.path.join(os.pardir, GM_EXPECTED_DIR) # Shouldn't assume we're in trunk...
try:
os.makedirs(gm_dir)
except:
pass
repo_to_modify = svn.Svn(gm_dir)
repo_to_modify.Checkout(GetBaselineSvnUrl(baseline_subdir), baseline_subdir)
# If there are any locally modified files in that directory, exit
# (so that we don't risk overwriting the user's previous work).
new_and_modified_files = repo_to_modify.GetNewAndModifiedFiles()
if not options.ignore_local_mods:
if new_and_modified_files:
raise Exception('Exiting because there are already new and/or '
'modified files in %s. To continue in spite of '
'that, run with %s option.' % (
baseline_subdir, OPTION_IGNORE_LOCAL_MODS))
# Download actual gm images into a separate repo in a temporary directory.
actual_dir = tempfile.mkdtemp()
actual_repo = svn.Svn(actual_dir)
print 'Using %s as a temp dir' % actual_dir
actual_url = GetLatestResultsSvnUrl(svn=actual_repo, baseline_subdir=baseline_subdir,
builder_suffix=options.builder_suffix)
print 'Reading actual buildbot GM results from %s' % actual_url
actual_repo.Checkout(actual_url, '.')
# Copy any of those files we are interested in into repo_to_modify,
# and then delete the temporary directory.
CopyMatchingFiles(source_dir=actual_dir,
dest_dir=os.path.join(gm_dir, baseline_subdir),
filename_pattern='*.png',
only_copy_updates=(not options.add_new_files))
shutil.rmtree(actual_dir)
actual_repo = None
# Add any new files to SVN control (if we are running with add_new_files).
if options.add_new_files:
new_files = repo_to_modify.GetNewFiles()
if new_files:
repo_to_modify.AddFiles(sorted(new_files))
# Set the mimetype property on any new/modified image files in
# baseline_subdir. (We used to set the mimetype property on *all* image
# files in the directory, even those whose content wasn't changing,
# but that caused confusion. See
# http://code.google.com/p/skia/issues/detail?id=618 .)
modified_files = repo_to_modify.GetNewAndModifiedFiles()
repo_to_modify.SetProperty(sorted(fnmatch.filter(modified_files, '*.png')),
svn.PROPERTY_MIMETYPE, 'image/png')
repo_to_modify.SetProperty(sorted(fnmatch.filter(modified_files, '*.pdf')),
svn.PROPERTY_MIMETYPE, 'application/pdf')
def RaiseUsageException():
raise Exception('%s\nRun with --help for more detail.' % (
USAGE_STRING % __file__))
def Main(options, args):
"""Allow other scripts to call this script with fake command-line args.
"""
# If no platforms are specified, do 'em all.
num_args = len(args)
if num_args == 0:
# TODO(epoger): automate the default set of platforms. We want to ensure
# that the user gets all of the platforms that the bots are running,
# not just whatever subdirectories he happens to have checked out...
# See http://code.google.com/p/skia/issues/detail?id=678
# Now that we have added Svn.ListSubdirs(), we should be able to do this
# pretty easily...
#
# For now, I generate this list using these Unix commands:
# svn ls http://skia.googlecode.com/svn/gm-expected | grep ^base | sort >/tmp/baselines
# svn ls http://skia-autogen.googlecode.com/svn/gm-actual | grep ^base | sort >/tmp/actual
# comm -1 -2 /tmp/baselines /tmp/actual
args = [
'base-android-galaxy-nexus',
'base-android-nexus-7',
'base-android-nexus-s',
'base-android-xoom',
'base-macmini',
'base-macmini-lion-float',
'base-shuttle-win7-intel-float',
'base-shuttle_ubuntu12_ati5770',
'base-shuttle-win7-intel-angle',
'base-shuttle-win7-intel-directwrite',
]
# Trim all subdir names.
baseline_subdirs = []
for arg in args:
baseline_subdirs.append(arg.rstrip(os.sep))
# Process the subdirs, one at a time.
for baseline_subdir in baseline_subdirs:
DownloadBaselinesForOnePlatform(baseline_subdir=baseline_subdir)
if __name__ == '__main__':
parser = optparse.OptionParser(USAGE_STRING % '%prog' + HELP_STRING)
parser.add_option(OPTION_ADD_NEW_FILES,
action='store_true', default=False,
help='in addition to downloading new versions of '
'existing baselines, also download baselines that are '
'not under SVN control yet')
parser.add_option(OPTION_BUILDER_SUFFIX,
action='store', type='string', default=DEFAULT_BUILDER_SUFFIX,
help='if multiple builders have uploaded actual GM images '
'for this platform, download the images uploaded by the '
'builder whose name ends in this suffix; defaults to '
'"%s".' % DEFAULT_BUILDER_SUFFIX)
parser.add_option(OPTION_IGNORE_LOCAL_MODS,
action='store_true', default=False,
help='allow tool to run even if there are already '
'local modifications in the baseline_subdir')
(options, args) = parser.parse_args()
Main(options, args)
|
