1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
|
#!/usr/bin/env python
# Copyright (c) 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""greenify.py: standalone script to correct flaky bench expectations.
Requires Rietveld credentials on the running machine.
Usage:
Copy script to a separate dir outside Skia repo. The script will create a
skia dir on the first run to host the repo, and will create/delete
temp dirs as needed.
./greenify.py --url <the stdio url from failed CheckForRegressions step>
"""
import argparse
import filecmp
import os
import re
import shutil
import subprocess
import time
import urllib2
# Regular expression for matching exception data.
EXCEPTION_RE = ('Bench (\S+) out of range \[(\d+.\d+), (\d+.\d+)\] \((\d+.\d+) '
'vs (\d+.\d+), ')
EXCEPTION_RE_COMPILED = re.compile(EXCEPTION_RE)
def clean_dir(d):
if os.path.exists(d):
shutil.rmtree(d)
os.makedirs(d)
def checkout_or_update_skia(repo_dir):
status = True
old_cwd = os.getcwd()
os.chdir(repo_dir)
print 'CHECK SKIA REPO...'
if subprocess.call(['git', 'pull'],
stderr=subprocess.PIPE):
print 'Checking out Skia from git, please be patient...'
os.chdir(old_cwd)
clean_dir(repo_dir)
os.chdir(repo_dir)
if subprocess.call(['git', 'clone', '-q', '--depth=50', '--single-branch',
'https://skia.googlesource.com/skia.git', '.']):
status = False
subprocess.call(['git', 'checkout', 'master'])
subprocess.call(['git', 'pull'])
os.chdir(old_cwd)
return status
def git_commit_expectations(repo_dir, exp_dir, bot, build, commit):
commit_msg = """Greenify bench bot %s at build %s
TBR=bsalomon@google.com
Bypassing trybots:
NOTRY=true""" % (bot, build)
old_cwd = os.getcwd()
os.chdir(repo_dir)
upload = ['git', 'cl', 'upload', '-f', '--bypass-hooks',
'--bypass-watchlists', '-m', commit_msg]
if commit:
upload.append('--use-commit-queue')
branch = exp_dir[exp_dir.rfind('/') + 1:]
filename = 'bench_expectations_%s.txt' % bot
cmds = ([['git', 'checkout', 'master'],
['git', 'pull'],
['git', 'checkout', '-b', branch, '-t', 'origin/master'],
['cp', '%s/%s' % (exp_dir, filename), 'expectations/bench'],
['git', 'add', 'expectations/bench/' + filename],
['git', 'commit', '-m', commit_msg],
upload,
['git', 'checkout', 'master'],
['git', 'branch', '-D', branch],
])
status = True
for cmd in cmds:
print 'Running ' + ' '.join(cmd)
if subprocess.call(cmd):
print 'FAILED. Please check if skia git repo is present.'
subprocess.call(['git', 'checkout', 'master'])
status = False
break
os.chdir(old_cwd)
return status
def delete_dirs(li):
for d in li:
print 'Deleting directory %s' % d
shutil.rmtree(d)
def widen_bench_ranges(url, bot, repo_dir, exp_dir):
fname = 'bench_expectations_%s.txt' % bot
src = os.path.join(repo_dir, 'expectations', 'bench', fname)
if not os.path.isfile(src):
print 'This bot has no expectations! %s' % bot
return False
row_dic = {}
for l in urllib2.urlopen(url).read().split('\n'):
data = EXCEPTION_RE_COMPILED.search(l)
if data:
row = data.group(1)
lb = float(data.group(2))
ub = float(data.group(3))
actual = float(data.group(4))
exp = float(data.group(5))
avg = (actual + exp) / 2
shift = avg - exp
lb = lb + shift
ub = ub + shift
# In case outlier really fluctuates a lot
if actual < lb:
lb = actual - abs(shift) * 0.1 + 0.5
elif actual > ub:
ub = actual + abs(shift) * 0.1 + 0.5
row_dic[row] = '%.2f,%.2f,%.2f' % (avg, lb, ub)
if not row_dic:
print 'NO out-of-range benches found at %s' % url
return False
changed = 0
li = []
for l in open(src).readlines():
parts = l.strip().split(',')
if parts[0].startswith('#') or len(parts) != 5:
li.append(l.strip())
continue
if ','.join(parts[:2]) in row_dic:
li.append(','.join(parts[:2]) + ',' + row_dic[','.join(parts[:2])])
changed += 1
else:
li.append(l.strip())
if not changed:
print 'Not in source file:\n' + '\n'.join(row_dic.keys())
return False
dst = os.path.join(exp_dir, fname)
with open(dst, 'w+') as f:
f.write('\n'.join(li))
return True
def main():
d = os.path.dirname(os.path.abspath(__file__))
os.chdir(d)
if not subprocess.call(['git', 'rev-parse'], stderr=subprocess.PIPE):
print 'Please copy script to a separate dir outside git repos to use.'
return
ts_str = '%s' % time.time()
parser = argparse.ArgumentParser()
parser.add_argument('--url',
help='Broken bench build CheckForRegressions page url.')
parser.add_argument('--commit', action='store_true',
help='Whether to commit changes automatically.')
args = parser.parse_args()
repo_dir = os.path.join(d, 'skia')
if not os.path.exists(repo_dir):
os.makedirs(repo_dir)
if not checkout_or_update_skia(repo_dir):
print 'ERROR setting up Skia repo at %s' % repo_dir
return 1
file_in_repo = os.path.join(d, 'skia/experimental/benchtools/greenify.py')
if not filecmp.cmp(__file__, file_in_repo):
shutil.copy(file_in_repo, __file__)
print 'Updated this script from repo; please run again.'
return
if not args.url:
raise Exception('Please provide a url with broken CheckForRegressions.')
path = args.url.split('/')
if len(path) != 11 or not path[6].isdigit():
raise Exception('Unexpected url format: %s' % args.url)
bot = path[4]
build = path[6]
commit = False
if args.commit:
commit = True
exp_dir = os.path.join(d, 'exp' + ts_str)
clean_dir(exp_dir)
if not widen_bench_ranges(args.url, bot, repo_dir, exp_dir):
print 'NO bench exceptions found! %s' % args.url
elif not git_commit_expectations(
repo_dir, exp_dir, bot, build, commit):
print 'ERROR uploading expectations using git.'
elif not commit:
print 'CL created. Please take a look at the link above.'
else:
print 'New bench baselines should be in CQ now.'
delete_dirs([exp_dir])
if __name__ == "__main__":
main()
|
