aboutsummaryrefslogtreecommitdiffhomepage
path: root/tools/submit_try
blob: e09e3049a93822e2ab74532f7789a18404cd692e (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
#!/usr/bin/python

# Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.


"""
submit_try: Submit a try request.

This is a thin wrapper around the try request utilities in depot_tools which
adds some validation and supports both git and svn.
"""


import httplib
import json
import os
import re
import subprocess
import svn
import sys

import buildbot_globals


# Alias which can be used to run a try on every builder.
ALL_BUILDERS = 'all'
# Alias which can be used to run a try on all compile builders.
COMPILE_BUILDERS = 'compile'
# Alias which can be used to run a try on all builders that are run in the CQ.
CQ_BUILDERS = 'cq'
# Alias which can be used to specify a regex to choose builders.
REGEX = 'regex'

ALL_ALIASES = [ALL_BUILDERS, COMPILE_BUILDERS, CQ_BUILDERS, REGEX]

# Contact information for the build master.
SKIA_BUILD_MASTER_HOST = str(buildbot_globals.Get('public_master_host'))
SKIA_BUILD_MASTER_PORT = str(buildbot_globals.Get('public_external_port'))

# All try builders have this suffix.
TRYBOT_SUFFIX = '-Trybot'

# Location of the codereview.settings file in the Skia repo.
SKIA_URL = 'skia.googlecode.com'
CODEREVIEW_SETTINGS = '/svn/codereview.settings'

# String for matching the svn url of the try server inside codereview.settings.
TRYSERVER_SVN_URL = 'TRYSERVER_SVN_URL: '

# Strings used for matching svn config properties.
URL_STR = 'URL'
REPO_ROOT_STR = 'Repository Root'


def FindDepotTools():
  """ Find depot_tools on the local machine and return its location. """
  which_cmd = 'where' if os.name == 'nt' else 'which'
  cmd = [which_cmd, 'gcl']
  proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
  if proc.wait() != 0:
    raise Exception('Couldn\'t find depot_tools in PATH!')
  gcl = proc.communicate()[0].split('\n')[0].rstrip()
  depot_tools_dir = os.path.dirname(gcl)
  return depot_tools_dir


def GetCheckoutRoot(is_svn=True):
  """ Determine where the local checkout is rooted.

  is_svn: boolean; whether we're in an SVN checkout. If False, assume we're in
      a git checkout.
  """
  if is_svn:
    repo = svn.Svn(os.curdir)
    svn_info = repo.GetInfo()
    url = svn_info.get(URL_STR, None)
    repo_root = svn_info.get(REPO_ROOT_STR, None)
    if not url or not repo_root:
      raise Exception('Couldn\'t find checkout root!')
    if url == repo_root:
      return 'svn'
    return url[len(repo_root)+1:]
  else:
    cmd = ['git', 'rev-parse', '--show-toplevel']
    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                            stderr=subprocess.STDOUT)
    if proc.wait() != 0:
      raise Exception('Couldn\'t find checkout root!')
    return os.path.basename(proc.communicate()[0])


def GetTryRepo():
  """ Determine the TRYSERVER_SVN_URL from the codereview.settings file in the
  Skia repo. """
  connection = httplib.HTTPConnection(SKIA_URL)
  connection.request('GET', CODEREVIEW_SETTINGS)
  content = connection.getresponse().read()
  for line in content.split('\n'):
    if line.startswith(TRYSERVER_SVN_URL):
      return line[len(TRYSERVER_SVN_URL):].rstrip()
  raise Exception('Couldn\'t determine the TRYSERVER_SVN_URL. Make sure it is '
                  'defined in the %s file.' % CODEREVIEW_SETTINGS)


def RetrieveTrybotList(json_filename):
  """ Retrieve the list of known trybots from the build master, stripping
  TRYBOT_SUFFIX from the name. """
  trybots = []
  connection = httplib.HTTPConnection(SKIA_BUILD_MASTER_HOST,  
                                      SKIA_BUILD_MASTER_PORT)
  connection.request('GET', '/json/%s' % json_filename)
  response = connection.getresponse()  
  builders = json.load(response)

  for builder in builders:
    if builder.endswith(TRYBOT_SUFFIX):
      trybots.append(builder[:-len(TRYBOT_SUFFIX)])
  return trybots


def ValidateArgs(argv, trybots, is_svn=True):
  """ Parse and validate command-line arguments. If the arguments are valid,
  returns a tuple of (<changelist name>, <list of trybots>).

  trybots: A list of the known try builders.
  """

  class CollectedArgs(object):
    def __init__(self, bots, changelist, revision):
      self._bots = bots
      self._changelist = changelist
      self._revision = revision

    @property
    def bots(self):
      for bot in self._bots:
        yield bot

    @property
    def changelist(self):
      return self._changelist

    @property
    def revision(self):
      return self._revision

  usage = (
"""submit_try: Submit a try request.
submit_try %s--bot <buildername> [<buildername> ...]

-b, --bot           Builder(s) or Alias on which to run the try. Required.
                    Allowed aliases: %s
-h, --help          Show this message.
-r <revision#>      Revision from which to run the try.
-l, --list_bots     List the available try builders and aliases and exit.
""" % ('<changelist> ' if is_svn else '', ALL_ALIASES))

  def Error(msg=None):
    if msg:
      print msg
    print usage
    sys.exit(1)

  using_bots = None
  changelist = None
  revision = None

  while argv:
    arg = argv.pop(0)
    if arg == '-h' or arg == '--help':
      Error()
    elif arg == '-l' or arg == '--list_bots':
      format_args = ['\n  '.join(trybots)] + ALL_ALIASES
      print (
"""
submit_try: Available builders:\n  %s

Can also use the following aliases to run on groups of builders-
  %s: Will run against all trybots.
  %s: Will run against all compile trybots.
  %s: Will run against the same trybots as the commit queue.
  %s: You will be prompted to enter a regex to select builders with.

""" % tuple(format_args))
      sys.exit(0)
    elif arg == '-b' or arg == '--bot':
      if using_bots:
        Error('--bot specified multiple times.')
      if len(argv) < 1:
        Error('You must specify a builder with "--bot".')
      using_bots = []
      while argv and not argv[0].startswith('-'):
        for bot in argv.pop(0).split(','):
          if bot in ALL_ALIASES:
            if using_bots:
              Error('Cannot specify "%s" with additional builder names or '
                    'aliases.' % bot)
            if bot == ALL_BUILDERS:
              are_you_sure = raw_input('Running a try on every bot is very '
                                       'expensive. You may be able to get '
                                       'enough information by running on a '
                                       'smaller set of bots. Are you sure you '
                                       'want to run your try job on all of the '
                                       'trybots? [y,n]: ')
              if are_you_sure == 'y':
                using_bots = trybots
            elif bot == COMPILE_BUILDERS:
              using_bots = [t for t in trybots if t.startswith('Build')]
            elif bot == CQ_BUILDERS:
              using_bots = RetrieveTrybotList(json_filename='cqtrybots')
            elif bot == REGEX:
              while True:
                regex = raw_input("Enter your trybot regex: ")
                p = re.compile(regex)
                using_bots = [t for t in trybots if p.match(t)]
                print '\n\nTrybots that match your regex:\n%s\n\n' % '\n'.join(
                    using_bots)
                if raw_input('Re-enter regex? [y,n]: ') == 'n':
                  break
            break
          else:
            if not bot in trybots:
              Error('Unrecognized builder: %s' % bot)
            using_bots.append(bot)
    elif arg == '-r':
      if len(argv) < 1:
        Error('You must specify a revision with "-r".')
      revision = argv.pop(0)
    else:
      if changelist or not is_svn:
        Error('Unknown argument: %s' % arg)
      changelist = arg
  if is_svn and not changelist:
    Error('You must specify a changelist name.')
  if not using_bots:
    Error('You must specify one or more builders using --bot.')
  return CollectedArgs(bots=using_bots, changelist=changelist,
                       revision=revision)


def SubmitTryRequest(args, is_svn=True):
  """ Submits a try request for the given changelist on the given list of
  trybots.

  args: Object whose properties are derived from command-line arguments. If
      is_svn is True, it should contain:
      - changelist: string; the name of the changelist to try.
      - bot: list of strings; the names of the try builders to run.
      - revision: optional, int; the revision number from which to run the try.
      If is_svn is False, it should contain:
      - bot: list of strings; the names of the try builders to run.
      - revision: optional, int; the revision number from which to run the try. 
  is_svn: boolean; are we in an SVN repo?
  """
  botlist = ','.join(['%s%s' % (bot, TRYBOT_SUFFIX) for bot in args.bots])
  if is_svn:
    gcl_cmd = 'gcl.bat' if os.name == 'nt' else 'gcl'
    try_args = [gcl_cmd, 'try', args.changelist,
                '--root', GetCheckoutRoot(is_svn),
                '--bot', botlist]
    if args.revision:
      try_args.extend(['-r', args.revision])
    print ' '.join(try_args)
    proc = subprocess.Popen(try_args, stdout=subprocess.PIPE,
                            stderr=subprocess.STDOUT)
    if proc.wait() != 0:
      raise Exception('Failed to submit try request: %s' % (
          proc.communicate()[0]))
    print proc.communicate()[0]
  else:
    # First, find depot_tools. This is needed to import trychange.
    sys.path.append(FindDepotTools())
    import trychange
    try_args = ['--use_svn',
                '--svn_repo', GetTryRepo(),
                '--root', GetCheckoutRoot(is_svn),
                '--bot', botlist]
    if args.revision:
      try_args.extend(['-r', args.revision])
    trychange.TryChange(try_args, None, False)


def main():
  # Retrieve the list of active try builders from the build master.
  trybots = RetrieveTrybotList(json_filename='trybots')

  # Determine if we're in an SVN checkout.
  is_svn = os.path.isdir('.svn')

  # Parse and validate the command-line arguments.
  args = ValidateArgs(sys.argv[1:], trybots=trybots, is_svn=is_svn)

  # Submit the try request.
  SubmitTryRequest(args, is_svn=is_svn)


if __name__ == '__main__':
  sys.exit(main())