1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
|
#!/usr/bin/env python
# Copyright 2016 The LUCI Authors. All rights reserved.
# Use of this source code is governed under the Apache License, Version 2.0
# that can be found in the LICENSE file.
"""Bootstrap script to clone and forward to the recipe engine tool.
***********************************************************************
** DO NOT MODIFY EXCEPT IN THE PER-REPO CONFIGURATION SECTION BELOW. **
***********************************************************************
This is a copy of https://github.com/luci/recipes-py/blob/master/doc/recipes.py.
To fix bugs, fix in the github repo then copy it back to here and fix the
PER-REPO CONFIGURATION section to look like this one.
"""
import os
#### PER-REPO CONFIGURATION (editable) ####
# The root of the repository relative to the directory of this file.
REPO_ROOT = os.path.join(os.pardir, os.pardir)
# The path of the recipes.cfg file relative to the root of the repository.
RECIPES_CFG = os.path.join('infra', 'config', 'recipes.cfg')
#### END PER-REPO CONFIGURATION ####
BOOTSTRAP_VERSION = 1
import ast
import logging
import random
import re
import subprocess
import sys
import time
import traceback
def parse_protobuf(fh):
"""Parse the protobuf text format just well enough to understand recipes.cfg.
We don't use the protobuf library because we want to be as self-contained
as possible in this bootstrap, so it can be simply vendored into a client
repo.
We assume all fields are repeated since we don't have a proto spec to work
with.
Args:
fh: a filehandle containing the text format protobuf.
Returns:
A recursive dictionary of lists.
"""
def parse_atom(text):
if text == 'true':
return True
if text == 'false':
return False
return ast.literal_eval(text)
ret = {}
for line in fh:
line = line.strip()
m = re.match(r'(\w+)\s*:\s*(.*)', line)
if m:
ret.setdefault(m.group(1), []).append(parse_atom(m.group(2)))
continue
m = re.match(r'(\w+)\s*{', line)
if m:
subparse = parse_protobuf(fh)
ret.setdefault(m.group(1), []).append(subparse)
continue
if line == '}':
return ret
if line == '':
continue
raise ValueError('Could not understand line: <%s>' % line)
return ret
def get_unique(things):
if len(things) == 1:
return things[0]
elif len(things) == 0:
raise ValueError("Expected to get one thing, but dinna get none.")
else:
logging.warn('Expected to get one thing, but got a bunch: %s\n%s' %
(things, traceback.format_stack()))
return things[0]
def _subprocess_call(argv, **kwargs):
logging.info('Running %r', argv)
return subprocess.call(argv, **kwargs)
def _subprocess_check_call(argv, **kwargs):
logging.info('Running %r', argv)
subprocess.check_call(argv, **kwargs)
def main():
if '--verbose' in sys.argv:
logging.getLogger().setLevel(logging.INFO)
if sys.platform.startswith(('win', 'cygwin')):
git = 'git.bat'
else:
git = 'git'
# Find the repository and config file to operate on.
repo_root = os.path.abspath(
os.path.join(os.path.dirname(__file__), REPO_ROOT))
recipes_cfg_path = os.path.join(repo_root, RECIPES_CFG)
with open(recipes_cfg_path, 'rU') as fh:
protobuf = parse_protobuf(fh)
engine_buf = get_unique([
b for b in protobuf['deps'] if b.get('project_id') == ['recipe_engine'] ])
engine_url = get_unique(engine_buf['url'])
engine_revision = get_unique(engine_buf['revision'])
engine_subpath = (get_unique(engine_buf.get('path_override', ['']))
.replace('/', os.path.sep))
recipes_path = os.path.join(repo_root,
get_unique(protobuf['recipes_path']).replace('/', os.path.sep))
deps_path = os.path.join(recipes_path, '.recipe_deps')
engine_path = os.path.join(deps_path, 'recipe_engine')
# Ensure that we have the recipe engine cloned.
def ensure_engine():
if not os.path.exists(deps_path):
os.makedirs(deps_path)
if not os.path.exists(engine_path):
_subprocess_check_call([git, 'clone', engine_url, engine_path])
needs_fetch = _subprocess_call(
[git, 'rev-parse', '--verify', '%s^{commit}' % engine_revision],
cwd=engine_path, stdout=open(os.devnull, 'w'))
if needs_fetch:
_subprocess_check_call([git, 'fetch'], cwd=engine_path)
_subprocess_check_call(
[git, 'checkout', '--quiet', engine_revision], cwd=engine_path)
try:
ensure_engine()
except subprocess.CalledProcessError:
logging.exception('ensure_engine failed')
# Retry errors.
time.sleep(random.uniform(2,5))
ensure_engine()
args = ['--package', recipes_cfg_path,
'--bootstrap-script', __file__] + sys.argv[1:]
return _subprocess_call([
sys.executable, '-u',
os.path.join(engine_path, engine_subpath, 'recipes.py')] + args)
if __name__ == '__main__':
sys.exit(main())
|
