1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
|
#!/usr/bin/env python
#
# Copyright 2016 Google Inc.
#
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Utilities for managing assets."""
import argparse
import os
import shlex
import shutil
import subprocess
import sys
SKIA_DIR = os.path.abspath(os.path.realpath(os.path.join(
os.path.dirname(os.path.abspath(__file__)),
os.pardir, os.pardir, os.pardir)))
INFRA_BOTS_DIR = os.path.join(SKIA_DIR, 'infra', 'bots')
sys.path.insert(0, INFRA_BOTS_DIR)
import utils
import zip_utils
ASSETS_DIR = os.path.join(INFRA_BOTS_DIR, 'assets')
DEFAULT_GS_BUCKET = 'skia-buildbots'
GS_SUBDIR_TMPL = 'gs://%s/assets/%s'
GS_PATH_TMPL = '%s/%s.zip'
VERSION_FILENAME = 'VERSION'
ZIP_BLACKLIST = ['.git', '.svn', '*.pyc', '.DS_STORE']
class _GSWrapper(object):
"""Wrapper object for interacting with Google Storage."""
def __init__(self, gsutil):
gsutil = os.path.abspath(gsutil) if gsutil else 'gsutil'
self._gsutil = [gsutil]
if gsutil.endswith('.py'):
self._gsutil = ['python', gsutil]
def copy(self, src, dst):
"""Copy src to dst."""
subprocess.check_call(self._gsutil + ['cp', src, dst])
def list(self, path):
"""List objects in the given path."""
try:
return subprocess.check_output(self._gsutil + ['ls', path]).splitlines()
except subprocess.CalledProcessError:
# If the prefix does not exist, we'll get an error, which is okay.
return []
def _prompt(prompt):
"""Prompt for input, return result."""
return raw_input(prompt)
class Asset(object):
def __init__(self, name, gs_bucket=DEFAULT_GS_BUCKET, gsutil=None):
self._gs = _GSWrapper(gsutil)
self._gs_subdir = GS_SUBDIR_TMPL % (gs_bucket, name)
self._name = name
self._dir = os.path.join(ASSETS_DIR, self._name)
@property
def version_file(self):
"""Return the path to the version file for this asset."""
return os.path.join(self._dir, VERSION_FILENAME)
def get_current_version(self):
"""Obtain the current version of the asset."""
if not os.path.isfile(self.version_file):
return -1
with open(self.version_file) as f:
return int(f.read())
def get_available_versions(self):
"""Return the existing version numbers for this asset."""
files = self._gs.list(self._gs_subdir)
bnames = [os.path.basename(f) for f in files]
suffix = '.zip'
versions = [int(f[:-len(suffix)]) for f in bnames if f.endswith(suffix)]
versions.sort()
return versions
def get_next_version(self):
"""Find the next available version number for the asset."""
versions = self.get_available_versions()
if len(versions) == 0:
return 0
return versions[-1] + 1
def download_version(self, version, target_dir):
"""Download the specified version of the asset."""
gs_path = GS_PATH_TMPL % (self._gs_subdir, str(version))
target_dir = os.path.abspath(target_dir)
with utils.tmp_dir():
zip_file = os.path.join(os.getcwd(), '%d.zip' % version)
self._gs.copy(gs_path, zip_file)
zip_utils.unzip(zip_file, target_dir)
def download_current_version(self, target_dir):
"""Download the version of the asset specified in its version file."""
v = self.get_current_version()
self.download_version(v, target_dir)
def upload_new_version(self, target_dir, commit=False):
"""Upload a new version and update the version file for the asset."""
version = self.get_next_version()
target_dir = os.path.abspath(target_dir)
with utils.tmp_dir():
zip_file = os.path.join(os.getcwd(), '%d.zip' % version)
zip_utils.zip(target_dir, zip_file, blacklist=ZIP_BLACKLIST)
gs_path = GS_PATH_TMPL % (self._gs_subdir, str(version))
self._gs.copy(zip_file, gs_path)
def _write_version():
with open(self.version_file, 'w') as f:
f.write(str(version))
subprocess.check_call([utils.GIT, 'add', self.version_file])
with utils.chdir(SKIA_DIR):
if commit:
with utils.git_branch():
_write_version()
subprocess.check_call([
utils.GIT, 'commit', '-m', 'Update %s version' % self._name])
subprocess.check_call([utils.GIT, 'cl', 'upload', '--bypass-hooks'])
else:
_write_version()
@classmethod
def add(cls, name, gs_bucket=DEFAULT_GS_BUCKET, gsutil=None):
"""Add an asset."""
asset = cls(name, gs_bucket=gs_bucket, gsutil=gsutil)
if os.path.isdir(asset._dir):
raise Exception('Asset %s already exists!' % asset._name)
print 'Creating asset in %s' % asset._dir
os.mkdir(asset._dir)
def copy_script(script):
src = os.path.join(ASSETS_DIR, 'scripts', script)
dst = os.path.join(asset._dir, script)
print 'Creating %s' % dst
shutil.copy(src, dst)
subprocess.check_call([utils.GIT, 'add', dst])
for script in ('download.py', 'upload.py', 'common.py'):
copy_script(script)
resp = _prompt('Add script to automate creation of this asset? (y/n) ')
if resp == 'y':
copy_script('create.py')
copy_script('create_and_upload.py')
print 'You will need to add implementation to the creation script.'
print 'Successfully created asset %s.' % asset._name
return asset
def remove(self):
"""Remove this asset."""
# Ensure that the asset exists.
if not os.path.isdir(self._dir):
raise Exception('Asset %s does not exist!' % self._name)
# Remove the asset.
subprocess.check_call([utils.GIT, 'rm', '-rf', self._dir])
if os.path.isdir(self._dir):
shutil.rmtree(self._dir)
# We *could* remove all uploaded versions of the asset in Google Storage but
# we choose not to be that destructive.
|
