1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
|
#!/bin/bash
# Copyright 2015 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -eu
# Main deploy functions for the continous build system
# Just source this file and use the various method:
# bazel_build build bazel and run all its test
# bazel_release use the artifact generated by bazel_build and push
# them to github for a release and to GCS for a release candidate.
# Also prepare an email for announcing the release.
# Load common.sh
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
source $(dirname ${SCRIPT_DIR})/release/common.sh
: ${GIT_REPOSITORY_URL:=https://github.com/bazelbuild/bazel}
: ${GCS_BASE_URL:=https://storage.googleapis.com}
: ${GCS_BUCKET:=bucket-o-bazel}
: ${GCS_APT_BUCKET:=bazel-apt}
: ${EMAIL_TEMPLATE_RC:=${SCRIPT_DIR}/rc_email.txt}
: ${EMAIL_TEMPLATE_RELEASE:=${SCRIPT_DIR}/release_email.txt}
: ${RELEASE_CANDIDATE_URL:="${GCS_BASE_URL}/${GCS_BUCKET}/%release_name%/rc%rc%/index.html"}
: ${RELEASE_URL="${GIT_REPOSITORY_URL}/releases/tag/%release_name%"}
: ${BOOTSTRAP_BAZEL:=bazel}
# Generate a string from a template and a list of substitutions.
# The first parameter is the template name and each subsequent parameter
# is taken as a couple: first is the string the substitute and the second
# is the result of the substitution.
function generate_from_template() {
local value="$1"
shift
while (( $# >= 2 )); do
value="${value//$1/$2}"
shift 2
done
echo "${value}"
}
# Generate the email for the release.
# The first line of the output will be the recipient, the second line
# the mail subjects and the subsequent lines the mail, its content.
# If no planed release, then this function output will be empty.
function generate_email() {
local release_name=$(get_release_name)
local rc=$(get_release_candidate)
local args=(
"%release_name%" "${release_name}"
"%rc%" "${rc}"
"%relnotes%" "# $(get_full_release_notes)"
)
if [ -n "${rc}" ]; then
args+=(
"%url%"
"$(generate_from_template "${RELEASE_CANDIDATE_URL}" "${args[@]}")"
)
generate_from_template "$(cat ${EMAIL_TEMPLATE_RC})" "${args[@]}"
elif [ -n "${release_name}" ]; then
args+=(
"%url%"
"$(generate_from_template "${RELEASE_URL}" "${args[@]}")"
)
generate_from_template "$(cat ${EMAIL_TEMPLATE_RELEASE})" "${args[@]}"
fi
}
function get_release_page() {
echo "# $(get_full_release_notes)"'
_Notice_: Bazel installers contain binaries licensed under the GPLv2 with
Classpath exception. Those installers should always be redistributed along with
the source code.
Some versions of Bazel contain a bundled version of OpenJDK. The license of the
bundled OpenJDK and other open-source components can be displayed by running
the command `bazel license`. The vendor and version information of the bundled
OpenJDK can be displayed by running the command `bazel info java-runtime`.
The binaries and source-code of the bundled OpenJDK can be
[downloaded from our mirror server](https://mirror.bazel.build/openjdk/index.html).
_Security_: All our binaries are signed with our
[public key](https://bazel.build/bazel-release.pub.gpg) 48457EE0.
'
}
# Deploy a github release using a third party tool:
# https://github.com/c4milo/github-release
# This methods expects the following arguments:
# $1..$n files generated by package_build (should not contains the README file)
# Please set GITHUB_TOKEN to talk to the Github API and GITHUB_RELEASE
# for the path to the https://github.com/c4milo/github-release tool.
# This method is also affected by GIT_REPOSITORY_URL which should be the
# URL to the github repository (defaulted to https://github.com/bazelbuild/bazel).
function release_to_github() {
local url="${GIT_REPOSITORY_URL}"
local release_name=$(get_release_name)
local rc=$(get_release_candidate)
local release_tool="${GITHUB_RELEASE:-$(which github-release 2>/dev/null || echo release-tool-not-found)}"
if [ "${release_tool}" = "release-tool-not-found" ]; then
echo "Please set GITHUB_RELEASE to the path to the github-release binary." >&2
echo "This probably means you haven't installed https://github.com/c4milo/github-release " >&2
echo "on this machine." >&2
return 1
fi
local github_repo="$(echo "$url" | sed -E 's|https?://github.com/([^/]*/[^/]*).*$|\1|')"
if [ -n "${release_name}" ] && [ -z "${rc}" ]; then
mkdir -p "${tmpdir}/to-github"
cp "${@}" "${tmpdir}/to-github"
"${release_tool}" "${github_repo}" "${release_name}" "" "$(get_release_page)" "${tmpdir}/to-github/"'*'
fi
}
# Creates an index of the files contained in folder $1 in mardown format
function create_index_md() {
# First, add the release notes
get_release_page
# Build log
if [ -f $1/build.log ]; then
echo
echo " [Build log](build.log)"
echo
fi
# Then, add the list of files
echo
echo "## Index of files"
echo
for f in $1/*.sha256; do # just list the sha256 ones
local filename=$(basename $f .sha256);
echo " - [${filename}](${filename}) [[SHA-256](${filename}.sha256)] [[SIG](${filename}.sig)]"
done
}
# Creates an index of the files contained in folder $1 in HTML format
# It supposes hoedown (https://github.com/hoedown/hoedown) is on the path,
# if not, set the HOEDOWN environment variable to the good path.
function create_index_html() {
local hoedown="${HOEDOWN:-$(which hoedown 2>/dev/null || true)}"
# Second line is to trick hoedown to behave as Github
create_index_md "${@}" \
| sed -E 's/^(Baseline.*)$/\1\
/' | sed 's/^ + / - /' | sed 's/_/\\_/g' \
| "${hoedown}"
}
function get_gsutil() {
local gs="${GSUTIL:-$(which gsutil 2>/dev/null || true) -m}"
if [ ! -x "${gs}" ]; then
echo "Please set GSUTIL to the path the gsutil binary." >&2
echo "gsutil (https://cloud.google.com/storage/docs/gsutil/) is the" >&2
echo "command-line interface to google cloud." >&2
exit 1
fi
echo "${gs}"
}
# Deploy a release candidate to Google Cloud Storage.
# It requires to have gsutil installed. You can force the path to gsutil
# by setting the GSUTIL environment variable. The GCS_BUCKET should be the
# name of the Google cloud bucket to deploy to.
# This methods expects the following arguments:
# $1..$n files generated by package_build
function release_to_gcs() {
local gs="$(get_gsutil)"
local release_name="$(get_release_name)"
local rc="$(get_release_candidate)"
if [ -z "${GCS_BUCKET-}" ]; then
echo "Please set GCS_BUCKET to the name of your Google Cloud Storage bucket." >&2
return 1
fi
if [ -n "${release_name}" ]; then
local release_path="${release_name}/release"
if [ -n "${rc}" ]; then
release_path="${release_name}/rc${rc}"
fi
# Make a temporary folder with the desired structure
local dir="$(mktemp -d ${TMPDIR:-/tmp}/tmp.XXXXXXXX)"
local prev_dir="$PWD"
trap "{ cd ${prev_dir}; rm -fr ${dir}; }" EXIT
mkdir -p "${dir}/${release_path}"
cp "${@}" "${dir}/${release_path}"
# Add a index.html file:
create_index_html "${dir}/${release_path}" \
>"${dir}/${release_path}"/index.html
cd ${dir}
"${gs}" -m cp -a public-read -r . "gs://${GCS_BUCKET}"
cd "${prev_dir}"
rm -fr "${dir}"
trap - EXIT
fi
}
function ensure_gpg_secret_key_imported() {
(gpg --list-secret-keys | grep "${APT_GPG_KEY_ID}" > /dev/null) || \
gpg --allow-secret-key-import --import "${APT_GPG_KEY_PATH}"
# Make sure we use stronger digest algorithm。
# We use reprepro to generate the debian repository,
# but there's no way to pass flags to gpg using reprepro, so writting it into
# ~/.gnupg/gpg.conf
(grep "digest-algo sha256" ~/.gnupg/gpg.conf > /dev/null) || \
echo "digest-algo sha256" >> ~/.gnupg/gpg.conf
}
function create_apt_repository() {
mkdir conf
cat > conf/distributions <<EOF
Origin: Bazel Authors
Label: Bazel
Codename: stable
Architectures: amd64 source
Components: jdk1.8
Description: Bazel APT Repository
DebOverride: override.stable
DscOverride: override.stable
SignWith: ${APT_GPG_KEY_ID}
Origin: Bazel Authors
Label: Bazel
Codename: testing
Architectures: amd64 source
Components: jdk1.8
Description: Bazel APT Repository
DebOverride: override.testing
DscOverride: override.testing
SignWith: ${APT_GPG_KEY_ID}
EOF
cat > conf/options <<EOF
verbose
ask-passphrase
basedir .
EOF
# TODO(#2264): this is a quick workaround #2256, figure out a correct fix.
cat > conf/override.stable <<EOF
bazel Section contrib/devel
bazel Priority optional
EOF
cat > conf/override.testing <<EOF
bazel Section contrib/devel
bazel Priority optional
EOF
ensure_gpg_secret_key_imported
local distribution="$1"
local deb_pkg_name_jdk8="$2"
local deb_dsc_name="$3"
debsign -k ${APT_GPG_KEY_ID} "${deb_dsc_name}"
reprepro -C jdk1.8 includedeb "${distribution}" "${deb_pkg_name_jdk8}"
reprepro -C jdk1.8 includedsc "${distribution}" "${deb_dsc_name}"
"${gs}" -m cp -a public-read -r dists "gs://${GCS_APT_BUCKET}/"
"${gs}" -m cp -a public-read -r pool "gs://${GCS_APT_BUCKET}/"
}
function release_to_apt() {
local gs="$(get_gsutil)"
local release_name="$(get_release_name)"
local rc="$(get_release_candidate)"
if [ -z "${GCS_APT_BUCKET-}" ]; then
echo "Please set GCS_APT_BUCKET to the name of your GCS bucket for apt repository." >&2
return 1
fi
if [ -z "${APT_GPG_KEY_ID-}" ]; then
echo "Please set APT_GPG_KEY_ID for apt repository." >&2
return 1
fi
if [ -n "${release_name}" ]; then
# Make a temporary folder with the desired structure
local dir="$(mktemp -d ${TMPDIR:-/tmp}/tmp.XXXXXXXX)"
local prev_dir="$PWD"
trap "{ cd ${prev_dir}; rm -fr ${dir}; }" EXIT
mkdir -p "${dir}/${release_name}"
local release_label="$(get_full_release_name)"
local deb_pkg_name_jdk8="${release_name}/bazel_${release_label}-linux-x86_64.deb"
local deb_dsc_name="${release_name}/bazel_${release_label}.dsc"
local deb_tar_name="${release_name}/bazel_${release_label}.tar.gz"
cp "${tmpdir}/bazel_${release_label}-linux-x86_64.deb" "${dir}/${deb_pkg_name_jdk8}"
cp "${tmpdir}/bazel.dsc" "${dir}/${deb_dsc_name}"
cp "${tmpdir}/bazel.tar.gz" "${dir}/${deb_tar_name}"
cd "${dir}"
if [ -n "${rc}" ]; then
create_apt_repository testing "${deb_pkg_name_jdk8}" "${deb_dsc_name}"
else
create_apt_repository stable "${deb_pkg_name_jdk8}" "${deb_dsc_name}"
fi
cd "${prev_dir}"
rm -fr "${dir}"
trap - EXIT
fi
}
# A wrapper around the release deployment methods.
function deploy_release() {
local github_args=()
for i in "$@"; do
if ! ( [[ "$i" =~ build.log ]] || [[ "$i" =~ bazel.dsc ]] || [[ "$i" =~ bazel.tar.gz ]] || [[ "$i" =~ .nobuild$ ]] ) ; then
github_args+=("$i")
fi
done
local gcs_args=()
# Filters out perf.bazel.*.nobuild
for i in "$@"; do
if ! [[ "$i" =~ .nobuild$ ]] ; then
gcs_args+=("$i")
fi
done
release_to_github "${github_args[@]}"
release_to_gcs "${gcs_args[@]}"
release_to_apt
}
# Use jekyll build to build the site and then gsutil to copy it to GCS
# Input: $1 tarball to the jekyll site
# $2 name of the bucket to deploy the site to
# $3 "nobuild" if only publish without build
# It requires to have gsutil installed. You can force the path to gsutil
# by setting the GSUTIL environment variable
function build_and_publish_site() {
tmpdir=$(mktemp -d ${TMPDIR:-/tmp}/tmp.XXXXXXXX)
trap 'rm -fr ${tmpdir}' EXIT
local gs="$(get_gsutil)"
local site="$1"
local bucket="$2"
local nobuild="$3"
if [ ! -f "${site}" ] || [ -z "${bucket}" ]; then
echo "Usage: build_and_publish_site <site-tarball> <bucket>" >&2
return 1
fi
local prod_dir="${tmpdir}"
tar xf "${site}" --exclude=CNAME -C "${tmpdir}"
if [ "$nobuild" != "nobuild" ]; then
jekyll build -s "${tmpdir}" -d "${tmpdir}/production"
prod_dir="${tmpdir}/production"
fi
# Rsync:
# -r: recursive
# -c: compute checksum even though the input is from the filesystem
"${gs}" rsync -r -c "${prod_dir}" "gs://${bucket}"
"${gs}" web set -m index.html -e 404.html "gs://${bucket}"
"${gs}" -m acl ch -R -u AllUsers:R "gs://${bucket}"
}
# Push json file to perf site, also add to file_list
# Input: $1 json file to push
# $2 name of the bucket to deploy the site to
function push_benchmark_output_to_site() {
tmpdir=$(mktemp -d ${TMPDIR:-/tmp}/tmp.XXXXXXXX)
trap 'rm -fr ${tmpdir}' EXIT
local gs="$(get_gsutil)"
local output_file="$1"
local output_file_basename="$(basename ${output_file})"
local bucket="$2"
if [ ! -f "${output_file}" ] || [ -z "${bucket}" ]; then
echo "Usage: push_benchmark_output_to_site <json-file-name> <bucket>" >&2
return 1
fi
# Upload json file
"${gs}" cp "${output_file}" "gs://${bucket}/data/${output_file_basename}"
# Download file_list (it might not exist)
"${gs}" cp "gs://${bucket}/file_list" "${tmpdir}" || true
# Update file_list
local list_file="${tmpdir}/file_list"
echo "${output_file_basename}" >> "${list_file}"
"${gs}" cp "${list_file}" "gs://${bucket}/file_list"
"${gs}" -m acl ch -R -u AllUsers:R "gs://${bucket}"
}
|