diff options
Diffstat (limited to 'scripts/ci')
-rwxr-xr-x | scripts/ci/build.sh | 265 | ||||
-rwxr-xr-x | scripts/ci/windows/compile_windows.sh | 57 |
2 files changed, 131 insertions, 191 deletions
diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh index ffcd0ed3b3..a2e45130c7 100755 --- a/scripts/ci/build.sh +++ b/scripts/ci/build.sh @@ -24,22 +24,47 @@ set -eu # Also prepare an email for announcing the release. # Load common.sh -SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -source $(dirname ${SCRIPT_DIR})/release/common.sh - -: ${GIT_REPOSITORY_URL:=https://github.com/bazelbuild/bazel} - -: ${GCS_BASE_URL:=https://storage.googleapis.com} -: ${GCS_BUCKET:=bucket-o-bazel} -: ${GCS_APT_BUCKET:=bazel-apt} - -: ${EMAIL_TEMPLATE_RC:=${SCRIPT_DIR}/rc_email.txt} -: ${EMAIL_TEMPLATE_RELEASE:=${SCRIPT_DIR}/release_email.txt} - -: ${RELEASE_CANDIDATE_URL:="https://releases.bazel.build/%release_name%/rc%rc%/index.html"} -: ${RELEASE_URL="${GIT_REPOSITORY_URL}/releases/tag/%release_name%"} - -: ${BOOTSTRAP_BAZEL:=bazel} +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$(dirname ${SCRIPT_DIR})/release/common.sh" + +if ! command -v gsutil &>/dev/null; then + echo "Required tool 'gsutil' not found. Please install it:" + echo "See https://cloud.google.com/sdk/downloads for instructions." + exit 1 +fi +if ! command -v github-release &>/dev/null; then + echo "Required tool 'github-release' not found. Download it from here:" + echo "https://github.com/c4milo/github-release/releases" + echo "Just extract the archive and put the binary on your PATH." + exit 1 +fi +if ! command -v debsign &>/dev/null; then + echo "Required tool 'debsign' not found. Please install it via apt-get:" + echo "apt-get install devscripts" + exit 1 +fi +if ! command -v reprepro &>/dev/null; then + echo "Required tool 'reprepro' not found. Please install it via apt-get:" + echo "apt-get install reprepro" + exit 1 +fi +if ! command -v gpg &>/dev/null; then + echo "Required tool 'gpg' not found. Please install it via apt-get:" + echo "apt-get install gnupg" + exit 1 +fi +if ! command -v pandoc &>/dev/null; then + echo "Required tool 'pandoc' not found. Please install it via apt-get:" + echo "apt-get install pandoc" + exit 1 +fi +# if ! command -v ssmtp &>/dev/null; then +# echo "Required tool 'ssmtp' not found. Please install it via apt-get:" +# echo "apt-get install ssmtp" +# exit 1 +# fi + +export APT_GPG_KEY_ID=$(gsutil cat gs://bazel-encrypted-secrets/release-key.gpg.id) # Generate a string from a template and a list of substitutions. # The first parameter is the template name and each subsequent parameter @@ -60,6 +85,9 @@ function generate_from_template() { # the mail subjects and the subsequent lines the mail, its content. # If no planed release, then this function output will be empty. function generate_email() { + RELEASE_CANDIDATE_URL="https://releases.bazel.build/%release_name%/rc%rc%/index.html" + RELEASE_URL="https://github.com/bazelbuild/bazel/releases/tag/%release_name%" + local release_name=$(get_release_name) local rc=$(get_release_candidate) local args=( @@ -69,16 +97,17 @@ function generate_email() { ) if [ -n "${rc}" ]; then args+=( - "%url%" - "$(generate_from_template "${RELEASE_CANDIDATE_URL}" "${args[@]}")" + "%url%" "$(generate_from_template "${RELEASE_CANDIDATE_URL}" "${args[@]}")" ) - generate_from_template "$(cat ${EMAIL_TEMPLATE_RC})" "${args[@]}" + generate_from_template \ + "$(cat "${SCRIPT_DIR}/rc_email.txt")" \ + "${args[@]}" elif [ -n "${release_name}" ]; then args+=( - "%url%" - "$(generate_from_template "${RELEASE_URL}" "${args[@]}")" + "%url%" "$(generate_from_template "${RELEASE_URL}" "${args[@]}")" ) - generate_from_template "$(cat ${EMAIL_TEMPLATE_RELEASE})" "${args[@]}" + generate_from_template \ + "$(cat "${SCRIPT_DIR}/release_email.txt")" "${args[@]}" fi } @@ -105,40 +134,25 @@ _Security_: All our binaries are signed with our # https://github.com/c4milo/github-release # This methods expects the following arguments: # $1..$n files generated by package_build (should not contains the README file) -# Please set GITHUB_TOKEN to talk to the Github API and GITHUB_RELEASE -# for the path to the https://github.com/c4milo/github-release tool. -# This method is also affected by GIT_REPOSITORY_URL which should be the -# URL to the github repository (defaulted to https://github.com/bazelbuild/bazel). +# Please set GITHUB_TOKEN to talk to the Github API. function release_to_github() { - local url="${GIT_REPOSITORY_URL}" + local artifact_dir="$1" + local release_name=$(get_release_name) local rc=$(get_release_candidate) - local release_tool="${GITHUB_RELEASE:-$(which github-release 2>/dev/null || echo release-tool-not-found)}" - if [ "${release_tool}" = "release-tool-not-found" ]; then - echo "Please set GITHUB_RELEASE to the path to the github-release binary." >&2 - echo "This probably means you haven't installed https://github.com/c4milo/github-release " >&2 - echo "on this machine." >&2 - return 1 - fi - local github_repo="$(echo "$url" | sed -E 's|https?://github.com/([^/]*/[^/]*).*$|\1|')" if [ -n "${release_name}" ] && [ -z "${rc}" ]; then - mkdir -p "${tmpdir}/to-github" - cp "${@}" "${tmpdir}/to-github" - "${release_tool}" "${github_repo}" "${release_name}" "" "$(get_release_page)" "${tmpdir}/to-github/"'*' + local github_token="$(gsutil cat gs://bazel-encrypted-secrets/github-token.enc | \ + gcloud kms decrypt --location global --keyring buildkite --key github-token --ciphertext-file - --plaintext-file -)" + + GITHUB_TOKEN=${github_token} github-release "bazelbuild/bazel" "${release_name}" "" "$(get_release_page)" "${artifact_dir}/*" fi } -# Creates an index of the files contained in folder $1 in mardown format +# Creates an index of the files contained in folder $1 in Markdown format. function create_index_md() { # First, add the release notes get_release_page - # Build log - if [ -f $1/build.log ]; then - echo - echo " [Build log](build.log)" - echo - fi # Then, add the list of files echo echo "## Index of files" @@ -149,74 +163,49 @@ function create_index_md() { done } -# Creates an index of the files contained in folder $1 in HTML format -# It supposes hoedown (https://github.com/hoedown/hoedown) is on the path, -# if not, set the HOEDOWN environment variable to the good path. +# Creates an index of the files contained in folder $1 in HTML format. function create_index_html() { - local hoedown="${HOEDOWN:-$(which hoedown 2>/dev/null || true)}" - # Second line is to trick hoedown to behave as Github - create_index_md "${@}" \ - | sed -E 's/^(Baseline.*)$/\1\ -/' | sed 's/^ + / - /' | sed 's/_/\\_/g' \ - | "${hoedown}" -} - -function get_gsutil() { - local gs="${GSUTIL:-$(which gsutil 2>/dev/null || true) -m}" - if [ ! -x "${gs}" ]; then - echo "Please set GSUTIL to the path the gsutil binary." >&2 - echo "gsutil (https://cloud.google.com/storage/docs/gsutil/) is the" >&2 - echo "command-line interface to google cloud." >&2 - exit 1 - fi - echo "${gs}" + create_index_md "${@}" | pandoc -f markdown -t html } # Deploy a release candidate to Google Cloud Storage. # It requires to have gsutil installed. You can force the path to gsutil -# by setting the GSUTIL environment variable. The GCS_BUCKET should be the -# name of the Google cloud bucket to deploy to. +# by setting the GSUTIL environment variable. # This methods expects the following arguments: # $1..$n files generated by package_build function release_to_gcs() { - local gs="$(get_gsutil)" + local artifact_dir="$1" + local release_name="$(get_release_name)" local rc="$(get_release_candidate)" - if [ -z "${GCS_BUCKET-}" ]; then - echo "Please set GCS_BUCKET to the name of your Google Cloud Storage bucket." >&2 - return 1 - fi + if [ -n "${release_name}" ]; then local release_path="${release_name}/release" if [ -n "${rc}" ]; then release_path="${release_name}/rc${rc}" fi - # Make a temporary folder with the desired structure - local dir="$(mktemp -d ${TMPDIR:-/tmp}/tmp.XXXXXXXX)" - local prev_dir="$PWD" - trap "{ cd ${prev_dir}; rm -fr ${dir}; }" EXIT - mkdir -p "${dir}/${release_path}" - cp "${@}" "${dir}/${release_path}" - # Add a index.html file: - create_index_html "${dir}/${release_path}" \ - >"${dir}/${release_path}"/index.html - cd ${dir} - "${gs}" -m cp -a public-read -r . "gs://${GCS_BUCKET}" - cd "${prev_dir}" - rm -fr "${dir}" - trap - EXIT + create_index_html "${artifact_dir}" > "${artifact_dir}/index.html" + gsutil -m cp -a public-read "${artifact_dir}/**" "gs://bazel/${release_path}" fi } function ensure_gpg_secret_key_imported() { - (gpg --list-secret-keys | grep "${APT_GPG_KEY_ID}" > /dev/null) || \ - gpg --allow-secret-key-import --import "${APT_GPG_KEY_PATH}" + if ! gpg --list-secret-keys | grep "${APT_GPG_KEY_ID}" > /dev/null; then + keyfile=$(mktemp --tmpdir) + chmod 0600 "${keyfile}" + gsutil cat "gs://bazel-encrypted-secrets/release-key.gpg.enc" | \ + gcloud kms decrypt --location "global" --keyring "buildkite" --key "bazel-release-key" --plaintext-file "-" --ciphertext-file "${keyfile}" + gpg --allow-secret-key-import --import "${keyfile}" + rm -f "${keyfile}" + fi + # Make sure we use stronger digest algorithm。 # We use reprepro to generate the debian repository, # but there's no way to pass flags to gpg using reprepro, so writting it into # ~/.gnupg/gpg.conf - (grep "digest-algo sha256" ~/.gnupg/gpg.conf > /dev/null) || \ - echo "digest-algo sha256" >> ~/.gnupg/gpg.conf + if ! grep "digest-algo sha256" ~/.gnupg/gpg.conf > /dev/null; then + echo "digest-algo sha256" >> ~/.gnupg/gpg.conf + fi } function create_apt_repository() { @@ -262,71 +251,79 @@ EOF ensure_gpg_secret_key_imported local distribution="$1" - local deb_pkg_name_jdk8="$2" + local deb_pkg_name="$2" local deb_dsc_name="$3" - debsign -k ${APT_GPG_KEY_ID} "${deb_dsc_name}" + debsign -k "${APT_GPG_KEY_ID}" "${deb_dsc_name}" - reprepro -C jdk1.8 includedeb "${distribution}" "${deb_pkg_name_jdk8}" + reprepro -C jdk1.8 includedeb "${distribution}" "${deb_pkg_name}" reprepro -C jdk1.8 includedsc "${distribution}" "${deb_dsc_name}" - "${gs}" -m cp -a public-read -r dists "gs://${GCS_APT_BUCKET}/" - "${gs}" -m cp -a public-read -r pool "gs://${GCS_APT_BUCKET}/" + gsutil -m cp -a public-read -r dists pool "gs://bazel-apt" } function release_to_apt() { - local gs="$(get_gsutil)" + local artifact_dir="$1" + local release_name="$(get_release_name)" local rc="$(get_release_candidate)" - if [ -z "${GCS_APT_BUCKET-}" ]; then - echo "Please set GCS_APT_BUCKET to the name of your GCS bucket for apt repository." >&2 - return 1 - fi - if [ -z "${APT_GPG_KEY_ID-}" ]; then - echo "Please set APT_GPG_KEY_ID for apt repository." >&2 - return 1 - fi + if [ -n "${release_name}" ]; then - # Make a temporary folder with the desired structure - local dir="$(mktemp -d ${TMPDIR:-/tmp}/tmp.XXXXXXXX)" - local prev_dir="$PWD" - trap "{ cd ${prev_dir}; rm -fr ${dir}; }" EXIT - mkdir -p "${dir}/${release_name}" local release_label="$(get_full_release_name)" - local deb_pkg_name_jdk8="${release_name}/bazel_${release_label}-linux-x86_64.deb" + local deb_pkg_name="${release_name}/bazel_${release_label}-linux-x86_64.deb" local deb_dsc_name="${release_name}/bazel_${release_label}.dsc" local deb_tar_name="${release_name}/bazel_${release_label}.tar.gz" - cp "${tmpdir}/bazel_${release_label}-linux-x86_64.deb" "${dir}/${deb_pkg_name_jdk8}" - cp "${tmpdir}/bazel.dsc" "${dir}/${deb_dsc_name}" - cp "${tmpdir}/bazel.tar.gz" "${dir}/${deb_tar_name}" - cd "${dir}" + + pushd "${artifact_dir}" if [ -n "${rc}" ]; then - create_apt_repository testing "${deb_pkg_name_jdk8}" "${deb_dsc_name}" + create_apt_repository testing "${deb_pkg_name}" "${deb_dsc_name}" else - create_apt_repository stable "${deb_pkg_name_jdk8}" "${deb_dsc_name}" + create_apt_repository stable "${deb_pkg_name}" "${deb_dsc_name}" fi - cd "${prev_dir}" - rm -fr "${dir}" - trap - EXIT + popd fi } # A wrapper around the release deployment methods. function deploy_release() { - local github_args=() - for i in "$@"; do - if ! ( [[ "$i" =~ build.log ]] || [[ "$i" =~ bazel.dsc ]] || [[ "$i" =~ bazel.tar.gz ]] || [[ "$i" =~ .nobuild$ ]] ) ; then - github_args+=("$i") - fi - done - local gcs_args=() - # Filters out perf.bazel.*.nobuild - for i in "$@"; do - if ! [[ "$i" =~ .nobuild$ ]] ; then - gcs_args+=("$i") - fi + local release_label="$(get_full_release_name)" + local release_name="$(get_release_name)" + + if [[ ! -d $1 ]]; then + echo "Usage: deploy_release ARTIFACT_DIR" + exit 1 + fi + artifact_dir="$1" + + if [[ -z $release_name ]]; then + echo "Could not get the release name - are you in a release branch directory?" + exit 1 + fi + + ensure_gpg_secret_key_imported + + rm -f "${artifact_dir}"/*.{sha256,sig} + for file in "${artifact_dir}"/*; do + (cd "${artifact_dir}" && sha256sum "$(basename "${file}")" > "${file}.sha256") + gpg --no-tty --detach-sign -u "${APT_GPG_KEY_ID}" "${file}" done - release_to_github "${github_args[@]}" - release_to_gcs "${gcs_args[@]}" - release_to_apt + + apt_working_dir="$(mktemp -d --tmpdir)" + echo "apt_working_dir = ${apt_working_dir}" + mkdir "${apt_working_dir}/${release_name}" + cp "${artifact_dir}/bazel_${release_label}-linux-x86_64.deb" "${apt_working_dir}/${release_name}" + cp "${artifact_dir}/bazel_${release_label}.dsc" "${apt_working_dir}/${release_name}" + cp "${artifact_dir}/bazel_${release_label}.tar.gz" "${apt_working_dir}/${release_name}" + release_to_apt "${apt_working_dir}" + + gcs_working_dir="$(mktemp -d --tmpdir)" + echo "gcs_working_dir = ${gcs_working_dir}" + cp "${artifact_dir}"/* "${gcs_working_dir}" + release_to_gcs "${gcs_working_dir}" + + github_working_dir="$(mktemp -d --tmpdir)" + echo "github_working_dir = ${github_working_dir}" + cp "${artifact_dir}"/* "${github_working_dir}" + rm -f "${github_working_dir}/bazel_${release_label}"*.{deb,dsc,tar.gz} + release_to_github "${github_working_dir}" } diff --git a/scripts/ci/windows/compile_windows.sh b/scripts/ci/windows/compile_windows.sh deleted file mode 100755 index ca2817a695..0000000000 --- a/scripts/ci/windows/compile_windows.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -# Copyright 2015 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Ideally we would call directly script/ci/build.sh just like we do -# for the linux script but we are not there yet. - -# Ensure we are in the root directory -cd $(dirname $0)/../../.. - -# Even though there are no quotes around $* in the .bat file, arguments -# containing spaces seem to be passed properly. -source ./scripts/ci/build.sh - -# Bazel still needs to know where bash is, take it from cygpath. -export BAZEL_SH="$(cygpath --windows /bin/bash)" -# Make sure JAVA_HOME is in Windows path style. -export JAVA_HOME="$(cygpath --windows "${JAVA_HOME}")" - -# TODO(bazel-team): we should replace ./compile.sh by the same script we use -# for other platform -release_label="$(get_full_release_name)" - -if [ -n "${release_label}" ]; then - export EMBED_LABEL="${release_label}" -fi - -export MSYS_NO_PATHCONV=1 -export MSYS2_ARG_CONV_EXCL="*" - -echo "BOOTSTRAP_BAZEL version:" -${BOOTSTRAP_BAZEL} --bazelrc=${BAZELRC:-/dev/null} --nomaster_bazelrc version - -${BOOTSTRAP_BAZEL} --bazelrc=${BAZELRC:-/dev/null} --nomaster_bazelrc build \ - --embed_label=${release_label} --stamp \ - //src:bazel //src:bazel_with_jdk - -# Copy the resulting artifacts. -mkdir -p output/ci -cp bazel-bin/src/bazel output/ci/bazel-$(get_full_release_name)-without-jdk.exe -cp bazel-bin/src/bazel_with_jdk output/ci/bazel-$(get_full_release_name).exe -cp bazel-bin/src/bazel output/bazel.exe -zip -j output/ci/bazel-$(get_full_release_name)-without-jdk.zip output/bazel.exe -cp -f bazel-bin/src/bazel_with_jdk output/bazel.exe -zip -j output/ci/bazel-$(get_full_release_name).zip output/bazel.exe |