CI scripts for the release process

It provides method to build, package and deploy artifacts, as
well as constructing announcement mails:
   - bazel_build build bazel with the good label, execute the tests,
     build the packages and copy the artifacts into a release directory.
   - bazel_release deploy the relase to GCS and Github and send an
     announcement mail.

--
Change-Id: I68f5a600c0718fac5fa12423d1cb8e0cc86728b5
Reviewed-on: https://bazel-review.googlesource.com/#/c/1811
MOS_MIGRATED_REVID=100805636

4 files changed, 344 insertions, 0 deletions

diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh
new file mode 100755
index 0000000000..0a0f09da1a
--- /dev/null
+++ b/scripts/ci/build.sh
@@ -0,0 +1,276 @@
+#!/bin/bash -eu
+
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Main deploy functions for the continous build system
+# Just source this file and use the various method:
+#   bazel_build build bazel and run all its test
+#   bazel_release use the artifact generated by bazel_build and push
+#     them to github for a release and to GCS for a release candidate.
+#     Also prepare an email for announcing the release.
+
+# Load common.sh
+SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+source $(dirname ${SCRIPT_DIR})/scripts/release/common.sh
+
+: ${GIT_REPOSITORY_URL:=https://github.com/google/bazel}
+
+: ${GCS_BASE_URL:=https://storage.googleapis.com}
+: ${GCS_BUCKET:=bucket-o-bazel}
+
+: ${EMAIL_TEMPLATE_RC:=${SCRIPT_DIR}/rc_email.txt}
+: ${EMAIL_TEMPLATE_RELEASE:=${SCRIPT_DIR}/release_email.txt}
+
+: ${RELEASE_CANDIDATE_URL:="${GCS_BASE_URL}/${GCS_BUCKET}/%release_name%/rc%rc%/index.html"}
+: ${RELEASE_URL="${GIT_REPOSITORY_URL}/releases/tag/%release_name%"}
+
+set -eu
+
+PLATFORM="$(uname -s | tr 'A-Z' 'a-z')"
+if [[ ${PLATFORM} == "darwin" ]]; then
+  function checksum() {
+    shasum -a 256 $1 | cut -f 1 -d " "
+  }
+else
+  function checksum() {
+    sha256sum $1 | cut -f 1 -d " "
+  }
+fi
+
+GIT_ROOT="$(git rev-parse --show-toplevel)"
+BUILD_SCRIPT_PATH="${GIT_ROOT}/compile.sh"
+
+# Returns the full release name in the form NAME(rcRC)?
+function get_full_release_name() {
+  local rc=$(get_release_candidate)
+  local name=$(get_release_name)
+  if [ -n "${rc}" ]; then
+    echo "${name}rc${rc}"
+  else
+    echo "${name}"
+  fi
+}
+
+# Main entry point for building bazel.
+# It sets the embed label to the release name if any, calls the whole
+# test suite, compile the various packages, then copy the artifacts
+# to the folder in $1
+function bazel_build() {
+  local release_label="$(get_full_release_name)"
+  local embed_label_opts=
+  if [ -n "${release_label}" ]; then
+    export EMBED_LABEL="${release_label}"
+  fi
+  ${BUILD_SCRIPT_PATH} ${BAZEL_COMPILE_TARGET:-all} || exit $?
+
+  # Build the packages
+  ./output/bazel --bazelrc=${BAZELRC:-/dev/null} --nomaster_bazelrc build \
+      --embed_label=${release_label} --stamp \
+      --workspace_status_command=scripts/ci/build_status_command.sh \
+      //scripts/packages/...
+
+  # Copy the results to the output directory
+  mkdir -p $1/packages
+  cp output/bazel $1/bazel
+  cp bazel-bin/scripts/packages/install.sh $1/install.sh
+  cp bazel-genfiles/scripts/packages/README.md $1/README.md
+}
+
+# Generate a string from a template and a list of substitutions.
+# The first parameter is the template name and each subsequent parameter
+# is taken as a couple: first is the string the substitute and the second
+# is the result of the substitution.
+function generate_from_template() {
+  local value="$1"
+  shift
+  while (( $# >= 2 )); do
+    value="${value//$1/$2}"
+    shift 2
+  done
+  echo "${value}"
+}
+
+# Generate the email for the release.
+# The first line of the output will be the recipient, the second line
+# the mail subjects and the subsequent lines the mail, its content.
+# If no planed release, then this function output will be empty.
+function generate_email() {
+  local release_name=$(get_release_name)
+  local rc=$(get_release_candidate)
+  local args=(
+      "%release_name%" "${release_name}"
+      "%rc%" "${rc}"
+      "%relnotes%" "# $(git_commit_msg)"
+  )
+  if [ -n "${rc}" ]; then
+    args+=(
+        "%url%"
+        "$(generate_from_template "${RELEASE_CANDIDATE_URL}" "${args[@]}")"
+    )
+    generate_from_template "$(cat ${EMAIL_TEMPLATE_RC})" "${args[@]}"
+  elif [ -n "${release_name}" ]; then
+    args+=(
+        "%url%"
+        "$(generate_from_template "${RELEASE_URL}" "${args[@]}")"
+    )
+    generate_from_template "$(cat ${EMAIL_TEMPLATE_RELEASE})" "${args[@]}"
+  fi
+}
+
+# Deploy a github release using a third party tool:
+#   https://github.com/c4milo/github-release
+# This methods expects the following arguments:
+#   $1..$n files generated by package_build (should not contains the README file)
+# Please set GITHUB_TOKEN to talk to the Github API and GITHUB_RELEASE
+# for the path to the https://github.com/c4milo/github-release tool.
+# This method is also affected by GIT_REPOSITORY_URL which should be the
+# URL to the github repository (defaulted to https://github.com/google/bazel).
+function release_to_github() {
+  local url="${GIT_REPOSITORY_URL}"
+  local release_name=$(get_release_name)
+  local rc=$(get_release_candidate)
+  local release_tool="${GITHUB_RELEASE:-$(which github-release 2>/dev/null || true)}"
+  if [ ! -x "${release_tool}" ]; then
+    echo "Please set GITHUB_RELEASE to the path to the github-release binary." >&2
+    echo "This probably means you haven't installed https://github.com/c4milo/github-release " >&2
+    echo "on this machine." >&2
+    return 1
+  fi
+  local github_repo="$(echo "$url" | sed -E 's|https?://github.com/([^/]*/[^/]*).*$|\1|')"
+  if [ -n "${release_name}" ] && [ -z "${rc}" ]; then
+    "${GITHUB_RELEASE}" "${github_repo}" "${release_name}" "" "# $(git_commit_msg)" "${@}"
+  fi
+}
+
+# Creates an index of the files contained in folder $1 in mardown format
+function create_index_md() {
+  # First, add the README.md
+  local file=$1/__temp.md
+  if [ -f $1/README.md ]; then
+    cat $1/README.md
+  fi
+  # Then, add the list of files
+  echo
+  echo "## Index of files"
+  echo
+  for f in $1/*.sha256; do  # just list the sha256 ones
+    local filename=$(basename $f .sha256);
+    echo " - [${filename}](${filename}) [[SHA-256](${filename}.sha256)]"
+  done
+}
+
+# Creates an index of the files contained in folder $1 in HTML format
+# It supposes hoedown (https://github.com/hoedown/hoedown) is on the path,
+# if not, set the HOEDOWN environment variable to the good path.
+function create_index_html() {
+  local hoedown="${HOEDOWN:-$(which hoedown 2>/dev/null || true)}"
+  # Second line is to trick hoedown to behave as Github
+  create_index_md "${@}" \
+      | sed -E 's/^(Baseline.*)$/\1\
+/' | sed 's/^   + / - /' \
+      | "${hoedown}"
+}
+
+# Deploy a release candidate to Google Cloud Storage.
+# It requires to have gsutil installed. You can force the path to gsutil
+# by setting the GSUTIL environment variable. The GCS_BUCKET should be the
+# name of the Google cloud bucket to deploy to.
+# This methods expects the following arguments:
+#   $1..$n files generated by package_build
+function release_to_gcs() {
+  local gs="${GSUTIL:-$(which gsutil 2>/dev/null || true) -m}"
+  local release_name=$(get_release_name)
+  local rc=$(get_release_candidate)
+  if [ ! -x "${gs}" ]; then
+    echo "Please set GSUTIL to the path the gsutil binary." >&2
+    echo "gsutil (https://cloud.google.com/storage/docs/gsutil/) is the" >&2
+    echo "command-line interface to google cloud." >&2
+    return 1
+  fi
+  if [ -z "${GCS_BUCKET-}" ]; then
+    echo "Please set GCS_BUCKET to the name of your Google Cloud Storage bucket." >&2
+    return 1
+  fi
+  if [ -n "${release_name}" ] && [ -n "${rc}" ]; then
+    # Make a temporary folder with the desired structure
+    local dir="$(mktemp -d ${TMPDIR:-/tmp}/tmp.XXXXXXXX)"
+    local prev_dir="$PWD"
+    trap "{ cd ${prev_dir}; rm -fr ${dir}; }" EXIT
+    mkdir -p "${dir}/${release_name}/rc${rc}"
+    cp "${@}" "${dir}/${release_name}/rc${rc}"
+    # Add a index.html file:
+    create_index_html "${dir}/${release_name}/rc${rc}" \
+        >"${dir}/${release_name}/rc${rc}"/index.html
+    cd ${dir}
+    "${gs}" cp -a public-read -r . "gs://${GCS_BUCKET}"
+    cd ${prev_dir}
+    rm -fr ${dir}
+    trap - EXIT
+  fi
+}
+
+# A wrapper around the release deployment methods.
+function deploy_release() {
+  local github_args=()
+  # Filters out README.md for github releases
+  for i in "$@"; do
+    if ! [[ "$i" =~ README.md$ ]]; then
+      github_args+=("$i")
+    fi
+  done
+  release_to_github "${github_args[@]}"
+  release_to_gcs "$@"
+}
+
+# A wrapper for the whole release phase:
+#   Compute the SHA-256, and arrange the input
+#   Deploy the release
+#   Generate the email
+# Input: $1 $2 [$3 $4 [$5 $6 ...]]
+#    Each pair denotes a couple (platform, folder) where the platform
+#    is the platform built for and the folder is the folder where the
+#    artifacts for this platform are.
+# Ouputs:
+#   RELEASE_EMAIL_RECIPIENT: who to send a mail to
+#   RELEASE_EMAIL_SUBJECT: the subject of the email to be sent
+#   RELEASE_EMAIL_CONTENT: the content of the email to be sent
+function bazel_release() {
+  local README=$2/README.md
+  local tmpdir=$(mktemp -d ${TMPDIR:-/tmp}/tmp.XXXXXXXX)
+  trap 'rm -fr ${tmpdir}' EXIT
+  while (( $# > 1 )); do
+    local platform=$1
+    local folder=$2
+    shift 2
+    for file in $folder/*; do
+      if [ $(basename $file) != README.md ]; then
+        if [[ "$file" =~ /([^/]*)(\.[^\./]+)$ ]]; then
+          local destfile=${tmpdir}/${BASH_REMATCH[1]}-${platform}${BASH_REMATCH[2]}
+        else
+          local destfile=${tmpdir}/$(basename $file)-${platform}
+        fi
+        mv $file $destfile
+        checksum $destfile > $destfile.sha256
+      fi
+    done
+  done
+  deploy_release $README $(find ${tmpdir} -type f)
+
+  export RELEASE_EMAIL="$(generate_email)"
+
+  export RELEASE_EMAIL_RECIPIENT="$(echo "${RELEASE_EMAIL}" | head -1)"
+  export RELEASE_EMAIL_SUBJECT="$(echo "${RELEASE_EMAIL}" | head -2 | tail -1)"
+  export RELEASE_EMAIL_CONTENT="$(echo "${RELEASE_EMAIL}" | tail -n +3)"
+}
diff --git a/scripts/ci/build_status_command.sh b/scripts/ci/build_status_command.sh
new file mode 100755
index 0000000000..6fb869fc92
--- /dev/null
+++ b/scripts/ci/build_status_command.sh
@@ -0,0 +1,43 @@
+#!/bin/bash -eu
+
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A build status command to provide the package info generator with
+# the information about the commit being built
+
+set -eu
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$(dirname ${SCRIPT_DIR})/scripts/release/common.sh"
+
+git_hash=$(git rev-parse --short HEAD)
+echo "RELEASE_GIT_HASH ${git_hash}"
+url="${GIT_REPOSITORY_URL:-https://github.com/google/bazel}"
+echo "RELEASE_COMMIT_URL ${url}/commit/${git_hash}"
+if [ -n "${BUILT_BY-}" ]; then
+  echo "RELEASE_BUILT_BY ${BUILT_BY}"
+fi
+if [ -n "${BUILD_LOG-}" ]; then
+  echo "RELEASE_BUILD_LOG ${BUILD_LOG}"
+fi
+echo "RELEASE_COMMIT_MSG $(git_commit_msg | tr '\n' '\f')"
+release_name=$(get_release_name)
+rc=$(get_release_candidate)
+if [ -n "${release_name}" ]; then
+  if [ -n "${rc}" ]; then
+    echo "RELEASE_NAME ${release_name}rc${rc}"
+  else
+    echo "RELEASE_NAME ${release_name}"
+  fi
+fi
diff --git a/scripts/ci/rc_email.txt b/scripts/ci/rc_email.txt
new file mode 100644
index 0000000000..74dbef2e8c
--- /dev/null
+++ b/scripts/ci/rc_email.txt
@@ -0,0 +1,14 @@
+bazel-discuss+release-candidate@googlegroups.com
+Bazel %release_name% release candidate %rc% is available for testing
+Bazel %release_name%rc%rc% is now available for those that want to try
+it out.
+
+You can download it from:
+  %url%
+
+Please report regressions to https://github.com/google/bazel/issues as soon
+as possible.
+
+The release notes for version %release_name% are:
+
+%relnotes%
diff --git a/scripts/ci/release_email.txt b/scripts/ci/release_email.txt
new file mode 100644
index 0000000000..4b0cb0165d
--- /dev/null
+++ b/scripts/ci/release_email.txt
@@ -0,0 +1,11 @@
+bazel-discuss+release@googlegroups.com
+Bazel release %release_name%
+Bazel %release_name% is now available for general usage at:
+  %url%
+
+If you observe any regression, please report it to
+https://github.com/google/bazel/issues as soon as possible.
+
+The release notes for version %release_name% are:
+
+%relnotes%