Simplify the release scripts in preparation of rewriting them in Python

These are the scripts I'm currently using for the Bazel 0.12.0 release.

Closes #4908.

PiperOrigin-RevId: 190598268

1 files changed, 131 insertions, 134 deletions

diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh
index ffcd0ed3b3..a2e45130c7 100755
--- a/scripts/ci/build.sh
+++ b/scripts/ci/build.sh
@@ -24,22 +24,47 @@ set -eu
 #     Also prepare an email for announcing the release.
 
 # Load common.sh
-SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-source $(dirname ${SCRIPT_DIR})/release/common.sh
-
-: ${GIT_REPOSITORY_URL:=https://github.com/bazelbuild/bazel}
-
-: ${GCS_BASE_URL:=https://storage.googleapis.com}
-: ${GCS_BUCKET:=bucket-o-bazel}
-: ${GCS_APT_BUCKET:=bazel-apt}
-
-: ${EMAIL_TEMPLATE_RC:=${SCRIPT_DIR}/rc_email.txt}
-: ${EMAIL_TEMPLATE_RELEASE:=${SCRIPT_DIR}/release_email.txt}
-
-: ${RELEASE_CANDIDATE_URL:="https://releases.bazel.build/%release_name%/rc%rc%/index.html"}
-: ${RELEASE_URL="${GIT_REPOSITORY_URL}/releases/tag/%release_name%"}
-
-: ${BOOTSTRAP_BAZEL:=bazel}
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$(dirname ${SCRIPT_DIR})/release/common.sh"
+
+if ! command -v gsutil &>/dev/null; then
+  echo "Required tool 'gsutil' not found. Please install it:"
+  echo "See https://cloud.google.com/sdk/downloads for instructions."
+  exit 1
+fi
+if ! command -v github-release &>/dev/null; then
+  echo "Required tool 'github-release' not found. Download it from here:"
+  echo "https://github.com/c4milo/github-release/releases"
+  echo "Just extract the archive and put the binary on your PATH."
+  exit 1
+fi
+if ! command -v debsign &>/dev/null; then
+  echo "Required tool 'debsign' not found. Please install it via apt-get:"
+  echo "apt-get install devscripts"
+  exit 1
+fi
+if ! command -v reprepro &>/dev/null; then
+  echo "Required tool 'reprepro' not found. Please install it via apt-get:"
+  echo "apt-get install reprepro"
+  exit 1
+fi
+if ! command -v gpg &>/dev/null; then
+  echo "Required tool 'gpg' not found. Please install it via apt-get:"
+  echo "apt-get install gnupg"
+  exit 1
+fi
+if ! command -v pandoc &>/dev/null; then
+  echo "Required tool 'pandoc' not found. Please install it via apt-get:"
+  echo "apt-get install pandoc"
+  exit 1
+fi
+# if ! command -v ssmtp &>/dev/null; then
+#   echo "Required tool 'ssmtp' not found. Please install it via apt-get:"
+#   echo "apt-get install ssmtp"
+#   exit 1
+# fi
+
+export APT_GPG_KEY_ID=$(gsutil cat gs://bazel-encrypted-secrets/release-key.gpg.id)
 
 # Generate a string from a template and a list of substitutions.
 # The first parameter is the template name and each subsequent parameter
@@ -60,6 +85,9 @@ function generate_from_template() {
 # the mail subjects and the subsequent lines the mail, its content.
 # If no planed release, then this function output will be empty.
 function generate_email() {
+  RELEASE_CANDIDATE_URL="https://releases.bazel.build/%release_name%/rc%rc%/index.html"
+  RELEASE_URL="https://github.com/bazelbuild/bazel/releases/tag/%release_name%"
+
   local release_name=$(get_release_name)
   local rc=$(get_release_candidate)
   local args=(
@@ -69,16 +97,17 @@ function generate_email() {
   )
   if [ -n "${rc}" ]; then
     args+=(
-        "%url%"
-        "$(generate_from_template "${RELEASE_CANDIDATE_URL}" "${args[@]}")"
+        "%url%" "$(generate_from_template "${RELEASE_CANDIDATE_URL}" "${args[@]}")"
     )
-    generate_from_template "$(cat ${EMAIL_TEMPLATE_RC})" "${args[@]}"
+    generate_from_template \
+        "$(cat "${SCRIPT_DIR}/rc_email.txt")" \
+        "${args[@]}"
   elif [ -n "${release_name}" ]; then
     args+=(
-        "%url%"
-        "$(generate_from_template "${RELEASE_URL}" "${args[@]}")"
+        "%url%" "$(generate_from_template "${RELEASE_URL}" "${args[@]}")"
     )
-    generate_from_template "$(cat ${EMAIL_TEMPLATE_RELEASE})" "${args[@]}"
+    generate_from_template \
+        "$(cat "${SCRIPT_DIR}/release_email.txt")" "${args[@]}"
   fi
 }
 
@@ -105,40 +134,25 @@ _Security_: All our binaries are signed with our
 #   https://github.com/c4milo/github-release
 # This methods expects the following arguments:
 #   $1..$n files generated by package_build (should not contains the README file)
-# Please set GITHUB_TOKEN to talk to the Github API and GITHUB_RELEASE
-# for the path to the https://github.com/c4milo/github-release tool.
-# This method is also affected by GIT_REPOSITORY_URL which should be the
-# URL to the github repository (defaulted to https://github.com/bazelbuild/bazel).
+# Please set GITHUB_TOKEN to talk to the Github API.
 function release_to_github() {
-  local url="${GIT_REPOSITORY_URL}"
+  local artifact_dir="$1"
+
   local release_name=$(get_release_name)
   local rc=$(get_release_candidate)
-  local release_tool="${GITHUB_RELEASE:-$(which github-release 2>/dev/null || echo release-tool-not-found)}"
 
-  if [ "${release_tool}" = "release-tool-not-found" ]; then
-    echo "Please set GITHUB_RELEASE to the path to the github-release binary." >&2
-    echo "This probably means you haven't installed https://github.com/c4milo/github-release " >&2
-    echo "on this machine." >&2
-    return 1
-  fi
-  local github_repo="$(echo "$url" | sed -E 's|https?://github.com/([^/]*/[^/]*).*$|\1|')"
   if [ -n "${release_name}" ] && [ -z "${rc}" ]; then
-    mkdir -p "${tmpdir}/to-github"
-    cp "${@}" "${tmpdir}/to-github"
-    "${release_tool}" "${github_repo}" "${release_name}" "" "$(get_release_page)" "${tmpdir}/to-github/"'*'
+    local github_token="$(gsutil cat gs://bazel-encrypted-secrets/github-token.enc | \
+        gcloud kms decrypt --location global --keyring buildkite --key github-token --ciphertext-file - --plaintext-file -)"
+
+    GITHUB_TOKEN=${github_token} github-release "bazelbuild/bazel" "${release_name}" "" "$(get_release_page)" "${artifact_dir}/*"
   fi
 }
 
-# Creates an index of the files contained in folder $1 in mardown format
+# Creates an index of the files contained in folder $1 in Markdown format.
 function create_index_md() {
   # First, add the release notes
   get_release_page
-  # Build log
-  if [ -f $1/build.log ]; then
-    echo
-    echo " [Build log](build.log)"
-    echo
-  fi
   # Then, add the list of files
   echo
   echo "## Index of files"
@@ -149,74 +163,49 @@ function create_index_md() {
   done
 }
 
-# Creates an index of the files contained in folder $1 in HTML format
-# It supposes hoedown (https://github.com/hoedown/hoedown) is on the path,
-# if not, set the HOEDOWN environment variable to the good path.
+# Creates an index of the files contained in folder $1 in HTML format.
 function create_index_html() {
-  local hoedown="${HOEDOWN:-$(which hoedown 2>/dev/null || true)}"
-  # Second line is to trick hoedown to behave as Github
-  create_index_md "${@}" \
-      | sed -E 's/^(Baseline.*)$/\1\
-/' | sed 's/^   + / - /' | sed 's/_/\\_/g' \
-      | "${hoedown}"
-}
-
-function get_gsutil() {
-  local gs="${GSUTIL:-$(which gsutil 2>/dev/null || true) -m}"
-  if [ ! -x "${gs}" ]; then
-    echo "Please set GSUTIL to the path the gsutil binary." >&2
-    echo "gsutil (https://cloud.google.com/storage/docs/gsutil/) is the" >&2
-    echo "command-line interface to google cloud." >&2
-    exit 1
-  fi
-  echo "${gs}"
+  create_index_md "${@}" | pandoc -f markdown -t html
 }
 
 # Deploy a release candidate to Google Cloud Storage.
 # It requires to have gsutil installed. You can force the path to gsutil
-# by setting the GSUTIL environment variable. The GCS_BUCKET should be the
-# name of the Google cloud bucket to deploy to.
+# by setting the GSUTIL environment variable.
 # This methods expects the following arguments:
 #   $1..$n files generated by package_build
 function release_to_gcs() {
-  local gs="$(get_gsutil)"
+  local artifact_dir="$1"
+
   local release_name="$(get_release_name)"
   local rc="$(get_release_candidate)"
-  if [ -z "${GCS_BUCKET-}" ]; then
-    echo "Please set GCS_BUCKET to the name of your Google Cloud Storage bucket." >&2
-    return 1
-  fi
+
   if [ -n "${release_name}" ]; then
     local release_path="${release_name}/release"
     if [ -n "${rc}" ]; then
       release_path="${release_name}/rc${rc}"
     fi
-    # Make a temporary folder with the desired structure
-    local dir="$(mktemp -d ${TMPDIR:-/tmp}/tmp.XXXXXXXX)"
-    local prev_dir="$PWD"
-    trap "{ cd ${prev_dir}; rm -fr ${dir}; }" EXIT
-    mkdir -p "${dir}/${release_path}"
-    cp "${@}" "${dir}/${release_path}"
-    # Add a index.html file:
-    create_index_html "${dir}/${release_path}" \
-        >"${dir}/${release_path}"/index.html
-    cd ${dir}
-    "${gs}" -m cp -a public-read -r . "gs://${GCS_BUCKET}"
-    cd "${prev_dir}"
-    rm -fr "${dir}"
-    trap - EXIT
+    create_index_html "${artifact_dir}" > "${artifact_dir}/index.html"
+    gsutil -m cp -a public-read "${artifact_dir}/**" "gs://bazel/${release_path}"
   fi
 }
 
 function ensure_gpg_secret_key_imported() {
-  (gpg --list-secret-keys | grep "${APT_GPG_KEY_ID}" > /dev/null) || \
-  gpg --allow-secret-key-import --import "${APT_GPG_KEY_PATH}"
+  if ! gpg --list-secret-keys | grep "${APT_GPG_KEY_ID}" > /dev/null; then
+    keyfile=$(mktemp --tmpdir)
+    chmod 0600 "${keyfile}"
+    gsutil cat "gs://bazel-encrypted-secrets/release-key.gpg.enc" | \
+        gcloud kms decrypt --location "global" --keyring "buildkite" --key "bazel-release-key" --plaintext-file "-" --ciphertext-file "${keyfile}"
+    gpg --allow-secret-key-import --import "${keyfile}"
+    rm -f "${keyfile}"
+  fi
+
   # Make sure we use stronger digest algorithm。
   # We use reprepro to generate the debian repository,
   # but there's no way to pass flags to gpg using reprepro, so writting it into
   # ~/.gnupg/gpg.conf
-  (grep "digest-algo sha256" ~/.gnupg/gpg.conf > /dev/null) || \
-  echo "digest-algo sha256" >> ~/.gnupg/gpg.conf
+  if ! grep "digest-algo sha256" ~/.gnupg/gpg.conf > /dev/null; then
+    echo "digest-algo sha256" >> ~/.gnupg/gpg.conf
+  fi
 }
 
 function create_apt_repository() {
@@ -262,71 +251,79 @@ EOF
   ensure_gpg_secret_key_imported
 
   local distribution="$1"
-  local deb_pkg_name_jdk8="$2"
+  local deb_pkg_name="$2"
   local deb_dsc_name="$3"
 
-  debsign -k ${APT_GPG_KEY_ID} "${deb_dsc_name}"
+  debsign -k "${APT_GPG_KEY_ID}" "${deb_dsc_name}"
 
-  reprepro -C jdk1.8 includedeb "${distribution}" "${deb_pkg_name_jdk8}"
+  reprepro -C jdk1.8 includedeb "${distribution}" "${deb_pkg_name}"
   reprepro -C jdk1.8 includedsc "${distribution}" "${deb_dsc_name}"
 
-  "${gs}" -m cp -a public-read -r dists "gs://${GCS_APT_BUCKET}/"
-  "${gs}" -m cp -a public-read -r pool "gs://${GCS_APT_BUCKET}/"
+  gsutil -m cp -a public-read -r dists pool "gs://bazel-apt"
 }
 
 function release_to_apt() {
-  local gs="$(get_gsutil)"
+  local artifact_dir="$1"
+
   local release_name="$(get_release_name)"
   local rc="$(get_release_candidate)"
-  if [ -z "${GCS_APT_BUCKET-}" ]; then
-    echo "Please set GCS_APT_BUCKET to the name of your GCS bucket for apt repository." >&2
-    return 1
-  fi
-  if [ -z "${APT_GPG_KEY_ID-}" ]; then
-    echo "Please set APT_GPG_KEY_ID for apt repository." >&2
-    return 1
-  fi
+
   if [ -n "${release_name}" ]; then
-    # Make a temporary folder with the desired structure
-    local dir="$(mktemp -d ${TMPDIR:-/tmp}/tmp.XXXXXXXX)"
-    local prev_dir="$PWD"
-    trap "{ cd ${prev_dir}; rm -fr ${dir}; }" EXIT
-    mkdir -p "${dir}/${release_name}"
     local release_label="$(get_full_release_name)"
-    local deb_pkg_name_jdk8="${release_name}/bazel_${release_label}-linux-x86_64.deb"
+    local deb_pkg_name="${release_name}/bazel_${release_label}-linux-x86_64.deb"
     local deb_dsc_name="${release_name}/bazel_${release_label}.dsc"
     local deb_tar_name="${release_name}/bazel_${release_label}.tar.gz"
-    cp "${tmpdir}/bazel_${release_label}-linux-x86_64.deb" "${dir}/${deb_pkg_name_jdk8}"
-    cp "${tmpdir}/bazel.dsc" "${dir}/${deb_dsc_name}"
-    cp "${tmpdir}/bazel.tar.gz" "${dir}/${deb_tar_name}"
-    cd "${dir}"
+
+    pushd "${artifact_dir}"
     if [ -n "${rc}" ]; then
-      create_apt_repository testing "${deb_pkg_name_jdk8}" "${deb_dsc_name}"
+      create_apt_repository testing "${deb_pkg_name}" "${deb_dsc_name}"
     else
-      create_apt_repository stable "${deb_pkg_name_jdk8}" "${deb_dsc_name}"
+      create_apt_repository stable "${deb_pkg_name}" "${deb_dsc_name}"
     fi
-    cd "${prev_dir}"
-    rm -fr "${dir}"
-    trap - EXIT
+    popd
   fi
 }
 
 # A wrapper around the release deployment methods.
 function deploy_release() {
-  local github_args=()
-  for i in "$@"; do
-    if ! ( [[ "$i" =~ build.log ]] || [[ "$i" =~ bazel.dsc ]] || [[ "$i" =~ bazel.tar.gz ]] || [[ "$i" =~ .nobuild$ ]] ) ; then
-      github_args+=("$i")
-    fi
-  done
-  local gcs_args=()
-  # Filters out perf.bazel.*.nobuild
-  for i in "$@"; do
-    if ! [[ "$i" =~ .nobuild$ ]] ; then
-      gcs_args+=("$i")
-    fi
+  local release_label="$(get_full_release_name)"
+  local release_name="$(get_release_name)"
+
+  if [[ ! -d $1 ]]; then
+    echo "Usage: deploy_release ARTIFACT_DIR"
+    exit 1
+  fi
+  artifact_dir="$1"
+
+  if [[ -z $release_name ]]; then
+    echo "Could not get the release name - are you in a release branch directory?"
+    exit 1
+  fi
+
+  ensure_gpg_secret_key_imported
+
+  rm -f "${artifact_dir}"/*.{sha256,sig}
+  for file in "${artifact_dir}"/*; do
+    (cd "${artifact_dir}" && sha256sum "$(basename "${file}")" > "${file}.sha256")
+    gpg --no-tty --detach-sign -u "${APT_GPG_KEY_ID}" "${file}"
   done
-  release_to_github "${github_args[@]}"
-  release_to_gcs "${gcs_args[@]}"
-  release_to_apt
+
+  apt_working_dir="$(mktemp -d --tmpdir)"
+  echo "apt_working_dir = ${apt_working_dir}"
+  mkdir "${apt_working_dir}/${release_name}"
+  cp "${artifact_dir}/bazel_${release_label}-linux-x86_64.deb" "${apt_working_dir}/${release_name}"
+  cp "${artifact_dir}/bazel_${release_label}.dsc" "${apt_working_dir}/${release_name}"
+  cp "${artifact_dir}/bazel_${release_label}.tar.gz" "${apt_working_dir}/${release_name}"
+  release_to_apt "${apt_working_dir}"
+
+  gcs_working_dir="$(mktemp -d --tmpdir)"
+  echo "gcs_working_dir = ${gcs_working_dir}"
+  cp "${artifact_dir}"/* "${gcs_working_dir}"
+  release_to_gcs "${gcs_working_dir}"
+
+  github_working_dir="$(mktemp -d --tmpdir)"
+  echo "github_working_dir = ${github_working_dir}"
+  cp "${artifact_dir}"/* "${github_working_dir}"
+  rm -f "${github_working_dir}/bazel_${release_label}"*.{deb,dsc,tar.gz}
+  release_to_github "${github_working_dir}"
 }