Bazel release notes creation

This script uses the RELNOTES: tag (RELNOTES for a
simple change, RELNOTES[NEW] for a new feature,
RELNOTES[INC] for an incompatible change) to create
the CHANGELOG.md file.

--
Change-Id: If457a0a85f4a9ceddf822393d0aeb8b60c54136b
Reviewed-on: https://bazel-review.googlesource.com/#/c/1583/
MOS_MIGRATED_REVID=99020942

5 files changed, 436 insertions, 0 deletions

diff --git a/BUILD b/BUILD
new file mode 100644
index 0000000000..0081bc6752
--- /dev/null
+++ b/BUILD
@@ -0,0 +1,6 @@
+package(default_visibility = ["//scripts/release:__pkg__"])
+
+filegroup(
+    name = "git",
+    srcs = glob([".git/**"]),
+)
diff --git a/scripts/release/BUILD b/scripts/release/BUILD
new file mode 100644
index 0000000000..b2fb039097
--- /dev/null
+++ b/scripts/release/BUILD
@@ -0,0 +1,22 @@
+# Scripts for building Bazel releases
+package(default_visibility = ["//visibility:private"])
+
+sh_library(
+    name = "relnotes",
+    srcs = ["relnotes.sh"],
+)
+
+sh_test(
+    name = "relnotes_test",
+    srcs = ["relnotes_test.sh"],
+    data = [
+        "testenv.sh",
+        "//:git",
+        "//src/test/shell:bashunit",
+    ],
+    shard_count = 2,
+    tags = ["need_git"],
+    deps = [
+        ":relnotes",
+    ],
+)
diff --git a/scripts/release/relnotes.sh b/scripts/release/relnotes.sh
new file mode 100755
index 0000000000..56888c574e
--- /dev/null
+++ b/scripts/release/relnotes.sh
@@ -0,0 +1,161 @@
+#!/bin/bash -eu
+
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Generate the release notes from the git history.
+
+# It uses the RELNOTES tag in the history to knows the important changes to
+# report:
+#   RELNOTES: indicates a change important the user.
+#   RELNOTES[NEW]: introduces a new feature.
+#   RELNOTES[INC]: indicates an incompatible change.
+# The previous releases base is detected using the CHANGELOG file from the
+# repository.
+RELNOTES_TYPES=("INC" "NEW" "")
+RELNOTES_DESC=("Incompatible changes" "New features" "Important changes")
+
+# Get the baseline version and cherry-picks of the previous release
+#  Parameter: $1 is the path to the changelog file
+#  Output: "${BASELINE} ${CHERRYPICKS}"
+#    BASELINE is the hash of the baseline commit of the latest release
+#    CHERRYPICKS is the list of hash of cherry-picked commits of the latest release
+#  return 1 if there is no initial release
+function get_last_release() {
+  local changelog=$1
+  [ -f "$changelog" ] || return 1  # No changelog = initial release
+  local BASELINE_LINE=$(grep -m 1 -n '^Baseline: ' "$changelog") || return 1
+  [ -n "${BASELINE_LINE}" ] || return 1  # No baseline = initial release
+  local BASELINE_LINENB=$(echo "${BASELINE_LINE}" | cut -d ":" -f 1)
+  BASELINE=$(echo "${BASELINE_LINE}" | cut -d " " -f 2)
+  local CHERRYPICK_LINE=$(($BASELINE_LINENB + 1))
+  # grep -B999 looks for all lines before the empty line and after that we
+  # restrict to only lines with the cherry picked hash then finally we cut
+  # the hash.
+  local CHERRY_PICKS=$(tail -n +${CHERRYPICK_LINE} "$changelog" \
+      | grep -m 1 "^$" -B999 \
+      | grep -E '^   \+ [a-z0-9]+:' \
+      | cut -d ":" -f 1 | cut -d "+" -f 2)
+  echo $BASELINE $CHERRY_PICKS
+  return 0
+}
+
+# Now get the list of commit with a RELNOTES since latest release baseline ($1)
+# discarding cherry_picks ($2..) and rollbacks. The returned list of commits is
+# from the oldest to the newest
+function get_release_notes_commits() {
+  local baseline=$1
+  shift
+  local cherry_picks="$@"
+  local rollback_commits=$(git log --oneline -E --grep='^Rollback of commit [a-z0-9]+.$' ${baseline}.. \
+      | grep -E '^[a-z0-9]+ Rollback of commit [a-z0-9]+.$')
+  local rollback_hashes=$(echo "$rollback_commits" | cut -d " " -f 1)
+  local rolledback_hashes=$(echo "$rollback_commits" | cut -d " " -f 5 | sed -E 's/^(.......).*$/\1/')
+  local exclude_hashes=$(echo $cherry_picks $rollback_hashes $rolledback_hashes | xargs echo | sed 's/ /|/g')
+  git log --reverse --pretty=format:%h ${baseline}.. -E --grep='^RELNOTES(\[[^\]+\])?:' \
+      | grep -Ev "^(${exclude_hashes})"
+}
+
+# Extract the release note from a commit hash ($1). It extracts
+# the RELNOTES([??]): lines. A new empty line ends the relnotes tag.
+# It adds the relnotes, if not "None" ("None.") or "n/a" ("n/a.") to
+# the correct array:
+#   RELNOTES_INC for incompatible changes
+#   RELNOTES_NEW for new features changes
+#   RELNOTES for other changes
+function extract_release_note() {
+  local relnote="$(git show -s $1 --pretty=format:%B | awk '/^RELNOTES(\[[^\]]+\])?:/,/^$/')"
+  local regex="^RELNOTES(\[([a-zA-Z]*)\])?:[[:space:]]*([^[:space:]].*[^[:space:]])[[:space:]]*$"
+  if [[ "$relnote" =~ $regex ]]; then
+      local relnote_kind=${BASH_REMATCH[2]}
+      local relnote_text="${BASH_REMATCH[3]}"
+      if [[ ! "$(echo $relnote_text | awk '{print tolower($0)}')" =~ ^(none|n/a)?.?$ ]]; then
+        eval "RELNOTES_${relnote_kind}+=(\"\${relnote_text}\")"
+      fi
+  fi
+}
+
+# Build release notes arrays from a list of commits ($@) and return the release
+# note in an array of array.
+function get_release_notes() {
+  for i in "${RELNOTES_TYPES[@]}"; do
+    eval "RELNOTES_${i}=()"
+  done
+  for i in $@; do
+    extract_release_note $i
+  done
+}
+
+# Returns the list of release notes in arguments into a list of points in
+# a markdown list. The release notes are wrapped to 70 characters so it
+# displays nicely in a git history.
+function format_release_notes() {
+  local i
+  for (( i=1; $i <= $#; i=$i+1 )); do
+    local relnote="${!i}"
+    local lines=$(echo "$relnote" | fmt -w 66)  # wrap to 70 counting the 4 leading spaces.
+    echo "  - $lines" | head -1
+    echo "$lines" | tail -n +2 | sed 's/^/    /'
+  done
+}
+
+# Create the release notes since commit $1 ($2...${[#]} are the cherry-picks,
+# so the commits to ignore.
+function release_notes() {
+  local i
+  local commits=$(get_release_notes_commits $@)
+  local length="${#RELNOTES_TYPES[@]}"
+  get_release_notes "$commits"
+  for (( i=0; $i < $length; i=$i+1 )); do
+    local relnotes_title="${RELNOTES_DESC[$i]}"
+    local relnotes_type=${RELNOTES_TYPES[$i]}
+    local relnotes="RELNOTES_${relnotes_type}[@]"
+    local nb_relnotes=$(eval "echo \${#$relnotes}")
+    if (( "${nb_relnotes}" > 0 )); then
+      echo "${relnotes_title}:"
+      echo
+      format_release_notes "${!relnotes}"
+      echo
+    fi
+  done
+}
+
+# A wrapper around all the previous function, using the CHANGELOG.md
+# file in $1 to compute the last release commit hash.
+function create_release_notes() {
+  local last_release=$(get_last_release "$1") || \
+      { echo "Initial release."; return 0; }
+  [ -n "${last_release}" ] || { echo "Initial release."; return 0; }
+  release_notes ${last_release}
+}
+
+# Create the revision information given a list of commits. The first
+# commit should be the baseline, and the other one are the cherry-picks.
+# The result is of the form:
+# Baseline: BASELINE_COMMIT
+#    + CHERRY_PICK1: commit message summary of the CHERRY_PICK1. This
+#                    message will be wrapped into 70 columns.
+#    + CHERRY_PICK2: commit message summary of the CHERRY_PICK2.
+function create_revision_information() {
+  echo "Baseline: $1"
+  shift
+  while [ -n "${1-}" ]; do
+    local hash="$1"
+    local subject=$(git show -s --pretty=format:%s $hash)
+    local lines=$(echo "$subject" | fmt -w 56)  # 14 leading spaces.
+    echo "   + $hash: $lines" | head -1
+    echo "$lines" | tail -n +2 | sed 's/^/              /'
+    shift
+  done
+}
diff --git a/scripts/release/relnotes_test.sh b/scripts/release/relnotes_test.sh
new file mode 100755
index 0000000000..32d571134e
--- /dev/null
+++ b/scripts/release/relnotes_test.sh
@@ -0,0 +1,204 @@
+#!/bin/bash
+
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Tests release notes generation (relnotes.sh)
+set -eu
+
+SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+source ${SCRIPT_DIR}/testenv.sh || { echo "testenv.sh not found!" >&2; exit 1; }
+
+### Setup a git repository
+setup_git_repository
+
+### Load the relnotes script
+source ${SCRIPT_DIR}/relnotes.sh || { echo "relnotes.sh not found!" >&2; exit 1; }
+
+### Tests method
+
+function set_up() {
+  cd ${MASTER_ROOT}
+}
+
+function test_format_release_notes() {
+  local expected='  - Lorem ipsus I do not know more of latin than that but I need to
+    type random text that spans multiple line so we can test that the
+    wrapping of lines works as intended.
+  - Another thing I must type.
+  - Yet another test that spans across multiple lines so I must type
+    some random stuff to test wrapping.'
+  local input=("Lorem ipsus I do not know more of latin \
+than that but I need to type random text that spans multiple line so we \
+can test that the wrapping of lines works as intended."
+"Another thing I must type."
+"Yet another test that spans across multiple lines so I must type \
+some random stuff to test wrapping.")
+  assert_equals "${expected}" "$(format_release_notes "${input[@]}")"
+}
+
+function test_get_release_notes_commits() {
+  # Generated with git log --grep RELNOTES.
+  # Only 6d98f6c 53c0748 are removed (rollback).
+  commits="0188971 957934c 7a99c7f b5ba24a c9041bf 8232d9b 422c731 e9029d4 \
+cc44636 06b09ce 29b05c8 67944d8 e8f6647 6d9fb36 f7c9922 5c0e4b2 9e387dd \
+98c9274 db4d861 a689f29 db487ce 965c392 bb59d88 d3461db cef25c4 14d905b"
+  assert_equals "$commits" "$(get_release_notes_commits 00d7223 | xargs)"
+  assert_equals "$(echo "$commits" | sed 's/957934c //')" \
+      "$(get_release_notes_commits 00d7223 957934c | xargs)"
+}
+
+TEST_INC_CHANGE='Incompatible changes:
+
+  - Remove deprecated "make var" INCDIR
+
+'
+TEST_NEW_CHANGE='New features:
+
+  - added --with_aspect_deps to blaze query, that prints additional
+    information about aspects of target when --output is set to {xml,
+    proto, record}.
+
+'
+TEST_CHANGE='Important changes:
+
+  - Use a default implementation of a progress message, rather than
+    defaulting to null for all SpawnActions.
+  - Attribute error messages related to Android resources are easier
+    to understand now.'
+
+function test_release_notes() {
+  assert_equals "$TEST_INC_CHANGE$(echo)$TEST_NEW_CHANGE$(echo)$TEST_CHANGE" \
+      "$(release_notes 965c392)"
+  assert_equals "$TEST_NEW_CHANGE$(echo)$TEST_CHANGE" \
+      "$(release_notes 965c392 bb59d88)"
+}
+
+function test_get_last_release() {
+  rm -f ${TEST_TMPDIR}/CHANGELOG.md
+  if (get_last_release "${TEST_TMPDIR}/CHANGELOG.md"); then
+    fail "Should have returned false for initial release"
+  fi
+  cat <<EOF >${TEST_TMPDIR}/CHANGELOG.md
+## No release
+EOF
+  if (get_last_release "${TEST_TMPDIR}/CHANGELOG.md"); then
+    fail "Should have returned false when no release exists"
+  fi
+  cat <<EOF >${TEST_TMPDIR}/CHANGELOG.md
+## New release
+
+Baseline: 965c392
+
+Initial release without cherry-picks
+
+EOF
+  assert_equals "965c392" \
+      "$(get_last_release "${TEST_TMPDIR}/CHANGELOG.md")"
+
+
+  mv ${TEST_TMPDIR}/CHANGELOG.md ${TEST_TMPDIR}/CHANGELOG.md.bak
+  cat <<EOF >${TEST_TMPDIR}/CHANGELOG.md
+## Cherry-picking bb59d88
+
+Baseline: 965c392
+   + bb59d88: RELNOTES[INC]: Remove deprecated "make var" INCDIR
+
+$TEST_INC_CHANGE
+EOF
+  cat ${TEST_TMPDIR}/CHANGELOG.md.bak >>${TEST_TMPDIR}/CHANGELOG.md
+  rm ${TEST_TMPDIR}/CHANGELOG.md.bak
+  assert_equals "965c392 bb59d88" \
+      "$(get_last_release "${TEST_TMPDIR}/CHANGELOG.md")"
+
+  mv ${TEST_TMPDIR}/CHANGELOG.md ${TEST_TMPDIR}/CHANGELOG.md.bak
+  cat <<EOF >${TEST_TMPDIR}/CHANGELOG.md
+## Cherry-picking bb59d88 and 14d905b
+
+Baseline: 965c392
+   + bb59d88: RELNOTES[INC]: Remove deprecated "make var" INCDIR
+   + 14d905b: Add --with_aspect_deps flag to blaze query. This flag
+              should produce additional information about aspect
+              dependencies when --output is set to {xml, proto}.
+
+$TEST_INC_CHANGE
+$TEST_NEW_CHANGE
+EOF
+  cat ${TEST_TMPDIR}/CHANGELOG.md.bak >>${TEST_TMPDIR}/CHANGELOG.md
+  rm ${TEST_TMPDIR}/CHANGELOG.md.bak
+  assert_equals "965c392 bb59d88 14d905b" \
+      "$(get_last_release "${TEST_TMPDIR}/CHANGELOG.md")"
+
+}
+
+function test_create_release_notes() {
+  cat <<EOF >${TEST_TMPDIR}/CHANGELOG.md
+## New release
+
+Baseline: 965c392
+
+Initial release without cherry-picks
+
+EOF
+  assert_equals "$TEST_INC_CHANGE$(echo)$TEST_NEW_CHANGE$(echo)$TEST_CHANGE" \
+      "$(create_release_notes ${TEST_TMPDIR}/CHANGELOG.md)"
+
+  cat <<'EOF' >${TEST_TMPDIR}/CHANGELOG.md
+## Cherry-picking bb59d88
+
+```
+Baseline: 965c392
+   + bb59d88: RELNOTES[INC]: Remove deprecated "make var" INCDIR
+```
+
+EOF
+  cat <<EOF >>${TEST_TMPDIR}/CHANGELOG.md
+$TEST_INC_CHANGE
+EOF
+  assert_equals "$TEST_NEW_CHANGE$(echo)$TEST_CHANGE" \
+      "$(create_release_notes ${TEST_TMPDIR}/CHANGELOG.md)"
+  assert_equals "965c392 bb59d88" \
+      "$(get_last_release "${TEST_TMPDIR}/CHANGELOG.md")"
+
+  cat <<'EOF' >${TEST_TMPDIR}/CHANGELOG.md
+## Cherry-picking bb59d88 and 14d905b
+
+```
+Baseline: 965c392
+   + bb59d88: RELNOTES[INC]: Remove deprecated "make var" INCDIR
+   + 14d905b: Add --with_aspect_deps flag to blaze query. This flag
+              should produce additional information about aspect
+              dependencies when --output is set to {xml, proto}.
+```
+
+EOF
+  cat <<EOF >>${TEST_TMPDIR}/CHANGELOG.md
+$TEST_INC_CHANGE
+$TEST_NEW_CHANGE
+EOF
+  assert_equals "$TEST_CHANGE" \
+      "$(create_release_notes ${TEST_TMPDIR}/CHANGELOG.md)"
+}
+
+function test_create_revision_information() {
+  expected='Baseline: 965c392
+   + bb59d88: RELNOTES[INC]: Remove deprecated "make var" INCDIR
+   + 14d905b: Add --with_aspect_deps flag to blaze query. This flag
+              should produce additional information about aspect
+              dependencies when --output is set to {xml, proto}.'
+   assert_equals "$expected" \
+		 "$(create_revision_information 965c392 bb59d88 14d905b)"
+}
+
+run_suite "Release notes generation tests"
diff --git a/scripts/release/testenv.sh b/scripts/release/testenv.sh
new file mode 100755
index 0000000000..9a57ebe753
--- /dev/null
+++ b/scripts/release/testenv.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+#
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Setting up the environment for Bazel release scripts test.
+
+[ -z "$TEST_SRCDIR" ] && { echo "TEST_SRCDIR not set!" >&2; exit 1; }
+
+# Load the unit-testing framework
+source "${TEST_SRCDIR}/src/test/shell/unittest.bash" || \
+  { echo "Failed to source unittest.bash" >&2; exit 1; }
+
+# Commit at which we cut the master to do the test so we always take the git
+# repository in a consistent state.
+: ${MASTER_COMMIT:=7d41d7417fc34f7fa8aac7130a0588b8557e4b57}
+
+# Set-up a copy of the git repository in ${MASTER_ROOT}, pointing master
+# to ${MASTER_COMMIT}.
+function setup_git_repository() {
+  local origin_git_root=${TEST_SRCDIR}
+  MASTER_ROOT=${TEST_TMPDIR}/git/root
+  local orig_dir=${PWD}
+  # Create a new origin with the good starting point
+  mkdir -p ${MASTER_ROOT}
+  cd ${MASTER_ROOT}
+  cp -RL ${origin_git_root}/.git .git
+  rm -f .git/hooks/*  # Do not keep custom hooks
+  git reset -q --hard HEAD
+  git checkout -q -B master ${MASTER_COMMIT}
+  cd ${orig_dir}
+}