unittest.bash: Correctly handle failures due to "errexit" in tests. This will get rid of all the "ghost flakes" where tests crashed with no apparant reason printed into our logs. Now a stack trace is printed and an easy to understand failure reason, too.

--
MOS_MIGRATED_REVID=110142957

1 files changed, 48 insertions, 3 deletions

diff --git a/src/test/shell/unittest.bash b/src/test/shell/unittest.bash
index dd31839c95..4caa93bc02 100644
--- a/src/test/shell/unittest.bash
+++ b/src/test/shell/unittest.bash
@@ -81,6 +81,23 @@
   { echo "unittest.bash only works with bash!" >&2; exit 1; }
 
 DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+
+#### Configuration variables (may be overridden by testenv.sh or the suite):
+
+# This function may be called by testenv.sh or a test suite to enable errexit
+# in a way that enables us to print pretty stack traces when something fails.
+function enable_errexit() {
+  set -o errtrace
+  set -eu
+  trap __test_terminated_err ERR
+}
+
+function disable_errexit() {
+  set +o errtrace
+  set +eu
+  trap - ERR
+}
+
 source ${DIR}/testenv.sh || { echo "testenv.sh not found!" >&2; exit 1; }
 
 #### Global variables:
@@ -122,8 +139,8 @@ TEST_script="$(pwd)/$0"         # Full path to test script
 #### Internal functions
 
 function __show_log() {
-    echo "-- Actual output: ------------------------------------------------------"
-    cat $TEST_log
+    echo "-- Test log: -----------------------------------------------------------"
+    [[ -e $TEST_log ]] && cat $TEST_log || echo "(Log file did not exist.)"
     echo "------------------------------------------------------------------------"
 }
 
@@ -404,7 +421,7 @@ function __update_shards() {
 # Usage: __test_terminated <signal-number>
 # Handler that is called when the test terminated unexpectedly
 function __test_terminated() {
-   __show_log >&2
+    __show_log >&2
     echo "$TEST_name FAILED: terminated by signal $1." >&2
     TEST_passed="false"
     __show_stack
@@ -412,6 +429,33 @@ function __test_terminated() {
     exit 1
 }
 
+# Usage: __test_terminated_err
+# Handler that is called when the test terminated unexpectedly due to "errexit".
+function __test_terminated_err() {
+    # When a subshell exits due to signal ERR, its parent shell also exits,
+    # thus the signal handler is called recursively and we print out the
+    # error message and stack trace multiple times. We're only interested
+    # in the first one though, as it contains the most information, so ignore
+    # all following.
+    if [[ -f $TEST_TMPDIR/__err_handled ]]; then
+      exit 1
+    fi
+    __show_log >&2
+    if [[ ! -z "$TEST_name" ]]; then
+      echo -n "$TEST_name "
+    fi
+    echo "FAILED: terminated because this command returned a non-zero status:" >&2
+    touch $TEST_TMPDIR/__err_handled
+    TEST_passed="false"
+    __show_stack
+    # If $TEST_name is still empty, the test suite failed before we even started
+    # to run tests, so we shouldn't call tear_down.
+    if [[ ! -z "$TEST_name" ]]; then
+      tear_down
+    fi
+    exit 1
+}
+
 # Usage: __trap_with_arg <handler> <signals ...>
 # Helper to install a trap handler for several signals preserving the signal
 # number, so that the signal number is available to the trap handler.
@@ -528,6 +572,7 @@ function run_suite() {
         ATEXIT=
 
         # Run test in a subshell.
+        rm -f $TEST_TMPDIR/__err_handled
         __trap_with_arg __test_terminated INT KILL PIPE TERM ABRT FPE ILL QUIT SEGV
         (
           timestamp >$TEST_TMPDIR/__ts_start