test: Better handling of stdout and stderr

Git-style tests (test_expect_success etc.) suppress stdout and stderr
unless -v is given. Notmuch-style tests (created by test_begin_subtest
and test_expect_equal) do not have this behavior so implement it the
same.

Additionally, for both test styles, the test-lib.sh is changed so that
the content of suppressed stdout and stderr is shown in case of failed
test.

Finally a test for this functionality is added to basic tests.

5 files changed, 106 insertions, 7 deletions

diff --git a/test/basic b/test/basic
index 725c7538..309779ca 100755
--- a/test/basic
+++ b/test/basic
@@ -52,9 +52,28 @@ test_expect_code 2 'failure to clean up causes the test to fail' '
 # Ensure that all tests are being run
 test_begin_subtest 'Ensure that all available tests will be run by notmuch-test'
 tests_in_suite=$(grep TESTS= ../notmuch-test | sed -e "s/TESTS=\"\(.*\)\"/\1/" | tr " " "\n" | sort)
-available=$(ls -1 ../ | grep -v -E "^(aggregate-results.sh|Makefile|Makefile.local|notmuch-test|README|test-lib.sh|test-results|tmp.*|valgrind|corpus*|emacs.expected-output|smtp-dummy|smtp-dummy.c)" | sort)
+available=$(ls -1 ../ | grep -v -E "^(aggregate-results.sh|Makefile|Makefile.local|notmuch-test|README|test-lib.sh|test-results|tmp.*|valgrind|corpus*|emacs.expected-output|smtp-dummy|smtp-dummy.c|test-verbose|test.expected-output)" | sort)
 test_expect_equal "$tests_in_suite" "$available"
 
+EXPECTED=../test.expected-output
+suppress_diff_date() {
+    sed -e 's/\(.*\-\-\- test-verbose\.4\.\expected\).*/\1/' \
+	-e 's/\(.*\+\+\+ test-verbose\.4\.\output\).*/\1/'
+}
+
+test_begin_subtest "Ensure that test output is suppressed unless the test fails"
+output=$(cd ..; ./test-verbose 2>&1 | suppress_diff_date)
+expected=$(cat $EXPECTED/test-verbose-no | suppress_diff_date)
+test_expect_equal "$output" "$expected"
+
+test_begin_subtest "Ensure that -v does not suppress test output"
+output=$(cd ..; ./test-verbose -v 2>&1 | suppress_diff_date)
+expected=$(cat $EXPECTED/test-verbose-yes | suppress_diff_date)
+# Do not include the results of test-verbose in totals
+rm $TEST_DIRECTORY/test-results/test-verbose-*
+test_expect_equal "$output" "$expected"
+
+
 ################################################################
 # Test mail store prepared in test-lib.sh
 
diff --git a/test/test-lib.sh b/test/test-lib.sh
index 418eaa7f..68c9cf8a 100644
--- a/test/test-lib.sh
+++ b/test/test-lib.sh
@@ -165,12 +165,6 @@ fi
 echo $(basename "$0"): "Testing ${test_description}"
 
 exec 5>&1
-if test "$verbose" = "t"
-then
-	exec 4>&2 3>&1
-else
-	exec 4>/dev/null 3>/dev/null
-fi
 
 test_failure=0
 test_count=0
@@ -403,6 +397,11 @@ add_email_corpus ()
 test_begin_subtest ()
 {
     test_subtest_name="$1"
+    # Remember stdout and stderr file descriptios and redirect test
+    # output to the previously prepared file descriptors 3 and 4 (see
+    # bellow)
+    if test "$verbose" != "t"; then exec 4>test.output 3>&4; fi
+    exec 6>&1 7>&2 >&3 2>&4
 }
 
 # Pass test if two arguments match
@@ -413,6 +412,7 @@ test_begin_subtest ()
 # name.
 test_expect_equal ()
 {
+	exec 1>&6 2>&7		# Restore stdout and stderr
 	test "$#" = 3 && { prereq=$1; shift; } || prereq=
 	test "$#" = 2 ||
 	error "bug in the test script: not 2 or 3 parameters to test_expect_equal"
@@ -508,6 +508,7 @@ test_failure_ () {
 	echo " $1"
 	shift
 	echo "$@" | sed -e 's/^/	/'
+	if test "$verbose" != "t"; then cat test.output; fi
 	test "$immediate" = "" || { GIT_EXIT_OK=t; exit 1; }
 }
 
@@ -529,6 +530,7 @@ test_debug () {
 
 test_run_ () {
 	test_cleanup=:
+	if test "$verbose" != "t"; then exec 4>test.output 3>&4; fi
 	eval >&3 2>&4 "$1"
 	eval_ret=$?
 	eval >&3 2>&4 "$test_cleanup"
@@ -918,6 +920,13 @@ EOF
 # in subprocesses like git equals our $PWD (for pathname comparisons).
 cd -P "$test" || error "Cannot setup test environment"
 
+if test "$verbose" = "t"
+then
+	exec 4>&2 3>&1
+else
+	exec 4>test.output 3>&4
+fi
+
 this_test=${0##*/}
 for skp in $NOTMUCH_SKIP_TESTS
 do
diff --git a/test/test-verbose b/test/test-verbose
new file mode 100755
index 00000000..f29a9c7d
--- /dev/null
+++ b/test/test-verbose
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+test_description='the verbosity options of the test framework itself.'
+
+. ./test-lib.sh
+
+test_expect_success 'print something in test_expect_success and pass' '
+  echo "hello stdout" &&
+  echo "hello stderr" >&2 &&
+  true
+'
+test_expect_success 'print something in test_expect_success and fail' '
+  echo "hello stdout" &&
+  echo "hello stderr" >&2 &&
+  false
+'
+test_begin_subtest 'print something between test_begin_subtest and test_expect_equal and pass'
+echo "hello stdout"
+echo "hello stderr" >&2
+test_expect_equal "a" "a"
+
+test_begin_subtest 'print something test_begin_subtest and test_expect_equal and fail'
+echo "hello stdout"
+echo "hello stderr" >&2
+test_expect_equal "a" "b"
+
+test_done
diff --git a/test/test.expected-output/test-verbose-no b/test/test.expected-output/test-verbose-no
new file mode 100644
index 00000000..0bca7540
--- /dev/null
+++ b/test/test.expected-output/test-verbose-no
@@ -0,0 +1,20 @@
+test-verbose: Testing the verbosity options of the test framework itself.
+ PASS   print something in test_expect_success and pass
+ FAIL   print something in test_expect_success and fail
+	
+	  echo "hello stdout" &&
+	  echo "hello stderr" >&2 &&
+	  false
+	
+hello stdout
+hello stderr
+ PASS   print something between test_begin_subtest and test_expect_equal and pass
+ FAIL   print something test_begin_subtest and test_expect_equal and fail
+	--- test-verbose.4.expected	2010-11-14 21:41:12.738189710 +0000
+	+++ test-verbose.4.output	2010-11-14 21:41:12.738189710 +0000
+	@@ -1 +1 @@
+	-b
+	+a
+hello stdout
+hello stderr
+
diff --git a/test/test.expected-output/test-verbose-yes b/test/test.expected-output/test-verbose-yes
new file mode 100644
index 00000000..ebe51874
--- /dev/null
+++ b/test/test.expected-output/test-verbose-yes
@@ -0,0 +1,24 @@
+test-verbose: Testing the verbosity options of the test framework itself.
+hello stdout
+hello stderr
+ PASS   print something in test_expect_success and pass
+hello stdout
+hello stderr
+ FAIL   print something in test_expect_success and fail
+	
+	  echo "hello stdout" &&
+	  echo "hello stderr" >&2 &&
+	  false
+	
+hello stdout
+hello stderr
+ PASS   print something between test_begin_subtest and test_expect_equal and pass
+hello stdout
+hello stderr
+ FAIL   print something test_begin_subtest and test_expect_equal and fail
+	--- test-verbose.4.expected	2010-11-14 21:41:06.650023289 +0000
+	+++ test-verbose.4.output	2010-11-14 21:41:06.650023289 +0000
+	@@ -1 +1 @@
+	-b
+	+a
+