Deflake --runs_per_test_detects_flakes.

Previously, if the first run failed (in iteration order, which I don't
think is necessarily execution order) then
--runs_per_test_detects_flakes would report FAILED instead of FLAKY.

--
Change-Id: Ice7889d46203e1598d94a4e3c0bcbe13a45b0fe1
Reviewed-on: https://bazel-review.googlesource.com/#/c/2210/
MOS_MIGRATED_REVID=106693725

1 files changed, 35 insertions, 0 deletions

diff --git a/src/test/shell/bazel/bazel_test_test.sh b/src/test/shell/bazel/bazel_test_test.sh
index a63f0b2439..9e7f609fe8 100755
--- a/src/test/shell/bazel/bazel_test_test.sh
+++ b/src/test/shell/bazel/bazel_test_test.sh
@@ -205,7 +205,42 @@ EOF
   bazel test --test_timeout=2 //dir:test &> $TEST_log && fail "should have timed out"
   expect_log "TIMEOUT"
   bazel test --test_timeout=4 //dir:test || fail "expected success"
+}
+
+# Makes sure that runs_per_test_detects_flakes detects FLAKY if any of the 5
+# attempts passes (which should cover all cases of being picky about the
+# first/last/etc ones only being counted).
+# We do this using an un-sandboxed test which keeps track of how many runs there
+# have been using files which are undeclared inputs/outputs.
+function test_runs_per_test_detects_flakes() {
+  # Directory for counters
+  local COUNTER_DIR="${TEST_TMPDIR}/counter_dir"
+  mkdir -p "${COUNTER_DIR}"
+
+  for (( i = 1 ; i <= 5 ; i++ )); do
+
+    # This file holds the number of the next run
+    echo 1 > "${COUNTER_DIR}/$i"
+    cat <<EOF > test$i.sh
+#!/bin/bash
+i=\$(< "${COUNTER_DIR}/$i")
+
+# increment the hidden state
+echo \$((i + 1)) > "${COUNTER_DIR}/$i"
 
+# succeed exactly once.
+exit \$((i != $i))
+}
+EOF
+    chmod +x test$i.sh
+    cat <<EOF > BUILD
+sh_test(name = "test$i", srcs = [ "test$i.sh" ])
+EOF
+    bazel test --spawn_strategy=standalone --jobs=1 \
+        --runs_per_test=5 --runs_per_test_detects_flakes \
+        //:test$i &> $TEST_log || fail "should have succeeded"
+    expect_log "FLAKY"
+  done
 }
 
 run_suite "test tests"