More resilient to remote spawn cache errors

Bazel used to fail the entire build if there is a single error with remote
spawn cache. This could happen on unreliable network or when server is under
load. This change will allow Bazel to fallback to local execution with a
warning.

Change-Id: I0bd41dc862b33a2ab2ab772d8ce9c943476edbf4
PiperOrigin-RevId: 175951699

2 files changed, 52 insertions, 13 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java
index c6c766454e..4a6ba9c61e 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java
@@ -114,18 +114,21 @@ final class RemoteSpawnCache implements SpawnCache {
         // We don't cache failed actions, so we know the outputs exist.
         // For now, download all outputs locally; in the future, we can reuse the digests to
         // just update the TreeNodeRepository and continue the build.
-        try {
-          remoteCache.download(result, execRoot, policy.getFileOutErr());
-          SpawnResult spawnResult =
-              new SpawnResult.Builder()
-                  .setStatus(Status.SUCCESS)
-                  .setExitCode(result.getExitCode())
-                  .build();
-          return SpawnCache.success(spawnResult);
-        } catch (CacheNotFoundException e) {
-          // There's a cache miss. Fall back to local execution.
-        }
+        remoteCache.download(result, execRoot, policy.getFileOutErr());
+        SpawnResult spawnResult =
+            new SpawnResult.Builder()
+                .setStatus(Status.SUCCESS)
+                .setExitCode(result.getExitCode())
+                .build();
+        return SpawnCache.success(spawnResult);
       }
+    } catch (CacheNotFoundException e) {
+      // There's a cache miss. Fall back to local execution.
+    } catch (IOException e) {
+      // There's an IO error. Fall back to local execution.
+      reportOnce(
+          Event.warn(
+              "Some artifacts failed be downloaded from the remote cache: " + e.getMessage()));
     } finally {
       withMetadata.detach(previous);
     }
@@ -157,7 +160,9 @@ final class RemoteSpawnCache implements SpawnCache {
             if (verboseFailures) {
               report(Event.debug("Upload to remote cache failed: " + e.getMessage()));
             } else {
-              reportOnce(Event.warn("Some artifacts failed be uploaded to the remote cache."));
+              reportOnce(
+                  Event.warn(
+                      "Some artifacts failed be uploaded to the remote cache: " + e.getMessage()));
             }
           } finally {
             withMetadata.detach(previous);
diff --git a/src/test/shell/bazel/remote_execution_test.sh b/src/test/shell/bazel/remote_execution_test.sh
index 054123e0aa..be74814850 100755
--- a/src/test/shell/bazel/remote_execution_test.sh
+++ b/src/test/shell/bazel/remote_execution_test.sh
@@ -216,7 +216,7 @@ EOF
 
   bazel clean --expunge >& $TEST_log
   bazel --host_jvm_args=-Dbazel.DigestFunction=SHA1 build \
-      --spawn_strategy=remote \
+      --experimental_remote_spawn_cache=true  \
       --remote_rest_cache=http://localhost:${hazelcast_port}/hazelcast/rest/maps \
       //a:test >& $TEST_log \
       || fail "Failed to build //a:test with remote REST cache service"
@@ -231,6 +231,40 @@ EOF
   fi
 }
 
+function test_cc_binary_rest_cache_bad_server() {
+  mkdir -p a
+  cat > a/BUILD <<EOF
+package(default_visibility = ["//visibility:public"])
+cc_binary(
+name = 'test',
+srcs = [ 'test.cc' ],
+)
+EOF
+  cat > a/test.cc <<EOF
+#include <iostream>
+int main() { std::cout << "Hello world!" << std::endl; return 0; }
+EOF
+  bazel build //a:test >& $TEST_log \
+    || fail "Failed to build //a:test without remote cache"
+  cp -f bazel-bin/a/test ${TEST_TMPDIR}/test_expected
+
+  bazel clean --expunge >& $TEST_log
+  bazel --host_jvm_args=-Dbazel.DigestFunction=SHA1 build \
+      --experimental_remote_spawn_cache=true \
+      --remote_rest_cache=http://bad.hostname/bad/cache \
+      //a:test >& $TEST_log \
+      || fail "Failed to build //a:test with remote REST cache service"
+  diff bazel-bin/a/test ${TEST_TMPDIR}/test_expected \
+      || fail "Remote cache generated different result"
+  # Check that persistent connections are closed after the build. Is there a good cross-platform way
+  # to check this?
+  if [[ "$PLATFORM" = "linux" ]]; then
+    if netstat -tn | grep -qE ":${hazelcast_port}\\s+ESTABLISHED$"; then
+      fail "connections to to cache not closed"
+    fi
+  fi
+}
+
 function test_py_test() {
   mkdir -p a
   cat > a/BUILD <<EOF