remote: don't fail build if upload fails

If the upload of local build artifacts fails, the build no longer fails
but instead a warning is printed once. If --verbose_failures is
specified, a detailed warning is printed for every failure.

This helps fixing #2964, however it doesn't fully fix it due to timeouts
and retries slowing the build significantly.

Also, add some other tests related to fallback behavior.

Change-Id: Ief49941f9bc7e0123b5d93456d77428686dd5268
PiperOrigin-RevId: 165938874

7 files changed, 259 insertions, 20 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java
index 826de367c8..8bb5a27e43 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java
@@ -51,7 +51,8 @@ final class RemoteActionContextProvider extends ActionContextProvider {
     RemoteOptions remoteOptions = checkNotNull(env.getOptions().getOptions(RemoteOptions.class));
 
     if (remoteOptions.experimentalRemoteSpawnCache) {
-      RemoteSpawnCache spawnCache = new RemoteSpawnCache(env.getExecRoot(), remoteOptions, cache);
+      RemoteSpawnCache spawnCache = new RemoteSpawnCache(env.getExecRoot(), remoteOptions, cache,
+          executionOptions.verboseFailures, env.getReporter());
       return ImmutableList.of(spawnCache);
     } else {
       RemoteSpawnRunner spawnRunner = new RemoteSpawnRunner(
@@ -59,6 +60,7 @@ final class RemoteActionContextProvider extends ActionContextProvider {
           remoteOptions,
           createFallbackRunner(env),
           executionOptions.verboseFailures,
+          env.getReporter(),
           cache,
           executor);
       return ImmutableList.of(new RemoteSpawnStrategy(spawnRunner));
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java
index 5d5c3e8d16..2b74e88b3f 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java
@@ -18,6 +18,8 @@ import com.google.devtools.build.lib.actions.ExecException;
 import com.google.devtools.build.lib.actions.ExecutionStrategy;
 import com.google.devtools.build.lib.actions.Spawn;
 import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.Reporter;
 import com.google.devtools.build.lib.exec.SpawnCache;
 import com.google.devtools.build.lib.exec.SpawnResult;
 import com.google.devtools.build.lib.exec.SpawnResult.Status;
@@ -34,6 +36,8 @@ import java.io.IOException;
 import java.util.Collection;
 import java.util.NoSuchElementException;
 import java.util.SortedMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import javax.annotation.Nullable;
 
 /**
  * A remote {@link SpawnCache} implementation.
@@ -50,12 +54,21 @@ final class RemoteSpawnCache implements SpawnCache {
   private final Platform platform;
 
   private final RemoteActionCache remoteCache;
+  private final boolean verboseFailures;
 
-  RemoteSpawnCache(Path execRoot, RemoteOptions options, RemoteActionCache remoteCache) {
+  @Nullable private final Reporter cmdlineReporter;
+
+  // Used to ensure that a warning is reported only once.
+  private final AtomicBoolean warningReported = new AtomicBoolean();
+
+  RemoteSpawnCache(Path execRoot, RemoteOptions options, RemoteActionCache remoteCache,
+      boolean verboseFailures, @Nullable Reporter cmdlineReporter) {
     this.execRoot = execRoot;
     this.options = options;
     this.platform = options.parseRemotePlatformOverride();
     this.remoteCache = remoteCache;
+    this.verboseFailures = verboseFailures;
+    this.cmdlineReporter = cmdlineReporter;
   }
 
   @Override
@@ -118,7 +131,15 @@ final class RemoteSpawnCache implements SpawnCache {
           if (result.status() != Status.SUCCESS || result.exitCode() != 0) {
             return;
           }
-          remoteCache.upload(actionKey, execRoot, files, policy.getFileOutErr());
+          try {
+            remoteCache.upload(actionKey, execRoot, files, policy.getFileOutErr());
+          } catch (IOException e) {
+            if (verboseFailures) {
+              report(Event.debug("Upload to remote cache failed: " + e.getMessage()));
+            } else {
+              reportOnce(Event.warn("Some artifacts failed be uploaded to the remote cache."));
+            }
+          }
         }
 
         @Override
@@ -129,4 +150,16 @@ final class RemoteSpawnCache implements SpawnCache {
       return SpawnCache.NO_RESULT_NO_STORE;
     }
   }
+
+  private void reportOnce(Event evt) {
+    if (warningReported.compareAndSet(false, true)) {
+      report(evt);
+    }
+  }
+
+  private void report(Event evt) {
+    if (cmdlineReporter != null) {
+      cmdlineReporter.handle(evt);
+    }
+  }
 }
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java
index 72c356b73f..26fba9d25c 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java
@@ -25,6 +25,8 @@ import com.google.devtools.build.lib.actions.Spawn;
 import com.google.devtools.build.lib.actions.Spawns;
 import com.google.devtools.build.lib.actions.cache.VirtualActionInput;
 import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.Reporter;
 import com.google.devtools.build.lib.exec.SpawnInputExpander;
 import com.google.devtools.build.lib.exec.SpawnResult;
 import com.google.devtools.build.lib.exec.SpawnResult.Status;
@@ -52,6 +54,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.SortedMap;
 import java.util.TreeSet;
+import java.util.concurrent.atomic.AtomicBoolean;
 import javax.annotation.Nullable;
 
 /** A client for the remote execution service. */
@@ -64,14 +67,19 @@ class RemoteSpawnRunner implements SpawnRunner {
   private final SpawnRunner fallbackRunner;
   private final boolean verboseFailures;
 
+  @Nullable private final Reporter cmdlineReporter;
   @Nullable private final RemoteActionCache remoteCache;
   @Nullable private final GrpcRemoteExecutor remoteExecutor;
 
+  // Used to ensure that a warning is reported only once.
+  private final AtomicBoolean warningReported = new AtomicBoolean();
+
   RemoteSpawnRunner(
       Path execRoot,
       RemoteOptions options,
       SpawnRunner fallbackRunner,
       boolean verboseFailures,
+      @Nullable Reporter cmdlineReporter,
       @Nullable RemoteActionCache remoteCache,
       @Nullable GrpcRemoteExecutor remoteExecutor) {
     this.execRoot = execRoot;
@@ -81,6 +89,7 @@ class RemoteSpawnRunner implements SpawnRunner {
     this.remoteCache = remoteCache;
     this.remoteExecutor = remoteExecutor;
     this.verboseFailures = verboseFailures;
+    this.cmdlineReporter = cmdlineReporter;
   }
 
   @Override
@@ -132,8 +141,8 @@ class RemoteSpawnRunner implements SpawnRunner {
         }
       }
     } catch (IOException e) {
-      return execLocallyOrFail(spawn, policy, inputMap, actionKey,
-          options.remoteUploadLocalResults, e);
+      return execLocallyOrFail(
+          spawn, policy, inputMap, actionKey, uploadLocalResults, e);
     }
 
     if (remoteExecutor == null) {
@@ -145,8 +154,8 @@ class RemoteSpawnRunner implements SpawnRunner {
       // Upload the command and all the inputs into the remote cache.
       remoteCache.ensureInputsPresent(repository, execRoot, inputRoot, command);
     } catch (IOException e) {
-      return execLocallyOrFail(spawn, policy, inputMap, actionKey,
-          options.remoteUploadLocalResults, e);
+      return execLocallyOrFail(
+          spawn, policy, inputMap, actionKey, uploadLocalResults, e);
     }
 
     final ActionResult result;
@@ -158,15 +167,14 @@ class RemoteSpawnRunner implements SpawnRunner {
 
     boolean executionFailed = result.getExitCode() != 0;
     if (options.remoteLocalFallback && executionFailed) {
-      return execLocally(spawn, policy, inputMap, options.remoteUploadLocalResults,
-          remoteCache, actionKey);
+      return execLocally(spawn, policy, inputMap, uploadLocalResults, remoteCache, actionKey);
     }
 
     try {
       return downloadRemoteResults(result, policy.getFileOutErr());
     } catch (IOException e) {
-      return execLocallyOrFail(spawn, policy, inputMap, actionKey,
-          options.remoteUploadLocalResults, e);
+      return execLocallyOrFail(
+          spawn, policy, inputMap, actionKey, uploadLocalResults, e);
     }
   }
 
@@ -197,8 +205,7 @@ class RemoteSpawnRunner implements SpawnRunner {
       boolean uploadLocalResults, IOException cause)
       throws ExecException, InterruptedException, IOException {
     if (options.remoteLocalFallback) {
-      return execLocally(spawn, policy, inputMap, uploadLocalResults,
-          remoteCache, actionKey);
+      return execLocally(spawn, policy, inputMap, uploadLocalResults, remoteCache, actionKey);
     }
     throw new EnvironmentalExecException(errorMessage(cause), cause, true);
   }
@@ -315,10 +322,30 @@ class RemoteSpawnRunner implements SpawnRunner {
       }
     }
     List<Path> outputFiles = listExistingOutputFiles(execRoot, spawn);
-    remoteCache.upload(actionKey, execRoot, outputFiles, policy.getFileOutErr());
+    try {
+      remoteCache.upload(actionKey, execRoot, outputFiles, policy.getFileOutErr());
+    } catch (IOException e) {
+      if (verboseFailures) {
+        report(Event.debug("Upload to remote cache failed: " + e.getMessage()));
+      } else {
+        reportOnce(Event.warn("Some artifacts failed be uploaded to the remote cache."));
+      }
+    }
     return result;
   }
 
+  private void reportOnce(Event evt) {
+    if (warningReported.compareAndSet(false, true)) {
+      report(evt);
+    }
+  }
+
+  private void report(Event evt) {
+    if (cmdlineReporter != null) {
+      cmdlineReporter.handle(evt);
+    }
+  }
+
   static List<Path> listExistingOutputFiles(Path execRoot, Spawn spawn) {
     ArrayList<Path> outputFiles = new ArrayList<>();
     for (ActionInput output : spawn.getOutputFiles()) {
diff --git a/src/test/java/com/google/devtools/build/lib/BUILD b/src/test/java/com/google/devtools/build/lib/BUILD
index 033da62bf1..cd0ea1419f 100644
--- a/src/test/java/com/google/devtools/build/lib/BUILD
+++ b/src/test/java/com/google/devtools/build/lib/BUILD
@@ -1081,6 +1081,7 @@ java_test(
         ":testutil",
         "//src/main/java/com/google/devtools/build/lib:auth_and_tls_options",
         "//src/main/java/com/google/devtools/build/lib:build-base",
+        "//src/main/java/com/google/devtools/build/lib:events",
         "//src/main/java/com/google/devtools/build/lib:inmemoryfs",
         "//src/main/java/com/google/devtools/build/lib:io",
         "//src/main/java/com/google/devtools/build/lib:preconditions",
diff --git a/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteExecutionClientTest.java b/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteExecutionClientTest.java
index daed01ac5a..1db31cc372 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteExecutionClientTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteExecutionClientTest.java
@@ -228,7 +228,8 @@ public class GrpcRemoteExecutionClientTest {
         GrpcUtils.newCallCredentials(Options.getDefaults(AuthAndTLSOptions.class));
     GrpcRemoteCache remoteCache =
         new GrpcRemoteCache(channel, creds, options, retrier);
-    client = new RemoteSpawnRunner(execRoot, options, null, true, remoteCache, executor);
+    client = new RemoteSpawnRunner(execRoot, options, null, true, /*cmdlineReporter=*/null,
+        remoteCache, executor);
     inputDigest = fakeFileCache.createScratchInput(simpleSpawn.getInputFiles().get(0), "xyz");
   }
 
diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java
index 930815bcf5..43e4de4f54 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java
@@ -16,12 +16,14 @@ package com.google.devtools.build.lib.remote;
 import static com.google.common.truth.Truth.assertThat;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.eventbus.EventBus;
 import com.google.devtools.build.lib.actions.ActionInput;
 import com.google.devtools.build.lib.actions.ActionInputFileCache;
 import com.google.devtools.build.lib.actions.ActionInputHelper;
@@ -29,6 +31,10 @@ import com.google.devtools.build.lib.actions.Artifact;
 import com.google.devtools.build.lib.actions.Artifact.ArtifactExpander;
 import com.google.devtools.build.lib.actions.ResourceSet;
 import com.google.devtools.build.lib.actions.SimpleSpawn;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventKind;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.events.StoredEventHandler;
 import com.google.devtools.build.lib.exec.SpawnCache.CacheHandle;
 import com.google.devtools.build.lib.exec.SpawnInputExpander;
 import com.google.devtools.build.lib.exec.SpawnResult;
@@ -77,6 +83,8 @@ public class RemoteSpawnCacheTest {
   private RemoteSpawnCache cache;
   private FileOutErr outErr;
 
+  private StoredEventHandler eventHandler = new StoredEventHandler();
+
   private final SpawnExecutionPolicy simplePolicy =
       new SpawnExecutionPolicy() {
         @Override
@@ -155,7 +163,10 @@ public class RemoteSpawnCacheTest {
     FileSystemUtils.createDirectoryAndParents(stderr.getParentDirectory());
     outErr = new FileOutErr(stdout, stderr);
     RemoteOptions options = Options.getDefaults(RemoteOptions.class);
-    cache = new RemoteSpawnCache(execRoot, options, remoteCache);
+    Reporter reporter = new Reporter(new EventBus());
+    eventHandler = new StoredEventHandler();
+    reporter.addHandler(eventHandler);
+    cache = new RemoteSpawnCache(execRoot, options, remoteCache, false, reporter);
     fakeFileCache.createScratchInput(simpleSpawn.getInputFiles().get(0), "xyz");
   }
 
@@ -200,4 +211,31 @@ public class RemoteSpawnCacheTest {
             eq(outputFiles),
             eq(outErr));
   }
+
+  @Test
+  public void printWarningIfUploadFails() throws Exception {
+    CacheHandle entry = cache.lookup(simpleSpawn, simplePolicy);
+    assertThat(entry.hasResult()).isFalse();
+    SpawnResult result = new SpawnResult.Builder().setExitCode(0).setStatus(Status.SUCCESS).build();
+    ImmutableList<Path> outputFiles = ImmutableList.of(fs.getPath("/random/file"));
+
+    doThrow(new IOException("cache down")).when(remoteCache).upload(any(ActionKey.class),
+        any(Path.class),
+        eq(outputFiles),
+        eq(outErr));
+
+    entry.store(result, outputFiles);
+    verify(remoteCache)
+        .upload(
+            any(ActionKey.class),
+            any(Path.class),
+            eq(outputFiles),
+            eq(outErr));
+
+    assertThat(eventHandler.getEvents()).hasSize(1);
+    Event evt = eventHandler.getEvents().get(0);
+    assertThat(evt.getKind()).isEqualTo(EventKind.WARNING);
+    assertThat(evt.getMessage()).contains("fail");
+    assertThat(evt.getMessage()).contains("upload");
+  }
 }
diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java
index 180a0dfcf1..b8e4f21ec0 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java
@@ -17,6 +17,8 @@ import static com.google.common.truth.Truth.assertThat;
 import static org.junit.Assert.fail;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.verify;
@@ -25,6 +27,7 @@ import static org.mockito.Mockito.when;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.eventbus.EventBus;
 import com.google.devtools.build.lib.actions.ActionInput;
 import com.google.devtools.build.lib.actions.ActionInputFileCache;
 import com.google.devtools.build.lib.actions.Artifact.ArtifactExpander;
@@ -33,6 +36,10 @@ import com.google.devtools.build.lib.actions.ExecutionRequirements;
 import com.google.devtools.build.lib.actions.ResourceSet;
 import com.google.devtools.build.lib.actions.SimpleSpawn;
 import com.google.devtools.build.lib.actions.Spawn;
+import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.events.EventKind;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.events.StoredEventHandler;
 import com.google.devtools.build.lib.exec.SpawnInputExpander;
 import com.google.devtools.build.lib.exec.SpawnResult;
 import com.google.devtools.build.lib.exec.SpawnResult.Status;
@@ -112,7 +119,8 @@ public class RemoteSpawnRunnerTest {
     options.remoteUploadLocalResults = true;
 
     RemoteSpawnRunner runner =
-        new RemoteSpawnRunner(execRoot, options, localRunner, true, cache, executor);
+        new RemoteSpawnRunner(execRoot, options, localRunner, true, /*cmdlineReporter=*/null,
+            cache, executor);
 
     ExecuteResponse succeeded = ExecuteResponse.newBuilder().setResult(
         ActionResult.newBuilder().setExitCode(0).build()).build();
@@ -154,7 +162,8 @@ public class RemoteSpawnRunnerTest {
     options.remoteUploadLocalResults = true;
 
     RemoteSpawnRunner runner =
-        new RemoteSpawnRunner(execRoot, options, localRunner, true, cache, null);
+        new RemoteSpawnRunner(execRoot, options, localRunner, true, /*cmdlineReporter=*/null,
+            cache, null);
 
     // Throw an IOException to trigger the local fallback.
     when(executor.executeRemotely(any(ExecuteRequest.class))).thenThrow(IOException.class);
@@ -190,7 +199,8 @@ public class RemoteSpawnRunnerTest {
     options.remoteUploadLocalResults = true;
 
     RemoteSpawnRunner runner =
-        spy(new RemoteSpawnRunner(execRoot, options, localRunner, true, cache, null));
+        spy(new RemoteSpawnRunner(execRoot, options, localRunner, true, /*cmdlineReporter=*/null,
+            cache, null));
 
     Spawn spawn = new SimpleSpawn(
         new FakeOwner("foo", "bar"),
@@ -236,7 +246,8 @@ public class RemoteSpawnRunnerTest {
     SpawnExecutionPolicy policy = new FakeSpawnExecutionPolicy(spawn);
 
     RemoteSpawnRunner runner =
-        spy(new RemoteSpawnRunner(execRoot, options, localRunner, true, cache, null));
+        spy(new RemoteSpawnRunner(execRoot, options, localRunner, true, /*cmdlineReporter=*/null,
+            cache, null));
 
     try {
       runner.exec(spawn, policy);
@@ -246,6 +257,132 @@ public class RemoteSpawnRunnerTest {
     }
   }
 
+  @Test
+  @SuppressWarnings("unchecked")
+  public void printWarningIfCacheIsDown() throws Exception {
+    // If we try to upload to a local cache, that is down a warning should be printed.
+
+    RemoteOptions options = Options.getDefaults(RemoteOptions.class);
+    options.remoteUploadLocalResults = true;
+    options.remoteLocalFallback = true;
+
+    Reporter reporter = new Reporter(new EventBus());
+    StoredEventHandler eventHandler = new StoredEventHandler();
+    reporter.addHandler(eventHandler);
+
+    RemoteSpawnRunner runner =
+        new RemoteSpawnRunner(execRoot, options, localRunner, false, reporter, cache, null);
+
+    Spawn spawn =
+        new SimpleSpawn(
+            new FakeOwner("foo", "bar"),
+            /*arguments=*/ ImmutableList.of(),
+            /*environment=*/ ImmutableMap.of(),
+            /*executionInfo=*/ ImmutableMap.of(),
+            /*inputs=*/ ImmutableList.of(),
+            /*outputs=*/ ImmutableList.<ActionInput>of(),
+            ResourceSet.ZERO);
+    SpawnExecutionPolicy policy = new FakeSpawnExecutionPolicy(spawn);
+
+    when(cache.getCachedActionResult(any(ActionKey.class)))
+        .thenThrow(new IOException("cache down"));
+
+    doThrow(new IOException("cache down")).when(cache)
+        .upload(any(ActionKey.class), any(Path.class), any(Collection.class),
+            any(FileOutErr.class));
+
+    SpawnResult res = new SpawnResult.Builder().setStatus(Status.SUCCESS).setExitCode(0).build();
+    when(localRunner.exec(eq(spawn), eq(policy))).thenReturn(res);
+
+    assertThat(runner.exec(spawn, policy)).isEqualTo(res);
+
+    verify(localRunner).exec(eq(spawn), eq(policy));
+
+    assertThat(eventHandler.getEvents()).hasSize(1);
+
+    Event evt = eventHandler.getEvents().get(0);
+    assertThat(evt.getKind()).isEqualTo(EventKind.WARNING);
+    assertThat(evt.getMessage()).contains("fail");
+    assertThat(evt.getMessage()).contains("upload");
+  }
+
+  @Test
+  public void fallbackFails() throws Exception {
+    // Errors from the fallback runner should be propogated out of the remote runner.
+
+    RemoteOptions options = Options.getDefaults(RemoteOptions.class);
+    options.remoteUploadLocalResults = true;
+    options.remoteLocalFallback = true;
+
+    RemoteSpawnRunner runner =
+        new RemoteSpawnRunner(execRoot, options, localRunner, true, /*cmdlineReporter=*/null,
+            cache, null);
+
+    Spawn spawn =
+        new SimpleSpawn(
+            new FakeOwner("foo", "bar"),
+            /*arguments=*/ ImmutableList.of(),
+            /*environment=*/ ImmutableMap.of(),
+            /*executionInfo=*/ ImmutableMap.of(),
+            /*inputs=*/ ImmutableList.of(),
+            /*outputs=*/ ImmutableList.<ActionInput>of(),
+            ResourceSet.ZERO);
+    SpawnExecutionPolicy policy = new FakeSpawnExecutionPolicy(spawn);
+
+    when(cache.getCachedActionResult(any(ActionKey.class))).thenReturn(null);
+
+    IOException err = new IOException("local execution error");
+    when(localRunner.exec(eq(spawn), eq(policy))).thenThrow(err);
+
+    try {
+      runner.exec(spawn, policy);
+      fail("expected IOException to be raised");
+    } catch (IOException e) {
+      assertThat(e).isSameAs(err);
+    }
+
+    verify(localRunner).exec(eq(spawn), eq(policy));
+  }
+
+  @Test
+  public void cacheDownloadFailureTriggersRemoteExecution() throws Exception {
+    // If downloading a cached action fails, remote execution should be tried.
+
+    RemoteOptions options = Options.getDefaults(RemoteOptions.class);
+
+    RemoteSpawnRunner runner =
+        new RemoteSpawnRunner(execRoot, options, localRunner, true, /*cmdlineReporter=*/null,
+            cache, executor);
+
+    ActionResult cachedResult = ActionResult.newBuilder().setExitCode(0).build();
+    when(cache.getCachedActionResult(any(ActionKey.class))).thenReturn(cachedResult);
+    doThrow(CacheNotFoundException.class)
+        .when(cache)
+        .download(eq(cachedResult), any(Path.class), any(FileOutErr.class));
+    ActionResult execResult = ActionResult.newBuilder().setExitCode(31).build();
+    ExecuteResponse succeeded = ExecuteResponse.newBuilder().setResult(execResult).build();
+    when(executor.executeRemotely(any(ExecuteRequest.class))).thenReturn(succeeded);
+    doNothing().when(cache).download(eq(execResult), any(Path.class), any(FileOutErr.class));
+
+    Spawn spawn =
+        new SimpleSpawn(
+            new FakeOwner("foo", "bar"),
+            /*arguments=*/ ImmutableList.of(),
+            /*environment=*/ ImmutableMap.of(),
+            /*executionInfo=*/ ImmutableMap.of(),
+            /*inputs=*/ ImmutableList.of(),
+            /*outputs=*/ ImmutableList.<ActionInput>of(),
+            ResourceSet.ZERO);
+
+    SpawnExecutionPolicy policy = new FakeSpawnExecutionPolicy(spawn);
+
+    SpawnResult res = runner.exec(spawn, policy);
+    assertThat(res.status()).isEqualTo(Status.SUCCESS);
+    assertThat(res.exitCode()).isEqualTo(31);
+
+    verify(executor).executeRemotely(any(ExecuteRequest.class));
+  }
+
   // TODO(buchgr): Extract a common class to be used for testing.
   class FakeSpawnExecutionPolicy implements SpawnExecutionPolicy {