Make the fallback strategy for Bazel's remote execution configurable.

RELNOTES: When using Bazel's remote execution feature and Bazel has to
fallback to local execution for an action, Bazel used non-sandboxed
local execution until now. From this release on, you can use the new
flag --remote_local_fallback_strategy=<strategy> to tell Bazel which
strategy to use in that case.
PiperOrigin-RevId: 206566380

5 files changed, 75 insertions, 39 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/exec/AbstractSpawnStrategy.java b/src/main/java/com/google/devtools/build/lib/exec/AbstractSpawnStrategy.java
index cb40215ad8..4be5e977c4 100644
--- a/src/main/java/com/google/devtools/build/lib/exec/AbstractSpawnStrategy.java
+++ b/src/main/java/com/google/devtools/build/lib/exec/AbstractSpawnStrategy.java
@@ -58,6 +58,17 @@ public abstract class AbstractSpawnStrategy implements SandboxedSpawnActionConte
     this.spawnRunner = spawnRunner;
   }
 
+  /**
+   * Get's the {@link SpawnRunner} that this {@link AbstractSpawnStrategy} uses to actually run
+   * spawns.
+   *
+   * <p>This is considered a stop-gap until we refactor the entire SpawnStrategy / SpawnRunner
+   * mechanism to no longer need Spawn strategies.
+   */
+  public SpawnRunner getSpawnRunner() {
+    return spawnRunner;
+  }
+
   @Override
   public List<SpawnResult> exec(Spawn spawn, ActionExecutionContext actionExecutionContext)
       throws ExecException, InterruptedException {
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java
index 88bc907a1e..2b44f481cd 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java
@@ -15,22 +15,24 @@ package com.google.devtools.build.lib.remote;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
+import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import com.google.devtools.build.lib.actions.ActionContext;
-import com.google.devtools.build.lib.actions.ResourceManager;
+import com.google.devtools.build.lib.actions.ExecutionStrategy;
+import com.google.devtools.build.lib.actions.ExecutorInitException;
+import com.google.devtools.build.lib.exec.AbstractSpawnStrategy;
 import com.google.devtools.build.lib.exec.ActionContextProvider;
 import com.google.devtools.build.lib.exec.ExecutionOptions;
 import com.google.devtools.build.lib.exec.SpawnRunner;
-import com.google.devtools.build.lib.exec.apple.XcodeLocalEnvProvider;
-import com.google.devtools.build.lib.exec.local.LocalEnvProvider;
-import com.google.devtools.build.lib.exec.local.LocalExecutionOptions;
-import com.google.devtools.build.lib.exec.local.LocalSpawnRunner;
-import com.google.devtools.build.lib.exec.local.PosixLocalEnvProvider;
-import com.google.devtools.build.lib.exec.local.WindowsLocalEnvProvider;
 import com.google.devtools.build.lib.remote.util.DigestUtil;
 import com.google.devtools.build.lib.runtime.CommandEnvironment;
-import com.google.devtools.build.lib.util.OS;
+import com.google.devtools.build.lib.util.ExitCode;
 import com.google.devtools.build.lib.vfs.Path;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.concurrent.atomic.AtomicReference;
 import javax.annotation.Nullable;
 
 /**
@@ -43,6 +45,7 @@ final class RemoteActionContextProvider extends ActionContextProvider {
   private final RemoteRetrier retrier;
   private final DigestUtil digestUtil;
   private final Path logDir;
+  private final AtomicReference<SpawnRunner> fallbackRunner = new AtomicReference<>();
 
   RemoteActionContextProvider(
       CommandEnvironment env,
@@ -64,7 +67,7 @@ final class RemoteActionContextProvider extends ActionContextProvider {
     ExecutionOptions executionOptions =
         checkNotNull(env.getOptions().getOptions(ExecutionOptions.class));
     RemoteOptions remoteOptions = checkNotNull(env.getOptions().getOptions(RemoteOptions.class));
-    String buildRequestId = env.getBuildRequestId().toString();
+    String buildRequestId = env.getBuildRequestId();
     String commandId = env.getCommandId().toString();
 
     if (executor == null && cache != null) {
@@ -84,7 +87,7 @@ final class RemoteActionContextProvider extends ActionContextProvider {
               env.getExecRoot(),
               remoteOptions,
               env.getOptions().getOptions(ExecutionOptions.class),
-              createFallbackRunner(env),
+              fallbackRunner,
               executionOptions.verboseFailures,
               env.getReporter(),
               buildRequestId,
@@ -98,21 +101,37 @@ final class RemoteActionContextProvider extends ActionContextProvider {
     }
   }
 
-  private static SpawnRunner createFallbackRunner(CommandEnvironment env) {
-    LocalExecutionOptions localExecutionOptions =
-        env.getOptions().getOptions(LocalExecutionOptions.class);
-    LocalEnvProvider localEnvProvider =
-        OS.getCurrent() == OS.DARWIN
-            ? new XcodeLocalEnvProvider(env.getClientEnv())
-            : (OS.getCurrent() == OS.WINDOWS
-                ? new WindowsLocalEnvProvider(env.getClientEnv())
-                : new PosixLocalEnvProvider(env.getClientEnv()));
-    return
-        new LocalSpawnRunner(
-            env.getExecRoot(),
-            localExecutionOptions,
-            ResourceManager.instance(),
-            localEnvProvider);
+  @Override
+  public void executorCreated(Iterable<ActionContext> usedContexts) throws ExecutorInitException {
+    SortedSet<String> validStrategies = new TreeSet<>();
+    fallbackRunner.set(null);
+
+    RemoteOptions remoteOptions = env.getOptions().getOptions(RemoteOptions.class);
+    String strategyName = remoteOptions.remoteLocalFallbackStrategy;
+
+    for (ActionContext context : usedContexts) {
+      if (context instanceof AbstractSpawnStrategy) {
+        ExecutionStrategy annotation = context.getClass().getAnnotation(ExecutionStrategy.class);
+        if (annotation != null) {
+          Collections.addAll(validStrategies, annotation.name());
+          if (!strategyName.equals("remote")
+              && Arrays.asList(annotation.name()).contains(strategyName)) {
+            AbstractSpawnStrategy spawnStrategy = (AbstractSpawnStrategy) context;
+            SpawnRunner spawnRunner = Preconditions.checkNotNull(spawnStrategy.getSpawnRunner());
+            fallbackRunner.set(spawnRunner);
+          }
+        }
+      }
+    }
+
+    if (fallbackRunner.get() == null) {
+      validStrategies.remove("remote");
+      throw new ExecutorInitException(
+          String.format(
+              "'%s' is an invalid value for --remote_local_fallback_strategy. Valid values are: %s",
+              strategyName, validStrategies),
+          ExitCode.COMMAND_LINE_ERROR);
+    }
   }
 
   @Override
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
index 7141d35621..14f373eb04 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
@@ -280,14 +280,8 @@ public final class RemoteModule extends BlazeModule {
   @Override
   public Iterable<Class<? extends OptionsBase>> getCommandOptions(Command command) {
     return "build".equals(command.name())
-        ? ImmutableList.<Class<? extends OptionsBase>>of(
-            RemoteOptions.class, AuthAndTLSOptions.class)
-        : ImmutableList.<Class<? extends OptionsBase>>of();
-  }
-
-  public static boolean remoteEnabled(RemoteOptions options) {
-    return SimpleBlobStoreFactory.isRemoteCacheOptions(options)
-        || GrpcRemoteCache.isRemoteCacheOptions(options);
+        ? ImmutableList.of(RemoteOptions.class, AuthAndTLSOptions.class)
+        : ImmutableList.of();
   }
 
   static RemoteRetrier createExecuteRetrier(
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteOptions.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteOptions.java
index 90f42077c4..b73784f529 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteOptions.java
@@ -103,6 +103,14 @@ public final class RemoteOptions extends OptionsBase {
   public boolean remoteLocalFallback;
 
   @Option(
+      name = "remote_local_fallback_strategy",
+      defaultValue = "local",
+      documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
+      effectTags = {OptionEffectTag.UNKNOWN},
+      help = "The strategy to use when remote execution has to fallback to local execution.")
+  public String remoteLocalFallbackStrategy;
+
+  @Option(
     name = "remote_upload_local_results",
     defaultValue = "true",
     documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java
index 95511d0599..e99cc6ecff 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java
@@ -72,6 +72,7 @@ import java.util.Map;
 import java.util.SortedMap;
 import java.util.TreeSet;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
 import javax.annotation.Nullable;
 
 /** A client for the remote execution service. */
@@ -82,7 +83,7 @@ class RemoteSpawnRunner implements SpawnRunner {
   private final Path execRoot;
   private final RemoteOptions remoteOptions;
   private final ExecutionOptions executionOptions;
-  private final SpawnRunner fallbackRunner;
+  private final AtomicReference<SpawnRunner> fallbackRunner;
   private final boolean verboseFailures;
 
   @Nullable private final Reporter cmdlineReporter;
@@ -101,7 +102,7 @@ class RemoteSpawnRunner implements SpawnRunner {
       Path execRoot,
       RemoteOptions remoteOptions,
       ExecutionOptions executionOptions,
-      SpawnRunner fallbackRunner,
+      AtomicReference<SpawnRunner> fallbackRunner,
       boolean verboseFailures,
       @Nullable Reporter cmdlineReporter,
       String buildRequestId,
@@ -135,7 +136,7 @@ class RemoteSpawnRunner implements SpawnRunner {
   public SpawnResult exec(Spawn spawn, SpawnExecutionContext context)
       throws ExecException, InterruptedException, IOException {
     if (!Spawns.mayBeExecutedRemotely(spawn) || remoteCache == null) {
-      return fallbackRunner.exec(spawn, context);
+      return fallbackRunner.get().exec(spawn, context);
     }
 
     context.report(ProgressStatus.EXECUTING, getName());
@@ -445,7 +446,7 @@ class RemoteSpawnRunner implements SpawnRunner {
     if (uploadToCache && remoteCache != null && actionKey != null) {
       return execLocallyAndUpload(spawn, context, inputMap, remoteCache, actionKey);
     }
-    return fallbackRunner.exec(spawn, context);
+    return fallbackRunner.get().exec(spawn, context);
   }
 
   @VisibleForTesting
@@ -457,7 +458,7 @@ class RemoteSpawnRunner implements SpawnRunner {
       ActionKey actionKey)
       throws ExecException, IOException, InterruptedException {
     Map<Path, Long> ctimesBefore = getInputCtimes(inputMap);
-    SpawnResult result = fallbackRunner.exec(spawn, context);
+    SpawnResult result = fallbackRunner.get().exec(spawn, context);
     Map<Path, Long> ctimesAfter = getInputCtimes(inputMap);
     for (Map.Entry<Path, Long> e : ctimesBefore.entrySet()) {
       // Skip uploading to remote cache, because an input was modified during execution.
@@ -494,7 +495,10 @@ class RemoteSpawnRunner implements SpawnRunner {
     }
   }
 
-  /** Resolve a collection of {@link com.google.build.lib.actions.ActionInput}s to {@link Path}s. */
+  /**
+   * Resolve a collection of {@link com.google.devtools.build.lib.actions.ActionInput}s to {@link
+   * Path}s.
+   */
   static Collection<Path> resolveActionInputs(
       Path execRoot, Collection<? extends ActionInput> actionInputs) {
     return actionInputs