Add --materialize_param_files option.

When set, any action parameter files are written locally upon action execution, even when the action is executed remotely. This is mainly useful for debugging.

This option is effectively implied by --subcommands and --verbose_failures, as it is likely that the user is debugging actions when using these flags.

RELNOTES: Add --materialize_param_files flag to write parameter files even when actions are executed remotely.
PiperOrigin-RevId: 201225566

2 files changed, 87 insertions, 5 deletions

diff --git a/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteExecutionClientTest.java b/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteExecutionClientTest.java
index 5bb5f6ab3c..89962f173b 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteExecutionClientTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteExecutionClientTest.java
@@ -41,6 +41,7 @@ import com.google.devtools.build.lib.analysis.BlazeVersionInfo;
 import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions;
 import com.google.devtools.build.lib.authandtls.GoogleAuthUtils;
 import com.google.devtools.build.lib.clock.JavaClock;
+import com.google.devtools.build.lib.exec.ExecutionOptions;
 import com.google.devtools.build.lib.exec.SpawnExecException;
 import com.google.devtools.build.lib.exec.SpawnInputExpander;
 import com.google.devtools.build.lib.exec.SpawnRunner.ProgressStatus;
@@ -247,21 +248,25 @@ public class GrpcRemoteExecutionClientTest {
     FileSystemUtils.createDirectoryAndParents(stdout.getParentDirectory());
     FileSystemUtils.createDirectoryAndParents(stderr.getParentDirectory());
     outErr = new FileOutErr(stdout, stderr);
-    RemoteOptions options = Options.getDefaults(RemoteOptions.class);
+    RemoteOptions remoteOptions = Options.getDefaults(RemoteOptions.class);
     RemoteRetrier retrier =
         new RemoteRetrier(
-            options, RemoteRetrier.RETRIABLE_GRPC_ERRORS, retryService, Retrier.ALLOW_ALL_CALLS);
+            remoteOptions,
+            RemoteRetrier.RETRIABLE_GRPC_ERRORS,
+            retryService,
+            Retrier.ALLOW_ALL_CALLS);
     Channel channel = InProcessChannelBuilder.forName(fakeServerName).directExecutor().build();
     GrpcRemoteExecutor executor =
-        new GrpcRemoteExecutor(channel, null, options.remoteTimeout, retrier);
+        new GrpcRemoteExecutor(channel, null, remoteOptions.remoteTimeout, retrier);
     CallCredentials creds =
         GoogleAuthUtils.newCallCredentials(Options.getDefaults(AuthAndTLSOptions.class));
     GrpcRemoteCache remoteCache =
-        new GrpcRemoteCache(channel, creds, options, retrier, DIGEST_UTIL);
+        new GrpcRemoteCache(channel, creds, remoteOptions, retrier, DIGEST_UTIL);
     client =
         new RemoteSpawnRunner(
             execRoot,
-            options,
+            remoteOptions,
+            Options.getDefaults(ExecutionOptions.class),
             null,
             true,
             /*cmdlineReporter=*/ null,
diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java
index 0b6e052c00..9eead3059d 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java
@@ -14,6 +14,7 @@
 package com.google.devtools.build.lib.remote;
 
 import static com.google.common.truth.Truth.assertThat;
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
 import static org.junit.Assert.fail;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.eq;
@@ -28,12 +29,15 @@ import static org.mockito.Mockito.when;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.eventbus.EventBus;
+import com.google.common.io.ByteStreams;
 import com.google.common.util.concurrent.SettableFuture;
 import com.google.devtools.build.lib.actions.ActionInput;
 import com.google.devtools.build.lib.actions.Artifact.ArtifactExpander;
+import com.google.devtools.build.lib.actions.CommandLines.ParamFileActionInput;
 import com.google.devtools.build.lib.actions.EnvironmentalExecException;
 import com.google.devtools.build.lib.actions.ExecutionRequirements;
 import com.google.devtools.build.lib.actions.MetadataProvider;
+import com.google.devtools.build.lib.actions.ParameterFile.ParameterFileType;
 import com.google.devtools.build.lib.actions.ResourceSet;
 import com.google.devtools.build.lib.actions.SimpleSpawn;
 import com.google.devtools.build.lib.actions.Spawn;
@@ -44,6 +48,7 @@ import com.google.devtools.build.lib.events.Event;
 import com.google.devtools.build.lib.events.EventKind;
 import com.google.devtools.build.lib.events.Reporter;
 import com.google.devtools.build.lib.events.StoredEventHandler;
+import com.google.devtools.build.lib.exec.ExecutionOptions;
 import com.google.devtools.build.lib.exec.SpawnExecException;
 import com.google.devtools.build.lib.exec.SpawnInputExpander;
 import com.google.devtools.build.lib.exec.SpawnRunner;
@@ -69,6 +74,8 @@ import com.google.devtools.remoteexecution.v1test.LogFile;
 import com.google.protobuf.ByteString;
 import com.google.rpc.Code;
 import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
 import java.time.Duration;
 import java.util.Collection;
 import java.util.SortedMap;
@@ -139,6 +146,7 @@ public class RemoteSpawnRunnerTest {
         new RemoteSpawnRunner(
             execRoot,
             options,
+            Options.getDefaults(ExecutionOptions.class),
             localRunner,
             true,
             /*cmdlineReporter=*/ null,
@@ -198,6 +206,7 @@ public class RemoteSpawnRunnerTest {
         new RemoteSpawnRunner(
             execRoot,
             options,
+            Options.getDefaults(ExecutionOptions.class),
             localRunner,
             true,
             /*cmdlineReporter=*/ null,
@@ -251,6 +260,7 @@ public class RemoteSpawnRunnerTest {
             new RemoteSpawnRunner(
                 execRoot,
                 options,
+                Options.getDefaults(ExecutionOptions.class),
                 localRunner,
                 true,
                 /*cmdlineReporter=*/ null,
@@ -300,6 +310,7 @@ public class RemoteSpawnRunnerTest {
             new RemoteSpawnRunner(
                 execRoot,
                 options,
+                Options.getDefaults(ExecutionOptions.class),
                 localRunner,
                 true,
                 /*cmdlineReporter=*/ null,
@@ -335,6 +346,7 @@ public class RemoteSpawnRunnerTest {
         new RemoteSpawnRunner(
             execRoot,
             options,
+            Options.getDefaults(ExecutionOptions.class),
             localRunner,
             false,
             reporter,
@@ -392,6 +404,7 @@ public class RemoteSpawnRunnerTest {
         new RemoteSpawnRunner(
             execRoot,
             options,
+            Options.getDefaults(ExecutionOptions.class),
             localRunner,
             true,
             /*cmdlineReporter=*/ null,
@@ -432,6 +445,7 @@ public class RemoteSpawnRunnerTest {
         new RemoteSpawnRunner(
             execRoot,
             options,
+            Options.getDefaults(ExecutionOptions.class),
             localRunner,
             true,
             /*cmdlineReporter=*/ null,
@@ -469,6 +483,7 @@ public class RemoteSpawnRunnerTest {
         new RemoteSpawnRunner(
             execRoot,
             options,
+            Options.getDefaults(ExecutionOptions.class),
             localRunner,
             true,
             /*cmdlineReporter=*/ null,
@@ -504,6 +519,7 @@ public class RemoteSpawnRunnerTest {
         new RemoteSpawnRunner(
             execRoot,
             Options.getDefaults(RemoteOptions.class),
+            Options.getDefaults(ExecutionOptions.class),
             localRunner,
             true,
             /*cmdlineReporter=*/ null,
@@ -544,6 +560,7 @@ public class RemoteSpawnRunnerTest {
         new RemoteSpawnRunner(
             execRoot,
             Options.getDefaults(RemoteOptions.class),
+            Options.getDefaults(ExecutionOptions.class),
             localRunner,
             true,
             /*cmdlineReporter=*/ null,
@@ -587,6 +604,7 @@ public class RemoteSpawnRunnerTest {
         new RemoteSpawnRunner(
             execRoot,
             Options.getDefaults(RemoteOptions.class),
+            Options.getDefaults(ExecutionOptions.class),
             localRunner,
             true,
             /*cmdlineReporter=*/ null,
@@ -625,6 +643,7 @@ public class RemoteSpawnRunnerTest {
         new RemoteSpawnRunner(
             execRoot,
             Options.getDefaults(RemoteOptions.class),
+            Options.getDefaults(ExecutionOptions.class),
             localRunner,
             true,
             /*cmdlineReporter=*/ null,
@@ -667,6 +686,7 @@ public class RemoteSpawnRunnerTest {
         new RemoteSpawnRunner(
             execRoot,
             options,
+            Options.getDefaults(ExecutionOptions.class),
             localRunner,
             true,
             /*cmdlineReporter=*/ null,
@@ -709,6 +729,7 @@ public class RemoteSpawnRunnerTest {
         new RemoteSpawnRunner(
             execRoot,
             options,
+            Options.getDefaults(ExecutionOptions.class),
             localRunner,
             true,
             /*cmdlineReporter=*/ null,
@@ -757,6 +778,7 @@ public class RemoteSpawnRunnerTest {
         new RemoteSpawnRunner(
             execRoot,
             options,
+            Options.getDefaults(ExecutionOptions.class),
             localRunner,
             true,
             /*cmdlineReporter=*/ null,
@@ -803,6 +825,7 @@ public class RemoteSpawnRunnerTest {
         new RemoteSpawnRunner(
             execRoot,
             options,
+            Options.getDefaults(ExecutionOptions.class),
             localRunner,
             true,
             /*cmdlineReporter=*/ null,
@@ -844,6 +867,7 @@ public class RemoteSpawnRunnerTest {
         new RemoteSpawnRunner(
             execRoot,
             options,
+            Options.getDefaults(ExecutionOptions.class),
             localRunner,
             true,
             /*cmdlineReporter=*/ null,
@@ -881,6 +905,7 @@ public class RemoteSpawnRunnerTest {
         new RemoteSpawnRunner(
             execRoot,
             options,
+            Options.getDefaults(ExecutionOptions.class),
             localRunner,
             true,
             /*cmdlineReporter=*/ null,
@@ -905,6 +930,58 @@ public class RemoteSpawnRunnerTest {
     }
   }
 
+  @Test
+  public void testMaterializeParamFiles() throws Exception {
+    ExecutionOptions executionOptions =
+        Options.parse(ExecutionOptions.class, "--materialize_param_files").getOptions();
+    executionOptions.materializeParamFiles = true;
+    RemoteSpawnRunner runner =
+        new RemoteSpawnRunner(
+            execRoot,
+            Options.getDefaults(RemoteOptions.class),
+            executionOptions,
+            localRunner,
+            true,
+            /*cmdlineReporter=*/ null,
+            "build-req-id",
+            "command-id",
+            cache,
+            executor,
+            digestUtil,
+            logDir);
+
+    ExecuteResponse succeeded =
+        ExecuteResponse.newBuilder()
+            .setResult(ActionResult.newBuilder().setExitCode(0).build())
+            .build();
+    when(executor.executeRemotely(any(ExecuteRequest.class))).thenReturn(succeeded);
+
+    ImmutableList<String> args = ImmutableList.of("--foo", "--bar");
+    ParamFileActionInput input =
+        new ParamFileActionInput(
+            PathFragment.create("out/param_file"), args, ParameterFileType.UNQUOTED, ISO_8859_1);
+    Spawn spawn =
+        new SimpleSpawn(
+            new FakeOwner("foo", "bar"),
+            /*arguments=*/ ImmutableList.of(),
+            /*environment=*/ ImmutableMap.of(),
+            /*executionInfo=*/ ImmutableMap.of(),
+            ImmutableList.of(input),
+            /*outputs=*/ ImmutableList.<ActionInput>of(),
+            ResourceSet.ZERO);
+    SpawnExecutionContext policy = new FakeSpawnExecutionContext(spawn);
+    SpawnResult res = runner.exec(spawn, policy);
+    assertThat(res.status()).isEqualTo(Status.SUCCESS);
+    Path paramFile = execRoot.getRelative("out/param_file");
+    assertThat(paramFile.exists()).isTrue();
+    try (InputStream inputStream = paramFile.getInputStream()) {
+      assertThat(
+              new String(ByteStreams.toByteArray(inputStream), StandardCharsets.UTF_8).split("\n"))
+          .asList()
+          .containsExactly("--foo", "--bar");
+    }
+  }
+
   private static Spawn newSimpleSpawn() {
     return new SimpleSpawn(
         new FakeOwner("foo", "bar"),