diff options
Diffstat (limited to 'src/main/java/com/google')
3 files changed, 44 insertions, 7 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java b/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java index 9f9e3351b4..ec006b9bbd 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java +++ b/src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java @@ -49,6 +49,16 @@ public class ExecutionOptions extends OptionsBase { public static final ExecutionOptions DEFAULTS = Options.getDefaults(ExecutionOptions.class); @Option( + name = "materialize_param_files", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.UNCATEGORIZED, + effectTags = {OptionEffectTag.UNKNOWN}, + help = + "Writes intermediate parameter files to output tree even when using " + + "remote action execution. Useful when debugging actions. ") + public boolean materializeParamFiles; + + @Option( name = "verbose_failures", defaultValue = "false", documentationCategory = OptionDocumentationCategory.UNCATEGORIZED, diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java index 4bcbb74ff1..e86b41ec6f 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java @@ -80,6 +80,7 @@ final class RemoteActionContextProvider extends ActionContextProvider { new RemoteSpawnRunner( env.getExecRoot(), remoteOptions, + env.getOptions().getOptions(ExecutionOptions.class), createFallbackRunner(env), executionOptions.verboseFailures, env.getReporter(), diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java index 5806b67660..1198c2f5d8 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.actions.ActionInput; import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.CommandLines.ParamFileActionInput; import com.google.devtools.build.lib.actions.EnvironmentalExecException; import com.google.devtools.build.lib.actions.ExecException; import com.google.devtools.build.lib.actions.MetadataProvider; @@ -35,6 +36,7 @@ import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.exec.ExecutionOptions; import com.google.devtools.build.lib.exec.SpawnExecException; import com.google.devtools.build.lib.exec.SpawnRunner; import com.google.devtools.build.lib.remote.Retrier.RetryException; @@ -59,6 +61,7 @@ import com.google.protobuf.TextFormat.ParseException; import io.grpc.Context; import io.grpc.Status.Code; import java.io.IOException; +import java.io.OutputStream; import java.time.Duration; import java.util.ArrayList; import java.util.Collection; @@ -77,7 +80,8 @@ class RemoteSpawnRunner implements SpawnRunner { private static final int POSIX_TIMEOUT_EXIT_CODE = /*SIGNAL_BASE=*/128 + /*SIGALRM=*/14; private final Path execRoot; - private final RemoteOptions options; + private final RemoteOptions remoteOptions; + private final ExecutionOptions executionOptions; private final SpawnRunner fallbackRunner; private final boolean verboseFailures; @@ -94,7 +98,8 @@ class RemoteSpawnRunner implements SpawnRunner { RemoteSpawnRunner( Path execRoot, - RemoteOptions options, + RemoteOptions remoteOptions, + ExecutionOptions executionOptions, SpawnRunner fallbackRunner, boolean verboseFailures, @Nullable Reporter cmdlineReporter, @@ -105,7 +110,8 @@ class RemoteSpawnRunner implements SpawnRunner { DigestUtil digestUtil, Path logDir) { this.execRoot = execRoot; - this.options = options; + this.remoteOptions = remoteOptions; + this.executionOptions = executionOptions; this.fallbackRunner = fallbackRunner; this.remoteCache = remoteCache; this.remoteExecutor = remoteExecutor; @@ -136,6 +142,7 @@ class RemoteSpawnRunner implements SpawnRunner { SortedMap<PathFragment, ActionInput> inputMap = context.getInputMapping(); TreeNode inputRoot = repository.buildFromActionInputs(inputMap); repository.computeMerkleDigests(inputRoot); + maybeWriteParamFilesLocally(spawn); Command command = buildCommand(spawn.getArguments(), spawn.getEnvironment()); Action action = buildAction( @@ -152,8 +159,8 @@ class RemoteSpawnRunner implements SpawnRunner { TracingMetadataUtils.contextWithMetadata(buildRequestId, commandId, actionKey); Context previous = withMetadata.attach(); try { - boolean acceptCachedResult = options.remoteAcceptCached && Spawns.mayBeCached(spawn); - boolean uploadLocalResults = options.remoteUploadLocalResults; + boolean acceptCachedResult = remoteOptions.remoteAcceptCached && Spawns.mayBeCached(spawn); + boolean uploadLocalResults = remoteOptions.remoteUploadLocalResults; try { // Try to lookup the action in the action cache. @@ -199,7 +206,7 @@ class RemoteSpawnRunner implements SpawnRunner { try { ExecuteRequest.Builder request = ExecuteRequest.newBuilder() - .setInstanceName(options.remoteInstanceName) + .setInstanceName(remoteOptions.remoteInstanceName) .setAction(action) .setSkipCacheLookup(!acceptCachedResult); ExecuteResponse reply = remoteExecutor.executeRemotely(request.build()); @@ -223,6 +230,25 @@ class RemoteSpawnRunner implements SpawnRunner { } } + private void maybeWriteParamFilesLocally(Spawn spawn) throws IOException { + if (!executionOptions.materializeParamFiles) { + return; + } + for (ActionInput actionInput : spawn.getInputFiles()) { + if (actionInput instanceof ParamFileActionInput) { + ParamFileActionInput paramFileActionInput = (ParamFileActionInput) actionInput; + Path outputPath = execRoot.getRelative(paramFileActionInput.getExecPath()); + if (outputPath.exists()) { + outputPath.delete(); + } + outputPath.getParentDirectory().createDirectoryAndParents(); + try (OutputStream out = outputPath.getOutputStream()) { + paramFileActionInput.writeTo(out); + } + } + } + } + private void maybeDownloadServerLogs(ExecuteResponse resp, ActionKey actionKey) throws InterruptedException { ActionResult result = resp.getResult(); @@ -271,7 +297,7 @@ class RemoteSpawnRunner implements SpawnRunner { if (Thread.currentThread().isInterrupted()) { throw new InterruptedException(); } - if (options.remoteLocalFallback + if (remoteOptions.remoteLocalFallback && !(cause instanceof RetryException && RemoteRetrierUtils.causedByExecTimeout((RetryException) cause))) { return execLocally(spawn, context, inputMap, uploadLocalResults, remoteCache, actionKey); |