Support deferred param files.

Design doc: https://docs.google.com/document/d/1JXqwwVHYosZOgmjN8xrfTalyhiUYJ99Qe2D0qBcqZ1c

The behaviour is gated on --defer_param_files (default off) and is controlled by --min_param_file_size.

This CL adds support for VirtualActionInputs to LocalSpawnRunner, and all remote runners already supports them. The sandboxed runners are not yet supported, but that can be added in a future CL.

This CL does not add support for spawn runner using different param file limits. This will require refactoring of the spawn strategies and runners to be viable.

RELNOTES: None
PiperOrigin-RevId: 194265291

1 files changed, 24 insertions, 2 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfiguration.java b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfiguration.java
index 80f35abd4f..9c33df8865 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfiguration.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/config/BuildConfiguration.java
@@ -34,6 +34,7 @@ import com.google.common.collect.MutableClassToInstanceMap;
 import com.google.devtools.build.lib.actions.ActionEnvironment;
 import com.google.devtools.build.lib.actions.ArtifactRoot;
 import com.google.devtools.build.lib.actions.BuildConfigurationEvent;
+import com.google.devtools.build.lib.actions.CommandLines.CommandLineLimits;
 import com.google.devtools.build.lib.analysis.BlazeDirectories;
 import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
 import com.google.devtools.build.lib.analysis.actions.FileWriteAction;
@@ -424,6 +425,21 @@ public class BuildConfiguration {
     public int minParamFileSize;
 
     @Option(
+        name = "defer_param_files",
+        defaultValue = "false",
+        documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+        effectTags = {
+          OptionEffectTag.LOADING_AND_ANALYSIS,
+          OptionEffectTag.EXECUTION,
+          OptionEffectTag.ACTION_COMMAND_LINES
+        },
+        help =
+            "Whether to use deferred param files. WHen set, param files will not be "
+                + "added to the action graph. Instead, they will be added as virtual action inputs "
+                + "and written at the same time as the action executes.")
+    public boolean deferParamFiles;
+
+    @Option(
       name = "experimental_extended_sanity_checks",
       defaultValue = "false",
       documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
@@ -993,6 +1009,7 @@ public class BuildConfiguration {
   private final String repositoryName;
   private final RepositoryName mainRepositoryName;
   private final ImmutableSet<String> reservedActionMnemonics;
+  private CommandLineLimits commandLineLimits;
 
   /**
    * Directories in the output tree.
@@ -1316,6 +1333,7 @@ public class BuildConfiguration {
 
     this.reservedActionMnemonics = reservedActionMnemonics;
     this.buildEventSupplier = Suppliers.memoize(this::createBuildEvent);
+    this.commandLineLimits = new CommandLineLimits(options.minParamFileSize);
   }
 
   /**
@@ -1752,8 +1770,12 @@ public class BuildConfiguration {
     return testEnv;
   }
 
-  public int getMinParamFileSize() {
-    return options.minParamFileSize;
+  public CommandLineLimits getCommandLineLimits() {
+    return commandLineLimits;
+  }
+
+  public boolean deferParamFiles() {
+    return options.deferParamFiles;
   }
 
   @SkylarkCallable(name = "coverage_enabled", structField = true,