remote: add support for directory inputs in runfiles

Add support for tree artifacts (ctx.action.declare_directory(...)) in
runfiles. Before this change we would throw away the information
about the files inside a tree artifact before executing an action.
That's fine for local execution where the sandbox just copies/symlinks
a directory and doesn't care much what's inside. However, in remote
execution we actually need to upload each individual file and so
we need to be aware of all individual files not just directories.

This change makes it so that this information is made available to a
SpawnRunner via the SpawnInputExpander.

RELNOTES: None
PiperOrigin-RevId: 207091668

1 files changed, 40 insertions, 11 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/exec/SpawnInputExpander.java b/src/main/java/com/google/devtools/build/lib/exec/SpawnInputExpander.java
index 444ffa51b6..0e965b32dd 100644
--- a/src/main/java/com/google/devtools/build/lib/exec/SpawnInputExpander.java
+++ b/src/main/java/com/google/devtools/build/lib/exec/SpawnInputExpander.java
@@ -20,6 +20,8 @@ import com.google.devtools.build.lib.actions.ActionInput;
 import com.google.devtools.build.lib.actions.ActionInputHelper;
 import com.google.devtools.build.lib.actions.Artifact;
 import com.google.devtools.build.lib.actions.Artifact.ArtifactExpander;
+import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact;
+import com.google.devtools.build.lib.actions.FileArtifactValue;
 import com.google.devtools.build.lib.actions.FilesetOutputSymlink;
 import com.google.devtools.build.lib.actions.MetadataProvider;
 import com.google.devtools.build.lib.actions.RunfilesSupplier;
@@ -29,6 +31,7 @@ import com.google.devtools.build.lib.exec.FilesetManifest.RelativeSymlinkBehavio
 import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.build.lib.vfs.PathFragment;
 import java.io.IOException;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.SortedMap;
@@ -40,8 +43,7 @@ import java.util.TreeMap;
  * laid out.
  */
 public class SpawnInputExpander {
-  @VisibleForTesting
-  static final ActionInput EMPTY_FILE = new EmptyActionInput("/dev/null");
+  @VisibleForTesting static final ActionInput EMPTY_FILE = new EmptyActionInput("/dev/null");
 
   private final Path execRoot;
   private final boolean strict;
@@ -103,10 +105,11 @@ public class SpawnInputExpander {
   void addRunfilesToInputs(
       Map<PathFragment, ActionInput> inputMap,
       RunfilesSupplier runfilesSupplier,
-      MetadataProvider actionFileCache)
+      MetadataProvider actionFileCache,
+      ArtifactExpander artifactExpander)
       throws IOException {
-    Map<PathFragment, Map<PathFragment, Artifact>> rootsAndMappings = null;
-    rootsAndMappings = runfilesSupplier.getMappings();
+    Map<PathFragment, Map<PathFragment, Artifact>> rootsAndMappings =
+        runfilesSupplier.getMappings();
 
     for (Map.Entry<PathFragment, Map<PathFragment, Artifact>> rootAndMappings :
         rootsAndMappings.entrySet()) {
@@ -116,10 +119,23 @@ public class SpawnInputExpander {
         PathFragment location = root.getRelative(mapping.getKey());
         Artifact localArtifact = mapping.getValue();
         if (localArtifact != null) {
-          if (strict && !actionFileCache.getMetadata(localArtifact).getType().isFile()) {
-            throw new IOException("Not a file: " + localArtifact.getPath().getPathString());
+          Preconditions.checkState(!localArtifact.isMiddlemanArtifact());
+          if (localArtifact.isTreeArtifact()) {
+            List<ActionInput> expandedInputs =
+                ActionInputHelper.expandArtifacts(
+                    Collections.singletonList(localArtifact), artifactExpander);
+            for (ActionInput input : expandedInputs) {
+              addMapping(
+                  inputMap,
+                  location.getRelative(((TreeFileArtifact) input).getParentRelativePath()),
+                  input);
+            }
+          } else {
+            if (strict) {
+              failIfDirectory(actionFileCache, localArtifact);
+            }
+            addMapping(inputMap, location, localArtifact);
           }
-          addMapping(inputMap, location, localArtifact);
         } else {
           addMapping(inputMap, location, EMPTY_FILE);
         }
@@ -127,6 +143,14 @@ public class SpawnInputExpander {
     }
   }
 
+  private static void failIfDirectory(MetadataProvider actionFileCache, ActionInput input)
+      throws IOException {
+    FileArtifactValue metadata = actionFileCache.getMetadata(input);
+    if (metadata != null && !metadata.getType().isFile()) {
+      throw new IOException("Not a file: " + input.getExecPathString());
+    }
+  }
+
   /**
    * Parses the fileset manifest file, adding to the inputMappings where appropriate. Lines
    * referring to directories are recursed.
@@ -175,10 +199,15 @@ public class SpawnInputExpander {
   }
 
   /**
-   * Convert the inputs of the given spawn to a map from exec-root relative paths to action inputs.
-   * The returned map never contains {@code null} values; it uses {@link #EMPTY_FILE} for empty
+   * Convert the inputs and runfiles of the given spawn to a map from exec-root relative paths to
+   * {@link ActionInput}s. The returned map does not contain tree artifacts as they are expanded
+   * to file artifacts.
+   *
+   * <p>The returned map never contains {@code null} values; it uses {@link #EMPTY_FILE} for empty
    * files, which is an instance of {@link
    * com.google.devtools.build.lib.actions.cache.VirtualActionInput}.
+   *
+   * <p>The returned map contains all runfiles, but not the {@code MANIFEST}.
    */
   public SortedMap<PathFragment, ActionInput> getInputMapping(
       Spawn spawn, ArtifactExpander artifactExpander, MetadataProvider actionInputFileCache)
@@ -186,7 +215,7 @@ public class SpawnInputExpander {
     TreeMap<PathFragment, ActionInput> inputMap = new TreeMap<>();
     addInputs(inputMap, spawn, artifactExpander);
     addRunfilesToInputs(
-        inputMap, spawn.getRunfilesSupplier(), actionInputFileCache);
+        inputMap, spawn.getRunfilesSupplier(), actionInputFileCache, artifactExpander);
     addFilesetManifests(spawn.getFilesetMappings(), inputMap);
     return inputMap;
   }