aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorGravatar Philipp Wollermann <philwo@google.com>2017-05-09 09:27:21 -0400
committerGravatar Kristina Chodorow <kchodorow@google.com>2017-05-09 10:55:21 -0400
commit3727645a000a3c4e34ea65f62836a1d0b53d94eb (patch)
treee7fc3ef5a8c7fe16cc30a9409a498f684f9c12dd
parentc240b4c2057e8b757377219b6087034bfcdbec93 (diff)
sandbox: Use the SpawnInputExpander everywhere and delete SpawnHelpers.
This unifies our code to use just one standard implementation to get the entire expanded input files for a Spawn, including from Filesets and Runfiles. Change-Id: I1e286508adf0a9aeddf70934b010e6fcc144c4a7 PiperOrigin-RevId: 155497273
-rw-r--r--src/main/java/com/google/devtools/build/lib/sandbox/DarwinSandboxedStrategy.java78
-rw-r--r--src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedStrategy.java9
-rw-r--r--src/main/java/com/google/devtools/build/lib/sandbox/ProcessWrapperSandboxedStrategy.java8
-rw-r--r--src/main/java/com/google/devtools/build/lib/sandbox/SandboxHelpers.java53
-rw-r--r--src/main/java/com/google/devtools/build/lib/sandbox/SandboxStrategy.java50
-rw-r--r--src/main/java/com/google/devtools/build/lib/sandbox/SpawnHelpers.java193
-rw-r--r--src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnStrategy.java6
-rw-r--r--src/test/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedStrategyTest.java87
8 files changed, 80 insertions, 404 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/DarwinSandboxedStrategy.java b/src/main/java/com/google/devtools/build/lib/sandbox/DarwinSandboxedStrategy.java
index 31fa4562e4..215356f3cb 100644
--- a/src/main/java/com/google/devtools/build/lib/sandbox/DarwinSandboxedStrategy.java
+++ b/src/main/java/com/google/devtools/build/lib/sandbox/DarwinSandboxedStrategy.java
@@ -16,35 +16,28 @@ package com.google.devtools.build.lib.sandbox;
import static java.nio.charset.StandardCharsets.UTF_8;
-import com.google.common.base.Predicates;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.devtools.build.lib.actions.ActionExecutionContext;
import com.google.devtools.build.lib.actions.ActionStatusMessage;
-import com.google.devtools.build.lib.actions.EnvironmentalExecException;
import com.google.devtools.build.lib.actions.ExecException;
import com.google.devtools.build.lib.actions.ExecutionStrategy;
import com.google.devtools.build.lib.actions.Executor;
import com.google.devtools.build.lib.actions.Spawn;
import com.google.devtools.build.lib.actions.SpawnActionContext;
-import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.buildtool.BuildRequest;
+import com.google.devtools.build.lib.exec.SpawnInputExpander;
import com.google.devtools.build.lib.runtime.CommandEnvironment;
import com.google.devtools.build.lib.shell.Command;
import com.google.devtools.build.lib.shell.CommandException;
import com.google.devtools.build.lib.shell.CommandResult;
import com.google.devtools.build.lib.standalone.StandaloneSpawnStrategy;
-import com.google.devtools.build.lib.util.Preconditions;
-import com.google.devtools.build.lib.vfs.FileStatus;
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
-import com.google.devtools.build.lib.vfs.Symlinks;
import java.io.IOException;
-import java.util.HashMap;
import java.util.HashSet;
-import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
@@ -55,12 +48,11 @@ import java.util.concurrent.atomic.AtomicReference;
)
public class DarwinSandboxedStrategy extends SandboxStrategy {
- private final BlazeDirectories blazeDirs;
private final Path execRoot;
private final boolean sandboxDebug;
private final boolean verboseFailures;
private final String productName;
- private final SpawnHelpers spawnHelpers;
+ private final SpawnInputExpander spawnInputExpander;
/**
* The set of directories that always should be writable, independent of the Spawn itself.
@@ -75,7 +67,6 @@ public class DarwinSandboxedStrategy extends SandboxStrategy {
Path sandboxBase,
boolean verboseFailures,
String productName,
- SpawnHelpers spawnHelpers,
ImmutableSet<Path> alwaysWritableDirs) {
super(
cmdEnv,
@@ -83,13 +74,12 @@ public class DarwinSandboxedStrategy extends SandboxStrategy {
sandboxBase,
verboseFailures,
buildRequest.getOptions(SandboxOptions.class));
- this.blazeDirs = cmdEnv.getDirectories();
- this.execRoot = blazeDirs.getExecRoot();
+ this.execRoot = cmdEnv.getExecRoot();
this.sandboxDebug = buildRequest.getOptions(SandboxOptions.class).sandboxDebug;
this.verboseFailures = verboseFailures;
this.productName = productName;
- this.spawnHelpers = spawnHelpers;
this.alwaysWritableDirs = alwaysWritableDirs;
+ spawnInputExpander = new SpawnInputExpander(false);
}
public static DarwinSandboxedStrategy create(
@@ -105,7 +95,6 @@ public class DarwinSandboxedStrategy extends SandboxStrategy {
sandboxBase,
verboseFailures,
productName,
- new SpawnHelpers(cmdEnv.getExecRoot()),
getAlwaysWritableDirs(cmdEnv.getDirectories().getFileSystem()));
}
@@ -189,7 +178,10 @@ public class DarwinSandboxedStrategy extends SandboxStrategy {
SymlinkedExecRoot symlinkedExecRoot = new SymlinkedExecRoot(sandboxExecRoot);
ImmutableSet<PathFragment> outputs = SandboxHelpers.getOutputFiles(spawn);
symlinkedExecRoot.createFileSystem(
- getMounts(spawn, actionExecutionContext), outputs, writableDirs);
+ SandboxHelpers.getInputFiles(
+ spawnInputExpander, this.execRoot, spawn, actionExecutionContext),
+ outputs,
+ writableDirs);
DarwinSandboxRunner runner =
new DarwinSandboxRunner(sandboxPath, sandboxExecRoot, writableDirs, verboseFailures);
@@ -218,58 +210,4 @@ public class DarwinSandboxedStrategy extends SandboxStrategy {
}
}
}
-
- @Override
- public Map<PathFragment, Path> getMounts(Spawn spawn, ActionExecutionContext executionContext)
- throws ExecException {
- try {
- Map<PathFragment, Path> mounts = new HashMap<>();
- spawnHelpers.mountInputs(mounts, spawn, executionContext);
-
- Map<PathFragment, Path> unfinalized = new HashMap<>();
- spawnHelpers.mountRunfilesFromSuppliers(unfinalized, spawn);
- spawnHelpers.mountFilesFromFilesetManifests(unfinalized, spawn, executionContext);
- mounts.putAll(finalizeLinks(unfinalized));
-
- return mounts;
- } catch (IllegalArgumentException | IOException e) {
- throw new EnvironmentalExecException("Could not prepare mounts for sandbox execution", e);
- }
- }
-
- private Map<PathFragment, Path> finalizeLinks(Map<PathFragment, Path> unfinalized)
- throws IOException {
- HashMap<PathFragment, Path> finalizedLinks = new HashMap<>();
- for (Map.Entry<PathFragment, Path> mount : unfinalized.entrySet()) {
- PathFragment target = mount.getKey();
- Path source = mount.getValue();
-
- // If the source is null, the target is supposed to be an empty file. In this case we don't
- // have to deal with finalizing the link.
- if (source == null) {
- finalizedLinks.put(target, source);
- continue;
- }
-
- FileStatus stat = source.statNullable(Symlinks.NOFOLLOW);
-
- if (stat != null && stat.isDirectory()) {
- for (Path subSource : FileSystemUtils.traverseTree(source, Predicates.alwaysTrue())) {
- PathFragment subTarget = target.getRelative(subSource.relativeTo(source));
- finalizeLinksPath(
- finalizedLinks, subTarget, subSource, subSource.statNullable(Symlinks.NOFOLLOW));
- }
- } else {
- finalizeLinksPath(finalizedLinks, target, source, stat);
- }
- }
- return finalizedLinks;
- }
-
- private void finalizeLinksPath(
- Map<PathFragment, Path> finalizedMounts, PathFragment target, Path source, FileStatus stat) {
- // The source must exist.
- Preconditions.checkArgument(stat != null, "%s does not exist", source.toString());
- finalizedMounts.put(target, source);
- }
}
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedStrategy.java b/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedStrategy.java
index c75f9fc763..764bcfd4e6 100644
--- a/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedStrategy.java
+++ b/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedStrategy.java
@@ -27,6 +27,7 @@ import com.google.devtools.build.lib.actions.SpawnActionContext;
import com.google.devtools.build.lib.actions.UserExecException;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.buildtool.BuildRequest;
+import com.google.devtools.build.lib.exec.SpawnInputExpander;
import com.google.devtools.build.lib.runtime.CommandEnvironment;
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
@@ -44,6 +45,7 @@ import java.util.concurrent.atomic.AtomicReference;
contextType = SpawnActionContext.class
)
public class LinuxSandboxedStrategy extends SandboxStrategy {
+
public static boolean isSupported(CommandEnvironment cmdEnv) {
return LinuxSandboxRunner.isSupported(cmdEnv);
}
@@ -52,6 +54,7 @@ public class LinuxSandboxedStrategy extends SandboxStrategy {
private final BlazeDirectories blazeDirs;
private final Path execRoot;
private final boolean verboseFailures;
+ private final SpawnInputExpander spawnInputExpander;
LinuxSandboxedStrategy(
CommandEnvironment cmdEnv,
@@ -68,6 +71,7 @@ public class LinuxSandboxedStrategy extends SandboxStrategy {
this.blazeDirs = cmdEnv.getDirectories();
this.execRoot = blazeDirs.getExecRoot();
this.verboseFailures = verboseFailures;
+ spawnInputExpander = new SpawnInputExpander(false);
}
@Override
@@ -90,7 +94,10 @@ public class LinuxSandboxedStrategy extends SandboxStrategy {
SymlinkedExecRoot symlinkedExecRoot = new SymlinkedExecRoot(sandboxExecRoot);
ImmutableSet<PathFragment> outputs = SandboxHelpers.getOutputFiles(spawn);
symlinkedExecRoot.createFileSystem(
- getMounts(spawn, actionExecutionContext), outputs, writableDirs);
+ SandboxHelpers.getInputFiles(
+ spawnInputExpander, this.execRoot, spawn, actionExecutionContext),
+ outputs,
+ writableDirs);
SandboxRunner runner =
new LinuxSandboxRunner(
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/ProcessWrapperSandboxedStrategy.java b/src/main/java/com/google/devtools/build/lib/sandbox/ProcessWrapperSandboxedStrategy.java
index 3ddaa683ae..64fecbcc4d 100644
--- a/src/main/java/com/google/devtools/build/lib/sandbox/ProcessWrapperSandboxedStrategy.java
+++ b/src/main/java/com/google/devtools/build/lib/sandbox/ProcessWrapperSandboxedStrategy.java
@@ -24,6 +24,7 @@ import com.google.devtools.build.lib.actions.Executor;
import com.google.devtools.build.lib.actions.Spawn;
import com.google.devtools.build.lib.actions.SpawnActionContext;
import com.google.devtools.build.lib.buildtool.BuildRequest;
+import com.google.devtools.build.lib.exec.SpawnInputExpander;
import com.google.devtools.build.lib.runtime.CommandEnvironment;
import com.google.devtools.build.lib.standalone.StandaloneSpawnStrategy;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
@@ -48,6 +49,7 @@ public class ProcessWrapperSandboxedStrategy extends SandboxStrategy {
private final Path execRoot;
private final boolean verboseFailures;
private final String productName;
+ private final SpawnInputExpander spawnInputExpander;
ProcessWrapperSandboxedStrategy(
CommandEnvironment cmdEnv,
@@ -65,6 +67,7 @@ public class ProcessWrapperSandboxedStrategy extends SandboxStrategy {
this.execRoot = cmdEnv.getExecRoot();
this.verboseFailures = verboseFailures;
this.productName = productName;
+ this.spawnInputExpander = new SpawnInputExpander(false);
}
@Override
@@ -92,7 +95,10 @@ public class ProcessWrapperSandboxedStrategy extends SandboxStrategy {
SymlinkedExecRoot symlinkedExecRoot = new SymlinkedExecRoot(sandboxExecRoot);
ImmutableSet<PathFragment> outputs = SandboxHelpers.getOutputFiles(spawn);
symlinkedExecRoot.createFileSystem(
- getMounts(spawn, actionExecutionContext), outputs, writableDirs);
+ SandboxHelpers.getInputFiles(
+ spawnInputExpander, this.execRoot, spawn, actionExecutionContext),
+ outputs,
+ writableDirs);
SandboxRunner runner = new ProcessWrapperRunner(sandboxExecRoot, verboseFailures);
try {
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxHelpers.java b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxHelpers.java
index 364ab38331..37f2ce8e4c 100644
--- a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxHelpers.java
+++ b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxHelpers.java
@@ -18,14 +18,23 @@ import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSet.Builder;
import com.google.devtools.build.lib.actions.ActionExecutionContext;
import com.google.devtools.build.lib.actions.ActionInput;
+import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.ExecException;
import com.google.devtools.build.lib.actions.Executor;
import com.google.devtools.build.lib.actions.Spawn;
import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
import com.google.devtools.build.lib.buildtool.BuildRequest;
+import com.google.devtools.build.lib.exec.SpawnInputExpander;
+import com.google.devtools.build.lib.rules.fileset.FilesetActionContext;
import com.google.devtools.build.lib.standalone.StandaloneSpawnStrategy;
import com.google.devtools.build.lib.util.Preconditions;
+import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
/** Helper methods that are shared by the different sandboxing strategies in this package. */
public final class SandboxHelpers {
@@ -44,6 +53,50 @@ public final class SandboxHelpers {
}
}
+ /**
+ * Returns the inputs of a Spawn as a map of PathFragments relative to an execRoot to paths in the
+ * host filesystem where the input files can be found.
+ */
+ public static Map<PathFragment, Path> getInputFiles(
+ SpawnInputExpander spawnInputExpander,
+ Path execRoot,
+ Spawn spawn,
+ ActionExecutionContext executionContext)
+ throws IOException {
+ Map<PathFragment, ActionInput> inputMap =
+ spawnInputExpander.getInputMapping(
+ spawn,
+ executionContext.getArtifactExpander(),
+ executionContext.getActionInputFileCache(),
+ executionContext.getExecutor().getContext(FilesetActionContext.class));
+ // SpawnInputExpander#getInputMapping uses ArtifactExpander#expandArtifacts to expand
+ // middlemen and tree artifacts, which expands empty tree artifacts to no entry. However,
+ // actions that accept TreeArtifacts as inputs generally expect that the empty directory is
+ // created. So we add those explicitly here.
+ // TODO(ulfjack): Move this code to SpawnInputExpander.
+ for (ActionInput input : spawn.getInputFiles()) {
+ if (input instanceof Artifact && ((Artifact) input).isTreeArtifact()) {
+ List<Artifact> containedArtifacts = new ArrayList<>();
+ executionContext.getArtifactExpander().expand((Artifact) input, containedArtifacts);
+ // Attempting to mount a non-empty directory results in ERR_DIRECTORY_NOT_EMPTY, so we
+ // only mount empty TreeArtifacts as directories.
+ if (containedArtifacts.isEmpty()) {
+ inputMap.put(input.getExecPath(), input);
+ }
+ }
+ }
+
+ Map<PathFragment, Path> inputFiles = new TreeMap<>();
+ for (Map.Entry<PathFragment, ActionInput> e : inputMap.entrySet()) {
+ Path inputPath =
+ e.getValue() == SpawnInputExpander.EMPTY_FILE
+ ? null
+ : execRoot.getRelative(e.getValue().getExecPath());
+ inputFiles.put(e.getKey(), inputPath);
+ }
+ return inputFiles;
+ }
+
public static ImmutableSet<PathFragment> getOutputFiles(Spawn spawn) {
Builder<PathFragment> outputFiles = ImmutableSet.builder();
for (ActionInput output : spawn.getOutputFiles()) {
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxStrategy.java b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxStrategy.java
index 48d125a3ae..82963df099 100644
--- a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxStrategy.java
+++ b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxStrategy.java
@@ -18,10 +18,7 @@ import com.google.common.collect.ImmutableSet;
import com.google.common.eventbus.EventBus;
import com.google.devtools.build.lib.actions.ActionExecutionContext;
import com.google.devtools.build.lib.actions.ActionExecutionMetadata;
-import com.google.devtools.build.lib.actions.ActionInput;
import com.google.devtools.build.lib.actions.ActionStatusMessage;
-import com.google.devtools.build.lib.actions.Artifact;
-import com.google.devtools.build.lib.actions.EnvironmentalExecException;
import com.google.devtools.build.lib.actions.ExecException;
import com.google.devtools.build.lib.actions.Executor;
import com.google.devtools.build.lib.actions.ResourceManager;
@@ -34,18 +31,13 @@ import com.google.devtools.build.lib.actions.UserExecException;
import com.google.devtools.build.lib.buildtool.BuildRequest;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.EventHandler;
-import com.google.devtools.build.lib.exec.SpawnInputExpander;
-import com.google.devtools.build.lib.rules.fileset.FilesetActionContext;
import com.google.devtools.build.lib.runtime.CommandEnvironment;
import com.google.devtools.build.lib.util.io.OutErr;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicReference;
/** Abstract common ancestor for sandbox strategies implementing the common parts. */
@@ -56,7 +48,6 @@ abstract class SandboxStrategy implements SandboxedSpawnActionContext {
private final Path execRoot;
private final boolean verboseFailures;
private final SandboxOptions sandboxOptions;
- private final SpawnInputExpander spawnInputExpander;
private final Path sandboxBase;
public SandboxStrategy(
@@ -71,7 +62,6 @@ abstract class SandboxStrategy implements SandboxedSpawnActionContext {
this.sandboxBase = sandboxBase;
this.verboseFailures = verboseFailures;
this.sandboxOptions = sandboxOptions;
- this.spawnInputExpander = new SpawnInputExpander(/*strict=*/false);
}
/** Executes the given {@code spawn}. */
@@ -202,44 +192,4 @@ abstract class SandboxStrategy implements SandboxedSpawnActionContext {
public boolean shouldPropagateExecException() {
return verboseFailures && sandboxOptions.sandboxDebug;
}
-
- public Map<PathFragment, Path> getMounts(Spawn spawn, ActionExecutionContext executionContext)
- throws ExecException {
- try {
- Map<PathFragment, ActionInput> inputMap = spawnInputExpander
- .getInputMapping(
- spawn,
- executionContext.getArtifactExpander(),
- executionContext.getActionInputFileCache(),
- executionContext.getExecutor().getContext(FilesetActionContext.class));
- // SpawnInputExpander#getInputMapping uses ArtifactExpander#expandArtifacts to expand
- // middlemen and tree artifacts, which expands empty tree artifacts to no entry. However,
- // actions that accept TreeArtifacts as inputs generally expect that the empty directory is
- // created. So we add those explicitly here.
- // TODO(ulfjack): Move this code to SpawnInputExpander.
- for (ActionInput input : spawn.getInputFiles()) {
- if (input instanceof Artifact && ((Artifact) input).isTreeArtifact()) {
- List<Artifact> containedArtifacts = new ArrayList<>();
- executionContext.getArtifactExpander().expand((Artifact) input, containedArtifacts);
- // Attempting to mount a non-empty directory results in ERR_DIRECTORY_NOT_EMPTY, so we
- // only mount empty TreeArtifacts as directories.
- if (containedArtifacts.isEmpty()) {
- inputMap.put(input.getExecPath(), input);
- }
- }
- }
-
- Map<PathFragment, Path> mounts = new TreeMap<>();
- for (Map.Entry<PathFragment, ActionInput> e : inputMap.entrySet()) {
- Path inputPath = e.getValue() == SpawnInputExpander.EMPTY_FILE
- ? null
- : execRoot.getRelative(e.getValue().getExecPath());
- mounts.put(e.getKey(), inputPath);
- }
- return mounts;
- } catch (IOException e) {
- throw new EnvironmentalExecException("Could not prepare mounts for sandbox execution", e);
- }
- }
-
}
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/SpawnHelpers.java b/src/main/java/com/google/devtools/build/lib/sandbox/SpawnHelpers.java
deleted file mode 100644
index b6bd3c017e..0000000000
--- a/src/main/java/com/google/devtools/build/lib/sandbox/SpawnHelpers.java
+++ /dev/null
@@ -1,193 +0,0 @@
-// Copyright 2016 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package com.google.devtools.build.lib.sandbox;
-
-import com.google.common.io.Files;
-import com.google.devtools.build.lib.actions.ActionExecutionContext;
-import com.google.devtools.build.lib.actions.ActionInput;
-import com.google.devtools.build.lib.actions.ActionInputHelper;
-import com.google.devtools.build.lib.actions.Artifact;
-import com.google.devtools.build.lib.actions.Spawn;
-import com.google.devtools.build.lib.analysis.AnalysisUtils;
-import com.google.devtools.build.lib.rules.fileset.FilesetActionContext;
-import com.google.devtools.build.lib.util.Preconditions;
-import com.google.devtools.build.lib.vfs.FileSystem;
-import com.google.devtools.build.lib.vfs.Path;
-import com.google.devtools.build.lib.vfs.PathFragment;
-import java.io.File;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/** Contains common helper methods that extract information from {@link Spawn} objects. */
-public final class SpawnHelpers {
-
- private final Path execRoot;
-
- public SpawnHelpers(Path execRoot) {
- this.execRoot = execRoot;
- }
-
- /**
- * Returns the inputs of a Spawn as a map of PathFragments relative to an execRoot to paths in the
- * host filesystem where the input files can be found.
- */
- public Map<PathFragment, Path> getMounts(Spawn spawn, ActionExecutionContext executionContext)
- throws IOException {
- Map<PathFragment, Path> mounts = new HashMap<>();
- mountRunfilesFromSuppliers(mounts, spawn);
- mountFilesFromFilesetManifests(mounts, spawn, executionContext);
- mountInputs(mounts, spawn, executionContext);
- return mounts;
- }
-
- /** Mount all files that the spawn needs as specified in its fileset manifests. */
- void mountFilesFromFilesetManifests(
- Map<PathFragment, Path> mounts, Spawn spawn, ActionExecutionContext executionContext)
- throws IOException {
- final FilesetActionContext filesetContext =
- executionContext.getExecutor().getContext(FilesetActionContext.class);
- for (Artifact fileset : spawn.getFilesetManifests()) {
- File manifestFile =
- new File(
- execRoot.getPathString(),
- AnalysisUtils.getManifestPathFromFilesetPath(fileset.getExecPath()).getPathString());
- PathFragment targetDirectory = fileset.getExecPath();
-
- parseManifestFile(
- execRoot.getFileSystem(),
- mounts,
- targetDirectory,
- manifestFile,
- true,
- filesetContext.getWorkspaceName());
- }
- }
-
- /** A parser for the MANIFEST files used by Filesets and runfiles. */
- static void parseManifestFile(
- FileSystem fs,
- Map<PathFragment, Path> mounts,
- PathFragment targetDirectory,
- File manifestFile,
- boolean isFilesetManifest,
- String workspaceName)
- throws IOException {
- int lineNum = 0;
- for (String line : Files.readLines(manifestFile, StandardCharsets.UTF_8)) {
- if (isFilesetManifest && (++lineNum % 2 == 0)) {
- continue;
- }
- if (line.isEmpty()) {
- continue;
- }
-
- String[] fields = line.trim().split(" ");
-
- // The "target" field is always a relative path that is to be interpreted in this way:
- // (1) If this is a fileset manifest and our workspace name is not empty, the first segment
- // of each "target" path must be the workspace name, which is then stripped before further
- // processing.
- // (2) The "target" path is then appended to the "targetDirectory", which is a path relative
- // to the execRoot. Together, this results in the full path in the execRoot in which place a
- // symlink referring to "source" has to be created (see below).
- PathFragment targetPath;
- if (isFilesetManifest) {
- PathFragment targetPathFragment = PathFragment.create(fields[0]);
- if (!workspaceName.isEmpty()) {
- Preconditions.checkState(
- targetPathFragment.getSegment(0).equals(workspaceName),
- "Fileset manifest line must start with workspace name");
- targetPathFragment = targetPathFragment.subFragment(1, targetPathFragment.segmentCount());
- }
- targetPath = targetDirectory.getRelative(targetPathFragment);
- } else {
- targetPath = targetDirectory.getRelative(fields[0]);
- }
-
- // The "source" field, if it exists, is always an absolute path and may point to any file in
- // the filesystem (it is not limited to files in the workspace or execroot).
- Path source;
- switch (fields.length) {
- case 1:
- source = null;
- break;
- case 2:
- source = fs.getPath(fields[1]);
- break;
- default:
- throw new IllegalStateException("'" + line + "' splits into more than 2 parts");
- }
-
- mounts.put(targetPath, source);
- }
- }
-
- /** Mount all runfiles that the spawn needs as specified via its runfiles suppliers. */
- void mountRunfilesFromSuppliers(Map<PathFragment, Path> mounts, Spawn spawn) throws IOException {
- Map<PathFragment, Map<PathFragment, Artifact>> rootsAndMappings =
- spawn.getRunfilesSupplier().getMappings();
- for (Map.Entry<PathFragment, Map<PathFragment, Artifact>> rootAndMappings :
- rootsAndMappings.entrySet()) {
- PathFragment root = rootAndMappings.getKey();
- Preconditions.checkState(!root.isAbsolute());
- for (Map.Entry<PathFragment, Artifact> mapping : rootAndMappings.getValue().entrySet()) {
- Artifact sourceArtifact = mapping.getValue();
- Path source =
- (sourceArtifact != null) ? execRoot.getRelative(sourceArtifact.getExecPath()) : null;
-
- Preconditions.checkArgument(!mapping.getKey().isAbsolute());
- PathFragment target = root.getRelative(mapping.getKey());
- mounts.put(target, source);
- }
- }
- }
-
- /** Mount all inputs of the spawn. */
- void mountInputs(
- Map<PathFragment, Path> mounts, Spawn spawn, ActionExecutionContext actionExecutionContext) {
- List<ActionInput> inputs =
- ActionInputHelper.expandArtifacts(
- spawn.getInputFiles(), actionExecutionContext.getArtifactExpander());
-
- // ActionInputHelper#expandArtifacts above expands empty TreeArtifacts into an empty list.
- // However, actions that accept TreeArtifacts as inputs generally expect that the empty
- // directory is created. So here we explicitly mount the directories of the TreeArtifacts as
- // inputs.
- for (ActionInput input : spawn.getInputFiles()) {
- if (input instanceof Artifact && ((Artifact) input).isTreeArtifact()) {
- List<Artifact> containedArtifacts = new ArrayList<>();
- actionExecutionContext.getArtifactExpander().expand((Artifact) input, containedArtifacts);
- // Attempting to mount a non-empty directory results in ERR_DIRECTORY_NOT_EMPTY, so we only
- // mount empty TreeArtifacts as directories.
- if (containedArtifacts.isEmpty()) {
- PathFragment mount = PathFragment.create(input.getExecPathString());
- mounts.put(mount, execRoot.getRelative(mount));
- }
- }
- }
-
- for (ActionInput input : inputs) {
- if (input.getExecPathString().contains("internal/_middlemen/")) {
- continue;
- }
- PathFragment mount = PathFragment.create(input.getExecPathString());
- mounts.put(mount, execRoot.getRelative(mount));
- }
- }
-}
diff --git a/src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnStrategy.java b/src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnStrategy.java
index 9a681ab5ea..2bd4fb1e2d 100644
--- a/src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnStrategy.java
+++ b/src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnStrategy.java
@@ -39,8 +39,8 @@ import com.google.devtools.build.lib.actions.SpawnActionContext;
import com.google.devtools.build.lib.actions.UserExecException;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.events.Event;
+import com.google.devtools.build.lib.exec.SpawnInputExpander;
import com.google.devtools.build.lib.sandbox.SandboxHelpers;
-import com.google.devtools.build.lib.sandbox.SpawnHelpers;
import com.google.devtools.build.lib.standalone.StandaloneSpawnStrategy;
import com.google.devtools.build.lib.util.CommandFailureUtils;
import com.google.devtools.build.lib.util.Preconditions;
@@ -85,6 +85,7 @@ public final class WorkerSpawnStrategy implements SandboxedSpawnActionContext {
private final Path execRoot;
private final boolean verboseFailures;
private final Multimap<String, String> extraFlags;
+ private final SpawnInputExpander spawnInputExpander;
public WorkerSpawnStrategy(
BlazeDirectories blazeDirs,
@@ -96,6 +97,7 @@ public final class WorkerSpawnStrategy implements SandboxedSpawnActionContext {
this.execRoot = blazeDirs.getExecRoot();
this.verboseFailures = verboseFailures;
this.extraFlags = extraFlags;
+ this.spawnInputExpander = new SpawnInputExpander(false);
}
@Override
@@ -162,7 +164,7 @@ public final class WorkerSpawnStrategy implements SandboxedSpawnActionContext {
HashCode workerFilesHash = WorkerFilesHash.getWorkerFilesHash(
spawn.getToolFiles(), actionExecutionContext);
Map<PathFragment, Path> inputFiles =
- new SpawnHelpers(execRoot).getMounts(spawn, actionExecutionContext);
+ SandboxHelpers.getInputFiles(spawnInputExpander, execRoot, spawn, actionExecutionContext);
Set<PathFragment> outputFiles = SandboxHelpers.getOutputFiles(spawn);
WorkerKey key =
diff --git a/src/test/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedStrategyTest.java b/src/test/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedStrategyTest.java
deleted file mode 100644
index 4fac68d1b4..0000000000
--- a/src/test/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedStrategyTest.java
+++ /dev/null
@@ -1,87 +0,0 @@
-// Copyright 2015 The Bazel Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-package com.google.devtools.build.lib.sandbox;
-
-import static com.google.common.truth.Truth.assertThat;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.devtools.build.lib.vfs.FileSystemUtils;
-import com.google.devtools.build.lib.vfs.Path;
-import com.google.devtools.build.lib.vfs.PathFragment;
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.TreeMap;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/** Tests for {@code LinuxSandboxedStrategy}. */
-@RunWith(JUnit4.class)
-public class LinuxSandboxedStrategyTest extends SandboxTestCase {
- private Path workspaceDir;
-
- @Before
- public final void createWorkspaceDir() throws IOException {
- workspaceDir = testRoot.getRelative("workspace");
- workspaceDir.createDirectory();
- }
-
- @Test
- public void testParseManifestFile() throws Exception {
- PathFragment targetDir = PathFragment.create("runfiles");
-
- Path testFile = workspaceDir.getRelative("testfile");
- FileSystemUtils.createEmptyFile(testFile);
-
- Path manifestFile = workspaceDir.getRelative("MANIFEST");
- FileSystemUtils.writeContent(
- manifestFile,
- Charset.defaultCharset(),
- String.format("x/testfile %s\nx/emptyfile \n", testFile.getPathString()));
-
- Map<PathFragment, Path> mounts = new TreeMap<>();
- SpawnHelpers.parseManifestFile(
- fileSystem, mounts, targetDir, manifestFile.getPathFile(), false, "");
-
- Map<PathFragment, Path> expected = new HashMap<>();
- expected.put(PathFragment.create("runfiles/x/testfile"), testFile);
- expected.put(PathFragment.create("runfiles/x/emptyfile"), null);
-
- assertThat(mounts).isEqualTo(expected);
- }
-
- @Test
- public void testParseFilesetManifestFile() throws Exception {
- PathFragment targetDir = PathFragment.create("fileset");
-
- Path testFile = workspaceDir.getRelative("testfile");
- FileSystemUtils.createEmptyFile(testFile);
-
- Path manifestFile = workspaceDir.getRelative("MANIFEST");
- FileSystemUtils.writeContent(
- manifestFile,
- Charset.defaultCharset(),
- String.format("workspace/x/testfile %s\n0\n", testFile.getPathString()));
-
- Map<PathFragment, Path> mounts = new HashMap<>();
- SpawnHelpers.parseManifestFile(
- fileSystem, mounts, targetDir, manifestFile.getPathFile(), true, "workspace");
-
- assertThat(mounts).isEqualTo(
- ImmutableMap.of(PathFragment.create("fileset/x/testfile"), testFile));
- }
-}