diff options
author | Philipp Wollermann <philwo@google.com> | 2017-05-09 09:27:21 -0400 |
---|---|---|
committer | Kristina Chodorow <kchodorow@google.com> | 2017-05-09 10:55:21 -0400 |
commit | 3727645a000a3c4e34ea65f62836a1d0b53d94eb (patch) | |
tree | e7fc3ef5a8c7fe16cc30a9409a498f684f9c12dd /src/main | |
parent | c240b4c2057e8b757377219b6087034bfcdbec93 (diff) |
sandbox: Use the SpawnInputExpander everywhere and delete SpawnHelpers.
This unifies our code to use just one standard implementation to get the
entire expanded input files for a Spawn, including from Filesets and
Runfiles.
Change-Id: I1e286508adf0a9aeddf70934b010e6fcc144c4a7
PiperOrigin-RevId: 155497273
Diffstat (limited to 'src/main')
7 files changed, 80 insertions, 317 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/DarwinSandboxedStrategy.java b/src/main/java/com/google/devtools/build/lib/sandbox/DarwinSandboxedStrategy.java index 31fa4562e4..215356f3cb 100644 --- a/src/main/java/com/google/devtools/build/lib/sandbox/DarwinSandboxedStrategy.java +++ b/src/main/java/com/google/devtools/build/lib/sandbox/DarwinSandboxedStrategy.java @@ -16,35 +16,28 @@ package com.google.devtools.build.lib.sandbox; import static java.nio.charset.StandardCharsets.UTF_8; -import com.google.common.base.Predicates; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.devtools.build.lib.actions.ActionExecutionContext; import com.google.devtools.build.lib.actions.ActionStatusMessage; -import com.google.devtools.build.lib.actions.EnvironmentalExecException; import com.google.devtools.build.lib.actions.ExecException; import com.google.devtools.build.lib.actions.ExecutionStrategy; import com.google.devtools.build.lib.actions.Executor; import com.google.devtools.build.lib.actions.Spawn; import com.google.devtools.build.lib.actions.SpawnActionContext; -import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.buildtool.BuildRequest; +import com.google.devtools.build.lib.exec.SpawnInputExpander; import com.google.devtools.build.lib.runtime.CommandEnvironment; import com.google.devtools.build.lib.shell.Command; import com.google.devtools.build.lib.shell.CommandException; import com.google.devtools.build.lib.shell.CommandResult; import com.google.devtools.build.lib.standalone.StandaloneSpawnStrategy; -import com.google.devtools.build.lib.util.Preconditions; -import com.google.devtools.build.lib.vfs.FileStatus; import com.google.devtools.build.lib.vfs.FileSystem; import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; -import com.google.devtools.build.lib.vfs.Symlinks; import java.io.IOException; -import java.util.HashMap; import java.util.HashSet; -import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; @@ -55,12 +48,11 @@ import java.util.concurrent.atomic.AtomicReference; ) public class DarwinSandboxedStrategy extends SandboxStrategy { - private final BlazeDirectories blazeDirs; private final Path execRoot; private final boolean sandboxDebug; private final boolean verboseFailures; private final String productName; - private final SpawnHelpers spawnHelpers; + private final SpawnInputExpander spawnInputExpander; /** * The set of directories that always should be writable, independent of the Spawn itself. @@ -75,7 +67,6 @@ public class DarwinSandboxedStrategy extends SandboxStrategy { Path sandboxBase, boolean verboseFailures, String productName, - SpawnHelpers spawnHelpers, ImmutableSet<Path> alwaysWritableDirs) { super( cmdEnv, @@ -83,13 +74,12 @@ public class DarwinSandboxedStrategy extends SandboxStrategy { sandboxBase, verboseFailures, buildRequest.getOptions(SandboxOptions.class)); - this.blazeDirs = cmdEnv.getDirectories(); - this.execRoot = blazeDirs.getExecRoot(); + this.execRoot = cmdEnv.getExecRoot(); this.sandboxDebug = buildRequest.getOptions(SandboxOptions.class).sandboxDebug; this.verboseFailures = verboseFailures; this.productName = productName; - this.spawnHelpers = spawnHelpers; this.alwaysWritableDirs = alwaysWritableDirs; + spawnInputExpander = new SpawnInputExpander(false); } public static DarwinSandboxedStrategy create( @@ -105,7 +95,6 @@ public class DarwinSandboxedStrategy extends SandboxStrategy { sandboxBase, verboseFailures, productName, - new SpawnHelpers(cmdEnv.getExecRoot()), getAlwaysWritableDirs(cmdEnv.getDirectories().getFileSystem())); } @@ -189,7 +178,10 @@ public class DarwinSandboxedStrategy extends SandboxStrategy { SymlinkedExecRoot symlinkedExecRoot = new SymlinkedExecRoot(sandboxExecRoot); ImmutableSet<PathFragment> outputs = SandboxHelpers.getOutputFiles(spawn); symlinkedExecRoot.createFileSystem( - getMounts(spawn, actionExecutionContext), outputs, writableDirs); + SandboxHelpers.getInputFiles( + spawnInputExpander, this.execRoot, spawn, actionExecutionContext), + outputs, + writableDirs); DarwinSandboxRunner runner = new DarwinSandboxRunner(sandboxPath, sandboxExecRoot, writableDirs, verboseFailures); @@ -218,58 +210,4 @@ public class DarwinSandboxedStrategy extends SandboxStrategy { } } } - - @Override - public Map<PathFragment, Path> getMounts(Spawn spawn, ActionExecutionContext executionContext) - throws ExecException { - try { - Map<PathFragment, Path> mounts = new HashMap<>(); - spawnHelpers.mountInputs(mounts, spawn, executionContext); - - Map<PathFragment, Path> unfinalized = new HashMap<>(); - spawnHelpers.mountRunfilesFromSuppliers(unfinalized, spawn); - spawnHelpers.mountFilesFromFilesetManifests(unfinalized, spawn, executionContext); - mounts.putAll(finalizeLinks(unfinalized)); - - return mounts; - } catch (IllegalArgumentException | IOException e) { - throw new EnvironmentalExecException("Could not prepare mounts for sandbox execution", e); - } - } - - private Map<PathFragment, Path> finalizeLinks(Map<PathFragment, Path> unfinalized) - throws IOException { - HashMap<PathFragment, Path> finalizedLinks = new HashMap<>(); - for (Map.Entry<PathFragment, Path> mount : unfinalized.entrySet()) { - PathFragment target = mount.getKey(); - Path source = mount.getValue(); - - // If the source is null, the target is supposed to be an empty file. In this case we don't - // have to deal with finalizing the link. - if (source == null) { - finalizedLinks.put(target, source); - continue; - } - - FileStatus stat = source.statNullable(Symlinks.NOFOLLOW); - - if (stat != null && stat.isDirectory()) { - for (Path subSource : FileSystemUtils.traverseTree(source, Predicates.alwaysTrue())) { - PathFragment subTarget = target.getRelative(subSource.relativeTo(source)); - finalizeLinksPath( - finalizedLinks, subTarget, subSource, subSource.statNullable(Symlinks.NOFOLLOW)); - } - } else { - finalizeLinksPath(finalizedLinks, target, source, stat); - } - } - return finalizedLinks; - } - - private void finalizeLinksPath( - Map<PathFragment, Path> finalizedMounts, PathFragment target, Path source, FileStatus stat) { - // The source must exist. - Preconditions.checkArgument(stat != null, "%s does not exist", source.toString()); - finalizedMounts.put(target, source); - } } diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedStrategy.java b/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedStrategy.java index c75f9fc763..764bcfd4e6 100644 --- a/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedStrategy.java +++ b/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedStrategy.java @@ -27,6 +27,7 @@ import com.google.devtools.build.lib.actions.SpawnActionContext; import com.google.devtools.build.lib.actions.UserExecException; import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.buildtool.BuildRequest; +import com.google.devtools.build.lib.exec.SpawnInputExpander; import com.google.devtools.build.lib.runtime.CommandEnvironment; import com.google.devtools.build.lib.vfs.FileSystem; import com.google.devtools.build.lib.vfs.FileSystemUtils; @@ -44,6 +45,7 @@ import java.util.concurrent.atomic.AtomicReference; contextType = SpawnActionContext.class ) public class LinuxSandboxedStrategy extends SandboxStrategy { + public static boolean isSupported(CommandEnvironment cmdEnv) { return LinuxSandboxRunner.isSupported(cmdEnv); } @@ -52,6 +54,7 @@ public class LinuxSandboxedStrategy extends SandboxStrategy { private final BlazeDirectories blazeDirs; private final Path execRoot; private final boolean verboseFailures; + private final SpawnInputExpander spawnInputExpander; LinuxSandboxedStrategy( CommandEnvironment cmdEnv, @@ -68,6 +71,7 @@ public class LinuxSandboxedStrategy extends SandboxStrategy { this.blazeDirs = cmdEnv.getDirectories(); this.execRoot = blazeDirs.getExecRoot(); this.verboseFailures = verboseFailures; + spawnInputExpander = new SpawnInputExpander(false); } @Override @@ -90,7 +94,10 @@ public class LinuxSandboxedStrategy extends SandboxStrategy { SymlinkedExecRoot symlinkedExecRoot = new SymlinkedExecRoot(sandboxExecRoot); ImmutableSet<PathFragment> outputs = SandboxHelpers.getOutputFiles(spawn); symlinkedExecRoot.createFileSystem( - getMounts(spawn, actionExecutionContext), outputs, writableDirs); + SandboxHelpers.getInputFiles( + spawnInputExpander, this.execRoot, spawn, actionExecutionContext), + outputs, + writableDirs); SandboxRunner runner = new LinuxSandboxRunner( diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/ProcessWrapperSandboxedStrategy.java b/src/main/java/com/google/devtools/build/lib/sandbox/ProcessWrapperSandboxedStrategy.java index 3ddaa683ae..64fecbcc4d 100644 --- a/src/main/java/com/google/devtools/build/lib/sandbox/ProcessWrapperSandboxedStrategy.java +++ b/src/main/java/com/google/devtools/build/lib/sandbox/ProcessWrapperSandboxedStrategy.java @@ -24,6 +24,7 @@ import com.google.devtools.build.lib.actions.Executor; import com.google.devtools.build.lib.actions.Spawn; import com.google.devtools.build.lib.actions.SpawnActionContext; import com.google.devtools.build.lib.buildtool.BuildRequest; +import com.google.devtools.build.lib.exec.SpawnInputExpander; import com.google.devtools.build.lib.runtime.CommandEnvironment; import com.google.devtools.build.lib.standalone.StandaloneSpawnStrategy; import com.google.devtools.build.lib.vfs.FileSystemUtils; @@ -48,6 +49,7 @@ public class ProcessWrapperSandboxedStrategy extends SandboxStrategy { private final Path execRoot; private final boolean verboseFailures; private final String productName; + private final SpawnInputExpander spawnInputExpander; ProcessWrapperSandboxedStrategy( CommandEnvironment cmdEnv, @@ -65,6 +67,7 @@ public class ProcessWrapperSandboxedStrategy extends SandboxStrategy { this.execRoot = cmdEnv.getExecRoot(); this.verboseFailures = verboseFailures; this.productName = productName; + this.spawnInputExpander = new SpawnInputExpander(false); } @Override @@ -92,7 +95,10 @@ public class ProcessWrapperSandboxedStrategy extends SandboxStrategy { SymlinkedExecRoot symlinkedExecRoot = new SymlinkedExecRoot(sandboxExecRoot); ImmutableSet<PathFragment> outputs = SandboxHelpers.getOutputFiles(spawn); symlinkedExecRoot.createFileSystem( - getMounts(spawn, actionExecutionContext), outputs, writableDirs); + SandboxHelpers.getInputFiles( + spawnInputExpander, this.execRoot, spawn, actionExecutionContext), + outputs, + writableDirs); SandboxRunner runner = new ProcessWrapperRunner(sandboxExecRoot, verboseFailures); try { diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxHelpers.java b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxHelpers.java index 364ab38331..37f2ce8e4c 100644 --- a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxHelpers.java +++ b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxHelpers.java @@ -18,14 +18,23 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet.Builder; import com.google.devtools.build.lib.actions.ActionExecutionContext; import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.actions.ExecException; import com.google.devtools.build.lib.actions.Executor; import com.google.devtools.build.lib.actions.Spawn; import com.google.devtools.build.lib.analysis.config.BuildConfiguration; import com.google.devtools.build.lib.buildtool.BuildRequest; +import com.google.devtools.build.lib.exec.SpawnInputExpander; +import com.google.devtools.build.lib.rules.fileset.FilesetActionContext; import com.google.devtools.build.lib.standalone.StandaloneSpawnStrategy; import com.google.devtools.build.lib.util.Preconditions; +import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; /** Helper methods that are shared by the different sandboxing strategies in this package. */ public final class SandboxHelpers { @@ -44,6 +53,50 @@ public final class SandboxHelpers { } } + /** + * Returns the inputs of a Spawn as a map of PathFragments relative to an execRoot to paths in the + * host filesystem where the input files can be found. + */ + public static Map<PathFragment, Path> getInputFiles( + SpawnInputExpander spawnInputExpander, + Path execRoot, + Spawn spawn, + ActionExecutionContext executionContext) + throws IOException { + Map<PathFragment, ActionInput> inputMap = + spawnInputExpander.getInputMapping( + spawn, + executionContext.getArtifactExpander(), + executionContext.getActionInputFileCache(), + executionContext.getExecutor().getContext(FilesetActionContext.class)); + // SpawnInputExpander#getInputMapping uses ArtifactExpander#expandArtifacts to expand + // middlemen and tree artifacts, which expands empty tree artifacts to no entry. However, + // actions that accept TreeArtifacts as inputs generally expect that the empty directory is + // created. So we add those explicitly here. + // TODO(ulfjack): Move this code to SpawnInputExpander. + for (ActionInput input : spawn.getInputFiles()) { + if (input instanceof Artifact && ((Artifact) input).isTreeArtifact()) { + List<Artifact> containedArtifacts = new ArrayList<>(); + executionContext.getArtifactExpander().expand((Artifact) input, containedArtifacts); + // Attempting to mount a non-empty directory results in ERR_DIRECTORY_NOT_EMPTY, so we + // only mount empty TreeArtifacts as directories. + if (containedArtifacts.isEmpty()) { + inputMap.put(input.getExecPath(), input); + } + } + } + + Map<PathFragment, Path> inputFiles = new TreeMap<>(); + for (Map.Entry<PathFragment, ActionInput> e : inputMap.entrySet()) { + Path inputPath = + e.getValue() == SpawnInputExpander.EMPTY_FILE + ? null + : execRoot.getRelative(e.getValue().getExecPath()); + inputFiles.put(e.getKey(), inputPath); + } + return inputFiles; + } + public static ImmutableSet<PathFragment> getOutputFiles(Spawn spawn) { Builder<PathFragment> outputFiles = ImmutableSet.builder(); for (ActionInput output : spawn.getOutputFiles()) { diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxStrategy.java b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxStrategy.java index 48d125a3ae..82963df099 100644 --- a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxStrategy.java +++ b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxStrategy.java @@ -18,10 +18,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.eventbus.EventBus; import com.google.devtools.build.lib.actions.ActionExecutionContext; import com.google.devtools.build.lib.actions.ActionExecutionMetadata; -import com.google.devtools.build.lib.actions.ActionInput; import com.google.devtools.build.lib.actions.ActionStatusMessage; -import com.google.devtools.build.lib.actions.Artifact; -import com.google.devtools.build.lib.actions.EnvironmentalExecException; import com.google.devtools.build.lib.actions.ExecException; import com.google.devtools.build.lib.actions.Executor; import com.google.devtools.build.lib.actions.ResourceManager; @@ -34,18 +31,13 @@ import com.google.devtools.build.lib.actions.UserExecException; import com.google.devtools.build.lib.buildtool.BuildRequest; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.EventHandler; -import com.google.devtools.build.lib.exec.SpawnInputExpander; -import com.google.devtools.build.lib.rules.fileset.FilesetActionContext; import com.google.devtools.build.lib.runtime.CommandEnvironment; import com.google.devtools.build.lib.util.io.OutErr; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; import java.util.Map; import java.util.Set; -import java.util.TreeMap; import java.util.concurrent.atomic.AtomicReference; /** Abstract common ancestor for sandbox strategies implementing the common parts. */ @@ -56,7 +48,6 @@ abstract class SandboxStrategy implements SandboxedSpawnActionContext { private final Path execRoot; private final boolean verboseFailures; private final SandboxOptions sandboxOptions; - private final SpawnInputExpander spawnInputExpander; private final Path sandboxBase; public SandboxStrategy( @@ -71,7 +62,6 @@ abstract class SandboxStrategy implements SandboxedSpawnActionContext { this.sandboxBase = sandboxBase; this.verboseFailures = verboseFailures; this.sandboxOptions = sandboxOptions; - this.spawnInputExpander = new SpawnInputExpander(/*strict=*/false); } /** Executes the given {@code spawn}. */ @@ -202,44 +192,4 @@ abstract class SandboxStrategy implements SandboxedSpawnActionContext { public boolean shouldPropagateExecException() { return verboseFailures && sandboxOptions.sandboxDebug; } - - public Map<PathFragment, Path> getMounts(Spawn spawn, ActionExecutionContext executionContext) - throws ExecException { - try { - Map<PathFragment, ActionInput> inputMap = spawnInputExpander - .getInputMapping( - spawn, - executionContext.getArtifactExpander(), - executionContext.getActionInputFileCache(), - executionContext.getExecutor().getContext(FilesetActionContext.class)); - // SpawnInputExpander#getInputMapping uses ArtifactExpander#expandArtifacts to expand - // middlemen and tree artifacts, which expands empty tree artifacts to no entry. However, - // actions that accept TreeArtifacts as inputs generally expect that the empty directory is - // created. So we add those explicitly here. - // TODO(ulfjack): Move this code to SpawnInputExpander. - for (ActionInput input : spawn.getInputFiles()) { - if (input instanceof Artifact && ((Artifact) input).isTreeArtifact()) { - List<Artifact> containedArtifacts = new ArrayList<>(); - executionContext.getArtifactExpander().expand((Artifact) input, containedArtifacts); - // Attempting to mount a non-empty directory results in ERR_DIRECTORY_NOT_EMPTY, so we - // only mount empty TreeArtifacts as directories. - if (containedArtifacts.isEmpty()) { - inputMap.put(input.getExecPath(), input); - } - } - } - - Map<PathFragment, Path> mounts = new TreeMap<>(); - for (Map.Entry<PathFragment, ActionInput> e : inputMap.entrySet()) { - Path inputPath = e.getValue() == SpawnInputExpander.EMPTY_FILE - ? null - : execRoot.getRelative(e.getValue().getExecPath()); - mounts.put(e.getKey(), inputPath); - } - return mounts; - } catch (IOException e) { - throw new EnvironmentalExecException("Could not prepare mounts for sandbox execution", e); - } - } - } diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/SpawnHelpers.java b/src/main/java/com/google/devtools/build/lib/sandbox/SpawnHelpers.java deleted file mode 100644 index b6bd3c017e..0000000000 --- a/src/main/java/com/google/devtools/build/lib/sandbox/SpawnHelpers.java +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright 2016 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.devtools.build.lib.sandbox; - -import com.google.common.io.Files; -import com.google.devtools.build.lib.actions.ActionExecutionContext; -import com.google.devtools.build.lib.actions.ActionInput; -import com.google.devtools.build.lib.actions.ActionInputHelper; -import com.google.devtools.build.lib.actions.Artifact; -import com.google.devtools.build.lib.actions.Spawn; -import com.google.devtools.build.lib.analysis.AnalysisUtils; -import com.google.devtools.build.lib.rules.fileset.FilesetActionContext; -import com.google.devtools.build.lib.util.Preconditions; -import com.google.devtools.build.lib.vfs.FileSystem; -import com.google.devtools.build.lib.vfs.Path; -import com.google.devtools.build.lib.vfs.PathFragment; -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** Contains common helper methods that extract information from {@link Spawn} objects. */ -public final class SpawnHelpers { - - private final Path execRoot; - - public SpawnHelpers(Path execRoot) { - this.execRoot = execRoot; - } - - /** - * Returns the inputs of a Spawn as a map of PathFragments relative to an execRoot to paths in the - * host filesystem where the input files can be found. - */ - public Map<PathFragment, Path> getMounts(Spawn spawn, ActionExecutionContext executionContext) - throws IOException { - Map<PathFragment, Path> mounts = new HashMap<>(); - mountRunfilesFromSuppliers(mounts, spawn); - mountFilesFromFilesetManifests(mounts, spawn, executionContext); - mountInputs(mounts, spawn, executionContext); - return mounts; - } - - /** Mount all files that the spawn needs as specified in its fileset manifests. */ - void mountFilesFromFilesetManifests( - Map<PathFragment, Path> mounts, Spawn spawn, ActionExecutionContext executionContext) - throws IOException { - final FilesetActionContext filesetContext = - executionContext.getExecutor().getContext(FilesetActionContext.class); - for (Artifact fileset : spawn.getFilesetManifests()) { - File manifestFile = - new File( - execRoot.getPathString(), - AnalysisUtils.getManifestPathFromFilesetPath(fileset.getExecPath()).getPathString()); - PathFragment targetDirectory = fileset.getExecPath(); - - parseManifestFile( - execRoot.getFileSystem(), - mounts, - targetDirectory, - manifestFile, - true, - filesetContext.getWorkspaceName()); - } - } - - /** A parser for the MANIFEST files used by Filesets and runfiles. */ - static void parseManifestFile( - FileSystem fs, - Map<PathFragment, Path> mounts, - PathFragment targetDirectory, - File manifestFile, - boolean isFilesetManifest, - String workspaceName) - throws IOException { - int lineNum = 0; - for (String line : Files.readLines(manifestFile, StandardCharsets.UTF_8)) { - if (isFilesetManifest && (++lineNum % 2 == 0)) { - continue; - } - if (line.isEmpty()) { - continue; - } - - String[] fields = line.trim().split(" "); - - // The "target" field is always a relative path that is to be interpreted in this way: - // (1) If this is a fileset manifest and our workspace name is not empty, the first segment - // of each "target" path must be the workspace name, which is then stripped before further - // processing. - // (2) The "target" path is then appended to the "targetDirectory", which is a path relative - // to the execRoot. Together, this results in the full path in the execRoot in which place a - // symlink referring to "source" has to be created (see below). - PathFragment targetPath; - if (isFilesetManifest) { - PathFragment targetPathFragment = PathFragment.create(fields[0]); - if (!workspaceName.isEmpty()) { - Preconditions.checkState( - targetPathFragment.getSegment(0).equals(workspaceName), - "Fileset manifest line must start with workspace name"); - targetPathFragment = targetPathFragment.subFragment(1, targetPathFragment.segmentCount()); - } - targetPath = targetDirectory.getRelative(targetPathFragment); - } else { - targetPath = targetDirectory.getRelative(fields[0]); - } - - // The "source" field, if it exists, is always an absolute path and may point to any file in - // the filesystem (it is not limited to files in the workspace or execroot). - Path source; - switch (fields.length) { - case 1: - source = null; - break; - case 2: - source = fs.getPath(fields[1]); - break; - default: - throw new IllegalStateException("'" + line + "' splits into more than 2 parts"); - } - - mounts.put(targetPath, source); - } - } - - /** Mount all runfiles that the spawn needs as specified via its runfiles suppliers. */ - void mountRunfilesFromSuppliers(Map<PathFragment, Path> mounts, Spawn spawn) throws IOException { - Map<PathFragment, Map<PathFragment, Artifact>> rootsAndMappings = - spawn.getRunfilesSupplier().getMappings(); - for (Map.Entry<PathFragment, Map<PathFragment, Artifact>> rootAndMappings : - rootsAndMappings.entrySet()) { - PathFragment root = rootAndMappings.getKey(); - Preconditions.checkState(!root.isAbsolute()); - for (Map.Entry<PathFragment, Artifact> mapping : rootAndMappings.getValue().entrySet()) { - Artifact sourceArtifact = mapping.getValue(); - Path source = - (sourceArtifact != null) ? execRoot.getRelative(sourceArtifact.getExecPath()) : null; - - Preconditions.checkArgument(!mapping.getKey().isAbsolute()); - PathFragment target = root.getRelative(mapping.getKey()); - mounts.put(target, source); - } - } - } - - /** Mount all inputs of the spawn. */ - void mountInputs( - Map<PathFragment, Path> mounts, Spawn spawn, ActionExecutionContext actionExecutionContext) { - List<ActionInput> inputs = - ActionInputHelper.expandArtifacts( - spawn.getInputFiles(), actionExecutionContext.getArtifactExpander()); - - // ActionInputHelper#expandArtifacts above expands empty TreeArtifacts into an empty list. - // However, actions that accept TreeArtifacts as inputs generally expect that the empty - // directory is created. So here we explicitly mount the directories of the TreeArtifacts as - // inputs. - for (ActionInput input : spawn.getInputFiles()) { - if (input instanceof Artifact && ((Artifact) input).isTreeArtifact()) { - List<Artifact> containedArtifacts = new ArrayList<>(); - actionExecutionContext.getArtifactExpander().expand((Artifact) input, containedArtifacts); - // Attempting to mount a non-empty directory results in ERR_DIRECTORY_NOT_EMPTY, so we only - // mount empty TreeArtifacts as directories. - if (containedArtifacts.isEmpty()) { - PathFragment mount = PathFragment.create(input.getExecPathString()); - mounts.put(mount, execRoot.getRelative(mount)); - } - } - } - - for (ActionInput input : inputs) { - if (input.getExecPathString().contains("internal/_middlemen/")) { - continue; - } - PathFragment mount = PathFragment.create(input.getExecPathString()); - mounts.put(mount, execRoot.getRelative(mount)); - } - } -} diff --git a/src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnStrategy.java b/src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnStrategy.java index 9a681ab5ea..2bd4fb1e2d 100644 --- a/src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnStrategy.java +++ b/src/main/java/com/google/devtools/build/lib/worker/WorkerSpawnStrategy.java @@ -39,8 +39,8 @@ import com.google.devtools.build.lib.actions.SpawnActionContext; import com.google.devtools.build.lib.actions.UserExecException; import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.exec.SpawnInputExpander; import com.google.devtools.build.lib.sandbox.SandboxHelpers; -import com.google.devtools.build.lib.sandbox.SpawnHelpers; import com.google.devtools.build.lib.standalone.StandaloneSpawnStrategy; import com.google.devtools.build.lib.util.CommandFailureUtils; import com.google.devtools.build.lib.util.Preconditions; @@ -85,6 +85,7 @@ public final class WorkerSpawnStrategy implements SandboxedSpawnActionContext { private final Path execRoot; private final boolean verboseFailures; private final Multimap<String, String> extraFlags; + private final SpawnInputExpander spawnInputExpander; public WorkerSpawnStrategy( BlazeDirectories blazeDirs, @@ -96,6 +97,7 @@ public final class WorkerSpawnStrategy implements SandboxedSpawnActionContext { this.execRoot = blazeDirs.getExecRoot(); this.verboseFailures = verboseFailures; this.extraFlags = extraFlags; + this.spawnInputExpander = new SpawnInputExpander(false); } @Override @@ -162,7 +164,7 @@ public final class WorkerSpawnStrategy implements SandboxedSpawnActionContext { HashCode workerFilesHash = WorkerFilesHash.getWorkerFilesHash( spawn.getToolFiles(), actionExecutionContext); Map<PathFragment, Path> inputFiles = - new SpawnHelpers(execRoot).getMounts(spawn, actionExecutionContext); + SandboxHelpers.getInputFiles(spawnInputExpander, execRoot, spawn, actionExecutionContext); Set<PathFragment> outputFiles = SandboxHelpers.getOutputFiles(spawn); WorkerKey key = |