diff options
Diffstat (limited to 'src/main/java')
3 files changed, 262 insertions, 21 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxedSpawn.java b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxedSpawn.java index 431a4b1d43..e6408dc3cc 100644 --- a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxedSpawn.java +++ b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxedSpawn.java @@ -14,8 +14,12 @@ package com.google.devtools.build.lib.sandbox; +import com.google.common.io.Files; +import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; import java.io.IOException; +import java.util.Collection; import java.util.List; import java.util.Map; @@ -56,4 +60,37 @@ interface SandboxedSpawn { * Deletes the sandbox directory. */ void delete(); + + /** + * Moves all given outputs from a root to another. + * + * <p>This is a support function to help with the implementation of {@link #copyOutputs(Path)}. + * + * @param outputs outputs to move as relative paths to a root + * @param sourceRoot source directory from which to resolve outputs + * @param targetRoot target directory to which to move the resolved outputs from the source + * @throws IOException if any of the moves fails + */ + static void moveOutputs(Collection<PathFragment> outputs, Path sourceRoot, Path targetRoot) + throws IOException { + for (PathFragment output : outputs) { + Path source = sourceRoot.getRelative(output); + Path target = targetRoot.getRelative(output); + if (source.isFile() || source.isSymbolicLink()) { + // Ensure the target directory exists in the target. The directories for the action outputs + // have already been created, but the spawn outputs may be different from the overall action + // outputs. This is the case for test actions. + target.getParentDirectory().createDirectoryAndParents(); + Files.move(source.getPathFile(), target.getPathFile()); + } else if (source.isDirectory()) { + try { + source.renameTo(target); + } catch (IOException e) { + // Failed to move directory directly, thus move it recursively. + target.createDirectory(); + FileSystemUtils.moveTreesBelow(source, target); + } + } + } + } } diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxfsSandboxedSpawn.java b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxfsSandboxedSpawn.java new file mode 100644 index 0000000000..a33785f648 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxfsSandboxedSpawn.java @@ -0,0 +1,224 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.sandbox; + +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.devtools.build.lib.sandbox.SandboxfsProcess.Mapping; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Logger; + +/** + * Creates an execRoot for a Spawn that contains all required input files by mounting a sandboxfs + * FUSE filesystem on the provided path. + */ +class SandboxfsSandboxedSpawn implements SandboxedSpawn { + private static final Logger log = Logger.getLogger(SandboxfsSandboxedSpawn.class.getName()); + + /** An empty file to use when mappings don't specify their target. */ + private static final PathFragment EMPTY_FILE = PathFragment.create("/dev/null"); + + /** Sequence number to assign a unique subtree to each action within the mount point. */ + private static final AtomicInteger lastId = new AtomicInteger(); + + /** The sandboxfs instance to use for this spawn. */ + private final SandboxfsProcess process; + + /** Arguments to pass to the spawn, including the binary name. */ + private final List<String> arguments; + + /** Environment variables to pass to the spawn. */ + private final Map<String, String> environment; + + /** Collection of input files to be made available to the spawn in read-only mode. */ + private final Map<PathFragment, Path> inputs; + + /** Collection of output files to expect from the spawn. */ + private final Collection<PathFragment> outputs; + + /** Collection of directories where the spawn can write files to relative to {@link #execRoot}. */ + private final Set<PathFragment> writableDirs; + + /** + * Writable directory to support the writes performed by the command. This acts as the target + * of all writable mappings in the sandboxfs instance. + */ + private final Path sandboxScratchDir; + + /** Path to the working directory of the command. */ + private final Path execRoot; + + /** + * Path to the working directory of the command, seen as an absolute path that starts at + * the sandboxfs's mount point. + */ + private final PathFragment innerExecRoot; + + /** + * Constructs a new sandboxfs-based spawn runner. + * + * @param process sandboxfs instance to use for this spawn + * @param outerDir writable directory where the spawn runner keeps control files + * @param arguments arguments to pass to the spawn, including the binary name + * @param environment environment variables to pass to the spawn + * @param inputs input files to be made available to the spawn in read-only mode + * @param outputs output files to expect from the spawn + * @param writableDirs directories where the spawn can write files to, relative to the sandbox's + * dynamically-allocated execroot + */ + SandboxfsSandboxedSpawn( + SandboxfsProcess process, + Path outerDir, + List<String> arguments, + Map<String, String> environment, + Map<PathFragment, Path> inputs, + Collection<PathFragment> outputs, + Set<PathFragment> writableDirs) { + this.process = process; + this.arguments = arguments; + this.environment = environment; + this.inputs = inputs; + for (PathFragment path : outputs) { + checkArgument(!path.isAbsolute(), "outputs %s must be relative", path); + } + this.outputs = outputs; + for (PathFragment path : writableDirs) { + checkArgument(!path.isAbsolute(), "writable directory %s must be relative", path); + } + this.writableDirs = writableDirs; + + this.sandboxScratchDir = outerDir.getRelative("scratch"); + + int id = lastId.getAndIncrement(); + this.execRoot = process.getMountPoint().getRelative("" + id); + this.innerExecRoot = PathFragment.create("/" + id); + } + + @Override + public Path getSandboxExecRoot() { + return execRoot; + } + + @Override + public List<String> getArguments() { + return arguments; + } + + @Override + public Map<String, String> getEnvironment() { + return environment; + } + + @Override + public void createFileSystem() throws IOException { + sandboxScratchDir.createDirectory(); + + reconfigure(inputs, writableDirs, outputs); + } + + @Override + public void copyOutputs(Path targetExecRoot) throws IOException { + // TODO(jmmv): If we knew the targetExecRoot when setting up the spawn, we may be able to + // configure sandboxfs so that the output files are written directly to their target locations. + // This would avoid having to move them after-the-fact. + SandboxedSpawn.moveOutputs(outputs, sandboxScratchDir, targetExecRoot); + } + + @Override + public void delete() { + try { + process.unmap(innerExecRoot); + } catch (IOException e) { + // We use independent subdirectories for each action, so a failure to unmap one, while + // annoying, is not a big deal. The sandboxfs instance will be unmounted anyway after + // the build, which will cause these to go away anyway. + log.warning("Cannot unmap " + innerExecRoot + ": " + e); + } + } + + /** + * Creates a new set of mappings to sandbox the given inputs. + * + * @param inputs collection of paths to expose within the sandbox as read-only mappings, given + * as a map of mapped path to target path. The target path may be null, in which case an empty + * read-only file is mapped. + * @return the collection of mappings to use for reconfiguration + */ + private List<Mapping> createMappings(Map<PathFragment, Path> inputs) { + List<Mapping> mappings = new ArrayList<>(); + + mappings.add(Mapping.builder() + .setPath(innerExecRoot) + .setTarget(sandboxScratchDir.asFragment()) + .setWritable(true) + .build()); + + for (Entry<PathFragment, Path> entry : inputs.entrySet()) { + mappings.add(Mapping.builder() + .setPath(innerExecRoot.getRelative(entry.getKey())) + .setTarget(entry.getValue() == null ? EMPTY_FILE : entry.getValue().asFragment()) + .setWritable(false) + .build()); + } + + return mappings; + } + + /** + * Pushes a new configuration to sandboxfs and waits for acceptance. + * + * @param inputs collection of paths to expose within the sandbox as read-only mappings, given + * as a map of mapped path to target path. The target path may be null, in which case an empty + * file is mapped. + * @param writableDirs collection of writable paths to create within the read-write portion of + * the sandbox + * @param outputs collection of outputs to expect within the read-write portion of the sandbox + * @throws IOException if reconfiguration fails + */ + private void reconfigure(Map<PathFragment, Path> inputs, Set<PathFragment> writableDirs, + Collection<PathFragment> outputs) throws IOException { + List<Mapping> mappings = createMappings(inputs); + + Set<PathFragment> dirsToCreate = new HashSet<>(writableDirs); + for (PathFragment output : outputs) { + dirsToCreate.add(output.getParentDirectory()); + } + for (PathFragment input : inputs.keySet()) { + // We must pre-create the directory layout for input files as well as for output files. + // The reason is that we map the root directory as writable within the sandbox and later + // map read-only files on top of it. We want the intermediate components on those paths + // to remain writable within the top-level root directory we have mounted, but that's + // only possible if we pre-create those. Otherwise, sandboxfs will see that those components + // do not exist when mapping the read-only file and will generate fake, in-memory, read-only + // components for them. + dirsToCreate.add(input.getParentDirectory()); + } + for (PathFragment dir : dirsToCreate) { + sandboxScratchDir.getRelative(dir).createDirectoryAndParents(); + } + + process.map(mappings); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/SymlinkedSandboxedSpawn.java b/src/main/java/com/google/devtools/build/lib/sandbox/SymlinkedSandboxedSpawn.java index 80db422dbf..9657d04203 100644 --- a/src/main/java/com/google/devtools/build/lib/sandbox/SymlinkedSandboxedSpawn.java +++ b/src/main/java/com/google/devtools/build/lib/sandbox/SymlinkedSandboxedSpawn.java @@ -15,7 +15,6 @@ package com.google.devtools.build.lib.sandbox; import com.google.common.base.Preconditions; -import com.google.common.io.Files; import com.google.devtools.build.lib.vfs.FileStatus; import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; @@ -170,28 +169,9 @@ public class SymlinkedSandboxedSpawn implements SandboxedSpawn { } } - /** Moves all {@code outputs} to {@code execRoot}. */ @Override public void copyOutputs(Path execRoot) throws IOException { - for (PathFragment output : outputs) { - Path source = sandboxExecRoot.getRelative(output); - Path target = execRoot.getRelative(output); - if (source.isFile() || source.isSymbolicLink()) { - // Ensure the target directory exists in the real execroot. The directories for the action - // outputs have already been created, but the spawn outputs may be different from the - // overall action outputs. This is the case for test actions. - FileSystemUtils.createDirectoryAndParents(target.getParentDirectory()); - Files.move(source.getPathFile(), target.getPathFile()); - } else if (source.isDirectory()) { - try { - source.renameTo(target); - } catch (IOException e) { - // Failed to move directory directly, thus move it recursively. - target.createDirectory(); - FileSystemUtils.moveTreesBelow(source, target); - } - } - } + SandboxedSpawn.moveOutputs(outputs, sandboxExecRoot, execRoot); } @Override |