diff options
Diffstat (limited to 'src/main/java/com/google/devtools/build/lib/standalone')
6 files changed, 847 insertions, 0 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/standalone/LinuxSandboxedStrategy.java b/src/main/java/com/google/devtools/build/lib/standalone/LinuxSandboxedStrategy.java new file mode 100644 index 0000000000..dc32ddc806 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/standalone/LinuxSandboxedStrategy.java @@ -0,0 +1,227 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.standalone; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.ActionInputHelper; +import com.google.devtools.build.lib.actions.Actions; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.ExecutionStrategy; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.Spawn; +import com.google.devtools.build.lib.actions.SpawnActionContext; +import com.google.devtools.build.lib.actions.UserExecException; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.rules.cpp.CppCompileAction; +import com.google.devtools.build.lib.shell.CommandException; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.unix.FilesystemUtils; +import com.google.devtools.build.lib.util.CommandFailureUtils; +import com.google.devtools.build.lib.util.DependencySet; +import com.google.devtools.build.lib.util.io.FileOutErr; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.TreeSet; + +/** + * Strategy that uses sandboxing to execute a process. + */ +@ExecutionStrategy(name = {"sandboxed"}, + contextType = SpawnActionContext.class) +public class LinuxSandboxedStrategy implements SpawnActionContext { + private final boolean verboseFailures; + private final BlazeDirectories directories; + + public LinuxSandboxedStrategy(BlazeDirectories blazeDirectories, boolean verboseFailures) { + this.directories = blazeDirectories; + this.verboseFailures = verboseFailures; + } + + /** + * Executes the given {@code spawn}. + */ + @Override + public void exec(Spawn spawn, ActionExecutionContext actionExecutionContext) + throws ExecException { + Executor executor = actionExecutionContext.getExecutor(); + if (executor.reportsSubcommands()) { + executor.reportSubcommand(Label.print(spawn.getOwner().getLabel()), + spawn.asShellCommand(executor.getExecRoot())); + } + boolean processHeaders = spawn.getResourceOwner() instanceof CppCompileAction; + + Path execPath = this.directories.getExecRoot(); + List<String> spawnArguments = new ArrayList<>(); + + for (String arg : spawn.getArguments()) { + if (arg.startsWith(execPath.getPathString())) { + // make all paths relative for the sandbox + spawnArguments.add(arg.substring(execPath.getPathString().length())); + } else { + spawnArguments.add(arg); + } + } + + List<? extends ActionInput> expandedInputs = + ActionInputHelper.expandMiddlemen(spawn.getInputFiles(), + actionExecutionContext.getMiddlemanExpander()); + + String cwd = executor.getExecRoot().getPathString(); + + FileOutErr outErr = actionExecutionContext.getFileOutErr(); + try { + PathFragment includePrefix = null; // null when there's no include mangling to do + List<PathFragment> includeDirectories = ImmutableList.of(); + if (processHeaders) { + CppCompileAction cppAction = (CppCompileAction) spawn.getResourceOwner(); + // headers are mounted in the sandbox in a separate include dir, so their names are mangled + // when running the compilation and will have to be unmangled after it's done in the *.pic.d + includeDirectories = extractIncludeDirs(execPath, cppAction, spawnArguments); + includePrefix = getSandboxIncludeDir(cppAction); + } + + NamespaceSandboxRunner runner = new NamespaceSandboxRunner(directories, spawn, includePrefix, + includeDirectories, spawn.getRunfilesManifests()); + runner.setupSandbox(expandedInputs, spawn.getOutputFiles()); + runner.run(spawnArguments, spawn.getEnvironment(), new File(cwd), outErr); + runner.copyOutputs(spawn.getOutputFiles(), outErr); + if (processHeaders) { + CppCompileAction cppAction = (CppCompileAction) spawn.getResourceOwner(); + unmangleHeaderFiles(cppAction); + } + runner.cleanup(); + } catch (CommandException e) { + String message = CommandFailureUtils.describeCommandFailure(verboseFailures, + spawn.getArguments(), spawn.getEnvironment(), cwd); + throw new UserExecException(String.format("%s: %s", message, e)); + } catch (IOException e) { + throw new UserExecException(e.getMessage()); + } + } + + private void unmangleHeaderFiles(CppCompileAction cppCompileAction) throws IOException { + Path execPath = this.directories.getExecRoot(); + CppCompileAction.DotdFile dotdfile = cppCompileAction.getDotdFile(); + DependencySet depset = new DependencySet(execPath).read(dotdfile.getPath()); + DependencySet unmangled = new DependencySet(execPath); + PathFragment sandboxIncludeDir = getSandboxIncludeDir(cppCompileAction); + PathFragment prefix = sandboxIncludeDir.getRelative(execPath.asFragment().relativeTo("/")); + for (PathFragment dep : depset.getDependencies()) { + if (dep.startsWith(prefix)) { + dep = dep.relativeTo(prefix); + } + unmangled.addDependency(dep); + } + unmangled.write(execPath.getRelative(depset.getOutputFileName()), ".d"); + } + + private PathFragment getSandboxIncludeDir(CppCompileAction cppCompileAction) { + return new PathFragment( + "include-" + Actions.escapedPath(cppCompileAction.getPrimaryOutput().toString())); + } + + private ImmutableList<PathFragment> extractIncludeDirs(Path execPath, + CppCompileAction cppCompileAction, List<String> spawnArguments) throws IOException { + List<PathFragment> includes = new ArrayList<>(); + includes.addAll(cppCompileAction.getQuoteIncludeDirs()); + includes.addAll(cppCompileAction.getIncludeDirs()); + includes.addAll(cppCompileAction.getSystemIncludeDirs()); + + // gcc implicitly includes headers in the same dir as .cc file + PathFragment sourceDirectory = + cppCompileAction.getSourceFile().getPath().getParentDirectory().asFragment(); + includes.add(sourceDirectory); + spawnArguments.add("-iquote"); + spawnArguments.add(sourceDirectory.toString()); + + TreeSet<PathFragment> processedIncludes = new TreeSet<>(); + for (int i = 0; i < includes.size(); i++) { + PathFragment absolutePath; + if (!includes.get(i).isAbsolute()) { + absolutePath = execPath.getRelative(includes.get(i)).asFragment(); + } else { + absolutePath = includes.get(i); + } + // CppCompileAction may provide execPath as one of the include directories. This is a big + // overestimation of what is actually needed and doesn't make for very hermetic sandbox + // (since everything from the workspace will be somehow accessed in the sandbox). To have + // some more hermeticity in this situation we mount all the include dirs in: + // sandbox-directory/include-prefix/actual-include-dir + // (where include-prefix is obtained from this.getSandboxIncludeDir(cppCompileAction)) + // and make so gcc looks there for includes. This should prevent the user from accessing + // files that technically should not be in the sandbox. + // TODO(bazel-team): change CppCompileAction so that include dirs contain only subsets of the + // execPath + if (absolutePath.equals(execPath.asFragment())) { + // we can't mount execPath because it will lead to a circular mount; instead mount its + // subdirs inside (other than the ones containing sandbox) + String[] subdirs = FilesystemUtils.readdir(absolutePath.toString()); + for (String dirName : subdirs) { + if (dirName.equals("_bin") || dirName.equals("bazel-out")) { + continue; + } + PathFragment child = absolutePath.getChild(dirName); + processedIncludes.add(child); + } + } else { + processedIncludes.add(absolutePath); + } + } + + // pseudo random name for include directory inside sandbox, so it won't be accessed by accident + String prefix = getSandboxIncludeDir(cppCompileAction).toString(); + + // change names in the invocation + for (int i = 0; i < spawnArguments.size(); i++) { + if (spawnArguments.get(i).startsWith("-I")) { + String argument = spawnArguments.get(i).substring(2); + spawnArguments.set(i, setIncludeDirSandboxPath(execPath, argument, "-I" + prefix)); + } + if (spawnArguments.get(i).equals("-iquote") || spawnArguments.get(i).equals("-isystem")) { + spawnArguments.set(i + 1, setIncludeDirSandboxPath(execPath, + spawnArguments.get(i + 1), prefix)); + } + } + return ImmutableList.copyOf(processedIncludes); + } + + private String setIncludeDirSandboxPath(Path execPath, String argument, String prefix) { + StringBuilder builder = new StringBuilder(prefix); + if (argument.charAt(0) != '/') { + // relative path + builder.append(execPath); + builder.append('/'); + } + builder.append(argument); + + return builder.toString(); + } + + @Override + public String strategyLocality(String mnemonic, boolean remotable) { + return "linux-sandboxing"; + } + + @Override + public boolean isRemotable(String mnemonic, boolean remotable) { + return false; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/standalone/LocalSpawnStrategy.java b/src/main/java/com/google/devtools/build/lib/standalone/LocalSpawnStrategy.java new file mode 100644 index 0000000000..fc2387c896 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/standalone/LocalSpawnStrategy.java @@ -0,0 +1,111 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.standalone; + +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.ExecutionStrategy; +import com.google.devtools.build.lib.actions.Executor; +import com.google.devtools.build.lib.actions.Spawn; +import com.google.devtools.build.lib.actions.SpawnActionContext; +import com.google.devtools.build.lib.actions.UserExecException; +import com.google.devtools.build.lib.shell.Command; +import com.google.devtools.build.lib.shell.CommandException; +import com.google.devtools.build.lib.syntax.Label; +import com.google.devtools.build.lib.util.CommandFailureUtils; +import com.google.devtools.build.lib.util.OS; +import com.google.devtools.build.lib.util.OsUtils; +import com.google.devtools.build.lib.util.io.FileOutErr; +import com.google.devtools.build.lib.vfs.Path; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +/** + * Strategy that uses subprocessing to execute a process. + */ +@ExecutionStrategy(name = { "standalone" }, contextType = SpawnActionContext.class) +public class LocalSpawnStrategy implements SpawnActionContext { + private final boolean verboseFailures; + + private final Path processWrapper; + + public LocalSpawnStrategy(Path execRoot, boolean verboseFailures) { + this.verboseFailures = verboseFailures; + this.processWrapper = execRoot.getRelative( + "_bin/process-wrapper" + OsUtils.executableExtension()); + } + + /** + * Executes the given {@code spawn}. + */ + @Override + public void exec(Spawn spawn, + ActionExecutionContext actionExecutionContext) + throws ExecException { + Executor executor = actionExecutionContext.getExecutor(); + if (executor.reportsSubcommands()) { + executor.reportSubcommand(Label.print(spawn.getOwner().getLabel()), + spawn.asShellCommand(executor.getExecRoot())); + } + + // We must wrap the subprocess with process-wrapper to kill the process tree. + // All actions therefore depend on the process-wrapper file. Since it's embedded, + // we don't bother with declaring it as an input. + List<String> args = new ArrayList<>(); + if (OS.getCurrent() != OS.WINDOWS) { + // TODO(bazel-team): process-wrapper seems to work on Windows, but requires + // additional setup as it is an msys2 binary, so it needs msys2 DLLs on %PATH%. + // Disable it for now to make the setup easier and to avoid further PATH hacks. + // Ideally we should have a native implementation of process-wrapper for Windows. + args.add(processWrapper.getPathString()); + args.add("-1"); /* timeout */ + args.add("0"); /* kill delay. */ + + // TODO(bazel-team): use process-wrapper redirection so we don't have to + // pass test logs through the Java heap. + args.add("-"); /* stdout. */ + args.add("-"); /* stderr. */ + } + args.addAll(spawn.getArguments()); + + String cwd = executor.getExecRoot().getPathString(); + Command cmd = new Command(args.toArray(new String[]{}), spawn.getEnvironment(), new File(cwd)); + + FileOutErr outErr = actionExecutionContext.getFileOutErr(); + try { + cmd.execute( + /* stdin */ new byte[]{}, + Command.NO_OBSERVER, + outErr.getOutputStream(), + outErr.getErrorStream(), + /*killSubprocessOnInterrupt*/ true); + } catch (CommandException e) { + String message = CommandFailureUtils.describeCommandFailure( + verboseFailures, spawn.getArguments(), spawn.getEnvironment(), cwd); + throw new UserExecException(String.format("%s: %s", message, e)); + } + } + + @Override + public String strategyLocality(String mnemonic, boolean remotable) { + return "standalone"; + } + + @Override + public boolean isRemotable(String mnemonic, boolean remotable) { + return false; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/standalone/NamespaceSandboxRunner.java b/src/main/java/com/google/devtools/build/lib/standalone/NamespaceSandboxRunner.java new file mode 100644 index 0000000000..3c7a8e0ad8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/standalone/NamespaceSandboxRunner.java @@ -0,0 +1,267 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.standalone; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.io.Files; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Spawn; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.shell.Command; +import com.google.devtools.build.lib.shell.CommandException; +import com.google.devtools.build.lib.unix.FilesystemUtils; +import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.util.io.FileOutErr; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map.Entry; + +/** + * Helper class for running the namespace sandbox. This runner prepares environment inside the + * sandbox (copies inputs, creates file structure), handles sandbox output, performs cleanup and + * changes invocation if necessary. + */ +public class NamespaceSandboxRunner { + private final boolean debug = true; + private final PathFragment sandboxDirectory; + private final Path sandboxPath; + private final List<String> mounts; + private final Path embeddedBinaries; + private final Path tools; + private final ImmutableList<PathFragment> includeDirectories; + private final PathFragment includePrefix; + private final ImmutableMap<PathFragment, Artifact> manifests; + private final Path execRoot; + + public NamespaceSandboxRunner(BlazeDirectories directories, Spawn spawn, + PathFragment includePrefix, List<PathFragment> includeDirectories, + ImmutableMap<PathFragment, Artifact> manifests) { + String md5sum = Fingerprint.md5Digest(spawn.getResourceOwner().getPrimaryOutput().toString()); + this.sandboxDirectory = new PathFragment("sandbox-root-" + md5sum); + this.sandboxPath = + directories.getExecRoot().getRelative("sandboxes").getRelative(sandboxDirectory); + this.mounts = new ArrayList<>(); + this.tools = directories.getExecRoot().getChild("tools"); + this.embeddedBinaries = directories.getEmbeddedBinariesRoot(); + this.includePrefix = includePrefix; + this.includeDirectories = ImmutableList.copyOf(includeDirectories); + this.manifests = manifests; + this.execRoot = directories.getExecRoot(); + } + + private void createFileSystem(Collection<? extends ActionInput> outputs) throws IOException { + // create the sandboxes' parent directory if needed + // TODO(bazel-team): create this with rest of the workspace dirs + if (!sandboxPath.getParentDirectory().isDirectory()) { + FilesystemUtils.mkdir(sandboxPath.getParentDirectory().getPathString(), 0755); + } + + FilesystemUtils.mkdir(sandboxPath.getPathString(), 0755); + String[] dirs = { "bin", "etc" }; + for (String dir : dirs) { + FilesystemUtils.mkdir(sandboxPath.getChild(dir).getPathString(), 0755); + mounts.add("/" + dir); + } + + // usr + String[] dirsUsr = { "bin", "include" }; + FilesystemUtils.mkdir(sandboxPath.getChild("usr").getPathString(), 0755); + Path usr = sandboxPath.getChild("usr"); + for (String dir : dirsUsr) { + FilesystemUtils.mkdir(usr.getChild(dir).getPathString(), 0755); + mounts.add("/usr/" + dir); + } + FileSystemUtils.createDirectoryAndParents(usr.getChild("local").getChild("include")); + mounts.add("/usr/local/include"); + + // shared libs + String[] rootDirs = FilesystemUtils.readdir("/"); + for (String entry : rootDirs) { + if (entry.startsWith("lib")) { + FilesystemUtils.mkdir(sandboxPath.getChild(entry).getPathString(), 0755); + mounts.add("/" + entry); + } + } + + String[] usrDirs = FilesystemUtils.readdir("/usr/"); + for (String entry : usrDirs) { + if (entry.startsWith("lib")) { + String lib = usr.getChild(entry).getPathString(); + FilesystemUtils.mkdir(lib, 0755); + mounts.add("/usr/" + entry); + } + } + + if (this.includePrefix != null) { + FilesystemUtils.mkdir(sandboxPath.getRelative(includePrefix).getPathString(), 0755); + + for (PathFragment fullPath : includeDirectories) { + // includeDirectories should be absolute paths like /usr/include/foo.h. we want to combine + // them into something like sandbox/include-prefix/usr/include/foo.h - for that we remove + // the leading '/' from the path string and concatenate with sandbox/include/prefix + FileSystemUtils.createDirectoryAndParents(sandboxPath.getRelative(includePrefix) + .getRelative(fullPath.getPathString().substring(1))); + } + } + + // output directories + for (ActionInput output : outputs) { + PathFragment parentDirectory = + new PathFragment(output.getExecPathString()).getParentDirectory(); + FileSystemUtils.createDirectoryAndParents(sandboxPath.getRelative(parentDirectory)); + } + } + + public void setupSandbox(List<? extends ActionInput> inputs, + Collection<? extends ActionInput> outputs) throws IOException { + createFileSystem(outputs); + setupBlazeUtils(); + includeManifests(); + copyInputs(inputs); + } + + private void copyInputs(List<? extends ActionInput> inputs) throws IOException { + for (ActionInput input : inputs) { + if (input.getExecPathString().contains("internal/_middlemen/")) { + continue; + } + // entire tools will be mounted in the sandbox, so don't copy parts of it + if (input.getExecPathString().startsWith("tools/")) { + continue; + } + Path target = sandboxPath.getRelative(input.getExecPathString()); + Path source = execRoot.getRelative(input.getExecPathString()); + FileSystemUtils.createDirectoryAndParents(target.getParentDirectory()); + File targetFile = new File(target.getPathString()); + // TODO(bazel-team): mount inputs inside sandbox instead of copying + Files.copy(new File(source.getPathString()), targetFile); + FilesystemUtils.chmod(targetFile, 0755); + } + } + + private void includeManifests() throws IOException { + for (Entry<PathFragment, Artifact> manifest : this.manifests.entrySet()) { + String path = manifest.getValue().getPath().getPathString(); + for (String line : Files.readLines(new File(path), Charset.defaultCharset())) { + String[] fields = line.split(" "); + String targetPath = sandboxPath.getPathString() + PathFragment.SEPARATOR_CHAR + fields[0]; + String sourcePath = fields[1]; + File source = new File(sourcePath); + File target = new File(targetPath); + Files.createParentDirs(target); + Files.copy(source, target); + } + } + } + + private void setupBlazeUtils() throws IOException { + Path bin = this.sandboxPath.getChild("_bin"); + if (!bin.isDirectory()) { + FilesystemUtils.mkdir(bin.getPathString(), 0755); + } + Files.copy(new File(this.embeddedBinaries.getChild("build-runfiles").getPathString()), + new File(bin.getChild("build-runfiles").getPathString())); + FilesystemUtils.chmod(bin.getChild("build-runfiles").getPathString(), 0755); + // TODO(bazel-team) filter tools out of input files instead + // some of the tools could be in inputs; we will mount entire tools anyway so it's just + // easier to remove them and remount inside sandbox + FilesystemUtils.rmTree(sandboxPath.getChild("tools").getPathString()); + } + + + /** + * Runs given + * + * @param spawnArguments - arguments of spawn to run inside the sandbox + * @param env - environment to run sandbox in + * @param cwd - current working directory + * @param outErr - error output to capture sandbox's and command's stderr + * @throws CommandException + */ + public void run(List<String> spawnArguments, ImmutableMap<String, String> env, File cwd, + FileOutErr outErr) throws CommandException { + List<String> args = new ArrayList<>(); + args.add(execRoot.getRelative("_bin/namespace-sandbox").getPathString()); + + // Only for c++ compilation + if (includePrefix != null) { + for (PathFragment include : includeDirectories) { + args.add("-n"); + args.add(include.getPathString()); + } + + args.add("-N"); + args.add(includePrefix.getPathString()); + } + + if (debug) { + args.add("-D"); + } + args.add("-t"); + args.add(tools.getPathString()); + + args.add("-S"); + args.add(sandboxPath.getPathString()); + for (String mount : mounts) { + args.add("-m"); + args.add(mount); + } + + args.add("-C"); + args.addAll(spawnArguments); + Command cmd = new Command(args.toArray(new String[] {}), env, cwd); + + cmd.execute( + /* stdin */new byte[] {}, + Command.NO_OBSERVER, + outErr.getOutputStream(), + outErr.getErrorStream(), + /* killSubprocessOnInterrupt */true); + } + + + public void cleanup() throws IOException { + FilesystemUtils.rmTree(sandboxPath.getPathString()); + } + + + public void copyOutputs(Collection<? extends ActionInput> outputs, FileOutErr outErr) + throws IOException { + for (ActionInput output : outputs) { + Path source = this.sandboxPath.getRelative(output.getExecPathString()); + Path target = this.execRoot.getRelative(output.getExecPathString()); + FileSystemUtils.createDirectoryAndParents(target.getParentDirectory()); + // TODO(bazel-team): eliminate cases when there are excessive outputs in spawns + // (java compilation expects "srclist" file in its outputs which is sometimes not produced) + if (source.isFile()) { + Files.move(new File(source.getPathString()), new File(target.getPathString())); + } else { + outErr.getErrorStream().write(("Output wasn't created by action: " + output + "\n") + .getBytes(StandardCharsets.UTF_8)); + } + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/standalone/StandaloneContextConsumer.java b/src/main/java/com/google/devtools/build/lib/standalone/StandaloneContextConsumer.java new file mode 100644 index 0000000000..2011327d3f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/standalone/StandaloneContextConsumer.java @@ -0,0 +1,57 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.standalone; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMap.Builder; +import com.google.devtools.build.lib.actions.ActionContextConsumer; +import com.google.devtools.build.lib.actions.Executor.ActionContext; +import com.google.devtools.build.lib.actions.SpawnActionContext; +import com.google.devtools.build.lib.analysis.actions.FileWriteActionContext; +import com.google.devtools.build.lib.rules.cpp.CppCompileActionContext; +import com.google.devtools.build.lib.rules.cpp.IncludeScanningContext; +import com.google.devtools.build.lib.rules.cpp.LinkStrategy; +import com.google.devtools.build.lib.rules.test.TestStrategy; + +import java.util.Map; + +/** + * {@link ActionContextConsumer} that requests the action contexts necessary for standalone + * execution. + */ +public class StandaloneContextConsumer implements ActionContextConsumer { + + @Override + public Map<String, String> getSpawnActionContexts() { + return ImmutableMap.of(); + } + + @Override + public Map<Class<? extends ActionContext>, String> getActionContexts() { + Builder<Class<? extends ActionContext>, String> actionContexts = + new ImmutableMap.Builder<Class<? extends ActionContext>, String>(); + + actionContexts.put(SpawnActionContext.class, "standalone"); + + // C++. + actionContexts.put(LinkStrategy.class, ""); + actionContexts.put(IncludeScanningContext.class, ""); + actionContexts.put(CppCompileActionContext.class, ""); + actionContexts.put(TestStrategy.class, ""); + actionContexts.put(FileWriteActionContext.class, ""); + + return actionContexts.build(); + } + +} diff --git a/src/main/java/com/google/devtools/build/lib/standalone/StandaloneContextProvider.java b/src/main/java/com/google/devtools/build/lib/standalone/StandaloneContextProvider.java new file mode 100644 index 0000000000..dbfac2eb54 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/standalone/StandaloneContextProvider.java @@ -0,0 +1,126 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.standalone; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; +import com.google.devtools.build.lib.actions.ActionContextProvider; +import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionGraph; +import com.google.devtools.build.lib.actions.ActionInputFileCache; +import com.google.devtools.build.lib.actions.ActionMetadata; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.ArtifactResolver; +import com.google.devtools.build.lib.actions.ExecutionStrategy; +import com.google.devtools.build.lib.actions.Executor.ActionContext; +import com.google.devtools.build.lib.actions.ExecutorInitException; +import com.google.devtools.build.lib.buildtool.BuildRequest; +import com.google.devtools.build.lib.exec.ExecutionOptions; +import com.google.devtools.build.lib.exec.FileWriteStrategy; +import com.google.devtools.build.lib.rules.cpp.IncludeScanningContext; +import com.google.devtools.build.lib.rules.cpp.LocalGccStrategy; +import com.google.devtools.build.lib.rules.cpp.LocalLinkStrategy; +import com.google.devtools.build.lib.rules.test.ExclusiveTestStrategy; +import com.google.devtools.build.lib.rules.test.StandaloneTestStrategy; +import com.google.devtools.build.lib.rules.test.TestActionContext; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.util.OS; +import com.google.devtools.build.lib.vfs.FileSystemUtils; + +import java.io.IOException; + +/** + * Provide a standalone, local execution context. + */ +public class StandaloneContextProvider implements ActionContextProvider { + + /** + * a IncludeScanningContext that does nothing. Since local execution does not need to + * discover inclusion in advance, we do not need include scanning. + */ + @ExecutionStrategy(contextType = IncludeScanningContext.class) + class DummyIncludeScanningContext implements IncludeScanningContext { + @Override + public void extractIncludes(ActionExecutionContext actionExecutionContext, + ActionMetadata resourceOwner, Artifact primaryInput, Artifact primaryOutput) + throws IOException, InterruptedException { + FileSystemUtils.writeContent(primaryOutput.getPath(), new byte[]{}); + } + + @Override + public ArtifactResolver getArtifactResolver() { + return runtime.getView().getArtifactFactory(); + } + } + + @SuppressWarnings("unchecked") + private final ActionContext localSpawnStrategy; + private final ImmutableList<ActionContext> strategies; + private final BlazeRuntime runtime; + + public StandaloneContextProvider( + BlazeRuntime runtime, BuildRequest buildRequest) { + boolean verboseFailures = buildRequest.getOptions(ExecutionOptions.class).verboseFailures; + + localSpawnStrategy = new LocalSpawnStrategy( + runtime.getDirectories().getExecRoot(), verboseFailures); + this.runtime = runtime; + + TestActionContext testStrategy = new StandaloneTestStrategy(buildRequest, + runtime.getStartupOptionsProvider(), runtime.getBinTools(), runtime.getRunfilesPrefix()); + Builder<ActionContext> strategiesBuilder = ImmutableList.builder(); + // order of strategies passed to builder is significant - when there are many strategies that + // could potentially be used and a spawnActionContext doesn't specify which one it wants, the + // last one from strategies list will be used + + // put sandboxed strategy first, as we don't want it by default + if (OS.getCurrent() == OS.LINUX) { + LinuxSandboxedStrategy sandboxedLinuxStrategy = + new LinuxSandboxedStrategy(runtime.getDirectories(), verboseFailures); + strategiesBuilder.add(sandboxedLinuxStrategy); + } + strategiesBuilder.add( + localSpawnStrategy, + new DummyIncludeScanningContext(), + new LocalLinkStrategy(), + testStrategy, + new ExclusiveTestStrategy(testStrategy), + new LocalGccStrategy(buildRequest), + new FileWriteStrategy()); + + + this.strategies = strategiesBuilder.build(); + } + + @Override + public Iterable<ActionContext> getActionContexts() { + return strategies; + } + + @Override + public void executorCreated(Iterable<ActionContext> usedContexts) throws ExecutorInitException { + } + + @Override + public void executionPhaseStarting( + ActionInputFileCache actionInputFileCache, + ActionGraph actionGraph, + Iterable<Artifact> topLevelArtifacts) throws ExecutorInitException { + } + + @Override + public void executionPhaseEnding() {} +} + + diff --git a/src/main/java/com/google/devtools/build/lib/standalone/StandaloneModule.java b/src/main/java/com/google/devtools/build/lib/standalone/StandaloneModule.java new file mode 100644 index 0000000000..06bffd0bd8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/standalone/StandaloneModule.java @@ -0,0 +1,59 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.standalone; + +import com.google.common.eventbus.Subscribe; +import com.google.devtools.build.lib.actions.ActionContextConsumer; +import com.google.devtools.build.lib.actions.ActionContextProvider; +import com.google.devtools.build.lib.buildtool.BuildRequest; +import com.google.devtools.build.lib.buildtool.buildevent.BuildStartingEvent; +import com.google.devtools.build.lib.runtime.BlazeModule; +import com.google.devtools.build.lib.runtime.BlazeRuntime; +import com.google.devtools.build.lib.runtime.Command; + +/** + * StandaloneModule provides pluggable functionality for blaze. + */ +public class StandaloneModule extends BlazeModule { + private final ActionContextConsumer actionContextConsumer = new StandaloneContextConsumer(); + private BuildRequest buildRequest; + private BlazeRuntime runtime; + + /** + * Returns the action context provider the module contributes to Blaze, if any. + */ + @Override + public ActionContextProvider getActionContextProvider() { + return new StandaloneContextProvider(runtime, buildRequest); + } + + /** + * Returns the action context consumer the module contributes to Blaze, if any. + */ + @Override + public ActionContextConsumer getActionContextConsumer() { + return actionContextConsumer; + } + + @Override + public void beforeCommand(BlazeRuntime runtime, Command command) { + this.runtime = runtime; + runtime.getEventBus().register(this); + } + + @Subscribe + public void buildStarting(BuildStartingEvent event) { + buildRequest = event.getRequest(); + } +} |