diff options
author | ulfjack <ulfjack@google.com> | 2018-06-11 07:01:48 -0700 |
---|---|---|
committer | Copybara-Service <copybara-piper@google.com> | 2018-06-11 07:02:58 -0700 |
commit | 65173666790d6fff0190c196536d34743b419c12 (patch) | |
tree | b16cba3ef9419c4fb8ce6fd2cc775696cf8727ce /src | |
parent | 4cf2ebdcef7b5d4c46e533a560642b89eb3f02a5 (diff) |
Inline some ActionContextProvider classes into their modules
This also gets rid of some boilerplate. The ExecutionTool.addActionContext
method has been around for a while, but is underused.
There are still a few ActionContextProvider implementations left, which are
implementing other functionality besides adding action contexts.
As a side effect, this change reduces null build time with a hot server on
linux by about a quarter. We were running the linux sandbox twice on every
build, which takes about 70ms each (on my machine), with the total null build
time around 300ms.
PiperOrigin-RevId: 200045145
Diffstat (limited to 'src')
10 files changed, 290 insertions, 398 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/exec/local/LocalExecutionOptions.java b/src/main/java/com/google/devtools/build/lib/exec/local/LocalExecutionOptions.java index aa83c16400..d1eabcc1f3 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/local/LocalExecutionOptions.java +++ b/src/main/java/com/google/devtools/build/lib/exec/local/LocalExecutionOptions.java @@ -18,6 +18,7 @@ import com.google.devtools.common.options.Option; import com.google.devtools.common.options.OptionDocumentationCategory; import com.google.devtools.common.options.OptionEffectTag; import com.google.devtools.common.options.OptionsBase; +import java.time.Duration; import java.util.regex.Pattern; /** @@ -59,4 +60,9 @@ public class LocalExecutionOptions extends OptionsBase { + "locally executed actions which don't use sandboxing" ) public boolean collectLocalExecutionStatistics; + + public Duration getLocalSigkillGraceSeconds() { + // TODO(ulfjack): Change localSigkillGraceSeconds type to Duration. + return Duration.ofSeconds(localSigkillGraceSeconds); + } } diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/DarwinSandboxedSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/sandbox/DarwinSandboxedSpawnRunner.java index 7e92b696db..bde1a99692 100644 --- a/src/main/java/com/google/devtools/build/lib/sandbox/DarwinSandboxedSpawnRunner.java +++ b/src/main/java/com/google/devtools/build/lib/sandbox/DarwinSandboxedSpawnRunner.java @@ -65,6 +65,10 @@ final class DarwinSandboxedSpawnRunner extends AbstractSandboxSpawnRunner { @VisibleForTesting static String sandboxExecBinary = "/usr/bin/sandbox-exec"; + /** + * Returns whether the darwin sandbox is supported on the local machine by running a small command + * in it. This is expensive! + */ public static boolean isSupported(CommandEnvironment cmdEnv) { if (OS.getCurrent() != OS.DARWIN) { return false; diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/DockerSandboxedSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/sandbox/DockerSandboxedSpawnRunner.java index 67c495248a..bc609d22d0 100644 --- a/src/main/java/com/google/devtools/build/lib/sandbox/DockerSandboxedSpawnRunner.java +++ b/src/main/java/com/google/devtools/build/lib/sandbox/DockerSandboxedSpawnRunner.java @@ -66,6 +66,10 @@ final class DockerSandboxedSpawnRunner extends AbstractSandboxSpawnRunner { private static final String CONTAINER_IMAGE_ENTRY_NAME = "container-image"; private static final String DOCKER_IMAGE_PREFIX = "docker://"; + /** + * Returns whether the darwin sandbox is supported on the local machine by running docker info. + * This is expensive, and we have also reports of docker hanging for a long time! + */ public static boolean isSupported(CommandEnvironment cmdEnv, Path dockerClient) { boolean verbose = cmdEnv.getOptions().getOptions(SandboxOptions.class).dockerVerbose; diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedSpawnRunner.java index 7879399463..ce3c8a39c3 100644 --- a/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedSpawnRunner.java +++ b/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedSpawnRunner.java @@ -46,6 +46,10 @@ import javax.annotation.Nullable; /** Spawn runner that uses linux sandboxing APIs to execute a local subprocess. */ final class LinuxSandboxedSpawnRunner extends AbstractSandboxSpawnRunner { + /** + * Returns whether the linux sandbox is supported on the local machine by running a small command + * in it. This is expensive! + */ public static boolean isSupported(CommandEnvironment cmdEnv) { if (OS.getCurrent() != OS.LINUX) { return false; diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxActionContextProvider.java b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxActionContextProvider.java deleted file mode 100644 index 8d04bb5b92..0000000000 --- a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxActionContextProvider.java +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright 2015 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.devtools.build.lib.sandbox; - -import com.google.common.base.Splitter; -import com.google.common.collect.ImmutableList; -import com.google.devtools.build.lib.actions.ActionContext; -import com.google.devtools.build.lib.actions.ExecException; -import com.google.devtools.build.lib.actions.ResourceManager; -import com.google.devtools.build.lib.actions.Spawn; -import com.google.devtools.build.lib.actions.SpawnResult; -import com.google.devtools.build.lib.actions.Spawns; -import com.google.devtools.build.lib.events.Event; -import com.google.devtools.build.lib.exec.ActionContextProvider; -import com.google.devtools.build.lib.exec.SpawnRunner; -import com.google.devtools.build.lib.exec.apple.XcodeLocalEnvProvider; -import com.google.devtools.build.lib.exec.local.LocalEnvProvider; -import com.google.devtools.build.lib.exec.local.LocalExecutionOptions; -import com.google.devtools.build.lib.exec.local.LocalSpawnRunner; -import com.google.devtools.build.lib.exec.local.PosixLocalEnvProvider; -import com.google.devtools.build.lib.runtime.CommandEnvironment; -import com.google.devtools.build.lib.util.OS; -import com.google.devtools.build.lib.vfs.FileSystem; -import com.google.devtools.build.lib.vfs.Path; -import com.google.devtools.common.options.OptionsProvider; -import java.io.File; -import java.io.IOException; -import java.time.Duration; -import java.util.Optional; -import javax.annotation.Nullable; - -/** - * Provides the sandboxed spawn strategy. - */ -final class SandboxActionContextProvider extends ActionContextProvider { - private final ImmutableList<ActionContext> contexts; - - private SandboxActionContextProvider(ImmutableList<ActionContext> contexts) { - this.contexts = contexts; - } - - public static SandboxActionContextProvider create(CommandEnvironment cmdEnv, Path sandboxBase, - @Nullable SandboxfsProcess process) - throws IOException { - ImmutableList.Builder<ActionContext> contexts = ImmutableList.builder(); - - OptionsProvider options = cmdEnv.getOptions(); - Duration timeoutKillDelay = - Duration.ofSeconds( - options.getOptions(LocalExecutionOptions.class).localSigkillGraceSeconds); - - // This works on most platforms, but isn't the best choice, so we put it first and let later - // platform-specific sandboxing strategies become the default. - if (ProcessWrapperSandboxedSpawnRunner.isSupported(cmdEnv)) { - SpawnRunner spawnRunner = - withFallback( - cmdEnv, - new ProcessWrapperSandboxedSpawnRunner( - cmdEnv, sandboxBase, cmdEnv.getRuntime().getProductName(), timeoutKillDelay)); - contexts.add(new ProcessWrapperSandboxedStrategy(cmdEnv.getExecRoot(), spawnRunner)); - } - - SandboxOptions sandboxOptions = options.getOptions(SandboxOptions.class); - - if (sandboxOptions.enableDockerSandbox) { - // This strategy uses Docker to execute spawns. It should work on all platforms that support - // Docker. - getPathToDockerClient(cmdEnv) - .ifPresent( - dockerClient -> { - if (DockerSandboxedSpawnRunner.isSupported(cmdEnv, dockerClient)) { - String defaultImage = sandboxOptions.dockerImage; - boolean useCustomizedImages = sandboxOptions.dockerUseCustomizedImages; - SpawnRunner spawnRunner = - withFallback( - cmdEnv, - new DockerSandboxedSpawnRunner( - cmdEnv, - dockerClient, - sandboxBase, - defaultImage, - timeoutKillDelay, - useCustomizedImages)); - contexts.add(new DockerSandboxedStrategy(cmdEnv.getExecRoot(), spawnRunner)); - } - }); - } else if (sandboxOptions.dockerVerbose) { - cmdEnv.getReporter().handle(Event.info( - "Docker sandboxing disabled. Use the '--experimental_enable_docker_sandbox' command " - + "line option to enable it")); - } - - // This is the preferred sandboxing strategy on Linux. - if (LinuxSandboxedSpawnRunner.isSupported(cmdEnv)) { - SpawnRunner spawnRunner = - withFallback( - cmdEnv, - LinuxSandboxedStrategy.create(cmdEnv, sandboxBase, timeoutKillDelay, process)); - contexts.add(new LinuxSandboxedStrategy(cmdEnv.getExecRoot(), spawnRunner)); - } - - // This is the preferred sandboxing strategy on macOS. - if (DarwinSandboxedSpawnRunner.isSupported(cmdEnv)) { - SpawnRunner spawnRunner = - withFallback( - cmdEnv, - new DarwinSandboxedSpawnRunner(cmdEnv, sandboxBase, timeoutKillDelay, process)); - contexts.add(new DarwinSandboxedStrategy(cmdEnv.getExecRoot(), spawnRunner)); - } - - return new SandboxActionContextProvider(contexts.build()); - } - - private static Optional<Path> getPathToDockerClient(CommandEnvironment cmdEnv) { - String path = cmdEnv.getClientEnv().getOrDefault("PATH", ""); - - Splitter pathSplitter = - Splitter.on(OS.getCurrent() == OS.WINDOWS ? ';' : ':').trimResults().omitEmptyStrings(); - - FileSystem fs = cmdEnv.getRuntime().getFileSystem(); - - for (String pathElement : pathSplitter.split(path)) { - // Sometimes the PATH contains the non-absolute entry "." - this resolves it against the - // current working directory. - pathElement = new File(pathElement).getAbsolutePath(); - try { - for (Path dentry : fs.getPath(pathElement).getDirectoryEntries()) { - if (dentry.getBaseName().replace(".exe", "").equals("docker")) { - return Optional.of(dentry); - } - } - } catch (IOException e) { - continue; - } - } - - return Optional.empty(); - } - - private static SpawnRunner withFallback(CommandEnvironment env, SpawnRunner sandboxSpawnRunner) { - return new SandboxFallbackSpawnRunner(sandboxSpawnRunner, createFallbackRunner(env)); - } - - private static SpawnRunner createFallbackRunner(CommandEnvironment env) { - LocalExecutionOptions localExecutionOptions = - env.getOptions().getOptions(LocalExecutionOptions.class); - LocalEnvProvider localEnvProvider = - OS.getCurrent() == OS.DARWIN - ? new XcodeLocalEnvProvider(env.getClientEnv()) - : new PosixLocalEnvProvider(env.getClientEnv()); - return - new LocalSpawnRunner( - env.getExecRoot(), - localExecutionOptions, - ResourceManager.instance(), - localEnvProvider); - } - - @Override - public Iterable<? extends ActionContext> getActionContexts() { - return contexts; - } - - private static final class SandboxFallbackSpawnRunner implements SpawnRunner { - private final SpawnRunner sandboxSpawnRunner; - private final SpawnRunner fallbackSpawnRunner; - - SandboxFallbackSpawnRunner(SpawnRunner sandboxSpawnRunner, SpawnRunner fallbackSpawnRunner) { - this.sandboxSpawnRunner = sandboxSpawnRunner; - this.fallbackSpawnRunner = fallbackSpawnRunner; - } - - @Override - public String getName() { - return "sandbox-fallback"; - } - - @Override - public SpawnResult exec(Spawn spawn, SpawnExecutionContext context) - throws InterruptedException, IOException, ExecException { - if (!Spawns.mayBeSandboxed(spawn)) { - return fallbackSpawnRunner.exec(spawn, context); - } else { - return sandboxSpawnRunner.exec(spawn, context); - } - } - } -} diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxModule.java b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxModule.java index 14abac497b..a331f55b3d 100644 --- a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxModule.java +++ b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxModule.java @@ -17,26 +17,40 @@ package com.google.devtools.build.lib.sandbox; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; +import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.google.common.eventbus.Subscribe; +import com.google.devtools.build.lib.actions.ExecException; import com.google.devtools.build.lib.actions.ExecutorInitException; +import com.google.devtools.build.lib.actions.ResourceManager; +import com.google.devtools.build.lib.actions.Spawn; import com.google.devtools.build.lib.actions.SpawnActionContext; +import com.google.devtools.build.lib.actions.SpawnResult; +import com.google.devtools.build.lib.actions.Spawns; import com.google.devtools.build.lib.buildtool.BuildRequest; import com.google.devtools.build.lib.buildtool.buildevent.BuildCompleteEvent; import com.google.devtools.build.lib.buildtool.buildevent.BuildInterruptedEvent; import com.google.devtools.build.lib.events.Event; -import com.google.devtools.build.lib.exec.ActionContextProvider; import com.google.devtools.build.lib.exec.ExecutorBuilder; +import com.google.devtools.build.lib.exec.SpawnRunner; +import com.google.devtools.build.lib.exec.apple.XcodeLocalEnvProvider; +import com.google.devtools.build.lib.exec.local.LocalEnvProvider; +import com.google.devtools.build.lib.exec.local.LocalExecutionOptions; +import com.google.devtools.build.lib.exec.local.LocalSpawnRunner; +import com.google.devtools.build.lib.exec.local.PosixLocalEnvProvider; import com.google.devtools.build.lib.runtime.BlazeModule; import com.google.devtools.build.lib.runtime.Command; import com.google.devtools.build.lib.runtime.CommandEnvironment; import com.google.devtools.build.lib.util.Fingerprint; +import com.google.devtools.build.lib.util.OS; import com.google.devtools.build.lib.vfs.FileSystem; import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.common.options.OptionsBase; +import java.io.File; import java.io.IOException; +import java.time.Duration; import javax.annotation.Nullable; /** @@ -96,48 +110,106 @@ public final class SandboxModule extends BlazeModule { public void executorInit(CommandEnvironment cmdEnv, BuildRequest request, ExecutorBuilder builder) throws ExecutorInitException { checkNotNull(env, "env not initialized; was beforeCommand called?"); - - SandboxOptions options = env.getOptions().getOptions(SandboxOptions.class); - checkNotNull(options, "We were told to initialize the executor but the SandboxOptions are " - + "not present; were they registered for all build commands?"); - try { - sandboxBase = computeSandboxBase(options, env); + setup(cmdEnv, builder); } catch (IOException e) { - throw new ExecutorInitException( - "--experimental_sandbox_base points to an invalid directory", e); + throw new ExecutorInitException("Failed to initialize sandbox", e); } + } - ActionContextProvider provider; - try { - // Ensure that each build starts with a clean sandbox base directory. Otherwise using the `id` - // that is provided by SpawnExecutionPolicy#getId to compute a base directory for a sandbox - // might result in an already existing directory. - if (sandboxBase.exists()) { - FileSystemUtils.deleteTree(sandboxBase); - } + private void setup(CommandEnvironment cmdEnv, ExecutorBuilder builder) + throws IOException { + SandboxOptions options = checkNotNull(env.getOptions().getOptions(SandboxOptions.class)); + sandboxBase = computeSandboxBase(options, env); + + // Ensure that each build starts with a clean sandbox base directory. Otherwise using the `id` + // that is provided by SpawnExecutionPolicy#getId to compute a base directory for a sandbox + // might result in an already existing directory. + if (sandboxBase.exists()) { + FileSystemUtils.deleteTree(sandboxBase); + } - sandboxBase.createDirectoryAndParents(); - if (options.useSandboxfs) { - Path mountPoint = sandboxBase.getRelative("sandboxfs"); - mountPoint.createDirectory(); - Path logFile = sandboxBase.getRelative("sandboxfs.log"); + sandboxBase.createDirectoryAndParents(); + if (options.useSandboxfs) { + Path mountPoint = sandboxBase.getRelative("sandboxfs"); + mountPoint.createDirectory(); + Path logFile = sandboxBase.getRelative("sandboxfs.log"); - env.getReporter().handle(Event.info("Mounting sandboxfs instance on " + mountPoint)); - sandboxfsProcess = RealSandboxfsProcess.mount( - PathFragment.create(options.sandboxfsPath), mountPoint, logFile); - provider = SandboxActionContextProvider.create(cmdEnv, sandboxBase, sandboxfsProcess); - } else { - provider = SandboxActionContextProvider.create(cmdEnv, sandboxBase, null); + env.getReporter().handle(Event.info("Mounting sandboxfs instance on " + mountPoint)); + sandboxfsProcess = RealSandboxfsProcess.mount( + PathFragment.create(options.sandboxfsPath), mountPoint, logFile); + } + + Duration timeoutKillDelay = + cmdEnv.getOptions().getOptions(LocalExecutionOptions.class).getLocalSigkillGraceSeconds(); + + boolean processWrapperSupported = ProcessWrapperSandboxedSpawnRunner.isSupported(cmdEnv); + // LinuxSandboxedSpawnRunner.isSupported is expensive! It runs the sandbox as a subprocess. + boolean linuxSandboxSupported = LinuxSandboxedSpawnRunner.isSupported(cmdEnv); + // DarwinSandboxedSpawnRunner.isSupported is expensive! It runs the sandbox as a subprocess. + boolean darwinSandboxSupported = DarwinSandboxedSpawnRunner.isSupported(cmdEnv); + + // This works on most platforms, but isn't the best choice, so we put it first and let later + // platform-specific sandboxing strategies become the default. + if (processWrapperSupported) { + SpawnRunner spawnRunner = + withFallback( + cmdEnv, + new ProcessWrapperSandboxedSpawnRunner( + cmdEnv, sandboxBase, cmdEnv.getRuntime().getProductName(), timeoutKillDelay)); + builder.addActionContext( + new ProcessWrapperSandboxedStrategy(cmdEnv.getExecRoot(), spawnRunner)); + } + + if (options.enableDockerSandbox) { + // This strategy uses Docker to execute spawns. It should work on all platforms that support + // Docker. + Path pathToDocker = getPathToDockerClient(cmdEnv); + // DockerSandboxedSpawnRunner.isSupported is expensive! It runs docker as a subprocess, and + // docker hangs sometimes. + if (pathToDocker != null && DockerSandboxedSpawnRunner.isSupported(cmdEnv, pathToDocker)) { + String defaultImage = options.dockerImage; + boolean useCustomizedImages = options.dockerUseCustomizedImages; + SpawnRunner spawnRunner = + withFallback( + cmdEnv, + new DockerSandboxedSpawnRunner( + cmdEnv, + pathToDocker, + sandboxBase, + defaultImage, + timeoutKillDelay, + useCustomizedImages)); + builder.addActionContext( + new DockerSandboxedStrategy(cmdEnv.getExecRoot(), spawnRunner)); } - } catch (IOException e) { - throw new ExecutorInitException("Failed to initialize sandbox", e); + } else if (options.dockerVerbose) { + cmdEnv.getReporter().handle(Event.info( + "Docker sandboxing disabled. Use the '--experimental_enable_docker_sandbox' command " + + "line option to enable it")); + } + + // This is the preferred sandboxing strategy on Linux. + if (linuxSandboxSupported) { + SpawnRunner spawnRunner = + withFallback( + cmdEnv, + LinuxSandboxedStrategy.create( + cmdEnv, sandboxBase, timeoutKillDelay, sandboxfsProcess)); + builder.addActionContext(new LinuxSandboxedStrategy(cmdEnv.getExecRoot(), spawnRunner)); } - builder.addActionContextProvider(provider); - if (LinuxSandboxedSpawnRunner.isSupported(cmdEnv) - || DarwinSandboxedSpawnRunner.isSupported(cmdEnv) - || ProcessWrapperSandboxedSpawnRunner.isSupported(cmdEnv)) { + // This is the preferred sandboxing strategy on macOS. + if (darwinSandboxSupported) { + SpawnRunner spawnRunner = + withFallback( + cmdEnv, + new DarwinSandboxedSpawnRunner( + cmdEnv, sandboxBase, timeoutKillDelay, sandboxfsProcess)); + builder.addActionContext(new DarwinSandboxedStrategy(cmdEnv.getExecRoot(), spawnRunner)); + } + + if (processWrapperSupported || linuxSandboxSupported || darwinSandboxSupported) { // This makes the "sandboxed" strategy available via --spawn_strategy=sandboxed, // but it is not necessarily the default. builder.addStrategyByContext(SpawnActionContext.class, "sandboxed"); @@ -152,6 +224,78 @@ public final class SandboxModule extends BlazeModule { shouldCleanupSandboxBase = !options.sandboxDebug; } + private static Path getPathToDockerClient(CommandEnvironment cmdEnv) { + String path = cmdEnv.getClientEnv().getOrDefault("PATH", ""); + + // TODO(philwo): Does this return the correct result if one of the elements intentionally ends + // in white space? + Splitter pathSplitter = + Splitter.on(OS.getCurrent() == OS.WINDOWS ? ';' : ':').trimResults().omitEmptyStrings(); + + FileSystem fs = cmdEnv.getRuntime().getFileSystem(); + + for (String pathElement : pathSplitter.split(path)) { + // Sometimes the PATH contains the non-absolute entry "." - this resolves it against the + // current working directory. + pathElement = new File(pathElement).getAbsolutePath(); + try { + for (Path dentry : fs.getPath(pathElement).getDirectoryEntries()) { + if (dentry.getBaseName().replace(".exe", "").equals("docker")) { + return dentry; + } + } + } catch (IOException e) { + continue; + } + } + + return null; + } + + private static SpawnRunner withFallback(CommandEnvironment env, SpawnRunner sandboxSpawnRunner) { + return new SandboxFallbackSpawnRunner(sandboxSpawnRunner, createFallbackRunner(env)); + } + + private static SpawnRunner createFallbackRunner(CommandEnvironment env) { + LocalExecutionOptions localExecutionOptions = + env.getOptions().getOptions(LocalExecutionOptions.class); + LocalEnvProvider localEnvProvider = + OS.getCurrent() == OS.DARWIN + ? new XcodeLocalEnvProvider(env.getClientEnv()) + : new PosixLocalEnvProvider(env.getClientEnv()); + return + new LocalSpawnRunner( + env.getExecRoot(), + localExecutionOptions, + ResourceManager.instance(), + localEnvProvider); + } + + private static final class SandboxFallbackSpawnRunner implements SpawnRunner { + private final SpawnRunner sandboxSpawnRunner; + private final SpawnRunner fallbackSpawnRunner; + + SandboxFallbackSpawnRunner(SpawnRunner sandboxSpawnRunner, SpawnRunner fallbackSpawnRunner) { + this.sandboxSpawnRunner = sandboxSpawnRunner; + this.fallbackSpawnRunner = fallbackSpawnRunner; + } + + @Override + public String getName() { + return "sandbox-fallback"; + } + + @Override + public SpawnResult exec(Spawn spawn, SpawnExecutionContext context) + throws InterruptedException, IOException, ExecException { + if (!Spawns.mayBeSandboxed(spawn)) { + return fallbackSpawnRunner.exec(spawn, context); + } else { + return sandboxSpawnRunner.exec(spawn, context); + } + } + } + private void unmountSandboxfs(String reason) { if (sandboxfsProcess != null) { checkNotNull(env, "env not initialized; was beforeCommand called?"); @@ -163,12 +307,12 @@ public final class SandboxModule extends BlazeModule { } @Subscribe - public void buildComplete(BuildCompleteEvent event) { + public void buildComplete(@SuppressWarnings("unused") BuildCompleteEvent event) { unmountSandboxfs("Build complete; unmounting sandboxfs..."); } @Subscribe - public void buildInterrupted(BuildInterruptedEvent event) { + public void buildInterrupted(@SuppressWarnings("unused") BuildInterruptedEvent event) { unmountSandboxfs("Build interrupted; unmounting sandboxfs..."); } diff --git a/src/main/java/com/google/devtools/build/lib/standalone/StandaloneActionContextProvider.java b/src/main/java/com/google/devtools/build/lib/standalone/StandaloneActionContextProvider.java deleted file mode 100644 index 2b1c6fd7ef..0000000000 --- a/src/main/java/com/google/devtools/build/lib/standalone/StandaloneActionContextProvider.java +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2014 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package com.google.devtools.build.lib.standalone; - -import com.google.common.collect.ImmutableList; -import com.google.devtools.build.lib.actions.ActionContext; -import com.google.devtools.build.lib.actions.ResourceManager; -import com.google.devtools.build.lib.analysis.test.TestActionContext; -import com.google.devtools.build.lib.exec.ActionContextProvider; -import com.google.devtools.build.lib.exec.ExecutionOptions; -import com.google.devtools.build.lib.exec.FileWriteStrategy; -import com.google.devtools.build.lib.exec.SpawnRunner; -import com.google.devtools.build.lib.exec.StandaloneTestStrategy; -import com.google.devtools.build.lib.exec.TestStrategy; -import com.google.devtools.build.lib.exec.apple.XcodeLocalEnvProvider; -import com.google.devtools.build.lib.exec.local.LocalEnvProvider; -import com.google.devtools.build.lib.exec.local.LocalExecutionOptions; -import com.google.devtools.build.lib.exec.local.LocalSpawnRunner; -import com.google.devtools.build.lib.exec.local.PosixLocalEnvProvider; -import com.google.devtools.build.lib.exec.local.WindowsLocalEnvProvider; -import com.google.devtools.build.lib.rules.cpp.SpawnGccStrategy; -import com.google.devtools.build.lib.rules.test.ExclusiveTestStrategy; -import com.google.devtools.build.lib.runtime.CommandEnvironment; -import com.google.devtools.build.lib.util.OS; -import com.google.devtools.build.lib.vfs.Path; - -/** - * Provide a standalone, local execution context. - */ -public class StandaloneActionContextProvider extends ActionContextProvider { - private final CommandEnvironment env; - - public StandaloneActionContextProvider(CommandEnvironment env) { - this.env = env; - } - - @Override - public Iterable<? extends ActionContext> getActionContexts() { - ExecutionOptions executionOptions = env.getOptions().getOptions(ExecutionOptions.class); - Path testTmpRoot = - TestStrategy.getTmpRoot(env.getWorkspace(), env.getExecRoot(), executionOptions); - - TestActionContext testStrategy = - new StandaloneTestStrategy( - executionOptions, - env.getBlazeWorkspace().getBinTools(), - testTmpRoot); - // Order of strategies passed to builder is significant - when there are many strategies that - // could potentially be used and a spawnActionContext doesn't specify which one it wants, the - // last one from strategies list will be used - return ImmutableList.of( - new StandaloneSpawnStrategy(env.getExecRoot(), createLocalRunner(env)), - new SpawnGccStrategy(), - testStrategy, - new ExclusiveTestStrategy(testStrategy), - new FileWriteStrategy()); - } - - private static SpawnRunner createLocalRunner(CommandEnvironment env) { - LocalExecutionOptions localExecutionOptions = - env.getOptions().getOptions(LocalExecutionOptions.class); - LocalEnvProvider localEnvProvider = - OS.getCurrent() == OS.DARWIN - ? new XcodeLocalEnvProvider(env.getClientEnv()) - : (OS.getCurrent() == OS.WINDOWS - ? new WindowsLocalEnvProvider(env.getClientEnv()) - : new PosixLocalEnvProvider(env.getClientEnv())); - return - new LocalSpawnRunner( - env.getExecRoot(), - localExecutionOptions, - ResourceManager.instance(), - localEnvProvider); - } -} diff --git a/src/main/java/com/google/devtools/build/lib/standalone/StandaloneModule.java b/src/main/java/com/google/devtools/build/lib/standalone/StandaloneModule.java index 728dc4b6d7..2f7a97b8da 100644 --- a/src/main/java/com/google/devtools/build/lib/standalone/StandaloneModule.java +++ b/src/main/java/com/google/devtools/build/lib/standalone/StandaloneModule.java @@ -13,11 +13,28 @@ // limitations under the License. package com.google.devtools.build.lib.standalone; +import com.google.devtools.build.lib.actions.ResourceManager; import com.google.devtools.build.lib.actions.SpawnActionContext; +import com.google.devtools.build.lib.analysis.test.TestActionContext; import com.google.devtools.build.lib.buildtool.BuildRequest; +import com.google.devtools.build.lib.exec.ExecutionOptions; import com.google.devtools.build.lib.exec.ExecutorBuilder; +import com.google.devtools.build.lib.exec.FileWriteStrategy; +import com.google.devtools.build.lib.exec.SpawnRunner; +import com.google.devtools.build.lib.exec.StandaloneTestStrategy; +import com.google.devtools.build.lib.exec.TestStrategy; +import com.google.devtools.build.lib.exec.apple.XcodeLocalEnvProvider; +import com.google.devtools.build.lib.exec.local.LocalEnvProvider; +import com.google.devtools.build.lib.exec.local.LocalExecutionOptions; +import com.google.devtools.build.lib.exec.local.LocalSpawnRunner; +import com.google.devtools.build.lib.exec.local.PosixLocalEnvProvider; +import com.google.devtools.build.lib.exec.local.WindowsLocalEnvProvider; +import com.google.devtools.build.lib.rules.cpp.SpawnGccStrategy; +import com.google.devtools.build.lib.rules.test.ExclusiveTestStrategy; import com.google.devtools.build.lib.runtime.BlazeModule; import com.google.devtools.build.lib.runtime.CommandEnvironment; +import com.google.devtools.build.lib.util.OS; +import com.google.devtools.build.lib.vfs.Path; /** * StandaloneModule provides pluggable functionality for blaze. @@ -25,8 +42,28 @@ import com.google.devtools.build.lib.runtime.CommandEnvironment; public class StandaloneModule extends BlazeModule { @Override public void executorInit(CommandEnvironment env, BuildRequest request, ExecutorBuilder builder) { - builder.addActionContextProvider(new StandaloneActionContextProvider(env)); + // TODO(ulfjack): Move this to another module. builder.addActionContextProvider(new DummyIncludeScanningContextProvider(env)); + + ExecutionOptions executionOptions = env.getOptions().getOptions(ExecutionOptions.class); + Path testTmpRoot = + TestStrategy.getTmpRoot(env.getWorkspace(), env.getExecRoot(), executionOptions); + TestActionContext testStrategy = + new StandaloneTestStrategy( + executionOptions, + env.getBlazeWorkspace().getBinTools(), + testTmpRoot); + + // Order of strategies passed to builder is significant - when there are many strategies that + // could potentially be used and a spawnActionContext doesn't specify which one it wants, the + // last one from strategies list will be used + builder.addActionContext( + new StandaloneSpawnStrategy(env.getExecRoot(), createLocalRunner(env))); + builder.addActionContext(new SpawnGccStrategy()); + builder.addActionContext(testStrategy); + builder.addActionContext(new ExclusiveTestStrategy(testStrategy)); + builder.addActionContext(new FileWriteStrategy()); + // This makes the "sandboxed" strategy the default Spawn strategy, unless it is overridden by a // later BlazeModule. builder.addStrategyByMnemonic("", "standalone"); @@ -35,4 +72,21 @@ public class StandaloneModule extends BlazeModule { // necessarily the default. builder.addStrategyByContext(SpawnActionContext.class, "standalone"); } + + private static SpawnRunner createLocalRunner(CommandEnvironment env) { + LocalExecutionOptions localExecutionOptions = + env.getOptions().getOptions(LocalExecutionOptions.class); + LocalEnvProvider localEnvProvider = + OS.getCurrent() == OS.DARWIN + ? new XcodeLocalEnvProvider(env.getClientEnv()) + : (OS.getCurrent() == OS.WINDOWS + ? new WindowsLocalEnvProvider(env.getClientEnv()) + : new PosixLocalEnvProvider(env.getClientEnv())); + return + new LocalSpawnRunner( + env.getExecRoot(), + localExecutionOptions, + ResourceManager.instance(), + localEnvProvider); + } } diff --git a/src/main/java/com/google/devtools/build/lib/worker/WorkerActionContextProvider.java b/src/main/java/com/google/devtools/build/lib/worker/WorkerActionContextProvider.java deleted file mode 100644 index f65ee91991..0000000000 --- a/src/main/java/com/google/devtools/build/lib/worker/WorkerActionContextProvider.java +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2015 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package com.google.devtools.build.lib.worker; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMultimap; -import com.google.devtools.build.lib.actions.ActionContext; -import com.google.devtools.build.lib.actions.ResourceManager; -import com.google.devtools.build.lib.exec.ActionContextProvider; -import com.google.devtools.build.lib.exec.SpawnRunner; -import com.google.devtools.build.lib.exec.apple.XcodeLocalEnvProvider; -import com.google.devtools.build.lib.exec.local.LocalEnvProvider; -import com.google.devtools.build.lib.exec.local.LocalExecutionOptions; -import com.google.devtools.build.lib.exec.local.LocalSpawnRunner; -import com.google.devtools.build.lib.exec.local.PosixLocalEnvProvider; -import com.google.devtools.build.lib.exec.local.WindowsLocalEnvProvider; -import com.google.devtools.build.lib.runtime.CommandEnvironment; -import com.google.devtools.build.lib.util.OS; - -/** - * Factory for the Worker-based execution strategy. - */ -final class WorkerActionContextProvider extends ActionContextProvider { - private final ImmutableList<ActionContext> strategies; - - public WorkerActionContextProvider(CommandEnvironment env, WorkerPool workers) { - ImmutableMultimap<String, String> extraFlags = - ImmutableMultimap.copyOf(env.getOptions().getOptions(WorkerOptions.class).workerExtraFlags); - - WorkerSpawnRunner spawnRunner = - new WorkerSpawnRunner( - env.getExecRoot(), - workers, - extraFlags, - env.getReporter(), - createFallbackRunner(env)); - - WorkerSpawnStrategy workerSpawnStrategy = - new WorkerSpawnStrategy(env.getExecRoot(), spawnRunner); - this.strategies = ImmutableList.of(workerSpawnStrategy); - } - - private static SpawnRunner createFallbackRunner(CommandEnvironment env) { - LocalExecutionOptions localExecutionOptions = - env.getOptions().getOptions(LocalExecutionOptions.class); - LocalEnvProvider localEnvProvider = - OS.getCurrent() == OS.DARWIN - ? new XcodeLocalEnvProvider(env.getClientEnv()) - : (OS.getCurrent() == OS.WINDOWS - ? new WindowsLocalEnvProvider(env.getClientEnv()) - : new PosixLocalEnvProvider(env.getClientEnv())); - return new LocalSpawnRunner( - env.getExecRoot(), - localExecutionOptions, - ResourceManager.instance(), - localEnvProvider); - } - - @Override - public Iterable<? extends ActionContext> getActionContexts() { - return strategies; - } -} diff --git a/src/main/java/com/google/devtools/build/lib/worker/WorkerModule.java b/src/main/java/com/google/devtools/build/lib/worker/WorkerModule.java index b75702dfe8..4a2097d16d 100644 --- a/src/main/java/com/google/devtools/build/lib/worker/WorkerModule.java +++ b/src/main/java/com/google/devtools/build/lib/worker/WorkerModule.java @@ -16,7 +16,9 @@ package com.google.devtools.build.lib.worker; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; import com.google.common.eventbus.Subscribe; +import com.google.devtools.build.lib.actions.ResourceManager; import com.google.devtools.build.lib.actions.SpawnActionContext; import com.google.devtools.build.lib.buildtool.BuildRequest; import com.google.devtools.build.lib.buildtool.buildevent.BuildCompleteEvent; @@ -24,10 +26,18 @@ import com.google.devtools.build.lib.buildtool.buildevent.BuildInterruptedEvent; import com.google.devtools.build.lib.buildtool.buildevent.BuildStartingEvent; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.exec.ExecutorBuilder; +import com.google.devtools.build.lib.exec.SpawnRunner; +import com.google.devtools.build.lib.exec.apple.XcodeLocalEnvProvider; +import com.google.devtools.build.lib.exec.local.LocalEnvProvider; +import com.google.devtools.build.lib.exec.local.LocalExecutionOptions; +import com.google.devtools.build.lib.exec.local.LocalSpawnRunner; +import com.google.devtools.build.lib.exec.local.PosixLocalEnvProvider; +import com.google.devtools.build.lib.exec.local.WindowsLocalEnvProvider; import com.google.devtools.build.lib.runtime.BlazeModule; import com.google.devtools.build.lib.runtime.Command; import com.google.devtools.build.lib.runtime.CommandEnvironment; import com.google.devtools.build.lib.runtime.commands.CleanCommand.CleanStartingEvent; +import com.google.devtools.build.lib.util.OS; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.common.options.OptionsBase; import java.io.IOException; @@ -125,11 +135,37 @@ public class WorkerModule extends BlazeModule { @Override public void executorInit(CommandEnvironment env, BuildRequest request, ExecutorBuilder builder) { Preconditions.checkNotNull(workerPool); - builder.addActionContextProvider(new WorkerActionContextProvider(env, workerPool)); + ImmutableMultimap<String, String> extraFlags = + ImmutableMultimap.copyOf(env.getOptions().getOptions(WorkerOptions.class).workerExtraFlags); + WorkerSpawnRunner spawnRunner = + new WorkerSpawnRunner( + env.getExecRoot(), + workerPool, + extraFlags, + env.getReporter(), + createFallbackRunner(env)); + builder.addActionContext(new WorkerSpawnStrategy(env.getExecRoot(), spawnRunner)); + builder.addStrategyByContext(SpawnActionContext.class, "standalone"); builder.addStrategyByContext(SpawnActionContext.class, "worker"); } + private static SpawnRunner createFallbackRunner(CommandEnvironment env) { + LocalExecutionOptions localExecutionOptions = + env.getOptions().getOptions(LocalExecutionOptions.class); + LocalEnvProvider localEnvProvider = + OS.getCurrent() == OS.DARWIN + ? new XcodeLocalEnvProvider(env.getClientEnv()) + : (OS.getCurrent() == OS.WINDOWS + ? new WindowsLocalEnvProvider(env.getClientEnv()) + : new PosixLocalEnvProvider(env.getClientEnv())); + return new LocalSpawnRunner( + env.getExecRoot(), + localExecutionOptions, + ResourceManager.instance(), + localEnvProvider); + } + @Subscribe public void buildComplete(BuildCompleteEvent event) { if (options != null && options.workerQuitAfterBuild) { |