From acd291a520d392977a2efff17949b75c2fa80eef Mon Sep 17 00:00:00 2001 From: ulfjack Date: Fri, 16 Jun 2017 15:25:42 +0200 Subject: Rewrite StandaloneSpawnStrategy to use LocalSpawnRunner PiperOrigin-RevId: 159221067 --- src/main/java/com/google/devtools/build/lib/BUILD | 1 + .../build/lib/exec/ActionContextProvider.java | 2 +- .../build/lib/exec/FilesetActionContextImpl.java | 4 +- .../lib/exec/SimpleActionContextProvider.java | 2 +- .../build/lib/exec/SpawnInputExpander.java | 5 +- .../build/lib/exec/StandaloneTestStrategy.java | 7 +- .../devtools/build/lib/exec/TestStrategy.java | 5 +- .../com/google/devtools/build/lib/exec/local/BUILD | 1 - .../build/lib/exec/local/LocalSpawnRunner.java | 30 +++- .../com/google/devtools/build/lib/remote/BUILD | 1 + .../lib/remote/RemoteActionContextProvider.java | 55 ++++--- .../devtools/build/lib/remote/RemoteModule.java | 2 +- .../build/lib/remote/RemoteSpawnStrategy.java | 13 +- .../build/lib/runtime/commands/BuildCommand.java | 2 + .../lib/sandbox/SandboxActionContextProvider.java | 4 +- .../StandaloneActionContextProvider.java | 52 +++--- .../build/lib/standalone/StandaloneModule.java | 2 +- .../lib/standalone/StandaloneSpawnStrategy.java | 179 ++++++++++----------- .../google/devtools/build/lib/util/OsUtils.java | 6 +- .../lib/worker/WorkerActionContextProvider.java | 2 +- .../build/lib/worker/WorkerTestStrategy.java | 3 +- 21 files changed, 211 insertions(+), 167 deletions(-) (limited to 'src/main/java/com/google/devtools/build/lib') diff --git a/src/main/java/com/google/devtools/build/lib/BUILD b/src/main/java/com/google/devtools/build/lib/BUILD index cecc9b0713..1cef34f338 100644 --- a/src/main/java/com/google/devtools/build/lib/BUILD +++ b/src/main/java/com/google/devtools/build/lib/BUILD @@ -1170,6 +1170,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader", "//src/main/java/com/google/devtools/build/lib/buildeventstream/proto:build_event_stream_java_proto", "//src/main/java/com/google/devtools/build/lib/cmdline", + "//src/main/java/com/google/devtools/build/lib/exec/local", "//src/main/java/com/google/devtools/build/lib/query2", "//src/main/java/com/google/devtools/build/lib/query2:query-engine", "//src/main/java/com/google/devtools/build/lib/query2:query-output", diff --git a/src/main/java/com/google/devtools/build/lib/exec/ActionContextProvider.java b/src/main/java/com/google/devtools/build/lib/exec/ActionContextProvider.java index 2362fcc847..bcf7287530 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/ActionContextProvider.java +++ b/src/main/java/com/google/devtools/build/lib/exec/ActionContextProvider.java @@ -31,7 +31,7 @@ public abstract class ActionContextProvider { *

These may or may not actually end up in the executor depending on the command line options * and other factors influencing how the executor is set up. */ - public abstract Iterable getActionContexts(); + public abstract Iterable getActionContexts(); /** * Two-phase initialization. The input file cache and the input prefetcher usually come from a diff --git a/src/main/java/com/google/devtools/build/lib/exec/FilesetActionContextImpl.java b/src/main/java/com/google/devtools/build/lib/exec/FilesetActionContextImpl.java index 0be6549792..a234e8f825 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/FilesetActionContextImpl.java +++ b/src/main/java/com/google/devtools/build/lib/exec/FilesetActionContextImpl.java @@ -57,8 +57,8 @@ public final class FilesetActionContextImpl implements FilesetActionContext { } @Override - public Iterable getActionContexts() { - return ImmutableList.of(impl); + public Iterable getActionContexts() { + return ImmutableList.of(impl); } @Override diff --git a/src/main/java/com/google/devtools/build/lib/exec/SimpleActionContextProvider.java b/src/main/java/com/google/devtools/build/lib/exec/SimpleActionContextProvider.java index a0f8aaef37..a459ba5b61 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/SimpleActionContextProvider.java +++ b/src/main/java/com/google/devtools/build/lib/exec/SimpleActionContextProvider.java @@ -28,7 +28,7 @@ final class SimpleActionContextProvider extends ActionContextProvider { } @Override - public Iterable getActionContexts() { + public Iterable getActionContexts() { return actionContexts; } } diff --git a/src/main/java/com/google/devtools/build/lib/exec/SpawnInputExpander.java b/src/main/java/com/google/devtools/build/lib/exec/SpawnInputExpander.java index ef538a0c2d..dc0fde00ff 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/SpawnInputExpander.java +++ b/src/main/java/com/google/devtools/build/lib/exec/SpawnInputExpander.java @@ -198,7 +198,10 @@ public class SpawnInputExpander { FilesetActionContext filesetContext) throws IOException { return getInputMapping( - spawn, artifactExpander, actionInputFileCache, filesetContext.getWorkspaceName()); + spawn, + artifactExpander, + actionInputFileCache, + filesetContext == null ? null : filesetContext.getWorkspaceName()); } /** diff --git a/src/main/java/com/google/devtools/build/lib/exec/StandaloneTestStrategy.java b/src/main/java/com/google/devtools/build/lib/exec/StandaloneTestStrategy.java index 51aeabdbd5..cbbabe4b21 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/StandaloneTestStrategy.java +++ b/src/main/java/com/google/devtools/build/lib/exec/StandaloneTestStrategy.java @@ -46,7 +46,6 @@ import com.google.devtools.build.lib.view.test.TestStatus.BlazeTestStatus; import com.google.devtools.build.lib.view.test.TestStatus.TestCase; import com.google.devtools.build.lib.view.test.TestStatus.TestResultData; import com.google.devtools.build.lib.view.test.TestStatus.TestResultData.Builder; -import com.google.devtools.common.options.OptionsClassProvider; import java.io.Closeable; import java.io.IOException; import java.util.HashMap; @@ -80,10 +79,8 @@ public class StandaloneTestStrategy extends TestStrategy { protected final Path tmpDirRoot; public StandaloneTestStrategy( - OptionsClassProvider requestOptions, - BinTools binTools, - Path tmpDirRoot) { - super(requestOptions, binTools); + ExecutionOptions executionOptions, BinTools binTools, Path tmpDirRoot) { + super(executionOptions, binTools); this.tmpDirRoot = tmpDirRoot; } diff --git a/src/main/java/com/google/devtools/build/lib/exec/TestStrategy.java b/src/main/java/com/google/devtools/build/lib/exec/TestStrategy.java index 3e860a9760..8b0450a599 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/TestStrategy.java +++ b/src/main/java/com/google/devtools/build/lib/exec/TestStrategy.java @@ -44,7 +44,6 @@ import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.lib.view.test.TestStatus.TestCase; import com.google.devtools.common.options.Converters.RangeConverter; import com.google.devtools.common.options.EnumConverter; -import com.google.devtools.common.options.OptionsClassProvider; import com.google.devtools.common.options.OptionsParsingException; import java.io.Closeable; import java.io.IOException; @@ -137,8 +136,8 @@ public abstract class TestStrategy implements TestActionContext { protected final ExecutionOptions executionOptions; protected final BinTools binTools; - public TestStrategy(OptionsClassProvider requestOptionsProvider, BinTools binTools) { - this.executionOptions = requestOptionsProvider.getOptions(ExecutionOptions.class); + public TestStrategy(ExecutionOptions executionOptions, BinTools binTools) { + this.executionOptions = executionOptions; this.binTools = binTools; } diff --git a/src/main/java/com/google/devtools/build/lib/exec/local/BUILD b/src/main/java/com/google/devtools/build/lib/exec/local/BUILD index 79e5dbf6a4..74f4f7b76c 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/local/BUILD +++ b/src/main/java/com/google/devtools/build/lib/exec/local/BUILD @@ -14,7 +14,6 @@ java_library( "//src/main/java/com/google/devtools/build/lib:io", "//src/main/java/com/google/devtools/build/lib:packages-internal", "//src/main/java/com/google/devtools/build/lib:process_util", - "//src/main/java/com/google/devtools/build/lib:runtime", "//src/main/java/com/google/devtools/build/lib:shell", "//src/main/java/com/google/devtools/build/lib:util", "//src/main/java/com/google/devtools/build/lib:vfs", diff --git a/src/main/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunner.java index acbb578cdd..5cd191be3b 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunner.java +++ b/src/main/java/com/google/devtools/build/lib/exec/local/LocalSpawnRunner.java @@ -36,6 +36,8 @@ import com.google.devtools.build.lib.shell.Command; import com.google.devtools.build.lib.shell.CommandException; import com.google.devtools.build.lib.shell.CommandResult; import com.google.devtools.build.lib.util.NetUtil; +import com.google.devtools.build.lib.util.OS; +import com.google.devtools.build.lib.util.OsUtils; import com.google.devtools.build.lib.util.Preconditions; import com.google.devtools.build.lib.util.io.FileOutErr; import com.google.devtools.build.lib.vfs.Path; @@ -87,13 +89,17 @@ public final class LocalSpawnRunner implements SpawnRunner { LocalExecutionOptions localExecutionOptions, ResourceManager resourceManager, boolean useProcessWrapper, + OS localOs, String productName, LocalEnvProvider localEnvProvider) { this.logger = logger; this.execRoot = execRoot; this.actionInputPrefetcher = Preconditions.checkNotNull(actionInputPrefetcher); - this.processWrapper = execRoot.getRelative("_bin/process-wrapper").getPathString(); - this.localExecutionOptions = localExecutionOptions; + this.processWrapper = + execRoot + .getRelative("_bin/process-wrapper" + OsUtils.executableExtension(localOs)) + .getPathString(); + this.localExecutionOptions = Preconditions.checkNotNull(localExecutionOptions); this.hostName = NetUtil.findShortHostName(); this.execCount = execCount; this.resourceManager = resourceManager; @@ -102,6 +108,26 @@ public final class LocalSpawnRunner implements SpawnRunner { this.localEnvProvider = localEnvProvider; } + public LocalSpawnRunner( + Path execRoot, + ActionInputPrefetcher actionInputPrefetcher, + LocalExecutionOptions localExecutionOptions, + ResourceManager resourceManager, + String productName, + LocalEnvProvider localEnvProvider) { + this( + Logger.getLogger(LocalSpawnRunner.class.getName()), + new AtomicInteger(), + execRoot, + actionInputPrefetcher, + localExecutionOptions, + resourceManager, + /*useProcessWrapper=*/OS.getCurrent() != OS.WINDOWS, + OS.getCurrent(), + productName, + localEnvProvider); + } + @Override public SpawnResult exec( Spawn spawn, diff --git a/src/main/java/com/google/devtools/build/lib/remote/BUILD b/src/main/java/com/google/devtools/build/lib/remote/BUILD index cbe44c0383..3a5404e970 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/BUILD +++ b/src/main/java/com/google/devtools/build/lib/remote/BUILD @@ -22,6 +22,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib:util", "//src/main/java/com/google/devtools/build/lib:vfs", "//src/main/java/com/google/devtools/build/lib/actions", + "//src/main/java/com/google/devtools/build/lib/exec/local", "//src/main/java/com/google/devtools/build/lib/standalone", "//src/main/java/com/google/devtools/common/options", "//third_party:apache_httpclient", diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java index f85930618e..7966f9e63d 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java @@ -14,39 +14,56 @@ package com.google.devtools.build.lib.remote; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableList.Builder; +import com.google.devtools.build.lib.actions.ActionInputFileCache; import com.google.devtools.build.lib.actions.Executor.ActionContext; +import com.google.devtools.build.lib.actions.ResourceManager; +import com.google.devtools.build.lib.actions.SpawnActionContext; import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions; -import com.google.devtools.build.lib.buildtool.BuildRequest; import com.google.devtools.build.lib.exec.ActionContextProvider; +import com.google.devtools.build.lib.exec.ActionInputPrefetcher; import com.google.devtools.build.lib.exec.ExecutionOptions; +import com.google.devtools.build.lib.exec.local.LocalExecutionOptions; import com.google.devtools.build.lib.runtime.CommandEnvironment; +import com.google.devtools.build.lib.standalone.StandaloneSpawnStrategy; /** * Provide a remote execution context. */ final class RemoteActionContextProvider extends ActionContextProvider { - private final ImmutableList strategies; + private final CommandEnvironment env; + private ActionInputPrefetcher actionInputPrefetcher; - RemoteActionContextProvider( - CommandEnvironment env, - BuildRequest buildRequest) { - boolean verboseFailures = buildRequest.getOptions(ExecutionOptions.class).verboseFailures; - Builder strategiesBuilder = ImmutableList.builder(); - strategiesBuilder.add( - new RemoteSpawnStrategy( - env.getClientEnv(), - env.getExecRoot(), - buildRequest.getOptions(RemoteOptions.class), - buildRequest.getOptions(AuthAndTLSOptions.class), - verboseFailures, - env.getRuntime().getProductName())); - this.strategies = strategiesBuilder.build(); + RemoteActionContextProvider(CommandEnvironment env) { + this.env = env; + } + + @Override + public void init( + ActionInputFileCache actionInputFileCache, ActionInputPrefetcher actionInputPrefetcher) { + this.actionInputPrefetcher = Preconditions.checkNotNull(actionInputPrefetcher); } @Override - public Iterable getActionContexts() { - return strategies; + public Iterable getActionContexts() { + ExecutionOptions executionOptions = env.getOptions().getOptions(ExecutionOptions.class); + LocalExecutionOptions localExecutionOptions = + env.getOptions().getOptions(LocalExecutionOptions.class); + SpawnActionContext fallbackStrategy = + new StandaloneSpawnStrategy( + env.getExecRoot(), + actionInputPrefetcher, + localExecutionOptions, + executionOptions.verboseFailures, + env.getRuntime().getProductName(), + ResourceManager.instance()); + return ImmutableList.of( + new RemoteSpawnStrategy( + env.getExecRoot(), + env.getOptions().getOptions(RemoteOptions.class), + env.getOptions().getOptions(AuthAndTLSOptions.class), + executionOptions.verboseFailures, + fallbackStrategy)); } } diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java index 0bd677f395..6e5f7909cb 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java @@ -103,7 +103,7 @@ public final class RemoteModule extends BlazeModule { @Override public void executorInit(CommandEnvironment env, BuildRequest request, ExecutorBuilder builder) { - builder.addActionContextProvider(new RemoteActionContextProvider(env, request)); + builder.addActionContextProvider(new RemoteActionContextProvider(env)); } @Subscribe diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnStrategy.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnStrategy.java index 98af3056de..70623b19e5 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnStrategy.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnStrategy.java @@ -34,7 +34,6 @@ import com.google.devtools.build.lib.exec.SpawnInputExpander; import com.google.devtools.build.lib.remote.Digests.ActionKey; import com.google.devtools.build.lib.remote.TreeNodeRepository.TreeNode; import com.google.devtools.build.lib.rules.fileset.FilesetActionContext; -import com.google.devtools.build.lib.standalone.StandaloneSpawnStrategy; import com.google.devtools.build.lib.util.CommandFailureUtils; import com.google.devtools.build.lib.util.io.FileOutErr; import com.google.devtools.build.lib.vfs.Path; @@ -54,7 +53,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.List; -import java.util.Map; import java.util.SortedMap; import java.util.TreeSet; @@ -68,7 +66,7 @@ import java.util.TreeSet; ) final class RemoteSpawnStrategy implements SpawnActionContext { private final Path execRoot; - private final StandaloneSpawnStrategy standaloneStrategy; + private final SpawnActionContext fallbackStrategy; private final boolean verboseFailures; private final RemoteOptions remoteOptions; // TODO(olaola): This will be set on a per-action basis instead. @@ -77,14 +75,13 @@ final class RemoteSpawnStrategy implements SpawnActionContext { private final SpawnInputExpander spawnInputExpander = new SpawnInputExpander(/*strict=*/ false); RemoteSpawnStrategy( - Map clientEnv, Path execRoot, RemoteOptions remoteOptions, AuthAndTLSOptions authTlsOptions, boolean verboseFailures, - String productName) { + SpawnActionContext fallbackStrategy) { this.execRoot = execRoot; - this.standaloneStrategy = new StandaloneSpawnStrategy(execRoot, verboseFailures, productName); + this.fallbackStrategy = fallbackStrategy; this.verboseFailures = verboseFailures; this.remoteOptions = remoteOptions; channelOptions = ChannelOptions.create(authTlsOptions); @@ -144,7 +141,7 @@ final class RemoteSpawnStrategy implements SpawnActionContext { RemoteActionCache remoteCache, ActionKey actionKey) throws ExecException, InterruptedException { - standaloneStrategy.exec(spawn, actionExecutionContext); + fallbackStrategy.exec(spawn, actionExecutionContext); if (remoteOptions.remoteUploadLocalResults && remoteCache != null && actionKey != null) { ArrayList outputFiles = new ArrayList<>(); for (ActionInput output : spawn.getOutputFiles()) { @@ -253,7 +250,7 @@ final class RemoteSpawnStrategy implements SpawnActionContext { } } if (!spawn.isRemotable() || remoteCache == null) { - standaloneStrategy.exec(spawn, actionExecutionContext); + fallbackStrategy.exec(spawn, actionExecutionContext); return; } if (executor.reportsSubcommands()) { diff --git a/src/main/java/com/google/devtools/build/lib/runtime/commands/BuildCommand.java b/src/main/java/com/google/devtools/build/lib/runtime/commands/BuildCommand.java index cf487bb5c7..13f3f21891 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/commands/BuildCommand.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/commands/BuildCommand.java @@ -18,6 +18,7 @@ import com.google.devtools.build.lib.buildtool.BuildRequest; import com.google.devtools.build.lib.buildtool.BuildRequest.BuildRequestOptions; import com.google.devtools.build.lib.buildtool.BuildTool; import com.google.devtools.build.lib.exec.ExecutionOptions; +import com.google.devtools.build.lib.exec.local.LocalExecutionOptions; import com.google.devtools.build.lib.pkgcache.LoadingOptions; import com.google.devtools.build.lib.pkgcache.PackageCacheOptions; import com.google.devtools.build.lib.runtime.BlazeCommand; @@ -38,6 +39,7 @@ import java.util.List; builds = true, options = { BuildRequestOptions.class, ExecutionOptions.class, + LocalExecutionOptions.class, PackageCacheOptions.class, SkylarkSemanticsOptions.class, BuildView.Options.class, diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxActionContextProvider.java b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxActionContextProvider.java index 74084c365a..1c37cd3e34 100644 --- a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxActionContextProvider.java +++ b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxActionContextProvider.java @@ -28,8 +28,6 @@ import java.io.IOException; * Provides the sandboxed spawn strategy. */ final class SandboxActionContextProvider extends ActionContextProvider { - - @SuppressWarnings("unchecked") private final ImmutableList contexts; private SandboxActionContextProvider(ImmutableList contexts) { @@ -72,7 +70,7 @@ final class SandboxActionContextProvider extends ActionContextProvider { } @Override - public Iterable getActionContexts() { + public Iterable getActionContexts() { return contexts; } } diff --git a/src/main/java/com/google/devtools/build/lib/standalone/StandaloneActionContextProvider.java b/src/main/java/com/google/devtools/build/lib/standalone/StandaloneActionContextProvider.java index 12e508dbe0..c1dbbac6f5 100644 --- a/src/main/java/com/google/devtools/build/lib/standalone/StandaloneActionContextProvider.java +++ b/src/main/java/com/google/devtools/build/lib/standalone/StandaloneActionContextProvider.java @@ -13,20 +13,23 @@ // limitations under the License. package com.google.devtools.build.lib.standalone; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableList.Builder; import com.google.devtools.build.lib.actions.Action; import com.google.devtools.build.lib.actions.ActionExecutionContext; +import com.google.devtools.build.lib.actions.ActionInputFileCache; import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.actions.ArtifactResolver; import com.google.devtools.build.lib.actions.ExecutionStrategy; import com.google.devtools.build.lib.actions.Executor.ActionContext; -import com.google.devtools.build.lib.buildtool.BuildRequest; +import com.google.devtools.build.lib.actions.ResourceManager; import com.google.devtools.build.lib.exec.ActionContextProvider; +import com.google.devtools.build.lib.exec.ActionInputPrefetcher; import com.google.devtools.build.lib.exec.ExecutionOptions; import com.google.devtools.build.lib.exec.FileWriteStrategy; import com.google.devtools.build.lib.exec.StandaloneTestStrategy; import com.google.devtools.build.lib.exec.TestStrategy; +import com.google.devtools.build.lib.exec.local.LocalExecutionOptions; import com.google.devtools.build.lib.rules.cpp.IncludeScanningContext; import com.google.devtools.build.lib.rules.cpp.SpawnGccStrategy; import com.google.devtools.build.lib.rules.test.ExclusiveTestStrategy; @@ -64,39 +67,46 @@ public class StandaloneActionContextProvider extends ActionContextProvider { } private final CommandEnvironment env; - private final ImmutableList strategies; + private ActionInputPrefetcher actionInputPrefetcher; - public StandaloneActionContextProvider(CommandEnvironment env, BuildRequest buildRequest) { + public StandaloneActionContextProvider(CommandEnvironment env) { this.env = env; - ExecutionOptions options = buildRequest.getOptions(ExecutionOptions.class); - boolean verboseFailures = options.verboseFailures; + } - Path testTmpRoot = TestStrategy.getTmpRoot(env.getWorkspace(), env.getExecRoot(), options); - TestActionContext testStrategy = - new StandaloneTestStrategy( - buildRequest, env.getBlazeWorkspace().getBinTools(), testTmpRoot); + @Override + public void init( + ActionInputFileCache actionInputFileCache, ActionInputPrefetcher actionInputPrefetcher) { + this.actionInputPrefetcher = Preconditions.checkNotNull(actionInputPrefetcher); + } - Builder strategiesBuilder = ImmutableList.builder(); + @Override + public Iterable getActionContexts() { + ExecutionOptions executionOptions = env.getOptions().getOptions(ExecutionOptions.class); + LocalExecutionOptions localExecutionOptions = + env.getOptions().getOptions(LocalExecutionOptions.class); + Path testTmpRoot = + TestStrategy.getTmpRoot(env.getWorkspace(), env.getExecRoot(), executionOptions); + TestActionContext testStrategy = + new StandaloneTestStrategy( + executionOptions, + env.getBlazeWorkspace().getBinTools(), + testTmpRoot); // Order of strategies passed to builder is significant - when there are many strategies that // could potentially be used and a spawnActionContext doesn't specify which one it wants, the // last one from strategies list will be used - strategiesBuilder.add( + return ImmutableList.of( new StandaloneSpawnStrategy( env.getExecRoot(), - verboseFailures, - env.getRuntime().getProductName()), + actionInputPrefetcher, + localExecutionOptions, + executionOptions.verboseFailures, + env.getRuntime().getProductName(), + ResourceManager.instance()), new DummyIncludeScanningContext(), new SpawnGccStrategy(), testStrategy, new ExclusiveTestStrategy(testStrategy), new FileWriteStrategy()); - - this.strategies = strategiesBuilder.build(); - } - - @Override - public Iterable getActionContexts() { - return strategies; } } diff --git a/src/main/java/com/google/devtools/build/lib/standalone/StandaloneModule.java b/src/main/java/com/google/devtools/build/lib/standalone/StandaloneModule.java index b60f47aa1e..6f299fa616 100644 --- a/src/main/java/com/google/devtools/build/lib/standalone/StandaloneModule.java +++ b/src/main/java/com/google/devtools/build/lib/standalone/StandaloneModule.java @@ -24,7 +24,7 @@ import com.google.devtools.build.lib.runtime.CommandEnvironment; public class StandaloneModule extends BlazeModule { @Override public void executorInit(CommandEnvironment env, BuildRequest request, ExecutorBuilder builder) { - builder.addActionContextProvider(new StandaloneActionContextProvider(env, request)); + builder.addActionContextProvider(new StandaloneActionContextProvider(env)); builder.addActionContextConsumer(new StandaloneActionContextConsumer()); } } diff --git a/src/main/java/com/google/devtools/build/lib/standalone/StandaloneSpawnStrategy.java b/src/main/java/com/google/devtools/build/lib/standalone/StandaloneSpawnStrategy.java index 724542ff1b..de47fe42d2 100644 --- a/src/main/java/com/google/devtools/build/lib/standalone/StandaloneSpawnStrategy.java +++ b/src/main/java/com/google/devtools/build/lib/standalone/StandaloneSpawnStrategy.java @@ -15,32 +15,36 @@ package com.google.devtools.build.lib.standalone; import com.google.common.eventbus.EventBus; import com.google.devtools.build.lib.actions.ActionExecutionContext; -import com.google.devtools.build.lib.actions.ActionExecutionMetadata; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.ActionInputFileCache; import com.google.devtools.build.lib.actions.ActionStatusMessage; import com.google.devtools.build.lib.actions.ExecException; import com.google.devtools.build.lib.actions.ExecutionStrategy; import com.google.devtools.build.lib.actions.Executor; import com.google.devtools.build.lib.actions.ResourceManager; -import com.google.devtools.build.lib.actions.ResourceManager.ResourceHandle; import com.google.devtools.build.lib.actions.Spawn; import com.google.devtools.build.lib.actions.SpawnActionContext; import com.google.devtools.build.lib.actions.Spawns; import com.google.devtools.build.lib.actions.UserExecException; +import com.google.devtools.build.lib.exec.ActionInputPrefetcher; +import com.google.devtools.build.lib.exec.SpawnExecException; +import com.google.devtools.build.lib.exec.SpawnInputExpander; +import com.google.devtools.build.lib.exec.SpawnResult; +import com.google.devtools.build.lib.exec.SpawnResult.Status; +import com.google.devtools.build.lib.exec.SpawnRunner.ProgressStatus; +import com.google.devtools.build.lib.exec.SpawnRunner.SpawnExecutionPolicy; import com.google.devtools.build.lib.exec.apple.XCodeLocalEnvProvider; import com.google.devtools.build.lib.exec.local.LocalEnvProvider; -import com.google.devtools.build.lib.shell.AbnormalTerminationException; -import com.google.devtools.build.lib.shell.Command; -import com.google.devtools.build.lib.shell.CommandException; -import com.google.devtools.build.lib.shell.TerminationStatus; +import com.google.devtools.build.lib.exec.local.LocalExecutionOptions; +import com.google.devtools.build.lib.exec.local.LocalSpawnRunner; +import com.google.devtools.build.lib.rules.fileset.FilesetActionContext; import com.google.devtools.build.lib.util.CommandFailureUtils; import com.google.devtools.build.lib.util.OS; -import com.google.devtools.build.lib.util.OsUtils; import com.google.devtools.build.lib.util.io.FileOutErr; import com.google.devtools.build.lib.vfs.Path; -import java.io.File; +import com.google.devtools.build.lib.vfs.PathFragment; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; +import java.util.SortedMap; /** * Strategy that uses subprocessing to execute a process. @@ -48,112 +52,97 @@ import java.util.List; @ExecutionStrategy(name = { "standalone", "local" }, contextType = SpawnActionContext.class) public class StandaloneSpawnStrategy implements SpawnActionContext { private final boolean verboseFailures; - private final Path processWrapper; - private final Path execRoot; - private final String productName; - private final ResourceManager resourceManager; - private final LocalEnvProvider localEnvProvider; - - public StandaloneSpawnStrategy(Path execRoot, boolean verboseFailures, String productName) { - this(execRoot, verboseFailures, productName, ResourceManager.instance()); - } + private final LocalSpawnRunner localSpawnRunner; public StandaloneSpawnStrategy( - Path execRoot, boolean verboseFailures, String productName, ResourceManager resourceManager) { + Path execRoot, ActionInputPrefetcher actionInputPrefetcher, + LocalExecutionOptions localExecutionOptions, boolean verboseFailures, String productName, + ResourceManager resourceManager) { this.verboseFailures = verboseFailures; - this.execRoot = execRoot; - this.processWrapper = execRoot.getRelative( - "_bin/process-wrapper" + OsUtils.executableExtension()); - this.productName = productName; - this.resourceManager = resourceManager; - this.localEnvProvider = OS.getCurrent() == OS.DARWIN + LocalEnvProvider localEnvProvider = OS.getCurrent() == OS.DARWIN ? new XCodeLocalEnvProvider() : LocalEnvProvider.UNMODIFIED; + this.localSpawnRunner = new LocalSpawnRunner( + execRoot, + actionInputPrefetcher, + localExecutionOptions, + resourceManager, + productName, + localEnvProvider); } /** * Executes the given {@code spawn}. */ @Override - public void exec(Spawn spawn, - ActionExecutionContext actionExecutionContext) + public void exec(final Spawn spawn, final ActionExecutionContext actionExecutionContext) throws ExecException, InterruptedException { - EventBus eventBus = actionExecutionContext.getExecutor().getEventBus(); - ActionExecutionMetadata owner = spawn.getResourceOwner(); - eventBus.post(ActionStatusMessage.schedulingStrategy(owner)); - try (ResourceHandle handle = - resourceManager.acquireResources(owner, spawn.getLocalResources())) { - eventBus.post(ActionStatusMessage.runningStrategy(owner, "standalone")); - try { - actuallyExec(spawn, actionExecutionContext); - } catch (IOException e) { - throw new UserExecException("I/O exception during local execution", e); + final int timeoutSeconds = Spawns.getTimeoutSeconds(spawn); + final EventBus eventBus = actionExecutionContext.getExecutor().getEventBus(); + SpawnExecutionPolicy policy = new SpawnExecutionPolicy() { + @Override + public ActionInputFileCache getActionInputFileCache() { + return actionExecutionContext.getActionInputFileCache(); } - } - } - /** - * Executes the given {@code spawn}. - */ - private void actuallyExec(Spawn spawn, - ActionExecutionContext actionExecutionContext) - throws ExecException, IOException { - Executor executor = actionExecutionContext.getExecutor(); + @Override + public void lockOutputFiles() throws InterruptedException { + // Do nothing for now. + } - if (executor.reportsSubcommands()) { - executor.reportSubcommand(spawn); - } + @Override + public long getTimeoutMillis() { + return timeoutSeconds * 1000L; + } - int timeoutSeconds = Spawns.getTimeoutSeconds(spawn); + @Override + public FileOutErr getFileOutErr() { + return actionExecutionContext.getFileOutErr(); + } - // We must wrap the subprocess with process-wrapper to kill the process tree. - // All actions therefore depend on the process-wrapper file. Since it's embedded, - // we don't bother with declaring it as an input. - List args = new ArrayList<>(); - if (OS.getCurrent() != OS.WINDOWS) { - // TODO(bazel-team): process-wrapper seems to work on Windows, but requires - // additional setup as it is an msys2 binary, so it needs msys2 DLLs on %PATH%. - // Disable it for now to make the setup easier and to avoid further PATH hacks. - // Ideally we should have a native implementation of process-wrapper for Windows. - args.add(processWrapper.getPathString()); - args.add(Integer.toString(timeoutSeconds)); - args.add("5"); /* kill delay: give some time to print stacktraces and whatnot. */ + @Override + public SortedMap getInputMapping() throws IOException { + return new SpawnInputExpander(/*strict*/false) + .getInputMapping( + spawn, + actionExecutionContext.getArtifactExpander(), + actionExecutionContext.getActionInputFileCache(), + actionExecutionContext.getExecutor().getContext(FilesetActionContext.class)); + } - // TODO(bazel-team): use process-wrapper redirection so we don't have to - // pass test logs through the Java heap. - args.add("-"); /* stdout. */ - args.add("-"); /* stderr. */ - } - args.addAll(spawn.getArguments()); + @Override + public void report(ProgressStatus state) { + switch (state) { + case EXECUTING: + String strategyName = "local"; + eventBus.post( + ActionStatusMessage.runningStrategy(spawn.getResourceOwner(), strategyName)); + break; + case SCHEDULING: + eventBus.post(ActionStatusMessage.schedulingStrategy(spawn.getResourceOwner())); + break; + default: + break; + } + } + }; - String cwd = executor.getExecRoot().getPathString(); - Command cmd = - new Command( - args.toArray(new String[] {}), - localEnvProvider.rewriteLocalEnv(spawn.getEnvironment(), execRoot, productName), - new File(cwd), - OS.getCurrent() == OS.WINDOWS && timeoutSeconds >= 0 ? timeoutSeconds * 1000 : -1); + Executor executor = actionExecutionContext.getExecutor(); + if (executor.reportsSubcommands()) { + executor.reportSubcommand(spawn); + } - FileOutErr outErr = actionExecutionContext.getFileOutErr(); try { - cmd.execute( - /* stdin */ new byte[] {}, - Command.NO_OBSERVER, - outErr.getOutputStream(), - outErr.getErrorStream(), - /*killSubprocessOnInterrupt*/ true); - } catch (AbnormalTerminationException e) { - TerminationStatus status = e.getResult().getTerminationStatus(); - boolean timedOut = !status.exited() && ( - status.timedout() || status.getTerminatingSignal() == 14 /* SIGALRM */); - String message = - CommandFailureUtils.describeCommandFailure( - verboseFailures, spawn.getArguments(), spawn.getEnvironment(), cwd); - throw new UserExecException(String.format("%s: %s", message, e), timedOut); - } catch (CommandException e) { - String message = CommandFailureUtils.describeCommandFailure( - verboseFailures, spawn.getArguments(), spawn.getEnvironment(), cwd); - throw new UserExecException(message, e); + SpawnResult result = localSpawnRunner.exec(spawn, policy); + if (result.status() != Status.SUCCESS || result.exitCode() != 0) { + String message = + CommandFailureUtils.describeCommandFailure( + verboseFailures, spawn.getArguments(), spawn.getEnvironment(), null); + throw new SpawnExecException( + message, result, /*forciblyRunRemotely=*/false, /*catastrophe=*/false); + } + } catch (IOException e) { + throw new UserExecException("I/O exception during local execution", e); } } diff --git a/src/main/java/com/google/devtools/build/lib/util/OsUtils.java b/src/main/java/com/google/devtools/build/lib/util/OsUtils.java index a25e64bac2..d3d0f4eac8 100644 --- a/src/main/java/com/google/devtools/build/lib/util/OsUtils.java +++ b/src/main/java/com/google/devtools/build/lib/util/OsUtils.java @@ -19,12 +19,16 @@ package com.google.devtools.build.lib.util; */ public final class OsUtils { - private static final String EXECUTABLE_EXTENSION = OS.getCurrent() == OS.WINDOWS ? ".exe" : ""; + private static final String EXECUTABLE_EXTENSION = executableExtension(OS.getCurrent()); // Utility class. private OsUtils() { } + public static String executableExtension(OS os) { + return os == OS.WINDOWS ? ".exe" : ""; + } + /** * Returns the extension used for executables on the current platform (.exe * for Windows, empty string for others). diff --git a/src/main/java/com/google/devtools/build/lib/worker/WorkerActionContextProvider.java b/src/main/java/com/google/devtools/build/lib/worker/WorkerActionContextProvider.java index f480117e5e..609ebf7cc8 100644 --- a/src/main/java/com/google/devtools/build/lib/worker/WorkerActionContextProvider.java +++ b/src/main/java/com/google/devtools/build/lib/worker/WorkerActionContextProvider.java @@ -45,7 +45,7 @@ final class WorkerActionContextProvider extends ActionContextProvider { } @Override - public Iterable getActionContexts() { + public Iterable getActionContexts() { return strategies; } } diff --git a/src/main/java/com/google/devtools/build/lib/worker/WorkerTestStrategy.java b/src/main/java/com/google/devtools/build/lib/worker/WorkerTestStrategy.java index 1ab07664cf..d3846945b6 100644 --- a/src/main/java/com/google/devtools/build/lib/worker/WorkerTestStrategy.java +++ b/src/main/java/com/google/devtools/build/lib/worker/WorkerTestStrategy.java @@ -28,6 +28,7 @@ import com.google.devtools.build.lib.actions.Spawn; import com.google.devtools.build.lib.actions.TestExecException; import com.google.devtools.build.lib.actions.UserExecException; import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.exec.ExecutionOptions; import com.google.devtools.build.lib.exec.StandaloneTestStrategy; import com.google.devtools.build.lib.rules.test.TestActionContext; import com.google.devtools.build.lib.rules.test.TestRunnerAction; @@ -67,7 +68,7 @@ public class WorkerTestStrategy extends StandaloneTestStrategy { WorkerPool workerPool, Multimap extraFlags) { super( - requestOptions, + requestOptions.getOptions(ExecutionOptions.class), env.getBlazeWorkspace().getBinTools(), env.getWorkspace()); this.workerPool = workerPool; -- cgit v1.2.3