diff options
Diffstat (limited to 'src/main/java/com/google/devtools/build/lib')
10 files changed, 406 insertions, 21 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelActionContextConsumer.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelActionContextConsumer.java index c995f78fe5..9bf4f6376e 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelActionContextConsumer.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelActionContextConsumer.java @@ -21,6 +21,7 @@ import com.google.devtools.build.lib.actions.ActionContext; import com.google.devtools.build.lib.analysis.actions.FileWriteActionContext; import com.google.devtools.build.lib.bazel.rules.BazelStrategyModule.BazelExecutionOptions; import com.google.devtools.build.lib.exec.ActionContextConsumer; +import com.google.devtools.build.lib.exec.SpawnCache; import com.google.devtools.build.lib.rules.android.WriteAdbArgsActionContext; import com.google.devtools.build.lib.rules.cpp.CppCompileActionContext; import com.google.devtools.build.lib.rules.cpp.IncludeScanningContext; @@ -75,6 +76,7 @@ public class BazelActionContextConsumer implements ActionContextConsumer { .put(IncludeScanningContext.class, "") .put(FileWriteActionContext.class, "") .put(WriteAdbArgsActionContext.class, "") + .put(SpawnCache.class, "") .build(); } } diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java index 25f4d0a993..f0d17413c7 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java @@ -137,6 +137,7 @@ public class ExecutionTool { for (ActionContext strategy : provider.getActionContexts()) { ExecutionStrategy annotation = strategy.getClass().getAnnotation(ExecutionStrategy.class); + // TODO(ulfjack): Don't silently ignore action contexts without annotation. if (annotation != null) { defaultClassMap.put(annotation.contextType(), strategy); diff --git a/src/main/java/com/google/devtools/build/lib/exec/AbstractSpawnStrategy.java b/src/main/java/com/google/devtools/build/lib/exec/AbstractSpawnStrategy.java index 9ad2d468f3..45322e471c 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/AbstractSpawnStrategy.java +++ b/src/main/java/com/google/devtools/build/lib/exec/AbstractSpawnStrategy.java @@ -14,6 +14,7 @@ package com.google.devtools.build.lib.exec; +import com.google.common.base.Preconditions; import com.google.common.base.Predicates; import com.google.common.collect.Iterables; import com.google.common.eventbus.EventBus; @@ -28,15 +29,19 @@ import com.google.devtools.build.lib.actions.SandboxedSpawnActionContext; import com.google.devtools.build.lib.actions.Spawn; import com.google.devtools.build.lib.actions.SpawnActionContext; import com.google.devtools.build.lib.actions.Spawns; +import com.google.devtools.build.lib.exec.SpawnCache.CacheHandle; import com.google.devtools.build.lib.exec.SpawnResult.Status; import com.google.devtools.build.lib.exec.SpawnRunner.ProgressStatus; import com.google.devtools.build.lib.exec.SpawnRunner.SpawnExecutionPolicy; import com.google.devtools.build.lib.rules.fileset.FilesetActionContext; import com.google.devtools.build.lib.util.CommandFailureUtils; import com.google.devtools.build.lib.util.io.FileOutErr; +import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import java.io.IOException; import java.time.Duration; +import java.util.ArrayList; +import java.util.List; import java.util.SortedMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -71,9 +76,29 @@ public abstract class AbstractSpawnStrategy implements SandboxedSpawnActionConte SpawnExecutionPolicy policy = new SpawnExecutionPolicyImpl( spawn, actionExecutionContext, writeOutputFiles, timeout); + // TODO(ulfjack): Provide a way to disable the cache. We don't want the RemoteSpawnStrategy to + // check the cache twice. Right now that can't happen because this is hidden behind an + // experimental flag. + SpawnCache cache = actionExecutionContext.getContext(SpawnCache.class); + // In production, the getContext method guarantees that we never get null back. However, our + // integration tests don't set it up correctly, so cache may be null in testing. + if (cache == null || !Spawns.mayBeCached(spawn)) { + cache = SpawnCache.NO_CACHE; + } SpawnResult result; try { - result = spawnRunner.exec(spawn, policy); + try (CacheHandle cacheHandle = cache.lookup(spawn, policy)) { + if (cacheHandle.hasResult()) { + result = Preconditions.checkNotNull(cacheHandle.getResult()); + } else { + // Actual execution. + result = spawnRunner.exec(spawn, policy); + if (cacheHandle.willStore()) { + cacheHandle.store( + result, listExistingOutputFiles(spawn, actionExecutionContext.getExecRoot())); + } + } + } } catch (IOException e) { throw new EnvironmentalExecException("Unexpected IO error.", e); } @@ -91,6 +116,19 @@ public abstract class AbstractSpawnStrategy implements SandboxedSpawnActionConte } } + private List<Path> listExistingOutputFiles(Spawn spawn, Path execRoot) { + ArrayList<Path> outputFiles = new ArrayList<>(); + for (ActionInput output : spawn.getOutputFiles()) { + Path outputPath = execRoot.getRelative(output.getExecPathString()); + // TODO(ulfjack): Store the actual list of output files in SpawnResult and use that instead + // of statting the files here again. + if (outputPath.exists()) { + outputFiles.add(outputPath); + } + } + return outputFiles; + } + private final class SpawnExecutionPolicyImpl implements SpawnExecutionPolicy { private final Spawn spawn; private final ActionExecutionContext actionExecutionContext; diff --git a/src/main/java/com/google/devtools/build/lib/exec/SpawnCache.java b/src/main/java/com/google/devtools/build/lib/exec/SpawnCache.java new file mode 100644 index 0000000000..20ea4211a6 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/exec/SpawnCache.java @@ -0,0 +1,175 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.exec; + +import com.google.devtools.build.lib.actions.ActionContext; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.ExecutionStrategy; +import com.google.devtools.build.lib.actions.Spawn; +import com.google.devtools.build.lib.exec.SpawnRunner.SpawnExecutionPolicy; +import com.google.devtools.build.lib.vfs.Path; +import java.io.Closeable; +import java.io.IOException; +import java.util.Collection; +import java.util.NoSuchElementException; + +/** + * A cache that can lookup a {@link SpawnResult} given a {@link Spawn}, and can also upload the + * results of an executed spawn to the cache. + * + * <p>This is an experimental interface to implement caching with sandboxed local execution. + */ +public interface SpawnCache extends ActionContext { + /** A no-op implementation that has no result, and performs no upload. */ + public static CacheHandle NO_RESULT_NO_STORE = new CacheHandle() { + @Override + public boolean hasResult() { + return false; + } + + @Override + public SpawnResult getResult() { + throw new NoSuchElementException(); + } + + @Override + public boolean willStore() { + return false; + } + + @Override + public void store(SpawnResult result, Collection<Path> files) + throws InterruptedException, IOException { + // Do nothing. + } + + @Override + public void close() { + } + }; + + /** + * Helper method to create a {@link CacheHandle} from a successful {@link SpawnResult} instance. + */ + public static CacheHandle success(final SpawnResult result) { + return new CacheHandle() { + @Override + public boolean hasResult() { + return true; + } + + @Override + public SpawnResult getResult() { + return result; + } + + @Override + public boolean willStore() { + return false; + } + + @Override + public void store(SpawnResult result, Collection<Path> files) + throws InterruptedException, IOException { + throw new IllegalStateException(); + } + + @Override + public void close() { + } + }; + } + + /** A no-op spawn cache. */ + @ExecutionStrategy( + name = {"no-cache"}, + contextType = SpawnCache.class + ) + public static class NoSpawnCache implements SpawnCache { + @Override + public CacheHandle lookup(Spawn spawn, SpawnExecutionPolicy context) { + return SpawnCache.NO_RESULT_NO_STORE; + } + } + + /** A no-op implementation that has no results and performs no stores. */ + public static SpawnCache NO_CACHE = new NoSpawnCache(); + + /** + * This object represents both a successful and an unsuccessful cache lookup. If + * {@link #hasResult} returns true, then {@link #getResult} must successfully return a non-null + * instance (use the {@link #success} helper method). Otherwise {@link #getResult} should throw an + * {@link IllegalStateException}. + * + * <p>If {@link #hasResult} returns false, then {@link #store} may upload the result to the cache + * after successful execution. + * + * <p>Note that this interface extends {@link Closeable}, and callers must guarantee that + * {@link #close} is called on this entry (e.g., by using try-with-resources) to free up any + * acquired resources. + */ + interface CacheHandle extends Closeable { + /** Returns whether the cache lookup was successful. */ + boolean hasResult(); + + /** + * Returns the cached result. + * + * @throws NoSuchElementException if there is no result in this cache entry + */ + SpawnResult getResult(); + + /** + * Returns true if the store call will actually do work. Use this to avoid unnecessary work + * before store if it won't do anything. + */ + boolean willStore(); + + /** + * Called after successful {@link Spawn} execution, which may or may not store the result in the + * cache. + * + * <p>A cache may silently return from a failed store operation. We recommend to err on the side + * of raising an exception rather than returning silently, and to offer command-line flags to + * tweak this default policy as needed. + * + * <p>If the current thread is interrupted, then this method should return as quickly as + * possible with an {@link InterruptedException}. + */ + void store(SpawnResult result, Collection<Path> files) + throws InterruptedException, IOException; + } + + /** + * Perform a spawn lookup. This method is similar to {@link SpawnRunner#exec}, taking the same + * parameters and being allowed to throw the same exceptions. The intent for this method is to + * compute a cache lookup key for the given spawn, looking it up in an implementation-dependent + * cache (can be either on the local or remote machine), and returning a non-null + * {@link CacheHandle} instance. + * + * <p>If the lookup was successful, this method should write the cached outputs to their + * corresponding output locations in the output tree, as well as stdout and stderr, after + * notifying {@link SpawnExecutionPolicy#lockOutputFiles}. + * + * <p>If the lookup was unsuccessful, this method can return a {@link CacheHandle} instance that + * has no result, but uploads the results of the execution to the cache. The reason for a callback + * object is for the cache to store expensive intermediate values (such as the cache key) that are + * needed both for the lookup and the subsequent store operation. + * + * <p>Note that cache stores may be disabled, in which case the returned {@link CacheHandle} + * instance's {@link CacheHandle#store} is a no-op. + */ + CacheHandle lookup(Spawn spawn, SpawnExecutionPolicy context) + throws ExecException, IOException, InterruptedException; +} diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java index 8424fca0c2..826de367c8 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java @@ -37,8 +37,6 @@ final class RemoteActionContextProvider extends ActionContextProvider { private final RemoteActionCache cache; private final GrpcRemoteExecutor executor; - private RemoteSpawnRunner spawnRunner; - RemoteActionContextProvider(CommandEnvironment env, @Nullable RemoteActionCache cache, @Nullable GrpcRemoteExecutor executor) { this.env = env; @@ -52,14 +50,19 @@ final class RemoteActionContextProvider extends ActionContextProvider { checkNotNull(env.getOptions().getOptions(ExecutionOptions.class)); RemoteOptions remoteOptions = checkNotNull(env.getOptions().getOptions(RemoteOptions.class)); - spawnRunner = new RemoteSpawnRunner( - env.getExecRoot(), - remoteOptions, - createFallbackRunner(env), - executionOptions.verboseFailures, - cache, - executor); - return ImmutableList.of(new RemoteSpawnStrategy(spawnRunner)); + if (remoteOptions.experimentalRemoteSpawnCache) { + RemoteSpawnCache spawnCache = new RemoteSpawnCache(env.getExecRoot(), remoteOptions, cache); + return ImmutableList.of(spawnCache); + } else { + RemoteSpawnRunner spawnRunner = new RemoteSpawnRunner( + env.getExecRoot(), + remoteOptions, + createFallbackRunner(env), + executionOptions.verboseFailures, + cache, + executor); + return ImmutableList.of(new RemoteSpawnStrategy(spawnRunner)); + } } private static SpawnRunner createFallbackRunner(CommandEnvironment env) { @@ -79,9 +82,8 @@ final class RemoteActionContextProvider extends ActionContextProvider { @Override public void executionPhaseEnding() { - if (spawnRunner != null) { - spawnRunner.close(); + if (cache != null) { + cache.close(); } - spawnRunner = null; } } diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteOptions.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteOptions.java index 175b7b834d..8b6975257d 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteOptions.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteOptions.java @@ -218,6 +218,17 @@ public final class RemoteOptions extends OptionsBase { ) public double experimentalRemoteRetryJitter; + @Option( + name = "experimental_remote_spawn_cache", + defaultValue = "false", + category = "remote", + documentationCategory = OptionDocumentationCategory.UNCATEGORIZED, + effectTags = {OptionEffectTag.UNKNOWN}, + help = "Whether to use the experimental spawn cache infrastructure for remote caching. " + + "Enabling this flag makes Bazel ignore any setting for remote_executor." + ) + public boolean experimentalRemoteSpawnCache; + public Platform parseRemotePlatformOverride() { if (experimentalRemotePlatformOverride != null) { Platform.Builder platformBuilder = Platform.newBuilder(); diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java new file mode 100644 index 0000000000..5d5c3e8d16 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java @@ -0,0 +1,132 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.remote; + +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.ExecutionStrategy; +import com.google.devtools.build.lib.actions.Spawn; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.exec.SpawnCache; +import com.google.devtools.build.lib.exec.SpawnResult; +import com.google.devtools.build.lib.exec.SpawnResult.Status; +import com.google.devtools.build.lib.exec.SpawnRunner.SpawnExecutionPolicy; +import com.google.devtools.build.lib.remote.Digests.ActionKey; +import com.google.devtools.build.lib.remote.TreeNodeRepository.TreeNode; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.remoteexecution.v1test.Action; +import com.google.devtools.remoteexecution.v1test.ActionResult; +import com.google.devtools.remoteexecution.v1test.Command; +import com.google.devtools.remoteexecution.v1test.Platform; +import java.io.IOException; +import java.util.Collection; +import java.util.NoSuchElementException; +import java.util.SortedMap; + +/** + * A remote {@link SpawnCache} implementation. + */ +@ThreadSafe // If the RemoteActionCache implementation is thread-safe. +@ExecutionStrategy( + name = {"remote-cache"}, + contextType = SpawnCache.class +) +final class RemoteSpawnCache implements SpawnCache { + private final Path execRoot; + private final RemoteOptions options; + // TODO(olaola): This will be set on a per-action basis instead. + private final Platform platform; + + private final RemoteActionCache remoteCache; + + RemoteSpawnCache(Path execRoot, RemoteOptions options, RemoteActionCache remoteCache) { + this.execRoot = execRoot; + this.options = options; + this.platform = options.parseRemotePlatformOverride(); + this.remoteCache = remoteCache; + } + + @Override + public CacheHandle lookup(Spawn spawn, SpawnExecutionPolicy policy) + throws InterruptedException, IOException, ExecException { + // Temporary hack: the TreeNodeRepository should be created and maintained upstream! + TreeNodeRepository repository = + new TreeNodeRepository(execRoot, policy.getActionInputFileCache()); + SortedMap<PathFragment, ActionInput> inputMap = policy.getInputMapping(); + TreeNode inputRoot = repository.buildFromActionInputs(inputMap); + repository.computeMerkleDigests(inputRoot); + Command command = RemoteSpawnRunner.buildCommand(spawn.getArguments(), spawn.getEnvironment()); + Action action = + RemoteSpawnRunner.buildAction( + spawn.getOutputFiles(), + Digests.computeDigest(command), + repository.getMerkleDigest(inputRoot), + platform, + policy.getTimeout()); + + // Look up action cache, and reuse the action output if it is found. + final ActionKey actionKey = Digests.computeActionKey(action); + ActionResult result = + this.options.remoteAcceptCached ? remoteCache.getCachedActionResult(actionKey) : null; + if (result != null) { + // We don't cache failed actions, so we know the outputs exist. + // For now, download all outputs locally; in the future, we can reuse the digests to + // just update the TreeNodeRepository and continue the build. + try { + remoteCache.download(result, execRoot, policy.getFileOutErr()); + SpawnResult spawnResult = new SpawnResult.Builder() + .setStatus(Status.SUCCESS) + .setExitCode(result.getExitCode()) + .build(); + return SpawnCache.success(spawnResult); + } catch (CacheNotFoundException e) { + // There's a cache miss. Fall back to local execution. + } + } + if (options.remoteUploadLocalResults) { + return new CacheHandle() { + @Override + public boolean hasResult() { + return false; + } + + @Override + public SpawnResult getResult() { + throw new NoSuchElementException(); + } + + @Override + public boolean willStore() { + return true; + } + + @Override + public void store(SpawnResult result, Collection<Path> files) + throws InterruptedException, IOException { + if (result.status() != Status.SUCCESS || result.exitCode() != 0) { + return; + } + remoteCache.upload(actionKey, execRoot, files, policy.getFileOutErr()); + } + + @Override + public void close() { + } + }; + } else { + return SpawnCache.NO_RESULT_NO_STORE; + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java index 9fe489a7c4..0657f2bc24 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java @@ -330,11 +330,4 @@ class RemoteSpawnRunner implements SpawnRunner { } return outputFiles; } - - /** Release resources associated with this spawn runner. */ - public void close() { - if (remoteCache != null) { - remoteCache.close(); - } - } } diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java index 1cc587ab43..d7bd5941d8 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java @@ -964,6 +964,8 @@ public final class BlazeRuntime { } runtimeBuilder.addBlazeModule(new BuiltinCommandModule()); + // This module needs to be registered before any module providing a SpawnCache implementation. + runtimeBuilder.addBlazeModule(new NoSpawnCacheModule()); runtimeBuilder.addBlazeModule(new CommandLogModule()); for (BlazeModule blazeModule : blazeModules) { runtimeBuilder.addBlazeModule(blazeModule); diff --git a/src/main/java/com/google/devtools/build/lib/runtime/NoSpawnCacheModule.java b/src/main/java/com/google/devtools/build/lib/runtime/NoSpawnCacheModule.java new file mode 100644 index 0000000000..ffc2076490 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/runtime/NoSpawnCacheModule.java @@ -0,0 +1,29 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.runtime; + +import com.google.devtools.build.lib.buildtool.BuildRequest; +import com.google.devtools.build.lib.exec.ExecutorBuilder; +import com.google.devtools.build.lib.exec.SpawnCache; + +/** + * Module providing a default no-op spawn cache. + */ +public final class NoSpawnCacheModule extends BlazeModule { + + @Override + public void executorInit(CommandEnvironment env, BuildRequest request, ExecutorBuilder builder) { + builder.addActionContext(SpawnCache.NO_CACHE); + } +} |