// Copyright 2017 The Bazel Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package com.google.devtools.build.lib.remote; import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.actions.ActionInput; import com.google.devtools.build.lib.actions.ExecException; import com.google.devtools.build.lib.actions.Spawn; import com.google.devtools.build.lib.actions.Spawns; import com.google.devtools.build.lib.actions.UserExecException; import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; import com.google.devtools.build.lib.exec.SpawnResult; import com.google.devtools.build.lib.exec.SpawnResult.Status; import com.google.devtools.build.lib.exec.SpawnRunner; import com.google.devtools.build.lib.remote.Digests.ActionKey; import com.google.devtools.build.lib.remote.TreeNodeRepository.TreeNode; import com.google.devtools.build.lib.util.io.FileOutErr; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.remoteexecution.v1test.Action; import com.google.devtools.remoteexecution.v1test.ActionResult; import com.google.devtools.remoteexecution.v1test.Command; import com.google.devtools.remoteexecution.v1test.Digest; import com.google.devtools.remoteexecution.v1test.Platform; import com.google.protobuf.Duration; import com.google.protobuf.TextFormat; import com.google.protobuf.TextFormat.ParseException; import io.grpc.StatusRuntimeException; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.SortedMap; import java.util.TreeSet; /** * A {@link SpawnRunner} implementation that adds a remote cache on top of an underlying local * {@link SpawnRunner} implementation. */ @ThreadSafe // If RemoteActionCache and SpawnRunner implementations are thread-safe. final class CachedLocalSpawnRunner implements SpawnRunner { private final Path execRoot; private final RemoteOptions options; // TODO(olaola): This will be set on a per-action basis instead. private final Platform platform; private final RemoteActionCache remoteCache; private final SpawnRunner delegate; CachedLocalSpawnRunner( Path execRoot, RemoteOptions options, RemoteActionCache remoteCache, SpawnRunner delegate) { this.execRoot = execRoot; this.options = options; if (options.experimentalRemotePlatformOverride != null) { Platform.Builder platformBuilder = Platform.newBuilder(); try { TextFormat.getParser().merge(options.experimentalRemotePlatformOverride, platformBuilder); } catch (ParseException e) { throw new RuntimeException("Failed to parse --experimental_remote_platform_override", e); } platform = platformBuilder.build(); } else { platform = null; } this.remoteCache = remoteCache; this.delegate = delegate; } @Override public SpawnResult exec(Spawn spawn, SpawnExecutionPolicy policy) throws InterruptedException, IOException, ExecException { ActionKey actionKey = null; String mnemonic = spawn.getMnemonic(); try { // Temporary hack: the TreeNodeRepository should be created and maintained upstream! TreeNodeRepository repository = new TreeNodeRepository(execRoot, policy.getActionInputFileCache()); SortedMap inputMap = policy.getInputMapping(); TreeNode inputRoot = repository.buildFromActionInputs(inputMap); repository.computeMerkleDigests(inputRoot); Command command = buildCommand(spawn.getArguments(), spawn.getEnvironment()); Action action = buildAction( spawn.getOutputFiles(), Digests.computeDigest(command), repository.getMerkleDigest(inputRoot), // TODO(olaola): set sensible local and remote timeouts. Spawns.getTimeoutSeconds(spawn, 120)); // Look up action cache, and reuse the action output if it is found. actionKey = Digests.computeActionKey(action); ActionResult result = this.options.remoteAcceptCached ? remoteCache.getCachedActionResult(actionKey) : null; if (result != null) { // We don't cache failed actions, so we know the outputs exist. // For now, download all outputs locally; in the future, we can reuse the digests to // just update the TreeNodeRepository and continue the build. try { // TODO(ulfjack): Download stdout, stderr, and the output files in a single call. remoteCache.downloadAllResults(result, execRoot); passRemoteOutErr(result, policy.getFileOutErr()); return new SpawnResult.Builder() .setStatus(Status.SUCCESS) .setExitCode(result.getExitCode()) .build(); } catch (CacheNotFoundException e) { // TODO(ulfjack): Track down who throws this exception in what cases and double-check that // ignoring the exception here is acceptable. Possible change it so that we throw in some // cases - we don't want to hide failures in the remote cache from the user. } } SpawnResult spawnResult = delegate.exec(spawn, policy); if (options.remoteUploadLocalResults && spawnResult.status() == Status.SUCCESS && spawnResult.exitCode() == 0) { writeCacheEntry(spawn, policy.getFileOutErr(), actionKey); } return spawnResult; } catch (StatusRuntimeException e) { throw new UserExecException(mnemonic + " remote work failed (" + e + ")", e); } } private Action buildAction( Collection outputs, Digest command, Digest inputRoot, long timeoutSeconds) { Action.Builder action = Action.newBuilder(); action.setCommandDigest(command); action.setInputRootDigest(inputRoot); // Somewhat ugly: we rely on the stable order of outputs here for remote action caching. for (ActionInput output : outputs) { // TODO: output directories should be handled here, when they are supported. action.addOutputFiles(output.getExecPathString()); } if (platform != null) { action.setPlatform(platform); } action.setTimeout(Duration.newBuilder().setSeconds(timeoutSeconds)); return action.build(); } private Command buildCommand(List arguments, ImmutableMap environment) { Command.Builder command = Command.newBuilder(); command.addAllArguments(arguments); // Sorting the environment pairs by variable name. TreeSet variables = new TreeSet<>(environment.keySet()); for (String var : variables) { command.addEnvironmentVariablesBuilder().setName(var).setValue(environment.get(var)); } return command.build(); } private void passRemoteOutErr(ActionResult result, FileOutErr outErr) throws IOException { try { if (!result.getStdoutRaw().isEmpty()) { result.getStdoutRaw().writeTo(outErr.getOutputStream()); outErr.getOutputStream().flush(); } else if (result.hasStdoutDigest()) { byte[] stdoutBytes = remoteCache.downloadBlob(result.getStdoutDigest()); outErr.getOutputStream().write(stdoutBytes); outErr.getOutputStream().flush(); } if (!result.getStderrRaw().isEmpty()) { result.getStderrRaw().writeTo(outErr.getErrorStream()); outErr.getErrorStream().flush(); } else if (result.hasStderrDigest()) { byte[] stderrBytes = remoteCache.downloadBlob(result.getStderrDigest()); outErr.getErrorStream().write(stderrBytes); outErr.getErrorStream().flush(); } } catch (CacheNotFoundException e) { outErr.printOutLn("Failed to fetch remote stdout/err due to cache miss."); outErr.getOutputStream().flush(); } } private void writeCacheEntry(Spawn spawn, FileOutErr outErr, ActionKey actionKey) throws IOException, InterruptedException { ArrayList outputFiles = new ArrayList<>(); for (ActionInput output : spawn.getOutputFiles()) { Path outputPath = execRoot.getRelative(output.getExecPathString()); // TODO(ulfjack): Store the actual list of output files in SpawnResult and use that instead // of statting the files here again. if (outputPath.exists()) { outputFiles.add(outputPath); } } ActionResult.Builder result = ActionResult.newBuilder(); remoteCache.uploadAllResults(execRoot, outputFiles, result); Digest stderr = remoteCache.uploadFileContents(outErr.getErrorPath()); Digest stdout = remoteCache.uploadFileContents(outErr.getOutputPath()); result.setStderrDigest(stderr); result.setStdoutDigest(stdout); remoteCache.setCachedActionResult(actionKey, result.build()); } }