// Copyright 2016 The Bazel Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package com.google.devtools.build.lib.remote; import static java.nio.charset.StandardCharsets.UTF_8; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.actions.ActionExecutionContext; import com.google.devtools.build.lib.actions.ActionInput; import com.google.devtools.build.lib.actions.ActionInputHelper; import com.google.devtools.build.lib.actions.ExecException; import com.google.devtools.build.lib.actions.ExecutionStrategy; import com.google.devtools.build.lib.actions.Spawn; import com.google.devtools.build.lib.actions.SpawnActionContext; import com.google.devtools.build.lib.actions.Spawns; import com.google.devtools.build.lib.actions.UserExecException; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.EventHandler; import com.google.devtools.build.lib.remote.ContentDigests.ActionKey; import com.google.devtools.build.lib.remote.RemoteProtocol.Action; import com.google.devtools.build.lib.remote.RemoteProtocol.ActionResult; import com.google.devtools.build.lib.remote.RemoteProtocol.Command; import com.google.devtools.build.lib.remote.RemoteProtocol.ContentDigest; import com.google.devtools.build.lib.remote.RemoteProtocol.ExecuteReply; import com.google.devtools.build.lib.remote.RemoteProtocol.ExecuteRequest; import com.google.devtools.build.lib.remote.RemoteProtocol.ExecutionStatus; import com.google.devtools.build.lib.remote.TreeNodeRepository.TreeNode; import com.google.devtools.build.lib.standalone.StandaloneSpawnStrategy; import com.google.devtools.build.lib.util.io.FileOutErr; import com.google.devtools.build.lib.vfs.Path; import io.grpc.StatusRuntimeException; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map; import java.util.TreeSet; /** * Strategy that uses a distributed cache for sharing action input and output files. Optionally this * strategy also support offloading the work to a remote worker. */ @ExecutionStrategy( name = {"remote"}, contextType = SpawnActionContext.class ) final class RemoteSpawnStrategy implements SpawnActionContext { private final Path execRoot; private final StandaloneSpawnStrategy standaloneStrategy; private final RemoteActionCache remoteActionCache; private final RemoteWorkExecutor remoteWorkExecutor; RemoteSpawnStrategy( Map clientEnv, Path execRoot, RemoteOptions options, boolean verboseFailures, RemoteActionCache actionCache, RemoteWorkExecutor workExecutor, String productName) { this.execRoot = execRoot; this.standaloneStrategy = new StandaloneSpawnStrategy(execRoot, verboseFailures, productName); this.remoteActionCache = actionCache; this.remoteWorkExecutor = workExecutor; } private Action buildAction( Collection outputs, ContentDigest command, ContentDigest inputRoot) { Action.Builder action = Action.newBuilder(); action.setCommandDigest(command); action.setInputRootDigest(inputRoot); // Somewhat ugly: we rely on the stable order of outputs here for remote action caching. for (ActionInput output : outputs) { action.addOutputPath(output.getExecPathString()); } // TODO(olaola): Need to set platform as well! return action.build(); } private Command buildCommand(List arguments, ImmutableMap environment) { Command.Builder command = Command.newBuilder(); command.addAllArgv(arguments); // Sorting the environment pairs by variable name. TreeSet variables = new TreeSet<>(environment.keySet()); for (String var : variables) { command.addEnvironmentBuilder().setVariable(var).setValue(environment.get(var)); } return command.build(); } /** * Fallback: execute the spawn locally. If an ActionKey is provided, try to upload results to * remote action cache. */ private void execLocally( Spawn spawn, ActionExecutionContext actionExecutionContext, ActionKey actionKey) throws ExecException, InterruptedException { standaloneStrategy.exec(spawn, actionExecutionContext); if (remoteActionCache != null && actionKey != null) { ArrayList outputFiles = new ArrayList<>(); for (ActionInput output : spawn.getOutputFiles()) { outputFiles.add(execRoot.getRelative(output.getExecPathString())); } try { ActionResult.Builder result = ActionResult.newBuilder(); remoteActionCache.uploadAllResults(execRoot, outputFiles, result); remoteActionCache.setCachedActionResult(actionKey, result.build()); // Handle all cache errors here. } catch (IOException e) { throw new UserExecException("Unexpected IO error.", e); } catch (UnsupportedOperationException e) { actionExecutionContext .getExecutor() .getEventHandler() .handle( Event.warn( spawn.getMnemonic() + " unsupported operation for action cache (" + e + ")")); } } } private void passRemoteOutErr(ActionResult result, FileOutErr outErr) { if (remoteActionCache == null) { return; } try { ImmutableList streams = remoteActionCache.downloadBlobs( ImmutableList.of(result.getStdoutDigest(), result.getStderrDigest())); outErr.printOut(new String(streams.get(0), UTF_8)); outErr.printErr(new String(streams.get(1), UTF_8)); } catch (CacheNotFoundException e) { // Ignoring. } } /** Executes the given {@code spawn}. */ @Override public void exec(Spawn spawn, ActionExecutionContext actionExecutionContext) throws ExecException, InterruptedException { if (!spawn.isRemotable() || remoteActionCache == null) { standaloneStrategy.exec(spawn, actionExecutionContext); return; } ActionKey actionKey = null; String mnemonic = spawn.getMnemonic(); EventHandler eventHandler = actionExecutionContext.getExecutor().getEventHandler(); try { // Temporary hack: the TreeNodeRepository should be created and maintained upstream! TreeNodeRepository repository = new TreeNodeRepository(execRoot); List inputs = ActionInputHelper.expandArtifacts( spawn.getInputFiles(), actionExecutionContext.getArtifactExpander()); TreeNode inputRoot = repository.buildFromActionInputs(inputs); repository.computeMerkleDigests(inputRoot); Command command = buildCommand(spawn.getArguments(), spawn.getEnvironment()); Action action = buildAction( spawn.getOutputFiles(), ContentDigests.computeDigest(command), repository.getMerkleDigest(inputRoot)); // Look up action cache, and reuse the action output if it is found. actionKey = ContentDigests.computeActionKey(action); ActionResult result = remoteActionCache.getCachedActionResult(actionKey); boolean acceptCached = true; if (result != null) { // We don't cache failed actions, so we know the outputs exist. // For now, download all outputs locally; in the future, we can reuse the digests to // just update the TreeNodeRepository and continue the build. try { remoteActionCache.downloadAllResults(result, execRoot); return; } catch (CacheNotFoundException e) { acceptCached = false; // Retry the action remotely and invalidate the results. } } if (remoteWorkExecutor == null) { execLocally(spawn, actionExecutionContext, actionKey); return; } // Upload the command and all the inputs into the remote cache. remoteActionCache.uploadBlob(command.toByteArray()); // TODO(olaola): this should use the ActionInputFileCache for SHA1 digests! remoteActionCache.uploadTree(repository, execRoot, inputRoot); // TODO(olaola): set BuildInfo and input total bytes as well. ExecuteRequest.Builder request = ExecuteRequest.newBuilder() .setAction(action) .setAcceptCached(acceptCached) .setTotalInputFileCount(inputs.size()) .setTimeoutMillis(1000 * Spawns.getTimeoutSeconds(spawn, 120)); // TODO(olaola): set sensible local and remote timouts. ExecuteReply reply = remoteWorkExecutor.executeRemotely(request.build()); ExecutionStatus status = reply.getStatus(); result = reply.getResult(); // We do not want to pass on the remote stdout and strerr if we are going to retry the // action. if (status.getSucceeded()) { passRemoteOutErr(result, actionExecutionContext.getFileOutErr()); remoteActionCache.downloadAllResults(result, execRoot); return; } if (status.getError() == ExecutionStatus.ErrorCode.EXEC_FAILED) { passRemoteOutErr(result, actionExecutionContext.getFileOutErr()); throw new UserExecException(status.getErrorDetail()); } // For now, we retry locally on all other remote errors. // TODO(olaola): add remote retries on cache miss errors. execLocally(spawn, actionExecutionContext, actionKey); } catch (IOException e) { throw new UserExecException("Unexpected IO error.", e); } catch (InterruptedException e) { eventHandler.handle(Event.warn(mnemonic + " remote work interrupted (" + e + ")")); Thread.currentThread().interrupt(); throw e; } catch (StatusRuntimeException e) { eventHandler.handle(Event.warn(mnemonic + " remote work failed (" + e + ")")); execLocally(spawn, actionExecutionContext, actionKey); } catch (CacheNotFoundException e) { eventHandler.handle(Event.warn(mnemonic + " remote work results cache miss (" + e + ")")); execLocally(spawn, actionExecutionContext, actionKey); } catch (UnsupportedOperationException e) { eventHandler.handle( Event.warn(mnemonic + " unsupported operation for action cache (" + e + ")")); } } @Override public boolean willExecuteRemotely(boolean remotable) { // Returning true here just helps to estimate the cost of this computation is zero. return remotable; } @Override public boolean shouldPropagateExecException() { return false; } }