// Copyright 2014 The Bazel Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package com.google.devtools.build.lib.skyframe; import com.google.common.base.Preconditions; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.common.eventbus.EventBus; import com.google.common.util.concurrent.Striped; import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.devtools.build.lib.actions.Action; import com.google.devtools.build.lib.actions.ActionAnalysisMetadata; import com.google.devtools.build.lib.actions.ActionCacheChecker; import com.google.devtools.build.lib.actions.ActionCacheChecker.Token; import com.google.devtools.build.lib.actions.ActionCompletionEvent; import com.google.devtools.build.lib.actions.ActionContext; import com.google.devtools.build.lib.actions.ActionExecutedEvent; import com.google.devtools.build.lib.actions.ActionExecutedEvent.ErrorTiming; import com.google.devtools.build.lib.actions.ActionExecutionContext; import com.google.devtools.build.lib.actions.ActionExecutionException; import com.google.devtools.build.lib.actions.ActionExecutionStatusReporter; import com.google.devtools.build.lib.actions.ActionGraph; import com.google.devtools.build.lib.actions.ActionInput; import com.google.devtools.build.lib.actions.ActionInputMap; import com.google.devtools.build.lib.actions.ActionInputPrefetcher; import com.google.devtools.build.lib.actions.ActionKeyContext; import com.google.devtools.build.lib.actions.ActionLogBufferPathGenerator; import com.google.devtools.build.lib.actions.ActionLookupData; import com.google.devtools.build.lib.actions.ActionLookupValue; import com.google.devtools.build.lib.actions.ActionMiddlemanEvent; import com.google.devtools.build.lib.actions.ActionResult; import com.google.devtools.build.lib.actions.ActionResultReceivedEvent; import com.google.devtools.build.lib.actions.ActionStartedEvent; import com.google.devtools.build.lib.actions.ActionStatusMessage; import com.google.devtools.build.lib.actions.Actions; import com.google.devtools.build.lib.actions.AlreadyReportedActionExecutionException; import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.actions.Artifact.ArtifactExpander; import com.google.devtools.build.lib.actions.Artifact.OwnerlessArtifactWrapper; import com.google.devtools.build.lib.actions.ArtifactPathResolver; import com.google.devtools.build.lib.actions.ArtifactPrefixConflictException; import com.google.devtools.build.lib.actions.CachedActionEvent; import com.google.devtools.build.lib.actions.EnvironmentalExecException; import com.google.devtools.build.lib.actions.Executor; import com.google.devtools.build.lib.actions.FileArtifactValue; import com.google.devtools.build.lib.actions.FilesetOutputSymlink; import com.google.devtools.build.lib.actions.MapBasedActionGraph; import com.google.devtools.build.lib.actions.MetadataConsumer; import com.google.devtools.build.lib.actions.MetadataProvider; import com.google.devtools.build.lib.actions.MutableActionGraph; import com.google.devtools.build.lib.actions.MutableActionGraph.ActionConflictException; import com.google.devtools.build.lib.actions.NotifyOnActionCacheHit; import com.google.devtools.build.lib.actions.NotifyOnActionCacheHit.ActionCachedContext; import com.google.devtools.build.lib.actions.PackageRootResolver; import com.google.devtools.build.lib.actions.TargetOutOfDateException; import com.google.devtools.build.lib.actions.cache.MetadataHandler; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.concurrent.ExecutorUtil; import com.google.devtools.build.lib.concurrent.Sharder; import com.google.devtools.build.lib.concurrent.ThrowableRecordingRunnableWrapper; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.EventHandler; import com.google.devtools.build.lib.events.ExtendedEventHandler; import com.google.devtools.build.lib.events.Reporter; import com.google.devtools.build.lib.profiler.Profiler; import com.google.devtools.build.lib.profiler.ProfilerTask; import com.google.devtools.build.lib.profiler.SilentCloseable; import com.google.devtools.build.lib.rules.cpp.IncludeScannable; import com.google.devtools.build.lib.util.Pair; import com.google.devtools.build.lib.util.io.FileOutErr; import com.google.devtools.build.lib.vfs.FileSystem; import com.google.devtools.build.lib.vfs.OutputService; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.lib.vfs.Root; import com.google.devtools.build.lib.vfs.Symlinks; import com.google.devtools.build.skyframe.SkyFunction.Environment; import java.io.FileNotFoundException; import java.io.IOException; import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedMap; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.FutureTask; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.Lock; import java.util.function.Supplier; import java.util.logging.Logger; import javax.annotation.Nullable; /** * Action executor: takes care of preparing an action for execution, executing it, validating that * all output artifacts were created, error reporting, etc. */ public final class SkyframeActionExecutor { private static final Logger logger = Logger.getLogger(SkyframeActionExecutor.class.getName()); // Used to prevent check-then-act races in #createOutputDirectories. See the comment there for // more detail. private static final Striped outputDirectoryDeletionLock = Striped.lock(64); private final ActionKeyContext actionKeyContext; private Reporter reporter; private Map clientEnv = ImmutableMap.of(); private Executor executorEngine; private ActionLogBufferPathGenerator actionLogBufferPathGenerator; private ActionCacheChecker actionCacheChecker; private final Profiler profiler = Profiler.instance(); private boolean explain; // We keep track of actions already executed this build in order to avoid executing a shared // action twice. Note that we may still unnecessarily re-execute the action on a subsequent // build: say actions A and B are shared. If A is requested on the first build and then B is // requested on the second build, we will execute B even though its output files are up to date. // However, we will not re-execute A on a subsequent build. // We do not allow the shared action to re-execute in the same build, even after the first // action has finished execution, because a downstream action might be reading the output file // at the same time as the shared action was writing to it. // This map is also used for Actions that try to execute twice because they have discovered // headers -- the SkyFunction tries to declare a dep on the missing headers and has to restart. // We don't want to execute the action again on the second entry to the SkyFunction. // In both cases, we store the already-computed ActionExecutionValue to avoid having to compute it // again. private ConcurrentMap< OwnerlessArtifactWrapper, Pair>> buildActionMap; // Errors found when examining all actions in the graph are stored here, so that they can be // thrown when execution of the action is requested. This field is set during each call to // findAndStoreArtifactConflicts, and is preserved across builds otherwise. private ImmutableMap badActionMap = ImmutableMap.of(); private boolean keepGoing; private boolean hadExecutionError; private MetadataProvider perBuildFileCache; private ActionInputPrefetcher actionInputPrefetcher; /** These variables are nulled out between executions. */ private ProgressSupplier progressSupplier; private ActionCompletedReceiver completionReceiver; private final AtomicReference statusReporterRef; private OutputService outputService; private final Supplier> sourceRootSupplier; SkyframeActionExecutor( ActionKeyContext actionKeyContext, AtomicReference statusReporterRef, Supplier> sourceRootSupplier) { this.actionKeyContext = actionKeyContext; this.statusReporterRef = statusReporterRef; this.sourceRootSupplier = sourceRootSupplier; } /** * A typed union of {@link ActionConflictException}, which indicates two actions that generate * the same {@link Artifact}, and {@link ArtifactPrefixConflictException}, which indicates that * the path of one {@link Artifact} is a prefix of another. */ public static class ConflictException extends Exception { @Nullable private final ActionConflictException ace; @Nullable private final ArtifactPrefixConflictException apce; public ConflictException(ActionConflictException e) { super(e); this.ace = e; this.apce = null; } public ConflictException(ArtifactPrefixConflictException e) { super(e); this.ace = null; this.apce = e; } void rethrowTyped() throws ActionConflictException, ArtifactPrefixConflictException { if (ace == null) { throw Preconditions.checkNotNull(apce); } if (apce == null) { throw Preconditions.checkNotNull(ace); } throw new IllegalStateException(); } } /** * Return the map of mostly recently executed bad actions to their corresponding exception. * See {#findAndStoreArtifactConflicts()}. */ public ImmutableMap badActions() { // TODO(bazel-team): Move badActions() and findAndStoreArtifactConflicts() to SkyframeBuildView // now that it's done in the analysis phase. return badActionMap; } /** * Find conflicts between generated artifacts. There are two ways to have conflicts. First, if * two (unshareable) actions generate the same output artifact, this will result in an {@link * ActionConflictException}. Second, if one action generates an artifact whose path is a prefix of * another artifact's path, those two artifacts cannot exist simultaneously in the output tree. * This causes an {@link ArtifactPrefixConflictException}. The relevant exceptions are stored in * the executor in {@code badActionMap}, and will be thrown immediately when that action is * executed. Those exceptions persist, so that even if the action is not executed this build, the * first time it is executed, the correct exception will be thrown. * *

This method must be called if a new action was added to the graph this build, so * whenever a new configured target was analyzed this build. It is somewhat expensive (~1s * range for a medium build as of 2014), so it should only be called when necessary. * *

Conflicts found may not be requested this build, and so we may overzealously throw an error. * For instance, if actions A and B generate the same artifact foo, and the user first requests * A' depending on A, and then in a subsequent build B' depending on B, we will fail the second * build, even though it would have succeeded if it had been the only build. However, since * Skyframe does not know the transitive dependencies of the request, we err on the conservative * side. * *

If the user first runs one action on the first build, and on the second build adds a * conflicting action, only the second action's error may be reported (because the first action * will be cached), whereas if both actions were requested for the first time, both errors would * be reported. However, the first time an action is added to the build, we are guaranteed to find * any conflicts it has, since this method will compare it against all other actions. So there is * no sequence of builds that can evade the error. */ void findAndStoreArtifactConflicts(Iterable actionLookupValues) throws InterruptedException { ConcurrentMap temporaryBadActionMap = new ConcurrentHashMap<>(); Pair> result; result = constructActionGraphAndPathMap(actionKeyContext, actionLookupValues, temporaryBadActionMap); ActionGraph actionGraph = result.first; SortedMap artifactPathMap = result.second; Map actionsWithArtifactPrefixConflict = Actions.findArtifactPrefixConflicts(actionGraph, artifactPathMap); for (Map.Entry actionExceptionPair : actionsWithArtifactPrefixConflict.entrySet()) { temporaryBadActionMap.put( actionExceptionPair.getKey(), new ConflictException(actionExceptionPair.getValue())); } this.badActionMap = ImmutableMap.copyOf(temporaryBadActionMap); } /** * Simultaneously construct an action graph for all the actions in Skyframe and a map from {@link * PathFragment}s to their respective {@link Artifact}s. We do this in a threadpool to save around * 1.5 seconds on a mid-sized build versus a single-threaded operation. */ private static Pair> constructActionGraphAndPathMap( ActionKeyContext actionKeyContext, Iterable values, ConcurrentMap badActionMap) throws InterruptedException { MutableActionGraph actionGraph = new MapBasedActionGraph(actionKeyContext); ConcurrentNavigableMap artifactPathMap = new ConcurrentSkipListMap<>(Actions.comparatorForPrefixConflicts()); // Action graph construction is CPU-bound. int numJobs = Runtime.getRuntime().availableProcessors(); // No great reason for expecting 5000 action lookup values, but not worth counting size of // values. Sharder actionShards = new Sharder<>(numJobs, 5000); for (ActionLookupValue value : values) { actionShards.add(value); } ThrowableRecordingRunnableWrapper wrapper = new ThrowableRecordingRunnableWrapper( "SkyframeActionExecutor#constructActionGraphAndPathMap"); ExecutorService executor = Executors.newFixedThreadPool( numJobs, new ThreadFactoryBuilder().setNameFormat("ActionLookupValue Processor %d").build()); for (List shard : actionShards) { executor.execute( wrapper.wrap(actionRegistration(shard, actionGraph, artifactPathMap, badActionMap))); } boolean interrupted = ExecutorUtil.interruptibleShutdown(executor); Throwables.propagateIfPossible(wrapper.getFirstThrownError()); if (interrupted) { throw new InterruptedException(); } return Pair.>of(actionGraph, artifactPathMap); } private static Runnable actionRegistration( final List values, final MutableActionGraph actionGraph, final ConcurrentMap artifactPathMap, final ConcurrentMap badActionMap) { return new Runnable() { @Override public void run() { for (ActionLookupValue value : values) { Set registeredActions = new HashSet<>(); for (Map.Entry entry : value.getMapForConsistencyCheck().entrySet()) { ActionAnalysisMetadata action = entry.getValue(); // We have an entry for each pair. Only try to register each action // once. if (registeredActions.add(action)) { try { actionGraph.registerAction(action); } catch (ActionConflictException e) { Exception oldException = badActionMap.put(action, new ConflictException(e)); Preconditions.checkState( oldException == null, "%s | %s | %s", action, e, oldException); // We skip the rest of the loop, and do not add the path->artifact mapping for this // artifact below -- we don't need to check it since this action is already in // error. continue; } } artifactPathMap.put(entry.getKey().getExecPath(), entry.getKey()); } } } }; } void prepareForExecution(Reporter reporter, Executor executor, boolean keepGoing, boolean explain, ActionCacheChecker actionCacheChecker, OutputService outputService) { this.reporter = Preconditions.checkNotNull(reporter); this.executorEngine = Preconditions.checkNotNull(executor); // Start with a new map each build so there's no issue with internal resizing. this.buildActionMap = Maps.newConcurrentMap(); this.keepGoing = keepGoing; this.hadExecutionError = false; this.actionCacheChecker = Preconditions.checkNotNull(actionCacheChecker); // Don't cache possibly stale data from the last build. this.explain = explain; this.outputService = outputService; } public void setActionLogBufferPathGenerator( ActionLogBufferPathGenerator actionLogBufferPathGenerator) { this.actionLogBufferPathGenerator = actionLogBufferPathGenerator; } public void setClientEnv(Map clientEnv) { this.clientEnv = clientEnv; } boolean usesActionFileSystem() { return outputService != null && outputService.supportsActionFileSystem(); } Path getExecRoot() { return executorEngine.getExecRoot(); } /** REQUIRES: {@link #usesActionFileSystem()} is true */ FileSystem createActionFileSystem( String relativeOutputPath, ActionInputMap inputArtifactData, Iterable outputArtifacts) { return outputService.createActionFileSystem( executorEngine.getFileSystem(), executorEngine.getExecRoot().asFragment(), relativeOutputPath, sourceRootSupplier.get(), inputArtifactData, outputArtifacts); } void updateActionFileSystemContext( FileSystem actionFileSystem, Environment env, MetadataConsumer consumer, ImmutableMap> filesets) throws IOException { outputService.updateActionFileSystemContext(actionFileSystem, env, consumer, filesets); } void executionOver() { this.reporter = null; // This transitively holds a bunch of heavy objects, so it's important to clear it at the // end of a build. this.executorEngine = null; this.outputService = null; } boolean probeActionExecution(Action action) { return buildActionMap.containsKey(new OwnerlessArtifactWrapper(action.getPrimaryOutput())); } private boolean actionReallyExecuted(Action action, ActionLookupData actionLookupData) { Pair cachedRun = Preconditions.checkNotNull( buildActionMap.get(new OwnerlessArtifactWrapper(action.getPrimaryOutput())), "%s %s", action, actionLookupData); return actionLookupData.equals(cachedRun.getFirst()); } void noteActionEvaluationStarted(ActionLookupData actionLookupData, Action action) { this.completionReceiver.noteActionEvaluationStarted(actionLookupData, action); } /** * Executes the provided action on the current thread. Returns the ActionExecutionValue with the * result, either computed here or already computed on another thread. * *

For use from {@link ArtifactFunction} only. */ ActionExecutionValue executeAction( ExtendedEventHandler eventHandler, Action action, ActionMetadataHandler metadataHandler, long actionStartTime, ActionExecutionContext actionExecutionContext, ActionLookupData actionLookupData, boolean inputDiscoveryRan) throws ActionExecutionException, InterruptedException { Exception exception = badActionMap.get(action); if (exception != null) { // If action had a conflict with some other action in the graph, report it now. reportError(exception.getMessage(), exception, action, null); } Artifact primaryOutput = action.getPrimaryOutput(); FutureTask actionTask = new FutureTask<>( new ActionRunner( eventHandler, action, metadataHandler, actionStartTime, actionExecutionContext, actionLookupData)); // Check to see if another action is already executing/has executed this value. Pair> oldAction = buildActionMap.putIfAbsent( new OwnerlessArtifactWrapper(primaryOutput), Pair.of(actionLookupData, actionTask)); // true if this is a non-shared action or it's shared and to be executed. boolean isPrimaryActionForTheValue = oldAction == null; if (isPrimaryActionForTheValue) { actionTask.run(); } else { // Wait for other action to finish, so any actions that depend on its outputs can execute. actionTask = oldAction.second; } try { ActionExecutionValue value = actionTask.get(); return isPrimaryActionForTheValue ? value : value.transformForSharedAction(action.getOutputs()); } catch (ExecutionException e) { Throwables.propagateIfPossible(e.getCause(), ActionExecutionException.class, InterruptedException.class); throw new IllegalStateException(e); } finally { if (!isPrimaryActionForTheValue && action.discoversInputs() && inputDiscoveryRan) { /** * If this is a shared action that does input discovery, but was not executed, we need to * remove it from the active actions pool (it was added there by {@link * ActionRunner#call()}). */ // TODO(b/72764586): Cleanup once we can properly skip input discovery for shared actions statusReporterRef.get().remove(action); } String message = action.getProgressMessage(); if (message != null) { // Tell the receiver that the action has completed *before* telling the reporter. // This way the latter will correctly show the number of completed actions when task // completion messages are enabled (--show_task_finish). completionReceiver.actionCompleted(actionLookupData); reporter.finishTask(null, prependExecPhaseStats(message)); } } } private static class ArtifactExpanderImpl implements ArtifactExpander { private final Map> expandedInputs; private final Map> expandedFilesets; private ArtifactExpanderImpl(Map> expandedInputMiddlemen, Map> expandedFilesets) { this.expandedInputs = expandedInputMiddlemen; this.expandedFilesets = expandedFilesets; } @Override public void expand(Artifact artifact, Collection output) { Preconditions.checkState(artifact.isMiddlemanArtifact() || artifact.isTreeArtifact(), artifact); Collection result = expandedInputs.get(artifact); if (result != null) { output.addAll(result); } } @Override public ImmutableList getFileset(Artifact artifact) { Preconditions.checkState(artifact.isFileset()); return Preconditions.checkNotNull(expandedFilesets.get(artifact)); } } /** * Returns an ActionExecutionContext suitable for executing a particular action. The caller should * pass the returned context to {@link #executeAction}, and any other method that needs to execute * tasks related to that action. */ public ActionExecutionContext getContext( MetadataProvider graphFileCache, MetadataHandler metadataHandler, Map> expandedInputs, Map> expandedFilesets, ImmutableMap> topLevelFilesets, @Nullable FileSystem actionFileSystem, @Nullable Object skyframeDepsResult) { FileOutErr fileOutErr = actionLogBufferPathGenerator.generate( ArtifactPathResolver.createPathResolver(actionFileSystem, executorEngine.getExecRoot())); return new ActionExecutionContext( executorEngine, createFileCache(graphFileCache, actionFileSystem), actionInputPrefetcher, actionKeyContext, metadataHandler, fileOutErr, clientEnv, topLevelFilesets, new ArtifactExpanderImpl(expandedInputs, expandedFilesets), actionFileSystem, skyframeDepsResult); } /** * Checks the action cache to see if {@code action} needs to be executed, or is up to date. * Returns a token with the semantics of {@link ActionCacheChecker#getTokenIfNeedToExecute}: null * if the action is up to date, and non-null if it needs to be executed, in which case that token * should be provided to the ActionCacheChecker after execution. */ Token checkActionCache( ExtendedEventHandler eventHandler, Action action, MetadataHandler metadataHandler, long actionStartTime, Iterable resolvedCacheArtifacts, Map clientEnv) { Token token; try (SilentCloseable c = profiler.profile(ProfilerTask.ACTION_CHECK, action.describe())) { token = actionCacheChecker.getTokenIfNeedToExecute( action, resolvedCacheArtifacts, clientEnv, explain ? reporter : null, metadataHandler); } if (token == null) { boolean eventPosted = false; // Notify BlazeRuntimeStatistics about the action middleman 'execution'. if (action.getActionType().isMiddleman()) { eventHandler.post(new ActionMiddlemanEvent(action, actionStartTime)); eventPosted = true; } if (action instanceof NotifyOnActionCacheHit) { NotifyOnActionCacheHit notify = (NotifyOnActionCacheHit) action; ActionCachedContext context = new ActionCachedContext() { @Override public EventHandler getEventHandler() { return executorEngine.getEventHandler(); } @Override public EventBus getEventBus() { return executorEngine.getEventBus(); } @Override public FileSystem getFileSystem() { return executorEngine.getFileSystem(); } @Override public Path getExecRoot() { return executorEngine.getExecRoot(); } @Override public T getContext(Class type) { return executorEngine.getContext(type); } }; notify.actionCacheHit(context); } // We still need to check the outputs so that output file data is available to the value. checkOutputs(action, metadataHandler); if (!eventPosted) { eventHandler.post(new CachedActionEvent(action, actionStartTime)); } } return token; } void afterExecution( Action action, MetadataHandler metadataHandler, Token token, Map clientEnv, ActionLookupData actionLookupData) { if (!actionReallyExecuted(action, actionLookupData)) { // If an action shared with this one executed, then we need not update the action cache, since // the other action will do it. Moreover, this action is not aware of metadata acquired // during execution, so its metadata handler is likely unusable anyway. return; } try { actionCacheChecker.afterExecution(action, token, metadataHandler, clientEnv); } catch (IOException e) { // Skyframe has already done all the filesystem access needed for outputs and swallows // IOExceptions for inputs. So an IOException is impossible here. throw new IllegalStateException( "failed to update action cache for " + action.prettyPrint() + ", but all outputs should already have been checked", e); } } @Nullable Iterable getActionCachedInputs(Action action, PackageRootResolver resolver) throws InterruptedException { return actionCacheChecker.getCachedInputs(action, resolver); } /** * Perform dependency discovery for action, which must discover its inputs. * *

This method is just a wrapper around {@link Action#discoverInputs} that properly processes * any ActionExecutionException thrown before rethrowing it to the caller. */ Iterable discoverInputs( Action action, PerActionFileCache graphFileCache, MetadataHandler metadataHandler, Environment env, @Nullable FileSystem actionFileSystem) throws ActionExecutionException, InterruptedException { ActionExecutionContext actionExecutionContext = ActionExecutionContext.forInputDiscovery( executorEngine, createFileCache(graphFileCache, actionFileSystem), actionInputPrefetcher, actionKeyContext, metadataHandler, actionLogBufferPathGenerator.generate(ArtifactPathResolver.createPathResolver( actionFileSystem, executorEngine.getExecRoot())), clientEnv, env, actionFileSystem); try { actionExecutionContext.getEventBus().post(ActionStatusMessage.analysisStrategy(action)); return action.discoverInputs(actionExecutionContext); } catch (ActionExecutionException e) { throw processAndThrow( env.getListener(), actionExecutionContext, action, e, actionExecutionContext.getFileOutErr(), ErrorTiming.BEFORE_EXECUTION); } } private MetadataProvider createFileCache( MetadataProvider graphFileCache, @Nullable FileSystem actionFileSystem) { if (actionFileSystem instanceof MetadataProvider) { return (MetadataProvider) actionFileSystem; } return new DelegatingPairFileCache(graphFileCache, perBuildFileCache); } /** * This method should be called if the builder encounters an error during * execution. This allows the builder to record that it encountered at * least one error, and may make it swallow its output to prevent * spamming the user any further. */ private void recordExecutionError() { hadExecutionError = true; } /** * Returns true if the Builder is winding down (i.e. cancelling outstanding * actions and preparing to abort.) * The builder is winding down iff: *

    *
  • we had an execution error *
  • we are not running with --keep_going *
*/ private boolean isBuilderAborting() { return hadExecutionError && !keepGoing; } void configure(MetadataProvider fileCache, ActionInputPrefetcher actionInputPrefetcher) { this.perBuildFileCache = fileCache; this.actionInputPrefetcher = actionInputPrefetcher; } private class ActionRunner implements Callable { private final ExtendedEventHandler eventHandler; private final Action action; private final ActionMetadataHandler metadataHandler; private long actionStartTime; private ActionExecutionContext actionExecutionContext; private final ActionLookupData actionLookupData; ActionRunner( ExtendedEventHandler eventHandler, Action action, ActionMetadataHandler metadataHandler, long actionStartTime, ActionExecutionContext actionExecutionContext, ActionLookupData actionLookupData) { this.eventHandler = eventHandler; this.action = action; this.metadataHandler = metadataHandler; this.actionStartTime = actionStartTime; this.actionExecutionContext = actionExecutionContext; this.actionLookupData = actionLookupData; } @Override public ActionExecutionValue call() throws ActionExecutionException, InterruptedException { try (SilentCloseable c = profiler.profile(ProfilerTask.ACTION, action.describe())) { if (actionCacheChecker.isActionExecutionProhibited(action)) { // We can't execute an action (e.g. because --check_???_up_to_date option was used). Fail // the build instead. synchronized (reporter) { TargetOutOfDateException e = new TargetOutOfDateException(action); reporter.handle(Event.error(e.getMessage())); recordExecutionError(); throw e; } } String message = action.getProgressMessage(); if (message != null) { reporter.startTask(null, prependExecPhaseStats(message)); } statusReporterRef.get().setPreparing(action); Preconditions.checkState(actionExecutionContext.getMetadataHandler() == metadataHandler, "%s %s", actionExecutionContext.getMetadataHandler(), metadataHandler); prepareScheduleExecuteAndCompleteAction( eventHandler, actionExecutionContext, action, actionStartTime, actionLookupData); Preconditions.checkState( actionExecutionContext.getOutputSymlinks() == null || action instanceof SkyframeAwareAction, "Unexpected to find outputSymlinks set" + " in an action which is not a SkyframeAwareAction. Action: %s\n symlinks:%s", action, actionExecutionContext.getOutputSymlinks()); return ActionExecutionValue.create( metadataHandler.getOutputArtifactData(), metadataHandler.getOutputTreeArtifactData(), metadataHandler.getAdditionalOutputData(), actionExecutionContext.getOutputSymlinks(), (action instanceof IncludeScannable) ? ((IncludeScannable) action).getDiscoveredModules() : null, action instanceof NotifyOnActionCacheHit); } } } private void createOutputDirectories(Action action, ActionExecutionContext context) throws ActionExecutionException { try { Set done = new HashSet<>(); // avoid redundant calls for the same directory. for (Artifact outputFile : action.getOutputs()) { Path outputDir; if (outputFile.isTreeArtifact()) { outputDir = context.getPathResolver().toPath(outputFile); } else { outputDir = context.getPathResolver().toPath(outputFile).getParentDirectory(); } if (done.add(outputDir)) { try { outputDir.createDirectoryAndParents(); continue; } catch (IOException e) { /* Fall through to plan B. */ } // Possibly some direct ancestors are not directories. In that case, we traverse the // ancestors upward, deleting any non-directories, until we reach a directory, then try // again. This handles the case where a file becomes a directory, either from one build to // another, or within a single build. // // Symlinks should not be followed so in order to clean up symlinks pointing to Fileset // outputs from previous builds. See bug [incremental build of Fileset fails if // Fileset.out was changed to be a subdirectory of the old value]. try { Path p = outputDir; while (true) { // This lock ensures that the only thread that observes a filesystem transition in // which the path p first exists and then does not is the thread that calls // p.delete() and causes the transition. // // If it were otherwise, then some thread A could test p.exists(), see that it does, // then test p.isDirectory(), see that p isn't a directory (because, say, thread // B deleted it), and then call p.delete(). That could result in two different kinds // of failures: // // 1) In the time between when thread A sees that p is not a directory and when thread // A calls p.delete(), thread B may reach the call to createDirectoryAndParents // and create a directory at p, which thread A then deletes. Thread B would then try // adding outputs to the directory it thought was there, and fail. // // 2) In the time between when thread A sees that p is not a directory and when thread // A calls p.delete(), thread B may create a directory at p, and then either create a // subdirectory beneath it or add outputs to it. Then when thread A tries to delete p, // it would fail. Lock lock = outputDirectoryDeletionLock.get(p); lock.lock(); try { if (p.exists(Symlinks.NOFOLLOW)) { boolean isDirectory = p.isDirectory(Symlinks.NOFOLLOW); if (isDirectory) { break; } // p may be a file or dangling symlink, or a symlink to an old Fileset output p.delete(); // throws IOException } } finally { lock.unlock(); } p = p.getParentDirectory(); } outputDir.createDirectoryAndParents(); } catch (IOException e) { throw new ActionExecutionException( "failed to create output directory '" + outputDir + "'", e, action, false); } } } } catch (ActionExecutionException ex) { printError(ex.getMessage(), action, null); throw ex; } } private String prependExecPhaseStats(String message) { // Prints a progress message like: // [2608/6445] Compiling foo/bar.cc [host] return progressSupplier.getProgressString() + " " + message; } /** * Prepare, schedule, execute, and then complete the action. When this function is called, we know * that this action needs to be executed. This function will prepare for the action's execution * (i.e. delete the outputs); schedule its execution; execute the action; and then do some * post-execution processing to complete the action: set the outputs readonly and executable, and * insert the action results in the action cache. * * @param action The action to execute * @param context services in the scope of the action * @param actionStartTime time when we started the first phase of the action execution. * @param actionLookupData key for action * @throws ActionExecutionException if the execution of the specified action failed for any * reason. * @throws InterruptedException if the thread was interrupted. */ private void prepareScheduleExecuteAndCompleteAction( ExtendedEventHandler eventHandler, ActionExecutionContext context, Action action, long actionStartTime, ActionLookupData actionLookupData) throws ActionExecutionException, InterruptedException { // Delete the outputs before executing the action, just to ensure that // the action really does produce the outputs. try { if (!usesActionFileSystem()) { action.prepare(context.getFileSystem(), context.getExecRoot()); } else { try { context.getFileOutErr().getOutputPath().getParentDirectory().createDirectoryAndParents(); context.getFileOutErr().getErrorPath().getParentDirectory().createDirectoryAndParents(); } catch (IOException e) { throw new ActionExecutionException( "failed to create output directory for output streams'" + context.getFileOutErr().getErrorPath() + "'", e, action, false); } } createOutputDirectories(action, context); } catch (IOException e) { reportError("failed to delete output files before executing action", e, action, null); } eventHandler.post(new ActionStartedEvent(action, actionStartTime)); ActionExecutionStatusReporter statusReporter = statusReporterRef.get(); try { // Mark the current action as being prepared. statusReporter.updateStatus(ActionStatusMessage.preparingStrategy(action)); boolean outputDumped = executeActionTask(eventHandler, action, context); completeAction( eventHandler, context, action, context.getMetadataHandler(), context.getFileOutErr(), outputDumped); } finally { statusReporterRef.get().remove(action); eventHandler.post(new ActionCompletionEvent(actionStartTime, action, actionLookupData)); } } private ActionExecutionException processAndThrow( ExtendedEventHandler eventHandler, ActionExecutionContext actionExecutionContext, Action action, ActionExecutionException e, FileOutErr outErrBuffer, ErrorTiming errorTiming) throws ActionExecutionException { reportActionExecution( eventHandler, actionExecutionContext, action, e, outErrBuffer, errorTiming); boolean reported = reportErrorIfNotAbortingMode(e, outErrBuffer); ActionExecutionException toThrow = e; if (reported){ // If we already printed the error for the exception we mark it as already reported // so that we do not print it again in upper levels. // Note that we need to report it here since we want immediate feedback of the errors // and in some cases the upper-level printing mechanism only prints one of the errors. toThrow = new AlreadyReportedActionExecutionException(e); } // Now, rethrow the exception. // This can have two effects: // If we're still building, the exception will get retrieved by the // completor and rethrown. // If we're aborting, the exception will never be retrieved from the // completor, since the completor is waiting for all outstanding jobs // to finish. After they have finished, it will only rethrow the // exception that initially caused it to abort will and not check the // exit status of any actions that had finished in the meantime. throw toThrow; } /** * Execute the specified action, in a profiler task. * The caller is responsible for having already checked that we need to * execute it and for acquiring/releasing any scheduling locks needed. * *

This is thread-safe so long as you don't try to execute the same action * twice at the same time (or overlapping times). * May execute in a worker thread. * * @throws ActionExecutionException if the execution of the specified action * failed for any reason. * @throws InterruptedException if the thread was interrupted. * @return true if the action output was dumped, false otherwise. */ private boolean executeActionTask( ExtendedEventHandler eventHandler, Action action, ActionExecutionContext actionExecutionContext) throws ActionExecutionException, InterruptedException { // ActionExecutionExceptions that occur as the thread is interrupted are // assumed to be a result of that, so we throw InterruptedException // instead. FileOutErr outErrBuffer = actionExecutionContext.getFileOutErr(); try (SilentCloseable c = profiler.profile(ProfilerTask.ACTION_EXECUTE, action.describe())) { ActionResult actionResult = action.execute(actionExecutionContext); if (actionResult != ActionResult.EMPTY) { eventHandler.post(new ActionResultReceivedEvent(action, actionResult)); } // Action terminated fine, now report the output. // The .showOutput() method is not necessarily a quick check: in its // current implementation it uses regular expression matching. if (outErrBuffer.hasRecordedOutput() && (action.showsOutputUnconditionally() || reporter.showOutput(Label.print(action.getOwner().getLabel())))) { dumpRecordedOutErr(action, outErrBuffer); return true; } // Defer reporting action success until outputs are checked } catch (ActionExecutionException e) { throw processAndThrow( eventHandler, actionExecutionContext, action, e, outErrBuffer, ErrorTiming.AFTER_EXECUTION); } return false; } private void completeAction( ExtendedEventHandler eventHandler, ActionExecutionContext actionExecutionContext, Action action, MetadataHandler metadataHandler, FileOutErr fileOutErr, boolean outputAlreadyDumped) throws ActionExecutionException { try { Preconditions.checkState(action.inputsDiscovered(), "Action %s successfully executed, but inputs still not known", action); try (SilentCloseable c = profiler.profile(ProfilerTask.ACTION_COMPLETE, action.describe())) { if (!checkOutputs(action, metadataHandler)) { reportError("not all outputs were created or valid", null, action, outputAlreadyDumped ? null : fileOutErr); } } if (outputService != null) { try { outputService.finalizeAction(action, metadataHandler); } catch (EnvironmentalExecException | IOException e) { reportError("unable to finalize action", e, action, fileOutErr); } } reportActionExecution( eventHandler, actionExecutionContext, action, null, fileOutErr, ErrorTiming.NO_ERROR); } catch (ActionExecutionException actionException) { // Success in execution but failure in completion. reportActionExecution( eventHandler, actionExecutionContext, action, actionException, fileOutErr, ErrorTiming.AFTER_EXECUTION); throw actionException; } catch (IllegalStateException exception) { // More serious internal error, but failure still reported. reportActionExecution( eventHandler, actionExecutionContext, action, new ActionExecutionException(exception, action, true), fileOutErr, ErrorTiming.AFTER_EXECUTION); throw exception; } } private static void reportMissingOutputFile( Action action, Artifact output, Reporter reporter, boolean isSymlink, IOException exception) { boolean genrule = action.getMnemonic().equals("Genrule"); String prefix = (genrule ? "declared output '" : "output '") + output.prettyPrint() + "' "; logger.warning( String.format( "Error creating %s%s%s: %s", isSymlink ? "symlink " : "", prefix, genrule ? " by genrule" : "", exception.getMessage())); if (isSymlink) { String msg = prefix + "is a dangling symbolic link"; reporter.handle(Event.error(action.getOwner().getLocation(), msg)); } else { String suffix = genrule ? " by genrule. This is probably " + "because the genrule actually didn't create this output, or because the output was a " + "directory and the genrule was run remotely (note that only the contents of " + "declared file outputs are copied from genrules run remotely)" : ""; reporter.handle(Event.error( action.getOwner().getLocation(), prefix + "was not created" + suffix)); } } private static void reportOutputTreeArtifactErrors( Action action, Artifact output, Reporter reporter, IOException e) { String errorMessage; if (e instanceof FileNotFoundException) { errorMessage = String.format("TreeArtifact %s was not created", output.prettyPrint()); } else { errorMessage = String.format( "Error while validating output TreeArtifact %s : %s", output, e.getMessage()); } reporter.handle(Event.error(action.getOwner().getLocation(), errorMessage)); } /** * Validates that all action outputs were created or intentionally omitted. This can result in * chmod calls on the output files; see {@link ActionMetadataHandler}. * * @return false if some outputs are missing, true - otherwise. */ private boolean checkOutputs(Action action, MetadataHandler metadataHandler) { boolean success = true; for (Artifact output : action.getOutputs()) { // getMetadata has the side effect of adding the artifact to the cache if it's not there // already (e.g., due to a previous call to MetadataHandler.injectDigest), therefore we only // call it if we know the artifact is not omitted. if (!metadataHandler.artifactOmitted(output)) { try { metadataHandler.getMetadata(output); } catch (IOException e) { success = false; if (output.isTreeArtifact()) { reportOutputTreeArtifactErrors(action, output, reporter, e); } else { // Are all exceptions caught due to missing files? reportMissingOutputFile(action, output, reporter, output.getPath().isSymbolicLink(), e); } } } } return success; } /** * Convenience function for reporting that the action failed due to a * the exception cause, if there is an additional explanatory message that * clarifies the message of the exception. Combines the user-provided message * and the exceptions' message and reports the combination as error. * Then, throws an ActionExecutionException with the reported error as * message and the provided exception as the cause. * * @param message A small text that explains why the action failed * @param cause The exception that caused the action to fail * @param action The action that failed * @param actionOutput The output of the failed Action. * May be null, if there is no output to display */ private void reportError(String message, Throwable cause, Action action, FileOutErr actionOutput) throws ActionExecutionException { ActionExecutionException ex; if (cause == null) { ex = new ActionExecutionException(message, action, false); } else { ex = new ActionExecutionException(message, cause, action, false); } printError(ex.getMessage(), action, actionOutput); throw ex; } /** * For the action 'action' that failed due to 'ex' with the output * 'actionOutput', notify the user about the error. To notify the user, the * method first displays the output of the action and then reports an error * via the reporter. The method ensures that the two messages appear next to * each other by locking the outErr object where the output is displayed. * * @param message The reason why the action failed * @param action The action that failed, must not be null. * @param actionOutput The output of the failed Action. * May be null, if there is no output to display */ private void printError(String message, Action action, FileOutErr actionOutput) { synchronized (reporter) { if (keepGoing) { message = "Couldn't " + describeAction(action) + ": " + message; } Event event = Event.error(action.getOwner().getLocation(), message); dumpRecordedOutErr(event, actionOutput); recordExecutionError(); } } /** Describe an action, for use in error messages. */ private static String describeAction(Action action) { if (action.getOutputs().isEmpty()) { return "run " + action.prettyPrint(); } else if (action.getActionType().isMiddleman()) { return "build " + action.prettyPrint(); } else { return "build file " + action.getPrimaryOutput().prettyPrint(); } } /** * Dump the output from the action. * * @param action The action whose output is being dumped * @param outErrBuffer The OutErr that recorded the actions output */ private void dumpRecordedOutErr(Action action, FileOutErr outErrBuffer) { StringBuilder message = new StringBuilder(""); message.append("From "); message.append(action.describe()); message.append(":"); Event event = Event.info(message.toString()); dumpRecordedOutErr(event, outErrBuffer); } /** * Dump the output from the action. * * @param prefixEvent An event to post before dumping the output * @param outErrBuffer The OutErr that recorded the actions output */ private void dumpRecordedOutErr(Event prefixEvent, FileOutErr outErrBuffer) { // Only print the output if we're not winding down. if (isBuilderAborting()) { return; } if (outErrBuffer != null && outErrBuffer.hasRecordedOutput()) { // Bind the output to the prefix event. // Note: here we temporarily (until the event is handled by the UI) read all // output into memory; as the output of regular actions (as opposed to test runs) usually is // short, so this should not be a problem. If it does turn out to be a problem, we have to // pass the outErrbuffer instead. reporter.handle( prefixEvent.withStdoutStderr(outErrBuffer.outAsLatin1(), outErrBuffer.errAsLatin1())); } else { reporter.handle(prefixEvent); } } private void reportActionExecution( ExtendedEventHandler eventHandler, ActionExecutionContext actionExecutionContext, Action action, ActionExecutionException exception, FileOutErr outErr, ErrorTiming errorTiming) { Path stdout = null; Path stderr = null; if (outErr.hasRecordedStdout()) { stdout = outErr.getOutputPath(); } if (outErr.hasRecordedStderr()) { stderr = outErr.getErrorPath(); } eventHandler.post( new ActionExecutedEvent( action, exception, actionExecutionContext.getInputPath(action.getPrimaryOutput()), stdout, stderr, errorTiming)); } /** * Returns true if the exception was reported. False otherwise. Currently this is a copy of what * we did in pre-Skyframe execution. The main implication is that we are printing the error to the * top level reporter instead of the action reporter. Because of that Skyframe values do not know * about the errors happening in the execution phase. Even if we change in the future to log to * the action reporter (that would be done in ActionExecutionFunction.compute() when we get an * ActionExecutionException), we probably do not want to also store the StdErr output, so * dumpRecordedOutErr() should still be called here. */ private boolean reportErrorIfNotAbortingMode(ActionExecutionException ex, FileOutErr outErrBuffer) { // For some actions (e.g., many local actions) the pollInterruptedStatus() // won't notice that we had an interrupted job. It will continue. // For that reason we must take care to NOT report errors if we're // in the 'aborting' mode: Any cancelled action would show up here. synchronized (this.reporter) { if (!isBuilderAborting()) { // Oops. The action aborted. Report the problem. printError(ex.getMessage(), ex.getAction(), outErrBuffer); return true; } } return false; } /** An object supplying data for action execution progress reporting. */ public interface ProgressSupplier { /** Returns the progress string to prefix action execution messages with. */ String getProgressString(); } /** An object that can be notified about action completion. */ public interface ActionCompletedReceiver { /** Receives a completed action. */ void actionCompleted(ActionLookupData actionLookupData); /** Notes that an action has started, giving the key. */ void noteActionEvaluationStarted(ActionLookupData actionLookupData, Action action); } public void setActionExecutionProgressReportingObjects( @Nullable ProgressSupplier progressSupplier, @Nullable ActionCompletedReceiver completionReceiver) { this.progressSupplier = progressSupplier; this.completionReceiver = completionReceiver; } private static class DelegatingPairFileCache implements MetadataProvider { private final MetadataProvider perActionCache; private final MetadataProvider perBuildFileCache; private DelegatingPairFileCache( MetadataProvider mainCache, MetadataProvider perBuildFileCache) { this.perActionCache = mainCache; this.perBuildFileCache = perBuildFileCache; } @Override public FileArtifactValue getMetadata(ActionInput input) throws IOException { FileArtifactValue metadata = perActionCache.getMetadata(input); return (metadata != null) && (metadata != FileArtifactValue.MISSING_FILE_MARKER) ? metadata : perBuildFileCache.getMetadata(input); } @Override public ActionInput getInput(String execPath) { ActionInput input = perActionCache.getInput(execPath); return input != null ? input : perBuildFileCache.getInput(execPath); } } }