// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.skyframe;
import static com.google.devtools.build.skyframe.EvaluationProgressReceiver.EvaluationState.BUILT;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.common.eventbus.EventBus;
import com.google.devtools.build.lib.actions.ActionAnalysisMetadata;
import com.google.devtools.build.lib.actions.ActionKeyContext;
import com.google.devtools.build.lib.actions.ActionLookupValue;
import com.google.devtools.build.lib.actions.ArtifactFactory;
import com.google.devtools.build.lib.actions.ArtifactOwner;
import com.google.devtools.build.lib.actions.ArtifactPrefixConflictException;
import com.google.devtools.build.lib.actions.MutableActionGraph.ActionConflictException;
import com.google.devtools.build.lib.actions.PackageRoots;
import com.google.devtools.build.lib.analysis.AnalysisFailureEvent;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.analysis.CachingAnalysisEnvironment;
import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
import com.google.devtools.build.lib.analysis.ConfiguredTarget;
import com.google.devtools.build.lib.analysis.ConfiguredTargetFactory;
import com.google.devtools.build.lib.analysis.ToolchainContext;
import com.google.devtools.build.lib.analysis.ViewCreationFailedException;
import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory;
import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory.BuildInfoKey;
import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection;
import com.google.devtools.build.lib.analysis.config.BuildOptions;
import com.google.devtools.build.lib.analysis.config.BuildOptions.OptionsDiff;
import com.google.devtools.build.lib.analysis.config.ConfigMatchingProvider;
import com.google.devtools.build.lib.analysis.config.FragmentClassSet;
import com.google.devtools.build.lib.buildeventstream.BuildEventId;
import com.google.devtools.build.lib.causes.Cause;
import com.google.devtools.build.lib.causes.LabelCause;
import com.google.devtools.build.lib.causes.LoadingFailedCause;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.cmdline.PackageIdentifier;
import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.EventHandler;
import com.google.devtools.build.lib.events.ExtendedEventHandler;
import com.google.devtools.build.lib.packages.Attribute;
import com.google.devtools.build.lib.packages.NoSuchPackageException;
import com.google.devtools.build.lib.packages.NoSuchTargetException;
import com.google.devtools.build.lib.packages.Package;
import com.google.devtools.build.lib.packages.Target;
import com.google.devtools.build.lib.pkgcache.LoadingFailureEvent;
import com.google.devtools.build.lib.profiler.Profiler;
import com.google.devtools.build.lib.profiler.SilentCloseable;
import com.google.devtools.build.lib.skyframe.AspectFunction.AspectCreationException;
import com.google.devtools.build.lib.skyframe.AspectValue.AspectValueKey;
import com.google.devtools.build.lib.skyframe.ConfiguredTargetFunction.ConfiguredValueCreationException;
import com.google.devtools.build.lib.skyframe.SkyframeActionExecutor.ConflictException;
import com.google.devtools.build.lib.skyframe.SkylarkImportLookupFunction.SkylarkImportFailedException;
import com.google.devtools.build.lib.util.OrderedSetMultimap;
import com.google.devtools.build.lib.vfs.Root;
import com.google.devtools.build.skyframe.CycleInfo;
import com.google.devtools.build.skyframe.ErrorInfo;
import com.google.devtools.build.skyframe.EvaluationProgressReceiver;
import com.google.devtools.build.skyframe.EvaluationResult;
import com.google.devtools.build.skyframe.SkyFunction.Environment;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
import com.google.devtools.build.skyframe.WalkableGraph;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;
import javax.annotation.Nullable;
/**
* Skyframe-based driver of analysis.
*
*
Covers enough functionality to work as a substitute for {@code BuildView#configureTargets}.
*/
public final class SkyframeBuildView {
private final ConfiguredTargetFactory factory;
private final ArtifactFactory artifactFactory;
private final SkyframeExecutor skyframeExecutor;
private final SkyframeActionExecutor skyframeActionExecutor;
private boolean enableAnalysis = false;
// This hack allows us to see when a configured target has been invalidated, and thus when the set
// of artifact conflicts needs to be recomputed (whenever a configured target has been invalidated
// or newly evaluated).
private final EvaluationProgressReceiver progressReceiver =
new ConfiguredTargetValueProgressReceiver();
private final Set evaluatedConfiguredTargets = Sets.newConcurrentHashSet();
// Used to see if checks of graph consistency need to be done after analysis.
private volatile boolean someConfiguredTargetEvaluated = false;
// We keep the set of invalidated configuration target keys so that we can know if something
// has been invalidated after graph pruning has been executed.
private Set dirtiedConfiguredTargetKeys = Sets.newConcurrentHashSet();
private volatile boolean anyConfiguredTargetDeleted = false;
private final AtomicInteger evaluatedActionCount = new AtomicInteger();
private final ConfiguredRuleClassProvider ruleClassProvider;
// The host configuration containing all fragments used by this build's transitive closure.
private BuildConfiguration topLevelHostConfiguration;
// Fragment-limited versions of the host configuration. It's faster to create/cache these here
// than to store them in Skyframe.
private Map hostConfigurationCache =
Maps.newConcurrentMap();
private BuildConfigurationCollection configurations;
/**
* If the last build was executed with {@code Options#discard_analysis_cache} and we are not
* running Skyframe full, we should clear the legacy data since it is out-of-sync.
*/
private boolean skyframeAnalysisWasDiscarded;
private ImmutableSet largestTopLevelKeySetCheckedForConflicts = ImmutableSet.of();
public SkyframeBuildView(
BlazeDirectories directories,
SkyframeExecutor skyframeExecutor,
ConfiguredRuleClassProvider ruleClassProvider,
SkyframeActionExecutor skyframeActionExecutor) {
this.skyframeActionExecutor = skyframeActionExecutor;
this.factory =
new ConfiguredTargetFactory(ruleClassProvider, skyframeExecutor.getDefaultBuildOptions());
this.artifactFactory =
new ArtifactFactory(directories.getExecRoot(), directories.getRelativeOutputPath());
this.skyframeExecutor = skyframeExecutor;
this.ruleClassProvider = ruleClassProvider;
}
public void resetEvaluatedConfiguredTargetKeysSet() {
evaluatedConfiguredTargets.clear();
}
public Set getEvaluatedTargetKeys() {
return ImmutableSet.copyOf(evaluatedConfiguredTargets);
}
ConfiguredTargetFactory getConfiguredTargetFactory() {
return factory;
}
public int getEvaluatedActionCount() {
return evaluatedActionCount.get();
}
public void resetEvaluationActionCount() {
evaluatedActionCount.set(0);
}
private boolean areConfigurationsDifferent(BuildConfigurationCollection configurations) {
if (this.configurations == null) {
// no configurations currently, no need to drop anything
return false;
}
if (configurations.equals(this.configurations)) {
// exact same configurations, no need to drop anything
return false;
}
if (configurations.getTargetConfigurations().size()
!= this.configurations.getTargetConfigurations().size()) {
// some option that changes the number of configurations has changed, that's definitely not
// any of the options that are okay to change
return true;
}
// Here we assume that the configurations will appear in the same order between invocations -
// which is the case today, because the only way to have multiple configurations is to use
// --experimental_multi_cpu, which creates configurations in sorted order of cpu value.
// Those configurations are all identical except for their cpu values, so the configuration we
// compare against only matters for making sure the cpu matches. Which we do care about.
for (int configIndex = 0;
configIndex < configurations.getTargetConfigurations().size();
configIndex += 1) {
BuildConfiguration oldConfig = this.configurations.getTargetConfigurations().get(configIndex);
BuildConfiguration newConfig = configurations.getTargetConfigurations().get(configIndex);
OptionsDiff diff = BuildOptions.diff(oldConfig.getOptions(), newConfig.getOptions());
if (ruleClassProvider.shouldInvalidateCacheForDiff(diff, newConfig.getOptions())) {
return true;
}
}
// We don't need to check the host configuration because it's derived from the target options.
return false;
}
/** Sets the configurations. Not thread-safe. DO NOT CALL except from tests! */
@VisibleForTesting
public void setConfigurations(
EventHandler eventHandler, BuildConfigurationCollection configurations) {
// Clear all cached ConfiguredTargets on configuration change of if --discard_analysis_cache
// was set on the previous build. In the former case, it's not required for correctness, but
// prevents unbounded memory usage.
if (this.areConfigurationsDifferent(configurations) || skyframeAnalysisWasDiscarded) {
eventHandler.handle(Event.info("Build options have changed, discarding analysis cache."));
skyframeExecutor.handleConfiguredTargetChange();
}
skyframeAnalysisWasDiscarded = false;
this.configurations = configurations;
setTopLevelHostConfiguration(configurations.getHostConfiguration());
}
/**
* Sets the host configuration consisting of all fragments that will be used by the top level
* targets' transitive closures.
*
*
This is used to power {@link #getHostConfiguration} during analysis, which computes
* fragment-trimmed host configurations from the top-level one.
*/
private void setTopLevelHostConfiguration(BuildConfiguration topLevelHostConfiguration) {
if (topLevelHostConfiguration.equals(this.topLevelHostConfiguration)) {
return;
}
hostConfigurationCache.clear();
this.topLevelHostConfiguration = topLevelHostConfiguration;
}
/**
* Drops the analysis cache. If building with Skyframe, targets in {@code topLevelTargets} may
* remain in the cache for use during the execution phase.
*
* @see com.google.devtools.build.lib.analysis.AnalysisOptions#discardAnalysisCache
*/
public void clearAnalysisCache(
Collection topLevelTargets, Collection topLevelAspects) {
// TODO(bazel-team): Consider clearing packages too to save more memory.
skyframeAnalysisWasDiscarded = true;
skyframeExecutor.clearAnalysisCache(topLevelTargets, topLevelAspects);
}
/**
* Analyzes the specified targets using Skyframe as the driving framework.
*
* @return the configured targets that should be built along with a WalkableGraph of the analysis.
*/
public SkyframeAnalysisResult configureTargets(
ExtendedEventHandler eventHandler,
List values,
List aspectKeys,
EventBus eventBus,
boolean keepGoing,
int numThreads)
throws InterruptedException, ViewCreationFailedException {
enableAnalysis(true);
EvaluationResult result;
try (SilentCloseable c = Profiler.instance().profile("skyframeExecutor.configureTargets")) {
result =
skyframeExecutor.configureTargets(
eventHandler, values, aspectKeys, keepGoing, numThreads);
} finally {
enableAnalysis(false);
}
try (SilentCloseable c =
Profiler.instance().profile("skyframeExecutor.findArtifactConflicts")) {
ImmutableSet newKeys =
ImmutableSet.builderWithExpectedSize(values.size() + aspectKeys.size())
.addAll(values)
.addAll(aspectKeys)
.build();
if (someConfiguredTargetEvaluated
|| anyConfiguredTargetDeleted
|| !dirtiedConfiguredTargetKeys.isEmpty()
|| !largestTopLevelKeySetCheckedForConflicts.containsAll(newKeys)) {
largestTopLevelKeySetCheckedForConflicts = newKeys;
// This operation is somewhat expensive, so we only do it if the graph might have changed in
// some way -- either we analyzed a new target or we invalidated an old one or are building
// targets together that haven't been built before.
skyframeActionExecutor.findAndStoreArtifactConflicts(
// If we do not track incremental state we do not have graph edges,
// so we cannot traverse the graph and find only actions in the current build.
// In this case we can simply return all ActionLookupValues in the graph,
// since the graph's lifetime is a single build anyway.
skyframeExecutor.tracksStateForIncrementality()
? getActionLookupValuesInBuild(values, aspectKeys)
: getActionLookupValuesInGraph());
someConfiguredTargetEvaluated = false;
}
}
ImmutableMap badActions =
skyframeActionExecutor.badActions();
Collection goodAspects = Lists.newArrayListWithCapacity(values.size());
Root singleSourceRoot = skyframeExecutor.getForcedSingleSourceRootIfNoExecrootSymlinkCreation();
NestedSetBuilder packages =
singleSourceRoot == null ? NestedSetBuilder.stableOrder() : null;
for (AspectValueKey aspectKey : aspectKeys) {
AspectValue value = (AspectValue) result.get(aspectKey);
if (value == null) {
// Skip aspects that couldn't be applied to targets.
continue;
}
goodAspects.add(value);
if (packages != null) {
packages.addTransitive(value.getTransitivePackagesForPackageRootResolution());
}
}
// Filter out all CTs that have a bad action and convert to a list of configured targets. This
// code ensures that the resulting list of configured targets has the same order as the incoming
// list of values, i.e., that the order is deterministic.
Collection goodCts = Lists.newArrayListWithCapacity(values.size());
for (ConfiguredTargetKey value : values) {
ConfiguredTargetValue ctValue = (ConfiguredTargetValue) result.get(value);
if (ctValue == null) {
continue;
}
goodCts.add(ctValue.getConfiguredTarget());
if (packages != null) {
packages.addTransitive(ctValue.getTransitivePackagesForPackageRootResolution());
}
}
PackageRoots packageRoots =
singleSourceRoot == null
? new MapAsPackageRoots(collectPackageRoots(packages.build().toCollection()))
: new PackageRootsNoSymlinkCreation(singleSourceRoot);
if (!result.hasError() && badActions.isEmpty()) {
return new SkyframeAnalysisResult(
/*hasLoadingError=*/false, /*hasAnalysisError=*/false,
ImmutableList.copyOf(goodCts),
result.getWalkableGraph(),
ImmutableList.copyOf(goodAspects),
packageRoots);
}
// --nokeep_going so we fail with an exception for the first error.
// TODO(bazel-team): We might want to report the other errors through the event bus but
// for keeping this code in parity with legacy we just report the first error for now.
if (!keepGoing) {
for (Map.Entry bad : badActions.entrySet()) {
ConflictException ex = bad.getValue();
try {
ex.rethrowTyped();
} catch (ActionConflictException ace) {
ace.reportTo(eventHandler);
String errorMsg = "Analysis of target '" + bad.getKey().getOwner().getLabel()
+ "' failed; build aborted";
throw new ViewCreationFailedException(errorMsg);
} catch (ArtifactPrefixConflictException apce) {
eventHandler.handle(Event.error(apce.getMessage()));
}
throw new ViewCreationFailedException(ex.getMessage());
}
Map.Entry error = result.errorMap().entrySet().iterator().next();
SkyKey topLevel = error.getKey();
ErrorInfo errorInfo = error.getValue();
assertSaneAnalysisError(errorInfo, topLevel);
skyframeExecutor.getCyclesReporter().reportCycles(errorInfo.getCycleInfo(), topLevel,
eventHandler);
Throwable cause = errorInfo.getException();
Preconditions.checkState(cause != null || !Iterables.isEmpty(errorInfo.getCycleInfo()),
errorInfo);
String errorMsg = null;
if (topLevel.argument() instanceof ConfiguredTargetKey) {
errorMsg =
"Analysis of target '"
+ NonRuleConfiguredTargetValue.extractLabel(topLevel)
+ "' failed; build aborted";
} else if (topLevel.argument() instanceof AspectValueKey) {
AspectValueKey aspectKey = (AspectValueKey) topLevel.argument();
errorMsg = "Analysis of aspect '" + aspectKey.getDescription() + "' failed; build aborted";
} else {
assert false;
}
if (cause instanceof ActionConflictException) {
((ActionConflictException) cause).reportTo(eventHandler);
}
if (errorInfo.getException() != null) {
throw new ViewCreationFailedException(errorMsg, errorInfo.getException());
} else {
throw new ViewCreationFailedException(errorMsg);
}
}
boolean hasLoadingError = false;
// --keep_going : We notify the error and return a NonRuleConfiguredTargetValue
for (Map.Entry errorEntry : result.errorMap().entrySet()) {
// Only handle errors of configured targets, not errors of top-level aspects.
// TODO(ulfjack): this is quadratic - if there are a lot of CTs, this could be rather slow.
if (!values.contains(errorEntry.getKey().argument())) {
continue;
}
SkyKey errorKey = errorEntry.getKey();
ConfiguredTargetKey label = (ConfiguredTargetKey) errorKey.argument();
Label topLevelLabel = label.getLabel();
ErrorInfo errorInfo = errorEntry.getValue();
assertSaneAnalysisError(errorInfo, errorKey);
skyframeExecutor.getCyclesReporter().reportCycles(errorInfo.getCycleInfo(), errorKey,
eventHandler);
Exception cause = errorInfo.getException();
BuildEventId configuration = null;
Iterable rootCauses;
if (cause instanceof ConfiguredValueCreationException) {
ConfiguredValueCreationException ctCause = (ConfiguredValueCreationException) cause;
// Previously, the nested set was de-duplicating loading root cause labels. Now that we
// track Cause instances including a message, we get one event per label and message. In
// order to keep backwards compatibility, we de-duplicate root cause labels here.
// TODO(ulfjack): Remove this code once we've migrated to the BEP.
Set