// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.skyframe;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.common.eventbus.EventBus;
import com.google.devtools.build.lib.actions.ActionAnalysisMetadata;
import com.google.devtools.build.lib.actions.ActionLookupValue;
import com.google.devtools.build.lib.actions.ArtifactFactory;
import com.google.devtools.build.lib.actions.ArtifactOwner;
import com.google.devtools.build.lib.actions.ArtifactPrefixConflictException;
import com.google.devtools.build.lib.actions.MutableActionGraph;
import com.google.devtools.build.lib.actions.MutableActionGraph.ActionConflictException;
import com.google.devtools.build.lib.analysis.AnalysisFailureEvent;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.analysis.BuildView;
import com.google.devtools.build.lib.analysis.CachingAnalysisEnvironment;
import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
import com.google.devtools.build.lib.analysis.ConfiguredTarget;
import com.google.devtools.build.lib.analysis.ConfiguredTargetFactory;
import com.google.devtools.build.lib.analysis.LabelAndConfiguration;
import com.google.devtools.build.lib.analysis.ViewCreationFailedException;
import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory;
import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory.BuildInfoKey;
import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection;
import com.google.devtools.build.lib.analysis.config.ConfigMatchingProvider;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.cmdline.PackageIdentifier;
import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.ExtendedEventHandler;
import com.google.devtools.build.lib.packages.Attribute;
import com.google.devtools.build.lib.packages.NoSuchPackageException;
import com.google.devtools.build.lib.packages.NoSuchTargetException;
import com.google.devtools.build.lib.packages.Package;
import com.google.devtools.build.lib.packages.RuleClassProvider;
import com.google.devtools.build.lib.packages.Target;
import com.google.devtools.build.lib.pkgcache.LoadingFailureEvent;
import com.google.devtools.build.lib.pkgcache.LoadingPhaseRunner;
import com.google.devtools.build.lib.skyframe.AspectFunction.AspectCreationException;
import com.google.devtools.build.lib.skyframe.AspectValue.AspectValueKey;
import com.google.devtools.build.lib.skyframe.BuildInfoCollectionValue.BuildInfoKeyAndConfig;
import com.google.devtools.build.lib.skyframe.ConfiguredTargetFunction.ConfiguredValueCreationException;
import com.google.devtools.build.lib.skyframe.SkyframeActionExecutor.ConflictException;
import com.google.devtools.build.lib.skyframe.SkylarkImportLookupFunction.SkylarkImportFailedException;
import com.google.devtools.build.lib.util.OrderedSetMultimap;
import com.google.devtools.build.lib.util.Preconditions;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.skyframe.CycleInfo;
import com.google.devtools.build.skyframe.ErrorInfo;
import com.google.devtools.build.skyframe.EvaluationProgressReceiver;
import com.google.devtools.build.skyframe.EvaluationResult;
import com.google.devtools.build.skyframe.SkyFunction.Environment;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.logging.Logger;
import javax.annotation.Nullable;
/**
* Skyframe-based driver of analysis.
*
*
Covers enough functionality to work as a substitute for {@code BuildView#configureTargets}.
*/
public final class SkyframeBuildView {
private static Logger LOG = Logger.getLogger(BuildView.class.getName());
private final ConfiguredTargetFactory factory;
private final ArtifactFactory artifactFactory;
private final SkyframeExecutor skyframeExecutor;
private boolean enableAnalysis = false;
// This hack allows us to see when a configured target has been invalidated, and thus when the set
// of artifact conflicts needs to be recomputed (whenever a configured target has been invalidated
// or newly evaluated).
private final EvaluationProgressReceiver progressReceiver =
new ConfiguredTargetValueProgressReceiver();
private final Set evaluatedConfiguredTargets = Sets.newConcurrentHashSet();
// Used to see if checks of graph consistency need to be done after analysis.
private volatile boolean someConfiguredTargetEvaluated = false;
// We keep the set of invalidated configuration target keys so that we can know if something
// has been invalidated after graph pruning has been executed.
private Set dirtiedConfiguredTargetKeys = Sets.newConcurrentHashSet();
private volatile boolean anyConfiguredTargetDeleted = false;
private final RuleClassProvider ruleClassProvider;
// The host configuration containing all fragments used by this build's transitive closure.
private BuildConfiguration topLevelHostConfiguration;
// Fragment-limited versions of the host configuration. It's faster to create/cache these here
// than to store them in Skyframe.
private Map>, BuildConfiguration>
hostConfigurationCache = Maps.newConcurrentMap();
private BuildConfigurationCollection configurations;
/**
* If the last build was executed with {@code Options#discard_analysis_cache} and we are not
* running Skyframe full, we should clear the legacy data since it is out-of-sync.
*/
private boolean skyframeAnalysisWasDiscarded;
public SkyframeBuildView(BlazeDirectories directories,
SkyframeExecutor skyframeExecutor, ConfiguredRuleClassProvider ruleClassProvider) {
this.factory = new ConfiguredTargetFactory(ruleClassProvider);
this.artifactFactory = new ArtifactFactory(
directories.getExecRoot().getParentDirectory(), directories.getRelativeOutputPath());
this.skyframeExecutor = skyframeExecutor;
this.ruleClassProvider = ruleClassProvider;
}
public void resetEvaluatedConfiguredTargetKeysSet() {
evaluatedConfiguredTargets.clear();
}
public Set getEvaluatedTargetKeys() {
return ImmutableSet.copyOf(evaluatedConfiguredTargets);
}
ConfiguredTargetFactory getConfiguredTargetFactory() {
return factory;
}
/**
* Sets the configurations. Not thread-safe. DO NOT CALL except from tests!
*/
@VisibleForTesting
public void setConfigurations(BuildConfigurationCollection configurations) {
// Clear all cached ConfiguredTargets on configuration change of if --discard_analysis_cache
// was set on the previous build. In the former case, it's not required for correctness, but
// prevents unbounded memory usage.
if ((this.configurations != null && !configurations.equals(this.configurations))
|| skyframeAnalysisWasDiscarded) {
LOG.info("Discarding analysis cache: configurations have changed.");
skyframeExecutor.dropConfiguredTargets();
}
skyframeAnalysisWasDiscarded = false;
this.configurations = configurations;
setTopLevelHostConfiguration(configurations.getHostConfiguration());
}
/**
* Sets the host configuration consisting of all fragments that will be used by the top level
* targets' transitive closures.
*
*
This is used to power {@link #getHostConfiguration} during analysis, which computes
* fragment-trimmed host configurations from the top-level one.
*/
private void setTopLevelHostConfiguration(BuildConfiguration topLevelHostConfiguration) {
if (topLevelHostConfiguration.equals(this.topLevelHostConfiguration)) {
return;
}
hostConfigurationCache.clear();
this.topLevelHostConfiguration = topLevelHostConfiguration;
}
/**
* Drops the analysis cache. If building with Skyframe, targets in {@code topLevelTargets} may
* remain in the cache for use during the execution phase.
*
* @see com.google.devtools.build.lib.analysis.BuildView.Options#discardAnalysisCache
*/
public void clearAnalysisCache(
Collection topLevelTargets, Collection topLevelAspects) {
// TODO(bazel-team): Consider clearing packages too to save more memory.
skyframeAnalysisWasDiscarded = true;
skyframeExecutor.clearAnalysisCache(topLevelTargets, topLevelAspects);
}
/**
* Analyzes the specified targets using Skyframe as the driving framework.
*
* @return the configured targets that should be built along with a WalkableGraph of the analysis.
*/
public SkyframeAnalysisResult configureTargets(
ExtendedEventHandler eventHandler,
List values,
List aspectKeys,
EventBus eventBus,
boolean keepGoing,
int numThreads)
throws InterruptedException, ViewCreationFailedException {
enableAnalysis(true);
EvaluationResult result;
try {
result =
skyframeExecutor.configureTargets(
eventHandler, values, aspectKeys, keepGoing, numThreads);
} finally {
enableAnalysis(false);
}
ImmutableMap badActions =
skyframeExecutor.findArtifactConflicts();
Collection goodAspects = Lists.newArrayListWithCapacity(values.size());
NestedSetBuilder packages = NestedSetBuilder.stableOrder();
for (AspectValueKey aspectKey : aspectKeys) {
AspectValue value = (AspectValue) result.get(aspectKey.getSkyKey());
if (value == null) {
// Skip aspects that couldn't be applied to targets.
continue;
}
goodAspects.add(value);
packages.addTransitive(value.getTransitivePackages());
}
// Filter out all CTs that have a bad action and convert to a list of configured targets. This
// code ensures that the resulting list of configured targets has the same order as the incoming
// list of values, i.e., that the order is deterministic.
Collection goodCts = Lists.newArrayListWithCapacity(values.size());
for (ConfiguredTargetKey value : values) {
ConfiguredTargetValue ctValue =
(ConfiguredTargetValue) result.get(ConfiguredTargetValue.key(value));
if (ctValue == null) {
continue;
}
goodCts.add(ctValue.getConfiguredTarget());
packages.addTransitive(ctValue.getTransitivePackages());
}
ImmutableMap packageRoots =
LoadingPhaseRunner.collectPackageRoots(packages.build().toCollection());
if (!result.hasError() && badActions.isEmpty()) {
return new SkyframeAnalysisResult(
/*hasLoadingError=*/false, /*hasAnalysisError=*/false,
ImmutableList.copyOf(goodCts),
result.getWalkableGraph(),
ImmutableList.copyOf(goodAspects),
packageRoots);
}
// --nokeep_going so we fail with an exception for the first error.
// TODO(bazel-team): We might want to report the other errors through the event bus but
// for keeping this code in parity with legacy we just report the first error for now.
if (!keepGoing) {
for (Map.Entry bad : badActions.entrySet()) {
ConflictException ex = bad.getValue();
try {
ex.rethrowTyped();
} catch (MutableActionGraph.ActionConflictException ace) {
ace.reportTo(eventHandler);
String errorMsg = "Analysis of target '" + bad.getKey().getOwner().getLabel()
+ "' failed; build aborted";
throw new ViewCreationFailedException(errorMsg);
} catch (ArtifactPrefixConflictException apce) {
eventHandler.handle(Event.error(apce.getMessage()));
}
throw new ViewCreationFailedException(ex.getMessage());
}
Map.Entry error = result.errorMap().entrySet().iterator().next();
SkyKey topLevel = error.getKey();
ErrorInfo errorInfo = error.getValue();
assertSaneAnalysisError(errorInfo, topLevel);
skyframeExecutor.getCyclesReporter().reportCycles(errorInfo.getCycleInfo(), topLevel,
eventHandler);
Throwable cause = errorInfo.getException();
Preconditions.checkState(cause != null || !Iterables.isEmpty(errorInfo.getCycleInfo()),
errorInfo);
String errorMsg = null;
if (topLevel.argument() instanceof ConfiguredTargetKey) {
errorMsg =
"Analysis of target '"
+ ConfiguredTargetValue.extractLabel(topLevel)
+ "' failed; build aborted";
} else if (topLevel.argument() instanceof AspectValueKey) {
AspectValueKey aspectKey = (AspectValueKey) topLevel.argument();
errorMsg = "Analysis of aspect '" + aspectKey.getDescription() + "' failed; build aborted";
} else {
assert false;
}
if (cause instanceof ActionConflictException) {
((ActionConflictException) cause).reportTo(eventHandler);
}
throw new ViewCreationFailedException(errorMsg);
}
boolean hasLoadingError = false;
// --keep_going : We notify the error and return a ConfiguredTargetValue
for (Map.Entry errorEntry : result.errorMap().entrySet()) {
// Only handle errors of configured targets, not errors of top-level aspects.
// TODO(ulfjack): this is quadratic - if there are a lot of CTs, this could be rather slow.
if (!values.contains(errorEntry.getKey().argument())) {
continue;
}
SkyKey errorKey = errorEntry.getKey();
ConfiguredTargetKey label = (ConfiguredTargetKey) errorKey.argument();
Label topLevelLabel = label.getLabel();
ErrorInfo errorInfo = errorEntry.getValue();
assertSaneAnalysisError(errorInfo, errorKey);
skyframeExecutor.getCyclesReporter().reportCycles(errorInfo.getCycleInfo(), errorKey,
eventHandler);
Exception cause = errorInfo.getException();
Label analysisRootCause = null;
if (cause instanceof ConfiguredValueCreationException) {
ConfiguredValueCreationException ctCause = (ConfiguredValueCreationException) cause;
for (Label rootCause : ctCause.getRootCauses()) {
hasLoadingError = true;
eventBus.post(new LoadingFailureEvent(topLevelLabel, rootCause));
}
analysisRootCause = ctCause.getAnalysisRootCause();
} else if (!Iterables.isEmpty(errorInfo.getCycleInfo())) {
analysisRootCause = maybeGetConfiguredTargetCycleCulprit(
topLevelLabel, errorInfo.getCycleInfo());
} else if (cause instanceof ActionConflictException) {
((ActionConflictException) cause).reportTo(eventHandler);
}
eventHandler.handle(
Event.warn("errors encountered while analyzing target '"
+ topLevelLabel + "': it will not be built"));
if (analysisRootCause != null) {
eventBus.post(new AnalysisFailureEvent(
LabelAndConfiguration.of(topLevelLabel, label.getConfiguration()), analysisRootCause));
}
}
Collection reportedExceptions = Sets.newHashSet();
for (Map.Entry bad : badActions.entrySet()) {
ConflictException ex = bad.getValue();
try {
ex.rethrowTyped();
} catch (MutableActionGraph.ActionConflictException ace) {
ace.reportTo(eventHandler);
eventHandler
.handle(Event.warn("errors encountered while analyzing target '"
+ bad.getKey().getOwner().getLabel() + "': it will not be built"));
} catch (ArtifactPrefixConflictException apce) {
if (reportedExceptions.add(apce)) {
eventHandler.handle(Event.error(apce.getMessage()));
}
}
}
if (!badActions.isEmpty()) {
// In order to determine the set of configured targets transitively error free from action
// conflict issues, we run a post-processing update() that uses the bad action map.
EvaluationResult actionConflictResult =
skyframeExecutor.postConfigureTargets(eventHandler, values, keepGoing, badActions);
goodCts = Lists.newArrayListWithCapacity(values.size());
for (ConfiguredTargetKey value : values) {
PostConfiguredTargetValue postCt =
actionConflictResult.get(PostConfiguredTargetValue.key(value));
if (postCt != null) {
goodCts.add(postCt.getCt());
}
}
}
return new SkyframeAnalysisResult(
hasLoadingError,
result.hasError() || !badActions.isEmpty(),
ImmutableList.copyOf(goodCts),
result.getWalkableGraph(),
ImmutableList.copyOf(goodAspects),
packageRoots);
}
@Nullable
private static Label maybeGetConfiguredTargetCycleCulprit(
Label labelToLoad, Iterable cycleInfos) {
for (CycleInfo cycleInfo : cycleInfos) {
SkyKey culprit = Iterables.getFirst(cycleInfo.getCycle(), null);
if (culprit == null) {
continue;
}
if (culprit.functionName().equals(SkyFunctions.CONFIGURED_TARGET)) {
return ((ConfiguredTargetKey) culprit.argument()).getLabel();
} else if (culprit.functionName().equals(SkyFunctions.TRANSITIVE_TARGET)) {
return (Label) culprit.argument();
} else {
return labelToLoad;
}
}
return null;
}
private static void assertSaneAnalysisError(ErrorInfo errorInfo, SkyKey key) {
Throwable cause = errorInfo.getException();
if (cause != null) {
// We should only be trying to configure targets when the loading phase succeeds, meaning
// that the only errors should be analysis errors.
Preconditions.checkState(
cause instanceof ConfiguredValueCreationException
|| cause instanceof ActionConflictException
// For top-level aspects
|| cause instanceof AspectCreationException
|| cause instanceof SkylarkImportFailedException
// Only if we run the reduced loading phase and then analyze with --nokeep_going.
|| cause instanceof NoSuchTargetException
|| cause instanceof NoSuchPackageException,
"%s -> %s",
key,
errorInfo);
}
}
public ArtifactFactory getArtifactFactory() {
return artifactFactory;
}
/**
* Because we don't know what build-info artifacts this configured target may request, we
* conservatively register a dep on all of them.
*/
// TODO(bazel-team): Allow analysis to return null so the value builder can exit and wait for a
// restart deps are not present.
private static boolean getWorkspaceStatusValues(Environment env, BuildConfiguration config)
throws InterruptedException {
env.getValue(WorkspaceStatusValue.SKY_KEY);
Map buildInfoFactories =
PrecomputedValue.BUILD_INFO_FACTORIES.get(env);
if (buildInfoFactories == null) {
return false;
}
// These factories may each create their own build info artifacts, all depending on the basic
// build-info.txt and build-changelist.txt.
List depKeys = Lists.newArrayList();
for (BuildInfoKey key : buildInfoFactories.keySet()) {
if (buildInfoFactories.get(key).isEnabled(config)) {
depKeys.add(BuildInfoCollectionValue.key(new BuildInfoKeyAndConfig(key, config)));
}
}
env.getValues(depKeys);
return !env.valuesMissing();
}
/** Returns null if any build-info values are not ready. */
@Nullable
CachingAnalysisEnvironment createAnalysisEnvironment(
ArtifactOwner owner,
boolean isSystemEnv,
ExtendedEventHandler eventHandler,
Environment env,
BuildConfiguration config)
throws InterruptedException {
if (config != null && !getWorkspaceStatusValues(env, config)) {
return null;
}
boolean extendedSanityChecks = config != null && config.extendedSanityChecks();
boolean allowRegisteringActions = config == null || config.isActionsEnabled();
return new CachingAnalysisEnvironment(
artifactFactory, owner, isSystemEnv, extendedSanityChecks, eventHandler, env,
allowRegisteringActions);
}
/**
* Invokes the appropriate constructor to create a {@link ConfiguredTarget} instance.
*
*
For use in {@code ConfiguredTargetFunction}.
*
*
Returns null if Skyframe deps are missing or upon certain errors.
*/
@Nullable
ConfiguredTarget createConfiguredTarget(Target target, BuildConfiguration configuration,
CachingAnalysisEnvironment analysisEnvironment,
OrderedSetMultimap prerequisiteMap,
ImmutableMap