// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.analysis;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.base.Predicate;
import com.google.common.base.Verify;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.eventbus.EventBus;
import com.google.devtools.build.lib.actions.ActionAnalysisMetadata;
import com.google.devtools.build.lib.actions.ActionGraph;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.ArtifactFactory;
import com.google.devtools.build.lib.actions.ArtifactOwner;
import com.google.devtools.build.lib.actions.Root;
import com.google.devtools.build.lib.analysis.ExtraActionArtifactsProvider.ExtraArtifactSet;
import com.google.devtools.build.lib.analysis.config.BinTools;
import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection;
import com.google.devtools.build.lib.analysis.config.BuildOptions;
import com.google.devtools.build.lib.analysis.config.ConfigMatchingProvider;
import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.cmdline.PackageIdentifier;
import com.google.devtools.build.lib.collect.nestedset.NestedSet;
import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
import com.google.devtools.build.lib.collect.nestedset.Order;
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.EventHandler;
import com.google.devtools.build.lib.events.StoredEventHandler;
import com.google.devtools.build.lib.packages.Attribute;
import com.google.devtools.build.lib.packages.BuildType;
import com.google.devtools.build.lib.packages.NativeAspectClass;
import com.google.devtools.build.lib.packages.NoSuchThingException;
import com.google.devtools.build.lib.packages.PackageSpecification;
import com.google.devtools.build.lib.packages.RawAttributeMapper;
import com.google.devtools.build.lib.packages.Rule;
import com.google.devtools.build.lib.packages.Target;
import com.google.devtools.build.lib.packages.TargetUtils;
import com.google.devtools.build.lib.pkgcache.LoadingResult;
import com.google.devtools.build.lib.rules.test.CoverageReportActionFactory;
import com.google.devtools.build.lib.rules.test.CoverageReportActionFactory.CoverageReportActionsWrapper;
import com.google.devtools.build.lib.rules.test.InstrumentedFilesProvider;
import com.google.devtools.build.lib.skyframe.ActionLookupValue;
import com.google.devtools.build.lib.skyframe.AspectValue;
import com.google.devtools.build.lib.skyframe.AspectValue.AspectValueKey;
import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey;
import com.google.devtools.build.lib.skyframe.ConfiguredTargetValue;
import com.google.devtools.build.lib.skyframe.CoverageReportValue;
import com.google.devtools.build.lib.skyframe.SkyframeAnalysisResult;
import com.google.devtools.build.lib.skyframe.SkyframeBuildView;
import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
import com.google.devtools.build.lib.syntax.EvalException;
import com.google.devtools.build.lib.util.OrderedSetMultimap;
import com.google.devtools.build.lib.util.Pair;
import com.google.devtools.build.lib.util.Preconditions;
import com.google.devtools.build.lib.util.RegexFilter;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.WalkableGraph;
import com.google.devtools.common.options.Option;
import com.google.devtools.common.options.OptionsBase;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.logging.Logger;
import javax.annotation.Nullable;
/**
*
The BuildView presents a semantically-consistent and transitively-closed
* dependency graph for some set of packages.
*
*
Package design
*
*
This package contains the Blaze dependency analysis framework (aka
* "analysis phase"). The goal of this code is to perform semantic analysis of
* all of the build targets required for a given build, to report
* errors/warnings for any problems in the input, and to construct an "action
* graph" (see {@code lib.actions} package) correctly representing the work to
* be done during the execution phase of the build.
*
*
Configurations the inputs to a build come from two sources: the
* intrinsic inputs, specified in the BUILD file, are called targets.
* The environmental inputs, coming from the build tool, the command-line, or
* configuration files, are called the configuration. Only when a
* target and a configuration are combined is there sufficient information to
* perform a build.
*
*
Targets are implemented by the {@link Target} hierarchy in the {@code
* lib.packages} code. Configurations are implemented by {@link
* BuildConfiguration}. The pair of these together is represented by an
* instance of class {@link ConfiguredTarget}; this is the root of a hierarchy
* with different implementations for each kind of target: source file, derived
* file, rules, etc.
*
*
The framework code in this package (as opposed to its subpackages) is
* responsible for constructing the {@code ConfiguredTarget} graph for a given
* target and configuration, taking care of such issues as:
*
*
caching common subgraphs.
*
detecting and reporting cycles.
*
correct propagation of errors through the graph.
*
reporting universal errors, such as dependencies from production code
* to tests, or to experimental branches.
*
capturing and replaying errors.
*
maintaining the graph from one build to the next to
* avoid unnecessary recomputation.
*
checking software licenses.
*
*
*
See also {@link ConfiguredTarget} which documents some important
* invariants.
*/
public class BuildView {
/**
* Options that affect the mechanism of analysis. These are distinct from {@link
* com.google.devtools.build.lib.analysis.config.BuildOptions}, which affect the value
* of a BuildConfiguration.
*/
public static class Options extends OptionsBase {
@Option(name = "keep_going",
abbrev = 'k',
defaultValue = "false",
category = "strategy",
help = "Continue as much as possible after an error. While the"
+ " target that failed, and those that depend on it, cannot be"
+ " analyzed (or built), the other prerequisites of these"
+ " targets can be analyzed (or built) all the same.")
public boolean keepGoing;
@Option(name = "analysis_warnings_as_errors",
deprecationWarning = "analysis_warnings_as_errors is now a no-op and will be removed in"
+ " an upcoming Blaze release",
defaultValue = "false",
category = "strategy",
help = "Treat visible analysis warnings as errors.")
public boolean analysisWarningsAsErrors;
@Option(name = "discard_analysis_cache",
defaultValue = "false",
category = "strategy",
help = "Discard the analysis cache immediately after the analysis phase completes."
+ " Reduces memory usage by ~10%, but makes further incremental builds slower.")
public boolean discardAnalysisCache;
@Option(name = "experimental_extra_action_filter",
defaultValue = "",
category = "experimental",
converter = RegexFilter.RegexFilterConverter.class,
help = "Filters set of targets to schedule extra_actions for.")
public RegexFilter extraActionFilter;
@Option(name = "experimental_extra_action_top_level_only",
defaultValue = "false",
category = "experimental",
help = "Only schedules extra_actions for top level targets.")
public boolean extraActionTopLevelOnly;
@Option(name = "version_window_for_dirty_node_gc",
defaultValue = "0",
category = "undocumented",
help = "Nodes that have been dirty for more than this many versions will be deleted"
+ " from the graph upon the next update. Values must be non-negative long integers,"
+ " or -1 indicating the maximum possible window.")
public long versionWindowForDirtyNodeGc;
@Deprecated
@Option(
name = "experimental_interleave_loading_and_analysis",
defaultValue = "true",
category = "experimental",
help = "No-op."
)
public boolean interleaveLoadingAndAnalysis;
}
private static Logger LOG = Logger.getLogger(BuildView.class.getName());
private final BlazeDirectories directories;
private final SkyframeExecutor skyframeExecutor;
private final SkyframeBuildView skyframeBuildView;
private final ConfiguredRuleClassProvider ruleClassProvider;
/**
* A factory class to create the coverage report action. May be null.
*/
@Nullable private final CoverageReportActionFactory coverageReportActionFactory;
@VisibleForTesting
public Set getSkyframeEvaluatedTargetKeysForTesting() {
return skyframeBuildView.getEvaluatedTargetKeys();
}
/** The number of targets freshly evaluated in the last analysis run. */
public int getTargetsVisited() {
return skyframeBuildView.getEvaluatedTargetKeys().size();
}
public BuildView(BlazeDirectories directories,
ConfiguredRuleClassProvider ruleClassProvider,
SkyframeExecutor skyframeExecutor,
CoverageReportActionFactory coverageReportActionFactory) {
this.directories = directories;
this.coverageReportActionFactory = coverageReportActionFactory;
this.ruleClassProvider = ruleClassProvider;
this.skyframeExecutor = Preconditions.checkNotNull(skyframeExecutor);
this.skyframeBuildView = skyframeExecutor.getSkyframeBuildView();
}
/**
* Returns whether the given configured target has errors.
*/
@VisibleForTesting
public boolean hasErrors(ConfiguredTarget configuredTarget) {
return configuredTarget == null;
}
/**
* Sets the configurations. Not thread-safe. DO NOT CALL except from tests!
*/
@VisibleForTesting
public void setConfigurationsForTesting(BuildConfigurationCollection configurations) {
skyframeBuildView.setConfigurations(configurations);
}
public ArtifactFactory getArtifactFactory() {
return skyframeBuildView.getArtifactFactory();
}
@VisibleForTesting
WorkspaceStatusAction getLastWorkspaceBuildInfoActionForTesting() {
return skyframeExecutor.getLastWorkspaceStatusActionForTesting();
}
@Override
public int hashCode() {
throw new UnsupportedOperationException(); // avoid nondeterminism
}
/**
* Return value for {@link BuildView#update} and {@code BuildTool.prepareToBuild}.
*/
public static final class AnalysisResult {
private final ImmutableList targetsToBuild;
@Nullable private final ImmutableList targetsToTest;
@Nullable private final String error;
private final ActionGraph actionGraph;
private final ImmutableSet artifactsToBuild;
private final ImmutableSet parallelTests;
private final ImmutableSet exclusiveTests;
@Nullable private final TopLevelArtifactContext topLevelContext;
private final ImmutableList aspects;
private final ImmutableMap packageRoots;
private final String workspaceName;
private AnalysisResult(
Collection targetsToBuild,
Collection aspects,
Collection targetsToTest,
@Nullable String error,
ActionGraph actionGraph,
Collection artifactsToBuild,
Collection parallelTests,
Collection exclusiveTests,
TopLevelArtifactContext topLevelContext,
ImmutableMap packageRoots,
String workspaceName) {
this.targetsToBuild = ImmutableList.copyOf(targetsToBuild);
this.aspects = ImmutableList.copyOf(aspects);
this.targetsToTest = targetsToTest == null ? null : ImmutableList.copyOf(targetsToTest);
this.error = error;
this.actionGraph = actionGraph;
this.artifactsToBuild = ImmutableSet.copyOf(artifactsToBuild);
this.parallelTests = ImmutableSet.copyOf(parallelTests);
this.exclusiveTests = ImmutableSet.copyOf(exclusiveTests);
this.topLevelContext = topLevelContext;
this.packageRoots = packageRoots;
this.workspaceName = workspaceName;
}
/**
* Returns configured targets to build.
*/
public Collection getTargetsToBuild() {
return targetsToBuild;
}
/**
* The map from package names to the package root where each package was found; this is used to
* set up the symlink tree.
*/
public ImmutableMap getPackageRoots() {
return packageRoots;
}
/**
* Returns aspects of configured targets to build.
*
*
If this list is empty, build the targets returned by {@code getTargetsToBuild()}.
* Otherwise, only build these aspects of the targets returned by {@code getTargetsToBuild()}.
*/
public Collection getAspects() {
return aspects;
}
/**
* Returns the configured targets to run as tests, or {@code null} if testing was not
* requested (e.g. "build" command rather than "test" command).
*/
@Nullable
public Collection getTargetsToTest() {
return targetsToTest;
}
public ImmutableSet getAdditionalArtifactsToBuild() {
return artifactsToBuild;
}
public ImmutableSet getExclusiveTests() {
return exclusiveTests;
}
public ImmutableSet getParallelTests() {
return parallelTests;
}
/**
* Returns an error description (if any).
*/
@Nullable public String getError() {
return error;
}
public boolean hasError() {
return error != null;
}
/**
* Returns the action graph.
*/
public ActionGraph getActionGraph() {
return actionGraph;
}
public TopLevelArtifactContext getTopLevelContext() {
return topLevelContext;
}
public String getWorkspaceName() {
return workspaceName;
}
}
/**
* Returns the collection of configured targets corresponding to any of the provided targets.
*/
@VisibleForTesting
static Iterable extends ConfiguredTarget> filterTestsByTargets(
Collection extends ConfiguredTarget> targets,
final Set extends Target> allowedTargets) {
return Iterables.filter(
targets,
new Predicate() {
@Override
public boolean apply(ConfiguredTarget rule) {
return allowedTargets.contains(rule.getTarget());
}
});
}
@ThreadCompatible
public AnalysisResult update(
LoadingResult loadingResult,
BuildConfigurationCollection configurations,
List aspects,
Options viewOptions,
TopLevelArtifactContext topLevelOptions,
EventHandler eventHandler,
EventBus eventBus)
throws ViewCreationFailedException, InterruptedException {
LOG.info("Starting analysis");
pollInterruptedStatus();
skyframeBuildView.resetEvaluatedConfiguredTargetKeysSet();
Collection targets = loadingResult.getTargets();
eventBus.post(new AnalysisPhaseStartedEvent(targets));
skyframeBuildView.setConfigurations(configurations);
// Determine the configurations.
List topLevelTargetsWithConfigs =
nodesForTopLevelTargets(configurations, targets, eventHandler);
List topLevelCtKeys = Lists.transform(topLevelTargetsWithConfigs,
new Function() {
@Override
public ConfiguredTargetKey apply(TargetAndConfiguration node) {
return new ConfiguredTargetKey(node.getLabel(), node.getConfiguration());
}
});
List aspectKeys = new ArrayList<>();
for (String aspect : aspects) {
// Syntax: label%aspect
int delimiterPosition = aspect.indexOf('%');
if (delimiterPosition >= 0) {
// TODO(jfield): For consistency with Skylark loads, the aspect should be specified
// as an absolute path. Also, we probably need to do at least basic validation of
// path well-formedness here.
PathFragment bzlFile = new PathFragment("/" + aspect.substring(0, delimiterPosition));
String skylarkFunctionName = aspect.substring(delimiterPosition + 1);
for (TargetAndConfiguration targetSpec : topLevelTargetsWithConfigs) {
if (!(targetSpec.getTarget() instanceof Rule)) {
continue;
}
aspectKeys.add(
AspectValue.createSkylarkAspectKey(
targetSpec.getLabel(),
// For invoking top-level aspects, use the top-level configuration for both the
// aspect and the base target while the top-level configuration is untrimmed.
targetSpec.getConfiguration(),
targetSpec.getConfiguration(),
bzlFile,
skylarkFunctionName));
}
} else {
final NativeAspectClass aspectFactoryClass =
ruleClassProvider.getNativeAspectClassMap().get(aspect);
if (aspectFactoryClass != null) {
for (TargetAndConfiguration targetSpec : topLevelTargetsWithConfigs) {
if (!(targetSpec.getTarget() instanceof Rule)) {
continue;
}
aspectKeys.add(
AspectValue.createAspectKey(
targetSpec.getLabel(),
// For invoking top-level aspects, use the top-level configuration for both the
// aspect and the base target while the top-level configuration is untrimmed.
targetSpec.getConfiguration(),
targetSpec.getConfiguration(),
aspectFactoryClass));
}
} else {
throw new ViewCreationFailedException("Aspect '" + aspect + "' is unknown");
}
}
}
skyframeExecutor.injectWorkspaceStatusData(loadingResult.getWorkspaceName());
SkyframeAnalysisResult skyframeAnalysisResult;
try {
skyframeAnalysisResult =
skyframeBuildView.configureTargets(
eventHandler, topLevelCtKeys, aspectKeys, eventBus, viewOptions.keepGoing);
setArtifactRoots(skyframeAnalysisResult.getPackageRoots());
} finally {
skyframeBuildView.clearInvalidatedConfiguredTargets();
}
int numTargetsToAnalyze = topLevelTargetsWithConfigs.size();
int numSuccessful = skyframeAnalysisResult.getConfiguredTargets().size();
if (0 < numSuccessful && numSuccessful < numTargetsToAnalyze) {
String msg = String.format("Analysis succeeded for only %d of %d top-level targets",
numSuccessful, numTargetsToAnalyze);
eventHandler.handle(Event.info(msg));
LOG.info(msg);
}
AnalysisResult result =
createResult(
eventHandler,
loadingResult,
topLevelOptions,
viewOptions,
skyframeAnalysisResult);
LOG.info("Finished analysis");
return result;
}
private AnalysisResult createResult(
EventHandler eventHandler,
LoadingResult loadingResult,
TopLevelArtifactContext topLevelOptions,
BuildView.Options viewOptions,
SkyframeAnalysisResult skyframeAnalysisResult)
throws InterruptedException {
Collection testsToRun = loadingResult.getTestsToRun();
Collection configuredTargets = skyframeAnalysisResult.getConfiguredTargets();
Collection aspects = skyframeAnalysisResult.getAspects();
Collection allTargetsToTest = null;
if (testsToRun != null) {
// Determine the subset of configured targets that are meant to be run as tests.
// Do not remove : workaround for Java 7 type inference.
allTargetsToTest =
Lists.newArrayList(
filterTestsByTargets(configuredTargets, Sets.newHashSet(testsToRun)));
}
Set artifactsToBuild = new HashSet<>();
Set parallelTests = new HashSet<>();
Set exclusiveTests = new HashSet<>();
// build-info and build-changelist.
Collection buildInfoArtifacts =
skyframeExecutor.getWorkspaceStatusArtifacts(eventHandler);
Preconditions.checkState(buildInfoArtifacts.size() == 2, buildInfoArtifacts);
artifactsToBuild.addAll(buildInfoArtifacts);
// Extra actions
addExtraActionsIfRequested(viewOptions, configuredTargets, aspects, artifactsToBuild);
// Coverage
NestedSet baselineCoverageArtifacts = getBaselineCoverageArtifacts(configuredTargets);
Iterables.addAll(artifactsToBuild, baselineCoverageArtifacts);
if (coverageReportActionFactory != null) {
CoverageReportActionsWrapper actionsWrapper;
actionsWrapper = coverageReportActionFactory.createCoverageReportActionsWrapper(
eventHandler,
directories,
allTargetsToTest,
baselineCoverageArtifacts,
getArtifactFactory(),
CoverageReportValue.ARTIFACT_OWNER);
if (actionsWrapper != null) {
ImmutableList actions = actionsWrapper.getActions();
skyframeExecutor.injectCoverageReportData(actions);
artifactsToBuild.addAll(actionsWrapper.getCoverageOutputs());
}
}
// Tests. This must come last, so that the exclusive tests are scheduled after everything else.
scheduleTestsIfRequested(parallelTests, exclusiveTests, topLevelOptions, allTargetsToTest);
String error = createErrorMessage(loadingResult, skyframeAnalysisResult);
final WalkableGraph graph = skyframeAnalysisResult.getWalkableGraph();
final ActionGraph actionGraph =
new ActionGraph() {
@Nullable
@Override
public ActionAnalysisMetadata getGeneratingAction(Artifact artifact) {
ArtifactOwner artifactOwner = artifact.getArtifactOwner();
if (artifactOwner instanceof ActionLookupValue.ActionLookupKey) {
SkyKey key = ActionLookupValue.key((ActionLookupValue.ActionLookupKey) artifactOwner);
ActionLookupValue val;
try {
val = (ActionLookupValue) graph.getValue(key);
} catch (InterruptedException e) {
throw new IllegalStateException(
"Interruption not expected from this graph: " + key, e);
}
return val == null ? null : val.getGeneratingAction(artifact);
}
return null;
}
};
return new AnalysisResult(
configuredTargets,
aspects,
allTargetsToTest,
error,
actionGraph,
artifactsToBuild,
parallelTests,
exclusiveTests,
topLevelOptions,
skyframeAnalysisResult.getPackageRoots(),
loadingResult.getWorkspaceName());
}
@Nullable
public static String createErrorMessage(
LoadingResult loadingResult, @Nullable SkyframeAnalysisResult skyframeAnalysisResult) {
return loadingResult.hasTargetPatternError()
? "command succeeded, but there were errors parsing the target pattern"
: loadingResult.hasLoadingError()
|| (skyframeAnalysisResult != null && skyframeAnalysisResult.hasLoadingError())
? "command succeeded, but there were loading phase errors"
: (skyframeAnalysisResult != null && skyframeAnalysisResult.hasAnalysisError())
? "command succeeded, but not all targets were analyzed"
: null;
}
private static NestedSet getBaselineCoverageArtifacts(
Collection configuredTargets) {
NestedSetBuilder baselineCoverageArtifacts = NestedSetBuilder.stableOrder();
for (ConfiguredTarget target : configuredTargets) {
InstrumentedFilesProvider provider = target.getProvider(InstrumentedFilesProvider.class);
if (provider != null) {
baselineCoverageArtifacts.addTransitive(provider.getBaselineCoverageArtifacts());
}
}
return baselineCoverageArtifacts.build();
}
private void addExtraActionsIfRequested(Options viewOptions,
Collection configuredTargets,
Collection aspects,
Set artifactsToBuild) {
addExtraActionsIfRequested(viewOptions, artifactsToBuild,
Iterables.transform(configuredTargets,
new Function>() {
@Nullable
@Override
public Pair