// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.analysis;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimaps;
import com.google.common.collect.Sets;
import com.google.common.eventbus.EventBus;
import com.google.devtools.build.lib.actions.Action;
import com.google.devtools.build.lib.actions.ActionGraph;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.ArtifactFactory;
import com.google.devtools.build.lib.actions.PackageRootResolver;
import com.google.devtools.build.lib.actions.Root;
import com.google.devtools.build.lib.analysis.DependencyResolver.Dependency;
import com.google.devtools.build.lib.analysis.ExtraActionArtifactsProvider.ExtraArtifactSet;
import com.google.devtools.build.lib.analysis.config.BinTools;
import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection;
import com.google.devtools.build.lib.analysis.config.ConfigMatchingProvider;
import com.google.devtools.build.lib.collect.nestedset.NestedSet;
import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
import com.google.devtools.build.lib.collect.nestedset.Order;
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.EventHandler;
import com.google.devtools.build.lib.events.StoredEventHandler;
import com.google.devtools.build.lib.packages.Attribute;
import com.google.devtools.build.lib.packages.NoSuchPackageException;
import com.google.devtools.build.lib.packages.NoSuchTargetException;
import com.google.devtools.build.lib.packages.NoSuchThingException;
import com.google.devtools.build.lib.packages.PackageIdentifier;
import com.google.devtools.build.lib.packages.PackageSpecification;
import com.google.devtools.build.lib.packages.RawAttributeMapper;
import com.google.devtools.build.lib.packages.Rule;
import com.google.devtools.build.lib.packages.Target;
import com.google.devtools.build.lib.packages.TargetUtils;
import com.google.devtools.build.lib.packages.Type;
import com.google.devtools.build.lib.pkgcache.LoadingPhaseRunner.LoadingResult;
import com.google.devtools.build.lib.pkgcache.PackageManager;
import com.google.devtools.build.lib.rules.test.CoverageReportActionFactory;
import com.google.devtools.build.lib.rules.test.CoverageReportActionFactory.CoverageReportActionsWrapper;
import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey;
import com.google.devtools.build.lib.skyframe.CoverageReportValue;
import com.google.devtools.build.lib.skyframe.SkyframeBuildView;
import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
import com.google.devtools.build.lib.syntax.EvalException;
import com.google.devtools.build.lib.syntax.Label;
import com.google.devtools.build.lib.util.RegexFilter;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.common.options.Option;
import com.google.devtools.common.options.OptionsBase;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.logging.Logger;
import javax.annotation.Nullable;
/**
*
The BuildView presents a semantically-consistent and transitively-closed
* dependency graph for some set of packages.
*
*
Package design
*
*
This package contains the Blaze dependency analysis framework (aka
* "analysis phase"). The goal of this code is to perform semantic analysis of
* all of the build targets required for a given build, to report
* errors/warnings for any problems in the input, and to construct an "action
* graph" (see {@code lib.actions} package) correctly representing the work to
* be done during the execution phase of the build.
*
*
Configurations the inputs to a build come from two sources: the
* intrinsic inputs, specified in the BUILD file, are called targets.
* The environmental inputs, coming from the build tool, the command-line, or
* configuration files, are called the configuration. Only when a
* target and a configuration are combined is there sufficient information to
* perform a build.
*
*
Targets are implemented by the {@link Target} hierarchy in the {@code
* lib.packages} code. Configurations are implemented by {@link
* BuildConfiguration}. The pair of these together is represented by an
* instance of class {@link ConfiguredTarget}; this is the root of a hierarchy
* with different implementations for each kind of target: source file, derived
* file, rules, etc.
*
*
The framework code in this package (as opposed to its subpackages) is
* responsible for constructing the {@code ConfiguredTarget} graph for a given
* target and configuration, taking care of such issues as:
*
*
caching common subgraphs.
*
detecting and reporting cycles.
*
correct propagation of errors through the graph.
*
reporting universal errors, such as dependencies from production code
* to tests, or to experimental branches.
*
capturing and replaying errors.
*
maintaining the graph from one build to the next to
* avoid unnecessary recomputation.
*
checking software licenses.
*
*
*
See also {@link ConfiguredTarget} which documents some important
* invariants.
*/
public class BuildView {
/**
* Options that affect the mechanism of analysis. These are distinct from {@link
* com.google.devtools.build.lib.analysis.config.BuildOptions}, which affect the value
* of a BuildConfiguration.
*/
public static class Options extends OptionsBase {
@Option(name = "keep_going",
abbrev = 'k',
defaultValue = "false",
category = "strategy",
help = "Continue as much as possible after an error. While the "
+ "target that failed, and those that depend on it, cannot be "
+ "analyzed (or built), the other prerequisites of these "
+ "targets can be analyzed (or built) all the same.")
public boolean keepGoing;
@Option(name = "analysis_warnings_as_errors",
deprecationWarning = "analysis_warnings_as_errors is now a no-op and will be removed in"
+ " an upcoming Blaze release",
defaultValue = "false",
category = "strategy",
help = "Treat visible analysis warnings as errors.")
public boolean analysisWarningsAsErrors;
@Option(name = "discard_analysis_cache",
defaultValue = "false",
category = "strategy",
help = "Discard the analysis cache immediately after the analysis phase completes. "
+ "Reduces memory usage by ~10%, but makes further incremental builds slower.")
public boolean discardAnalysisCache;
@Option(name = "keep_forward_graph",
deprecationWarning = "keep_forward_graph is now a no-op and will be removed in an "
+ "upcoming Blaze release",
defaultValue = "false",
category = "undocumented",
help = "Cache the forward action graph across builds for faster "
+ "incremental rebuilds. May slightly increase memory while Blaze "
+ "server is idle."
)
public boolean keepForwardGraph;
@Option(name = "experimental_extra_action_filter",
defaultValue = "",
category = "experimental",
converter = RegexFilter.RegexFilterConverter.class,
help = "Filters set of targets to schedule extra_actions for.")
public RegexFilter extraActionFilter;
@Option(name = "experimental_extra_action_top_level_only",
defaultValue = "false",
category = "experimental",
help = "Only schedules extra_actions for top level targets.")
public boolean extraActionTopLevelOnly;
@Option(name = "version_window_for_dirty_node_gc",
defaultValue = "0",
category = "undocumented",
help = "Nodes that have been dirty for more than this many versions will be deleted"
+ " from the graph upon the next update. Values must be non-negative long integers,"
+ " or -1 indicating the maximum possible window.")
public long versionWindowForDirtyNodeGc;
}
private static Logger LOG = Logger.getLogger(BuildView.class.getName());
private final BlazeDirectories directories;
private final SkyframeExecutor skyframeExecutor;
private final SkyframeBuildView skyframeBuildView;
private final PackageManager packageManager;
private final BinTools binTools;
private BuildConfigurationCollection configurations;
private final ConfiguredRuleClassProvider ruleClassProvider;
private final ArtifactFactory artifactFactory;
/**
* A factory class to create the coverage report action. May be null.
*/
@Nullable private final CoverageReportActionFactory coverageReportActionFactory;
/**
* A union of package roots of all previous incremental analysis results. This is used to detect
* changes of package roots between incremental analysis instances.
*/
private final Map cumulativePackageRoots = new HashMap<>();
/**
* Used only for testing that we clear Skyframe caches correctly.
* TODO(bazel-team): Remove this once we get rid of legacy Skyframe synchronization.
*/
private boolean skyframeCacheWasInvalidated = false;
/**
* If the last build was executed with {@code Options#discard_analysis_cache} and we are not
* running Skyframe full, we should clear the legacy data since it is out-of-sync.
*/
private boolean skyframeAnalysisWasDiscarded = false;
@VisibleForTesting
public Set getSkyframeEvaluatedTargetKeysForTesting() {
return skyframeBuildView.getEvaluatedTargetKeys();
}
/** The number of targets freshly evaluated in the last analysis run. */
public int getTargetsVisited() {
return skyframeBuildView.getEvaluatedTargetKeys().size();
}
/**
* Returns true iff Skyframe was invalidated during the analysis phase.
* TODO(bazel-team): Remove this once we do not need to keep legacy in sync with Skyframe.
*/
@VisibleForTesting
boolean wasSkyframeCacheInvalidatedDuringAnalysis() {
return skyframeCacheWasInvalidated;
}
public BuildView(BlazeDirectories directories, PackageManager packageManager,
ConfiguredRuleClassProvider ruleClassProvider,
SkyframeExecutor skyframeExecutor,
BinTools binTools, CoverageReportActionFactory coverageReportActionFactory) {
this.directories = directories;
this.packageManager = packageManager;
this.binTools = binTools;
this.coverageReportActionFactory = coverageReportActionFactory;
this.artifactFactory = new ArtifactFactory(directories.getExecRoot());
this.ruleClassProvider = ruleClassProvider;
this.skyframeExecutor = Preconditions.checkNotNull(skyframeExecutor);
this.skyframeBuildView =
new SkyframeBuildView(
new ConfiguredTargetFactory(ruleClassProvider),
artifactFactory,
skyframeExecutor,
new Runnable() {
@Override
public void run() {
clear();
}
},
binTools);
skyframeExecutor.setSkyframeBuildView(skyframeBuildView);
}
/** Returns the action graph. */
public ActionGraph getActionGraph() {
return new ActionGraph() {
@Override
public Action getGeneratingAction(Artifact artifact) {
return skyframeExecutor.getGeneratingAction(artifact);
}
};
}
/**
* Returns whether the given configured target has errors.
*/
@VisibleForTesting
public boolean hasErrors(ConfiguredTarget configuredTarget) {
return configuredTarget == null;
}
/**
* Sets the configurations. Not thread-safe. DO NOT CALL except from tests!
*/
@VisibleForTesting
public void setConfigurationsForTesting(BuildConfigurationCollection configurations) {
this.configurations = configurations;
}
public BuildConfigurationCollection getConfigurationCollection() {
return configurations;
}
/**
* Clear the graphs of ConfiguredTargets and Artifacts.
*/
@VisibleForTesting
public void clear() {
cumulativePackageRoots.clear();
artifactFactory.clear();
}
public ArtifactFactory getArtifactFactory() {
return artifactFactory;
}
@VisibleForTesting
WorkspaceStatusAction getLastWorkspaceBuildInfoActionForTesting() {
return skyframeExecutor.getLastWorkspaceStatusActionForTesting();
}
/**
* Returns a corresponding ConfiguredTarget, if one exists; otherwise throws an {@link
* NoSuchConfiguredTargetException}.
*/
@ThreadSafe
private ConfiguredTarget getConfiguredTarget(Target target, BuildConfiguration config)
throws NoSuchConfiguredTargetException {
ConfiguredTarget result =
getExistingConfiguredTarget(target.getLabel(), config);
if (result == null) {
throw new NoSuchConfiguredTargetException(target.getLabel(), config);
}
return result;
}
/**
* Obtains a {@link ConfiguredTarget} given a {@code label}, by delegating
* to the package cache and
* {@link #getConfiguredTarget(Target, BuildConfiguration)}.
*/
public ConfiguredTarget getConfiguredTarget(Label label, BuildConfiguration config)
throws NoSuchPackageException, NoSuchTargetException, NoSuchConfiguredTargetException {
return getConfiguredTarget(packageManager.getLoadedTarget(label), config);
}
public Iterable getDirectPrerequisites(ConfiguredTarget ct) {
return getDirectPrerequisites(ct, null);
}
public Iterable getDirectPrerequisites(ConfiguredTarget ct,
@Nullable final LoadingCache