Add an overview comment about how the analysis phase works and some pointers to it.

--
MOS_MIGRATED_REVID=122718503

10 files changed, 174 insertions, 61 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredAspect.java b/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredAspect.java
index 7f3db2a01e..402668a10a 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredAspect.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredAspect.java
@@ -44,6 +44,12 @@ import javax.annotation.Nullable;
  * added.
  *
  * <p>Aspects are created alongside configured targets on request from dependents.
+ *
+ * <p>For more information about aspects, see
+ * {@link com.google.devtools.build.lib.packages.AspectClass}.
+ *
+ * @see com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory
+ * @see com.google.devtools.build.lib.packages.AspectClass
  */
 @Immutable
 public final class ConfiguredAspect implements Iterable<TransitiveInfoProvider> {
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/RuleConfiguredTarget.java b/src/main/java/com/google/devtools/build/lib/analysis/RuleConfiguredTarget.java
index f1980ec6a6..fc29d3e82e 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/RuleConfiguredTarget.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/RuleConfiguredTarget.java
@@ -28,8 +28,11 @@ import java.util.LinkedHashMap;
 import java.util.Map;
 
 /**
- * A generic implementation of RuleConfiguredTarget. Do not use directly. Use {@link
- * RuleConfiguredTargetBuilder} instead.
+ * A {@link ConfiguredTarget} that is produced by a rule.
+ *
+ * <p>Created by {@link RuleConfiguredTargetBuilder}. There is an instance of this class for every
+ * analyzed rule. For more information about how analysis works, see
+ * {@link com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory}.
  */
 public final class RuleConfiguredTarget extends AbstractConfiguredTarget {
   /**
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/RuleConfiguredTargetBuilder.java b/src/main/java/com/google/devtools/build/lib/analysis/RuleConfiguredTargetBuilder.java
index b6fd29c230..17617e7d07 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/RuleConfiguredTargetBuilder.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/RuleConfiguredTargetBuilder.java
@@ -46,7 +46,13 @@ import java.util.Map.Entry;
 import java.util.TreeMap;
 
 /**
- * Builder class for analyzed rule instances (i.e., instances of {@link ConfiguredTarget}).
+ * Builder class for analyzed rule instances.
+ *
+ * <p>This is used to tell Bazel which {@link TransitiveInfoProvider}s are produced by the analysis
+ * of a configured target. For more information about analysis, see
+ * {@link com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory}.
+ *
+ * @see com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory
  */
 public final class RuleConfiguredTargetBuilder {
   private final RuleContext ruleContext;
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/RuleContext.java b/src/main/java/com/google/devtools/build/lib/analysis/RuleContext.java
index eb457d0742..20ff73c903 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/RuleContext.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/RuleContext.java
@@ -89,14 +89,14 @@ import java.util.Set;
 import javax.annotation.Nullable;
 
 /**
- * A helper class for rule implementations building and initialization. Objects of this
- * class are intended to be passed to the builder for the configured target, which then creates the
- * configured target.
+ * The totality of data available during the analysis of a rule.
  *
  * <p>These objects should not outlast the analysis phase. Do not pass them to {@link Action}
  * objects or other persistent objects. There are internal tests to ensure that RuleContext objects
  * are not persisted that check the name of this class, so update those tests if you change this
  * class's name.
+ *
+ * @see com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory
  */
 public final class RuleContext extends TargetContext
     implements ActionConstructionContext, ActionRegistry, RuleErrorConsumer {
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/TransitiveInfoCollection.java b/src/main/java/com/google/devtools/build/lib/analysis/TransitiveInfoCollection.java
index 42b829785f..ab53315111 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/TransitiveInfoCollection.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/TransitiveInfoCollection.java
@@ -21,57 +21,11 @@ import com.google.devtools.build.lib.skylarkinterface.SkylarkModule;
 import javax.annotation.Nullable;
 
 /**
- * Objects that implement this interface bundle multiple {@link TransitiveInfoProvider} interfaces.
+ * Multiple {@link TransitiveInfoProvider}s bundled together.
  *
- * <p>This interface (together with {@link TransitiveInfoProvider} is the cornerstone of the data
- * model of the analysis phase.
- *
- * <p>The computation a configured target does is allowed to depend on the following things:
- * <ul>
- * <li>The associated Target (which will usually be a Rule)
- * <li>Its own configuration (the configured target does not have access to other configurations,
- * e.g. the host configuration)
- * <li>The transitive info providers and labels of its direct dependencies.
- * </ul>
- *
- * <p>And these are the only inputs. Notably, a configured target is not supposed to access
- * other configured targets, the transitive info collections of configured targets it does not
- * directly depend on, the actions created by anyone else or the contents of any input file. We
- * strive to make it impossible for configured targets to do these things.
- *
- * <p>A configured target is expected to produce the following data during its analysis:
- * <ul>
- * <li>A number of Artifacts and Actions generating them
- * <li>A set of {@link TransitiveInfoProvider}s that it passes on to the targets directly dependent
- * on it
- * </ul>
- *
- * <p>The information that can be passed on to dependent targets by way of
- * {@link TransitiveInfoProvider} is subject to constraints (which are detailed in the
- * documentation of that class).
- *
- * <p>Configured targets are currently allowed to create artifacts at any exec path. It would be
- * better if they could be constrained to a subtree based on the label of the configured target,
- * but this is currently not feasible because multiple rules violate this constraint and the
- * output format is part of its interface.
- *
- * <p>In principle, multiple configured targets should not create actions with conflicting
- * outputs. There are still a few exceptions to this rule that are slated to be eventually
- * removed, we have provisions to handle this case (Action instances that share at least one
- * output file are required to be exactly the same), but this does put some pressure on the design
- * and we are eventually planning to eliminate this option.
- *
- * <p>These restrictions together make it possible to:
- * <ul>
- * <li>Correctly cache the analysis phase; by tightly constraining what a configured target is
- * allowed to access and what it is not, we can know when it needs to invalidate a particular
- * one and when it can reuse an already existing one.
- * <li>Serialize / deserialize individual configured targets at will, making it possible for
- * example to swap out part of the analysis state if there is memory pressure or to move them in
- * persistent storage so that the state can be reconstructed at a different time or in a
- * different process. The stretch goal is to eventually facilitate cross-user caching of this
- * information.
- * </ul>
+ * Represents the information made available by a {@link ConfiguredTarget} to other ones that
+ * depend on it. For more information about the analysis phase, see
+ * {@link com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory}.
  *
  * <p>Implementations of build rules should <b>not</b> hold on to references to the
  * {@link TransitiveInfoCollection}s representing their direct prerequisites in order to reduce
@@ -79,6 +33,7 @@ import javax.annotation.Nullable;
  * dependencies in turn, thereby making the size of the objects reachable from a single instance
  * unbounded).
  *
+ * @see com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory
  * @see TransitiveInfoProvider
  */
 @SkylarkModule(name = "Target", doc =
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/TransitiveInfoProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/TransitiveInfoProvider.java
index 37f589d276..9967aa8baf 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/TransitiveInfoProvider.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/TransitiveInfoProvider.java
@@ -15,9 +15,10 @@
 package com.google.devtools.build.lib.analysis;
 
 /**
- * This marker interface must be extended by every interface that represents
- * rolled-up data about the transitive closure of a configured target.
+ * Contains rolled-up data about the transitive closure of a configured target.
  *
+ * For more information about how analysis works, see
+ * {@link com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory}.
  * TransitiveInfoProviders need to be serializable, and for that reason they must conform to
  * the following restrictions:
  *
@@ -49,13 +50,14 @@ package com.google.devtools.build.lib.analysis;
  * being O(n^2): in a long dependency chain, if every target adds one single artifact, storing the
  * transitive closures of every rule would take 1+2+3+...+n-1+n = O(n^2) memory.
  *
- * <p>In order to avoid this, we introduce the concept of nested sets, {@link com.google.devtools
- * .build.lib.collect.nestedset.NestedSet}. A nested set is an immutable
+ * <p>In order to avoid this, we introduce the concept of nested sets, {@link
+ * com.google.devtools.build.lib.collect.nestedset.NestedSet}. A nested set is an immutable
  * data structure that can contain direct members and other nested sets (recursively). Nested sets
  * are iterable and can be flattened into ordered sets, where the order depends on which
  * implementation of NestedSet you pick.
  *
  * @see TransitiveInfoCollection
+ * @see com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory
  */
 public interface TransitiveInfoProvider {
 }
diff --git a/src/main/java/com/google/devtools/build/lib/packages/AspectClass.java b/src/main/java/com/google/devtools/build/lib/packages/AspectClass.java
index 6a7d24902e..2d615e1f6f 100644
--- a/src/main/java/com/google/devtools/build/lib/packages/AspectClass.java
+++ b/src/main/java/com/google/devtools/build/lib/packages/AspectClass.java
@@ -16,7 +16,37 @@ package com.google.devtools.build.lib.packages;
 
 /**
  *  A class of aspects.
- *  <p>Aspects might be defined natively, in Java ({@link NativeAspectClass})
+ *
+ *  <p>An aspect allows a rule to create actions in its dependencies, without their knowledge.
+ *  It can be viewed as the ability to attach shadow targets to transitive dependencies or a way
+ *  to run visitations of certain parts of the transitive closure of a rule in such a way that can
+ *  be cached (even partially) and reused between different configured targets requiring the same
+ *  aspect. Some examples where aspects are useful:
+ *
+ *  <ul>
+ *    <li>Converting the .jar files in the transitive closure of an Android binary to dexes</li>
+ *    <li>Emitting Java sources for a <code>proto_library</code> and the messages it depends
+ *        on</li>
+ *    <li>Collecting all the dependencies of a rule to make sure that it does not contain a
+ *        forbidden one</li>
+ *  </ul>
+ *
+ *  <p>When a configured target requests that an aspect be attached to one of its dependencies,
+ *  the {@link com.google.devtools.build.lib.analysis.TransitiveInfoProvider}s generated by that
+ *  aspects are merged with those of the actual dependency, that is,
+ *  {@link com.google.devtools.build.lib.analysis.RuleContext#getPrerequisite(
+ *  String, com.google.devtools.build.lib.analysis.RuleConfiguredTarget.Mode)} will
+ *  contain the transitive info providers produced both by the dependency and the aspects that are
+ *  attached to it.
+ *
+ *  <p>Configured targets can specify which aspects should be attached to some of their dependencies
+ *  by specifying this in their {@link com.google.devtools.build.lib.analysis.RuleDefinition}: each
+ *  attribute can have a list of aspects to be applied to the rules in that attribute and each
+ *  aspect can specify which {@link com.google.devtools.build.lib.analysis.TransitiveInfoProvider}s
+ *  it needs on a rule so that it can do meaningful work (for example, dexing only makes sense for
+ *  configured targets that produce Java code).
+ *
+ *  <p>Aspects can be defined natively, in Java ({@link NativeAspectClass})
  *  or in Skylark ({@link SkylarkAspectClass}).
  *
  *  Bazel propagates aspects through a multistage process. The general pipeline is as follows:
@@ -64,6 +94,9 @@ package com.google.devtools.build.lib.packages;
  *  used in SkyKeys or in other contexts that need equality for aspects.
  *  See also {@link com.google.devtools.build.lib.skyframe.AspectFunction} for details
  *  on Skyframe treatment of Aspects.
+ *
+ *  @see com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory
+ *  @see com.google.devtools.build.lib.skyframe.AspectFunction
  */
 public interface AspectClass {
 
diff --git a/src/main/java/com/google/devtools/build/lib/rules/RuleConfiguredTargetFactory.java b/src/main/java/com/google/devtools/build/lib/rules/RuleConfiguredTargetFactory.java
index dd846833ba..6f5eb7cd90 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/RuleConfiguredTargetFactory.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/RuleConfiguredTargetFactory.java
@@ -13,12 +13,109 @@
 // limitations under the License.
 package com.google.devtools.build.lib.rules;
 
+import com.google.devtools.build.lib.actions.ActionAnalysisMetadata;
 import com.google.devtools.build.lib.analysis.ConfiguredTarget;
 import com.google.devtools.build.lib.analysis.RuleContext;
 import com.google.devtools.build.lib.packages.RuleClass;
 
 /**
  * A shortcut class to the appropriate specialization of {@code RuleClass.ConfiguredTargetFactory}.
+ *
+ * <p>Here follows an overview of how loading and analysis works in Bazel:
+ *
+ * <p>Actions (i.e. commands that are run during the build) are created by configured targets
+ * (see {@link ConfiguredTarget}), wich are a pair of a target (e.g. <code>//src:bazel</code>) and a
+ * {@link com.google.devtools.build.lib.analysis.config.BuildConfiguration}, which is a blob of
+ * data that contains extra information about how the target should be built (for example, for which
+ * platform or with which C++ preprocessor definitions). Accordingly, a target can give rise to
+ * multiple configured targets, for example, if it needs to be built both for the host and the
+ * target configuration.
+ *
+ * <p>The process of creating the appropriate {@link com.google.devtools.build.lib.actions.Action}s
+ * for a configured target is called "analysis". The analysis of a configured target is composed of
+ * the following steps (which process is orchestrated by
+ * {@link com.google.devtools.build.lib.skyframe.ConfiguredTargetFunction}):
+ * <ol>
+ *   <li>The corresponding {@link com.google.devtools.build.lib.packages.Target} is loaded, i.e.
+ *       the BUILD file is parsed.</li>
+ *   <li>Its direct dependencies are analyzed, during which in turn indirect dependencies are
+ *       also analyzed.</li>
+ *   <li>Aspects specified by the configured target are analyzed. These can be thought of as
+ *       visitations of the transitive dependencies of the target. For more information, see
+ *       {@link com.google.devtools.build.lib.packages.AspectClass}.
+ *   <li>The configured target and the actions it generates are created based on the data from the
+ *       previous two steps.</li>
+ * </ol>
+ *
+ * Targets can be of three main kinds (plus a few special ones which are not important for
+ * understanding the big picture):
+ * <p>
+ *   <li>Input and output files. These represent either a file that is in the source tree or a
+ *       file produced by during the build. Not every file produced during the build has a
+ *       corresponding output file target.</li>
+ *   <li>Rules. These describe things a build actually does. Each rule has a class (e.g.
+ *       <code>cc_binary</code>. Rule classes can be defined either in Skylark using the
+ *       <code>rule()</code> function or in Java code by implementing
+ *       {@link com.google.devtools.build.lib.analysis.RuleDefinition}.</li>
+ * </p>
+ * <p>During the analysis of a configured target, the following pieces of data are available:
+ * <ul>
+ *   <li>The corresponding target itself. This is necessary so that the analysis has access to e.g.
+ *       the attributes a rule has in the BUILD file.</li>
+ *   <li>The {@link com.google.devtools.build.lib.analysis.TransitiveInfoCollection}s of direct
+ *       dependencies. They are used to gather information from the transitive closure, for example,
+ *       the include path entries for C++ compilation or all the object files that need to be
+ *       compiled into a C++ binary.</li>
+ *   <li>The configuration, which is used to determine which compiler to use and to get access
+ *       to some command line options of Bazel that influence analysis.</li>
+ *   <li>Skyframe, for requesting arbitrary Skyframe nodes. This is an escape hatch that should be
+ *       used when other mechanisms provided are not suitable and allows one to e.g. read arbitrary
+ *       files. With great power...
+ * </ul>
+ *
+ * <p>Analysis of non-rule configured targets is special-cased and is not covered here.
+ *
+ * <p>The analysis of a rule itself is done by implementations {@link RuleConfiguredTargetFactory}
+ * (there should be one for each rule class). the data above is available using the
+ * {@link RuleContext} argument passed into its create() method. It should
+ * result in three things:
+ * <ul>
+ *   <li>A set of actions. These should be passed to
+ *   {@link RuleContext#registerAction(ActionAnalysisMetadata...)}, although for more
+ *       common cases (e.g. {@link com.google.devtools.build.lib.analysis.actions.SpawnAction)},
+ *       shortcuts are provided.</li>
+ *   <li>A set of artifacts (files produced by actions). These should be created using methods of
+ *       {@link RuleContext}. Each artifact thus created must have a generating action.</li>
+ *   <li>A set of {@link com.google.devtools.build.lib.analysis.TransitiveInfoProvider}s that are
+ *       passed on to direct dependencies. These must be registered using
+ *       {@link com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder#add(
+ *       Class, com.google.devtools.build.lib.analysis.TransitiveInfoProvider)}
+ *       </li>
+ * </ul>
+ *
+ * <p>Configured targets are currently allowed to create artifacts at any exec path. It would be
+ * better if they could be constrained to a subtree based on the label of the configured target,
+ * but this is currently not feasible because multiple rules violate this constraint and the
+ * output format is part of its interface.
+ *
+ * <p>In principle, multiple configured targets should not create actions with conflicting
+ * outputs. There are still a few exceptions to this rule that are slated to be eventually
+ * removed, we have provisions to handle this case (Action instances that share at least one
+ * output file are required to be exactly the same), but this does put some pressure on the design
+ * and we are eventually planning to eliminate this option.
+ *
+ * <p>These restrictions together make it possible to:
+ * <ul>
+ * <li>Correctly cache the analysis phase; by tightly constraining what a configured target is
+ * allowed to access and what it is not, we can know when it needs to invalidate a particular
+ * one and when it can reuse an already existing one.
+ * <li>Serialize / deserialize individual configured targets at will, making it possible for
+ * example to swap out part of the analysis state if there is memory pressure or to move them in
+ * persistent storage so that the state can be reconstructed at a different time or in a
+ * different process. The stretch goal is to eventually facilitate cross-user caching of this
+ * information.
+ * </ul>
+ *
  */
 public interface RuleConfiguredTargetFactory
     extends RuleClass.ConfiguredTargetFactory<ConfiguredTarget, RuleContext> {
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/AspectFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/AspectFunction.java
index 38e384c196..9de5f09b81 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/AspectFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/AspectFunction.java
@@ -63,6 +63,9 @@ import javax.annotation.Nullable;
 /**
  * The Skyframe function that generates aspects.
  *
+ * This class, together with {@link ConfiguredTargetFunction} drives the analysis phase. For more
+ * information, see {@link com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory}.
+ *
  * {@link AspectFunction} takes a SkyKey containing an {@link AspectKey} [a tuple of
  * (target label, configurations, aspect class and aspect parameters)],
  * loads an {@link Aspect} from aspect class and aspect parameters,
@@ -71,6 +74,9 @@ import javax.annotation.Nullable;
  *
  * See {@link com.google.devtools.build.lib.packages.AspectClass} documentation
  * for an overview of aspect-related classes
+ *
+ * @see com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory
+ * @see com.google.devtools.build.lib.packages.AspectClass
  */
 public final class AspectFunction implements SkyFunction {
   private final BuildViewProvider buildViewProvider;
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetFunction.java
index 33d68bba44..2107d56d89 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/ConfiguredTargetFunction.java
@@ -87,6 +87,11 @@ import javax.annotation.Nullable;
 
 /**
  * SkyFunction for {@link ConfiguredTargetValue}s.
+ *
+ * This class, together with {@link AspectFunction} drives the analysis phase. For more information,
+ * see {@link com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory}.
+ *
+ * @see com.google.devtools.build.lib.rules.RuleConfiguredTargetFactory
  */
 final class ConfiguredTargetFunction implements SkyFunction {
   // This construction is a bit funky, but guarantees that the Object reference here is globally