BEP: Add tags and output files to TargetComplete event

A target's tags and output files are reported as part of
the TargetComplete event of the build event protocol (BEP).

The output files are grouped by their corresponding output
group. If an output file belongs to more than one output group
it appears once in each output group.

--
Change-Id: Ia37db68709850d8550e478dcc30064dc7366bd1b
Reviewed-on: https://cr.bazel.build/8955
PiperOrigin-RevId: 148667599
MOS_MIGRATED_REVID=148667599

5 files changed, 180 insertions, 49 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/analysis/TargetCompleteEvent.java b/src/main/java/com/google/devtools/build/lib/analysis/TargetCompleteEvent.java
index 971c003d4a..1427e89799 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/TargetCompleteEvent.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/TargetCompleteEvent.java
@@ -16,8 +16,12 @@ package com.google.devtools.build.lib.analysis;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.analysis.TopLevelArtifactHelper.ArtifactsInOutputGroup;
 import com.google.devtools.build.lib.buildeventstream.BuildEventId;
 import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos;
+import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.File;
+import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.OutputGroup;
 import com.google.devtools.build.lib.buildeventstream.BuildEventWithOrderConstraint;
 import com.google.devtools.build.lib.buildeventstream.GenericBuildEvent;
 import com.google.devtools.build.lib.buildeventstream.PathConverter;
@@ -26,7 +30,9 @@ import com.google.devtools.build.lib.cmdline.Label;
 import com.google.devtools.build.lib.collect.nestedset.NestedSet;
 import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
 import com.google.devtools.build.lib.collect.nestedset.Order;
+import com.google.devtools.build.lib.packages.AttributeMap;
 import com.google.devtools.build.lib.rules.test.TestProvider;
+import com.google.devtools.build.lib.syntax.Type;
 import com.google.devtools.build.lib.util.Preconditions;
 import com.google.devtools.build.skyframe.SkyValue;
 import java.util.Collection;
@@ -37,10 +43,14 @@ public final class TargetCompleteEvent implements SkyValue, BuildEventWithOrderC
   private final ConfiguredTarget target;
   private final NestedSet<Cause> rootCauses;
   private final Collection<BuildEventId> postedAfter;
+  private final Iterable<ArtifactsInOutputGroup> outputs;
   private final boolean isTest;
 
   private TargetCompleteEvent(
-      ConfiguredTarget target, NestedSet<Cause> rootCauses, boolean isTest) {
+      ConfiguredTarget target,
+      NestedSet<Cause> rootCauses,
+      Iterable<ArtifactsInOutputGroup> outputs,
+      boolean isTest) {
     this.target = target;
     this.rootCauses =
         (rootCauses == null) ? NestedSetBuilder.<Cause>emptySet(Order.STABLE_ORDER) : rootCauses;
@@ -50,17 +60,19 @@ public final class TargetCompleteEvent implements SkyValue, BuildEventWithOrderC
       postedAfterBuilder.add(BuildEventId.fromCause(cause));
     }
     this.postedAfter = postedAfterBuilder.build();
+    this.outputs = outputs;
     this.isTest = isTest;
   }
 
   /** Construct a successful target completion event. */
-  public static TargetCompleteEvent createSuccessfulTarget(ConfiguredTarget ct) {
-    return new TargetCompleteEvent(ct, null, false);
+  public static TargetCompleteEvent createSuccessfulTarget(
+      ConfiguredTarget ct, NestedSet<ArtifactsInOutputGroup> outputs) {
+    return new TargetCompleteEvent(ct, null, outputs, false);
   }
 
   /** Construct a successful target completion event for a target that will be tested. */
   public static TargetCompleteEvent createSuccessfulTestTarget(ConfiguredTarget ct) {
-    return new TargetCompleteEvent(ct, null, true);
+    return new TargetCompleteEvent(ct, null, ImmutableList.<ArtifactsInOutputGroup>of(), true);
   }
 
 
@@ -69,7 +81,8 @@ public final class TargetCompleteEvent implements SkyValue, BuildEventWithOrderC
    */
   public static TargetCompleteEvent createFailed(ConfiguredTarget ct, NestedSet<Cause> rootCauses) {
     Preconditions.checkArgument(!Iterables.isEmpty(rootCauses));
-    return new TargetCompleteEvent(ct, rootCauses, false);
+    return new TargetCompleteEvent(
+        ct, rootCauses, ImmutableList.<ArtifactsInOutputGroup>of(), false);
   }
 
   /**
@@ -120,8 +133,14 @@ public final class TargetCompleteEvent implements SkyValue, BuildEventWithOrderC
 
   @Override
   public BuildEventStreamProtos.BuildEvent asStreamProto(PathConverter pathConverter) {
-    BuildEventStreamProtos.TargetComplete complete =
-        BuildEventStreamProtos.TargetComplete.newBuilder().setSuccess(!failed()).build();
+    BuildEventStreamProtos.TargetComplete.Builder builder =
+        BuildEventStreamProtos.TargetComplete.newBuilder();
+
+    builder.setSuccess(!failed());
+    builder.addAllTag(getTags());
+    builder.addAllOutputGroup(getOutputFilesByGroup(pathConverter));
+
+    BuildEventStreamProtos.TargetComplete complete = builder.build();
     return GenericBuildEvent.protoChaining(this).setCompleted(complete).build();
   }
 
@@ -129,4 +148,32 @@ public final class TargetCompleteEvent implements SkyValue, BuildEventWithOrderC
   public Collection<BuildEventId> postedAfter() {
     return postedAfter;
   }
+
+  private Iterable<String> getTags() {
+    // We are only interested in targets that are rules.
+    if (!(target instanceof RuleConfiguredTarget)) {
+      return ImmutableList.<String>of();
+    }
+    AttributeMap attributes = ConfiguredAttributeMapper.of((RuleConfiguredTarget) target);
+    // Every rule (implicitly) has a "tags" attribute.
+    return attributes.get("tags", Type.STRING_LIST);
+  }
+
+  private Iterable<OutputGroup> getOutputFilesByGroup(PathConverter pathConverter) {
+    ImmutableList.Builder<OutputGroup> groups = ImmutableList.builder();
+    for (ArtifactsInOutputGroup artifactsInOutputGroup : outputs) {
+      OutputGroup.Builder groupBuilder = OutputGroup.newBuilder();
+      groupBuilder.setName(artifactsInOutputGroup.getOutputGroup());
+
+      File.Builder fileBuilder = File.newBuilder();
+      for (Artifact artifact : artifactsInOutputGroup.getArtifacts()) {
+        String name = artifact.getFilename();
+        String uri = pathConverter.apply(artifact.getPath());
+        groupBuilder.addOutputFile(fileBuilder.setName(name).setUri(uri).build());
+      }
+
+      groups.add(groupBuilder.build());
+    }
+    return groups.build();
+  }
 }
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/TopLevelArtifactHelper.java b/src/main/java/com/google/devtools/build/lib/analysis/TopLevelArtifactHelper.java
index bb0922b368..6665918167 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/TopLevelArtifactHelper.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/TopLevelArtifactHelper.java
@@ -14,6 +14,8 @@
 
 package com.google.devtools.build.lib.analysis;
 
+import static com.google.devtools.build.lib.util.Preconditions.checkNotNull;
+
 import com.google.common.collect.ImmutableCollection;
 import com.google.common.collect.ImmutableList;
 import com.google.devtools.build.lib.actions.Artifact;
@@ -22,7 +24,6 @@ import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
 import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
 import com.google.devtools.build.lib.rules.test.TestProvider;
 import com.google.devtools.build.lib.skyframe.AspectValue;
-
 import javax.annotation.Nullable;
 
 /**
@@ -31,36 +32,82 @@ import javax.annotation.Nullable;
  */
 public final class TopLevelArtifactHelper {
 
-  /***
+  /** Set of {@link Artifact}s in an output group. */
+  @Immutable
+  public static final class ArtifactsInOutputGroup {
+    private final String outputGroup;
+    private final boolean important;
+    private final NestedSet<Artifact> artifacts;
+
+    private ArtifactsInOutputGroup(
+        String outputGroup, boolean important, NestedSet<Artifact> artifacts) {
+      this.outputGroup = checkNotNull(outputGroup);
+      this.important = important;
+      this.artifacts = checkNotNull(artifacts);
+    }
+
+    public String getOutputGroup() {
+      return outputGroup;
+    }
+
+    public NestedSet<Artifact> getArtifacts() {
+      return artifacts;
+    }
+
+    /** Returns {@code true} if the user should know about this output group. */
+    public boolean areImportant() {
+      return important;
+    }
+  }
+
+  /**
    * The set of artifacts to build.
    *
    * <p>There are two kinds: the ones that the user cares about (e.g. files to build) and the ones
    * she doesn't (e.g. baseline coverage artifacts). The latter type doesn't get reported on various
-   * outputs, e.g. on the console  output listing the output artifacts of targets on the command
+   * outputs, e.g. on the console output listing the output artifacts of targets on the command
    * line.
    */
   @Immutable
   public static final class ArtifactsToBuild {
-    private final NestedSet<Artifact> important;
-    private final NestedSet<Artifact> all;
+    private NestedSet<ArtifactsInOutputGroup> artifacts;
 
-    private ArtifactsToBuild(NestedSet<Artifact> important, NestedSet<Artifact> all) {
-      this.important = important;
-      this.all = all;
+    private ArtifactsToBuild(NestedSet<ArtifactsInOutputGroup> artifacts) {
+      this.artifacts = checkNotNull(artifacts);
     }
 
     /**
      * Returns the artifacts that the user should know about.
      */
     public NestedSet<Artifact> getImportantArtifacts() {
-      return important;
+      NestedSetBuilder<Artifact> builder = new NestedSetBuilder<>(artifacts.getOrder());
+      for (ArtifactsInOutputGroup artifactsInOutputGroup : artifacts) {
+        if (artifactsInOutputGroup.areImportant()) {
+          builder.addTransitive(artifactsInOutputGroup.getArtifacts());
+        }
+      }
+      return builder.build();
     }
 
     /**
      * Returns the actual set of artifacts that need to be built.
      */
     public NestedSet<Artifact> getAllArtifacts() {
-      return all;
+      NestedSetBuilder<Artifact> builder = new NestedSetBuilder<>(artifacts.getOrder());
+      for (ArtifactsInOutputGroup artifactsInOutputGroup : artifacts) {
+        builder.addTransitive(artifactsInOutputGroup.getArtifacts());
+      }
+      return builder.build();
+    }
+
+    /**
+     * Returns the set of all {@link Artifact}s grouped by their corresponding output group.
+     *
+     * <p>If an {@link Artifact} belongs to two or more output groups, it appears once in each
+     * output group.
+     */
+    public NestedSet<ArtifactsInOutputGroup> getAllArtifactsByOutputGroup() {
+      return artifacts;
     }
   }
 
@@ -88,14 +135,12 @@ public final class TopLevelArtifactHelper {
    */
   public static ArtifactsToBuild getAllArtifactsToBuild(
       Iterable<? extends TransitiveInfoCollection> targets, TopLevelArtifactContext context) {
-    NestedSetBuilder<Artifact> allArtifacts = NestedSetBuilder.stableOrder();
-    NestedSetBuilder<Artifact> importantArtifacts = NestedSetBuilder.stableOrder();
+    NestedSetBuilder<ArtifactsInOutputGroup> artifacts = NestedSetBuilder.stableOrder();
     for (TransitiveInfoCollection target : targets) {
       ArtifactsToBuild targetArtifacts = getAllArtifactsToBuild(target, context);
-      allArtifacts.addTransitive(targetArtifacts.getAllArtifacts());
-      importantArtifacts.addTransitive(targetArtifacts.getImportantArtifacts());
+      artifacts.addTransitive(targetArtifacts.getAllArtifactsByOutputGroup());
     }
-    return new ArtifactsToBuild(importantArtifacts.build(), allArtifacts.build());
+    return new ArtifactsToBuild(artifacts.build());
   }
 
   /**
@@ -103,14 +148,12 @@ public final class TopLevelArtifactHelper {
    */
   public static ArtifactsToBuild getAllArtifactsToBuildFromAspects(
       Iterable<AspectValue> aspects, TopLevelArtifactContext context) {
-    NestedSetBuilder<Artifact> allArtifacts = NestedSetBuilder.stableOrder();
-    NestedSetBuilder<Artifact> importantArtifacts = NestedSetBuilder.stableOrder();
+    NestedSetBuilder<ArtifactsInOutputGroup> artifacts = NestedSetBuilder.stableOrder();
     for (AspectValue aspect : aspects) {
       ArtifactsToBuild aspectArtifacts = getAllArtifactsToBuild(aspect, context);
-      allArtifacts.addTransitive(aspectArtifacts.getAllArtifacts());
-      importantArtifacts.addTransitive(aspectArtifacts.getImportantArtifacts());
+      artifacts.addTransitive(aspectArtifacts.getAllArtifactsByOutputGroup());
     }
-    return new ArtifactsToBuild(importantArtifacts.build(), allArtifacts.build());
+    return new ArtifactsToBuild(artifacts.build());
   }
 
 
@@ -145,31 +188,33 @@ public final class TopLevelArtifactHelper {
       @Nullable OutputGroupProvider outputGroupProvider,
       @Nullable FileProvider fileProvider,
       TopLevelArtifactContext context) {
-    NestedSetBuilder<Artifact> importantBuilder = NestedSetBuilder.stableOrder();
-    NestedSetBuilder<Artifact> allBuilder = NestedSetBuilder.stableOrder();
+    NestedSetBuilder<ArtifactsInOutputGroup> allBuilder = NestedSetBuilder.stableOrder();
 
     for (String outputGroup : context.outputGroups()) {
       NestedSetBuilder<Artifact> results = NestedSetBuilder.stableOrder();
 
-      if (outputGroup.equals(OutputGroupProvider.DEFAULT)) {
-        if (fileProvider != null) {
-          results.addTransitive(fileProvider.getFilesToBuild());
-        }
+      if (outputGroup.equals(OutputGroupProvider.DEFAULT) && fileProvider != null) {
+        results.addTransitive(fileProvider.getFilesToBuild());
       }
 
       if (outputGroupProvider != null) {
         results.addTransitive(outputGroupProvider.getOutputGroup(outputGroup));
       }
 
-      if (outputGroup.startsWith(OutputGroupProvider.HIDDEN_OUTPUT_GROUP_PREFIX)) {
-        allBuilder.addTransitive(results.build());
-      } else {
-        importantBuilder.addTransitive(results.build());
+      // Ignore output groups that have no artifacts.
+      if (results.isEmpty()) {
+        continue;
       }
+
+      boolean isImportantGroup =
+          !outputGroup.startsWith(OutputGroupProvider.HIDDEN_OUTPUT_GROUP_PREFIX);
+
+      ArtifactsInOutputGroup artifacts =
+          new ArtifactsInOutputGroup(outputGroup, isImportantGroup, results.build());
+
+      allBuilder.add(artifacts);
     }
 
-    NestedSet<Artifact> importantArtifacts = importantBuilder.build();
-    allBuilder.addTransitive(importantArtifacts);
-    return new ArtifactsToBuild(importantArtifacts, allBuilder.build());
+    return new ArtifactsToBuild(allBuilder.build());
   }
 }
diff --git a/src/main/java/com/google/devtools/build/lib/buildeventstream/proto/build_event_stream.proto b/src/main/java/com/google/devtools/build/lib/buildeventstream/proto/build_event_stream.proto
index fc50d9b358..f70c715781 100644
--- a/src/main/java/com/google/devtools/build/lib/buildeventstream/proto/build_event_stream.proto
+++ b/src/main/java/com/google/devtools/build/lib/buildeventstream/proto/build_event_stream.proto
@@ -198,11 +198,28 @@ message ActionExecuted {
   string label = 5;
 }
 
+// Collection of all output files belonging to that output group.
+message OutputGroup {
+  // Name of the output group
+  string name = 1;
+
+  // List of output files that belong to this output group.
+  repeated File output_file = 2;
+}
+
 // Payload of the event indicating the completion of a target. The target is
 // specified in the id. If the target failed the root causes are provided as
 // children events.
 message TargetComplete {
   bool success = 1;
+
+  // The output files are arranged by their output group. If an output file
+  // is part of multiple output groups, it appears once in each output
+  // group.
+  repeated OutputGroup output_group = 2;
+
+  // List of tags associated with this target.
+  repeated string tag = 3;
 }
 
 // Payload on events reporting about individual test action.
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionProgressReceiver.java b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionProgressReceiver.java
index cb182603a0..bb6fd0874f 100644
--- a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionProgressReceiver.java
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionProgressReceiver.java
@@ -21,6 +21,9 @@ import com.google.devtools.build.lib.actions.ActionExecutionStatusReporter;
 import com.google.devtools.build.lib.analysis.AspectCompleteEvent;
 import com.google.devtools.build.lib.analysis.ConfiguredTarget;
 import com.google.devtools.build.lib.analysis.TargetCompleteEvent;
+import com.google.devtools.build.lib.analysis.TopLevelArtifactContext;
+import com.google.devtools.build.lib.analysis.TopLevelArtifactHelper;
+import com.google.devtools.build.lib.analysis.TopLevelArtifactHelper.ArtifactsToBuild;
 import com.google.devtools.build.lib.skyframe.ActionExecutionInactivityWatchdog;
 import com.google.devtools.build.lib.skyframe.ActionExecutionValue;
 import com.google.devtools.build.lib.skyframe.AspectCompletionValue;
@@ -56,6 +59,7 @@ public final class ExecutionProgressReceiver
   private final int exclusiveTestsCount;
   private final Set<ConfiguredTarget> testedTargets;
   private final EventBus eventBus;
+  private final TopLevelArtifactContext topLevelArtifactContext;
 
   static {
     PROGRESS_MESSAGE_NUMBER_FORMATTER = NumberFormat.getIntegerInstance(Locale.ENGLISH);
@@ -70,11 +74,13 @@ public final class ExecutionProgressReceiver
       Set<ConfiguredTarget> builtTargets,
       int exclusiveTestsCount,
       Set<ConfiguredTarget> testedTargets,
+      TopLevelArtifactContext topLevelArtifactContext,
       EventBus eventBus) {
     this.builtTargets = Collections.synchronizedSet(builtTargets);
     this.exclusiveTestsCount = exclusiveTestsCount;
-    this.eventBus = eventBus;
     this.testedTargets = testedTargets;
+    this.topLevelArtifactContext = topLevelArtifactContext;
+    this.eventBus = eventBus;
   }
 
   @Override
@@ -99,14 +105,17 @@ public final class ExecutionProgressReceiver
     SkyFunctionName type = skyKey.functionName();
     if (type.equals(SkyFunctions.TARGET_COMPLETION)) {
       TargetCompletionValue value = (TargetCompletionValue) skyValueSupplier.get();
-      if (value != null) {
-        ConfiguredTarget target = value.getConfiguredTarget();
-        builtTargets.add(target);
-        if (testedTargets.contains(target)) {
-          eventBus.post(TargetCompleteEvent.createSuccessfulTestTarget(target));
-        } else {
-          eventBus.post(TargetCompleteEvent.createSuccessfulTarget(target));
-        }
+      if (value == null) {
+        return;
+      }
+
+      ConfiguredTarget target = value.getConfiguredTarget();
+      builtTargets.add(target);
+
+      if (testedTargets.contains(target)) {
+        postTestTargetComplete(target);
+      } else {
+        postBuildTargetComplete(target);
       }
     } else if (type.equals(SkyFunctions.ASPECT_COMPLETION)) {
       AspectCompletionValue value = (AspectCompletionValue) skyValueSupplier.get();
@@ -206,4 +215,16 @@ public final class ExecutionProgressReceiver
       }
     };
   }
+
+  private void postTestTargetComplete(ConfiguredTarget target) {
+    eventBus.post(TargetCompleteEvent.createSuccessfulTestTarget(target));
+  }
+
+  private void postBuildTargetComplete(ConfiguredTarget target) {
+    ArtifactsToBuild artifactsToBuild =
+        TopLevelArtifactHelper.getAllArtifactsToBuild(target, topLevelArtifactContext);
+    eventBus.post(
+        TargetCompleteEvent.createSuccessfulTarget(
+            target, artifactsToBuild.getAllArtifactsByOutputGroup()));
+  }
 }
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/SkyframeBuilder.java b/src/main/java/com/google/devtools/build/lib/buildtool/SkyframeBuilder.java
index 038d95110a..7dcd76f902 100644
--- a/src/main/java/com/google/devtools/build/lib/buildtool/SkyframeBuilder.java
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/SkyframeBuilder.java
@@ -115,6 +115,7 @@ public class SkyframeBuilder implements Builder {
                 .addAll(parallelTests)
                 .addAll(exclusiveTests)
                 .build(),
+            topLevelArtifactContext,
             skyframeExecutor.getEventBus());
     skyframeExecutor
         .getEventBus()