Include more information about build events' referenced local files.

Instead of just a path, events now include information about the type of file (output, source file, stdout/stderr, test logs, etc.). This information can be used by the uploaders to determine a) whether to upload, b) what kind of lease to give the files.

RELNOTES: None
PiperOrigin-RevId: 203285549

4 files changed, 85 insertions, 32 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEvent.java b/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEvent.java
index 6d6900687c..cee573cae2 100644
--- a/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEvent.java
+++ b/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEvent.java
@@ -14,10 +14,11 @@
 
 package com.google.devtools.build.lib.buildeventstream;
 
-import com.google.common.collect.ImmutableSet;
+import com.google.common.base.Objects;
+import com.google.common.collect.ImmutableList;
 import com.google.devtools.build.lib.events.ExtendedEventHandler;
 import com.google.devtools.build.lib.vfs.Path;
-import java.util.Set;
+import java.util.Collection;
 
 /**
  * Interface for objects that can be posted on the public event stream.
@@ -28,6 +29,52 @@ import java.util.Set;
 public interface BuildEvent extends ChainableEvent, ExtendedEventHandler.Postable {
 
   /**
+   * A local file that is referenced by the build event. These can be uploaded to a separate backend
+   * storage.
+   */
+  final class LocalFile {
+
+    /**
+     * The type of the local file. This is used by uploaders to determine how long to store the
+     * associated files for.
+     */
+    public enum LocalFileType {
+      SOURCE,
+      OUTPUT,
+      SUCCESSFUL_TEST_OUTPUT,
+      FAILED_TEST_OUTPUT,
+      STDOUT,
+      STDERR,
+      LOG,
+    }
+
+    public final Path path;
+    public final LocalFileType type;
+
+    public LocalFile(Path path, LocalFileType type) {
+      this.path = path;
+      this.type = type;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      if (this == o) {
+        return true;
+      }
+      if (o == null || getClass() != o.getClass()) {
+        return false;
+      }
+      LocalFile localFile = (LocalFile) o;
+      return Objects.equal(path, localFile.path) && type == localFile.type;
+    }
+
+    @Override
+    public int hashCode() {
+      return Objects.hashCode(path, type);
+    }
+  }
+
+  /**
    * Returns a list of files that are referenced in the protobuf representation returned by {@link
    * #asStreamProto(BuildEventContext)}.
    *
@@ -35,13 +82,11 @@ public interface BuildEvent extends ChainableEvent, ExtendedEventHandler.Postabl
    * only returns files directly referenced in the protobuf returned by {@link
    * #asStreamProto(BuildEventContext)}.
    *
-   * <p>Note the consistency requirement - you must not attempt to pass Path objects to the
-   * {@link PathConverter} unless you have returned the Path object here.
+   * <p>Note the consistency requirement - you must not attempt to pass Path objects to the {@link
+   * PathConverter} unless you have returned a corresponding {@link LocalFile} object here.
    */
-  // TODO(ulfjack): Consider moving the upload call to the BuildEventContext and returning a map
-  // from Path to URI, rather than a callback.
-  default Set<Path> referencedLocalFiles() {
-    return ImmutableSet.of();
+  default Collection<LocalFile> referencedLocalFiles() {
+    return ImmutableList.of();
   }
 
   /**
diff --git a/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEventArtifactUploader.java b/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEventArtifactUploader.java
index 5c1fe9c5b2..3f536325db 100644
--- a/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEventArtifactUploader.java
+++ b/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildEventArtifactUploader.java
@@ -15,30 +15,32 @@ package com.google.devtools.build.lib.buildeventstream;
 
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
+import com.google.devtools.build.lib.buildeventstream.BuildEvent.LocalFile;
 import com.google.devtools.build.lib.buildeventstream.PathConverter.FileUriPathConverter;
 import com.google.devtools.build.lib.vfs.Path;
-import java.util.Set;
+import java.util.Map;
 
 /** Uploads artifacts referenced by the Build Event Protocol (BEP). */
 public interface BuildEventArtifactUploader {
-  BuildEventArtifactUploader LOCAL_FILES_UPLOADER = new BuildEventArtifactUploader() {
-    private final ListenableFuture<PathConverter> completedPathConverter =
-        Futures.immediateFuture(new FileUriPathConverter());
+  BuildEventArtifactUploader LOCAL_FILES_UPLOADER =
+      new BuildEventArtifactUploader() {
+        private final ListenableFuture<PathConverter> completedPathConverter =
+            Futures.immediateFuture(new FileUriPathConverter());
 
-    @Override
-    public ListenableFuture<PathConverter> upload(Set<Path> files) {
-      return completedPathConverter;
-    }
-  };
+        @Override
+        public ListenableFuture<PathConverter> upload(Map<Path, LocalFile> files) {
+          return completedPathConverter;
+        }
+      };
 
   /**
-   * Asynchronously uploads a set of files referenced by the protobuf representation of a
-   * {@link BuildEvent}. This method is expected to return quickly.
+   * Asynchronously uploads a set of files referenced by the protobuf representation of a {@link
+   * BuildEvent}. This method is expected to return quickly.
    *
    * <p>This method must not throw any exceptions.
-   * 
+   *
    * <p>Returns a future to a {@link PathConverter} that must provide a name for each uploaded file
    * as it should appear in the BEP.
    */
-  ListenableFuture<PathConverter> upload(Set<Path> files);
+  ListenableFuture<PathConverter> upload(Map<Path, LocalFile> files);
 }
diff --git a/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildToolLogs.java b/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildToolLogs.java
index c4eee48b28..8ac55d6db2 100644
--- a/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildToolLogs.java
+++ b/src/main/java/com/google/devtools/build/lib/buildeventstream/BuildToolLogs.java
@@ -14,7 +14,7 @@
 package com.google.devtools.build.lib.buildeventstream;
 
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
+import com.google.devtools.build.lib.buildeventstream.BuildEvent.LocalFile.LocalFileType;
 import com.google.devtools.build.lib.util.Pair;
 import com.google.devtools.build.lib.vfs.Path;
 import com.google.protobuf.ByteString;
@@ -38,16 +38,16 @@ public class BuildToolLogs implements BuildEventWithOrderConstraint {
 
   @Override
   public Collection<BuildEventId> getChildrenEvents() {
-    return ImmutableList.<BuildEventId>of();
+    return ImmutableList.of();
   }
 
   @Override
-  public ImmutableSet<Path> referencedLocalFiles() {
-    ImmutableSet.Builder<Path> artifacts = ImmutableSet.builder();
+  public Collection<LocalFile> referencedLocalFiles() {
+    ImmutableList.Builder<LocalFile> localFiles = ImmutableList.builder();
     for (Pair<String, Path> logFile : logFiles) {
-      artifacts.add(logFile.getSecond());
+      localFiles.add(new LocalFile(logFile.getSecond(), LocalFileType.LOG));
     }
-    return artifacts.build();
+    return localFiles.build();
   }
 
   @Override
diff --git a/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/FileTransport.java b/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/FileTransport.java
index c193ed094b..cce4f2d213 100644
--- a/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/FileTransport.java
+++ b/src/main/java/com/google/devtools/build/lib/buildeventstream/transports/FileTransport.java
@@ -19,12 +19,14 @@ import static com.google.common.base.Preconditions.checkNotNull;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Function;
 import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.util.concurrent.FutureCallback;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.MoreExecutors;
 import com.google.devtools.build.lib.buildeventstream.ArtifactGroupNamer;
 import com.google.devtools.build.lib.buildeventstream.BuildEvent;
+import com.google.devtools.build.lib.buildeventstream.BuildEvent.LocalFile;
 import com.google.devtools.build.lib.buildeventstream.BuildEventArtifactUploader;
 import com.google.devtools.build.lib.buildeventstream.BuildEventContext;
 import com.google.devtools.build.lib.buildeventstream.BuildEventProtocolOptions;
@@ -37,7 +39,7 @@ import com.google.devtools.build.lib.util.io.AsynchronousFileOutputStream;
 import com.google.devtools.build.lib.vfs.Path;
 import com.google.protobuf.Message;
 import java.io.IOException;
-import java.util.Set;
+import java.util.Collection;
 import java.util.function.Consumer;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -147,10 +149,14 @@ abstract class FileTransport implements BuildEventTransport {
    * Returns a {@link PathConverter} for the uploaded files, or {@code null} when the uploaded
    * failed.
    */
-  private ListenableFuture<PathConverter> uploadReferencedFiles(Set<Path> artifacts) {
-    checkNotNull(artifacts);
-
-    ListenableFuture<PathConverter> upload = uploader.upload(artifacts);
+  private ListenableFuture<PathConverter> uploadReferencedFiles(Collection<LocalFile> localFiles) {
+    checkNotNull(localFiles);
+    ImmutableMap.Builder<Path, LocalFile> localFileMap =
+        ImmutableMap.builderWithExpectedSize(localFiles.size());
+    for (LocalFile localFile : localFiles) {
+      localFileMap.put(localFile.path, localFile);
+    }
+    ListenableFuture<PathConverter> upload = uploader.upload(localFileMap.build());
     Futures.addCallback(
         upload,
         new FutureCallback<PathConverter>() {