Add a mechanism for build event protocol events to upload files

This should be a no-op, mostly replacing PathConverter with
BuildEventArtifactUploader, since none of the implementations perform any
upload yet.

PiperOrigin-RevId: 200685325

3 files changed, 76 insertions, 28 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/buildeventservice/BuildEventServiceModule.java b/src/main/java/com/google/devtools/build/lib/buildeventservice/BuildEventServiceModule.java
index 80c5599620..115808a66a 100644
--- a/src/main/java/com/google/devtools/build/lib/buildeventservice/BuildEventServiceModule.java
+++ b/src/main/java/com/google/devtools/build/lib/buildeventservice/BuildEventServiceModule.java
@@ -20,14 +20,16 @@ import static com.google.devtools.build.lib.buildeventservice.BuildEventServiceT
 import static java.lang.String.format;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions;
 import com.google.devtools.build.lib.buildeventservice.client.BuildEventServiceClient;
+import com.google.devtools.build.lib.buildeventstream.BuildEventArtifactUploader;
+import com.google.devtools.build.lib.buildeventstream.BuildEventArtifactUploaderMap;
 import com.google.devtools.build.lib.buildeventstream.BuildEventProtocolOptions;
 import com.google.devtools.build.lib.buildeventstream.BuildEventTransport;
-import com.google.devtools.build.lib.buildeventstream.PathConverter;
 import com.google.devtools.build.lib.buildeventstream.transports.BuildEventStreamOptions;
 import com.google.devtools.build.lib.buildeventstream.transports.BuildEventTransportFactory;
 import com.google.devtools.build.lib.clock.Clock;
@@ -72,8 +74,7 @@ public abstract class BuildEventServiceModule<T extends BuildEventServiceOptions
   }
 
   @Override
-  public void beforeCommand(CommandEnvironment commandEnvironment)
-      throws AbruptExitException {
+  public void beforeCommand(CommandEnvironment commandEnvironment) {
     // Reset to null in case afterCommand was not called.
     this.outErr = null;
     if (!whitelistedCommands().contains(commandEnvironment.getCommandName())) {
@@ -87,7 +88,7 @@ public abstract class BuildEventServiceModule<T extends BuildEventServiceOptions
             commandEnvironment.getReporter(),
             commandEnvironment.getBlazeModuleEnvironment(),
             commandEnvironment.getRuntime().getClock(),
-            commandEnvironment.getRuntime().getPathToUriConverter(),
+            commandEnvironment.getRuntime().getBuildEventArtifactUploaders(),
             commandEnvironment.getReporter(),
             commandEnvironment.getBuildRequestId().toString(),
             commandEnvironment.getCommandId().toString(),
@@ -129,9 +130,7 @@ public abstract class BuildEventServiceModule<T extends BuildEventServiceOptions
     this.outErr = null;
   }
 
-  /**
-   * Returns {@code null} if no stream could be created.
-   */
+  /** Returns {@code null} if no stream could be created. */
   @Nullable
   @VisibleForTesting
   BuildEventStreamer tryCreateStreamer(
@@ -140,11 +139,13 @@ public abstract class BuildEventServiceModule<T extends BuildEventServiceOptions
       EventHandler commandLineReporter,
       ModuleEnvironment moduleEnvironment,
       Clock clock,
-      PathConverter pathConverter,
+      BuildEventArtifactUploaderMap artifactUploaders,
       Reporter reporter,
       String buildRequestId,
       String invocationId,
       String commandName) {
+    Preconditions.checkNotNull(artifactUploaders);
+
     try {
       T besOptions =
           checkNotNull(
@@ -172,18 +173,20 @@ public abstract class BuildEventServiceModule<T extends BuildEventServiceOptions
                 moduleEnvironment,
                 clock,
                 protocolOptions,
-                pathConverter,
+                artifactUploaders,
                 commandLineReporter,
                 startupOptionsProvider);
       } catch (Exception e) {
         commandLineReporter.handle(Event.error(format(UPLOAD_FAILED_MESSAGE, e.getMessage())));
-        moduleEnvironment.exit(new AbruptExitException(
-            "Failed while creating BuildEventTransport", ExitCode.PUBLISH_ERROR));
+        moduleEnvironment.exit(
+            new AbruptExitException(
+                "Failed while creating BuildEventTransport", ExitCode.PUBLISH_ERROR));
         return null;
       }
 
       ImmutableSet<BuildEventTransport> bepTransports =
-          BuildEventTransportFactory.createFromOptions(bepOptions, protocolOptions, pathConverter);
+          BuildEventTransportFactory.createFromOptions(
+              bepOptions, protocolOptions, artifactUploaders, moduleEnvironment::exit);
 
       ImmutableSet.Builder<BuildEventTransport> transportsBuilder =
           ImmutableSet.<BuildEventTransport>builder().addAll(bepTransports);
@@ -213,7 +216,7 @@ public abstract class BuildEventServiceModule<T extends BuildEventServiceOptions
       ModuleEnvironment moduleEnvironment,
       Clock clock,
       BuildEventProtocolOptions protocolOptions,
-      PathConverter pathConverter,
+      BuildEventArtifactUploaderMap artifactUploaders,
       EventHandler commandLineReporter,
       OptionsProvider startupOptionsProvider)
       throws IOException, OptionsParsingException {
@@ -242,9 +245,13 @@ public abstract class BuildEventServiceModule<T extends BuildEventServiceOptions
                     besOptions.besBackend, buildRequestId, invocationId)));
       }
 
+      BuildEventServiceClient client = createBesClient(besOptions, authTlsOptions);
+      BuildEventArtifactUploader artifactUploader =
+          artifactUploaders.select(protocolOptions.buildEventUploadStrategy);
+
       BuildEventTransport besTransport =
           new BuildEventServiceTransport(
-              createBesClient(besOptions, authTlsOptions),
+              client,
               besOptions.besTimeout,
               besOptions.besLifecycleEvents,
               buildRequestId,
@@ -253,11 +260,11 @@ public abstract class BuildEventServiceModule<T extends BuildEventServiceOptions
               moduleEnvironment,
               clock,
               protocolOptions,
-              pathConverter,
               commandLineReporter,
               besOptions.projectId,
               keywords(besOptions, startupOptionsProvider),
-              besResultsUrl);
+              besResultsUrl,
+              artifactUploader);
       logger.fine("BuildEventServiceTransport was created successfully");
       return besTransport;
     }
diff --git a/src/main/java/com/google/devtools/build/lib/buildeventservice/BuildEventServiceProtoUtil.java b/src/main/java/com/google/devtools/build/lib/buildeventservice/BuildEventServiceProtoUtil.java
index 217fcbd419..dc6172cd08 100644
--- a/src/main/java/com/google/devtools/build/lib/buildeventservice/BuildEventServiceProtoUtil.java
+++ b/src/main/java/com/google/devtools/build/lib/buildeventservice/BuildEventServiceProtoUtil.java
@@ -107,6 +107,11 @@ public final class BuildEventServiceProtoUtil {
     return streamSequenceNumber.getAndIncrement();
   }
 
+  /** Creates a PublishBuildToolEventStreamRequest from a packed bazel event. */
+  public PublishBuildToolEventStreamRequest bazelEvent(Any packedEvent, int sequenceNumber) {
+    return bazelEvent(sequenceNumber, packedEvent);
+  }
+
   @VisibleForTesting
   public PublishBuildToolEventStreamRequest bazelEvent(int sequenceNumber, Any packedEvent) {
     return publishBuildToolEventStreamRequest(
@@ -114,7 +119,6 @@ public final class BuildEventServiceProtoUtil {
         com.google.devtools.build.v1.BuildEvent.newBuilder().setBazelEvent(packedEvent));
   }
 
-  @VisibleForTesting
   public PublishBuildToolEventStreamRequest streamFinished(int sequenceNumber) {
     return publishBuildToolEventStreamRequest(
         sequenceNumber,
diff --git a/src/main/java/com/google/devtools/build/lib/buildeventservice/BuildEventServiceTransport.java b/src/main/java/com/google/devtools/build/lib/buildeventservice/BuildEventServiceTransport.java
index 37355cc6f6..acf5686671 100644
--- a/src/main/java/com/google/devtools/build/lib/buildeventservice/BuildEventServiceTransport.java
+++ b/src/main/java/com/google/devtools/build/lib/buildeventservice/BuildEventServiceTransport.java
@@ -27,6 +27,7 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.ListeningExecutorService;
 import com.google.common.util.concurrent.SettableFuture;
@@ -34,6 +35,7 @@ import com.google.devtools.build.lib.buildeventservice.client.BuildEventServiceC
 import com.google.devtools.build.lib.buildeventstream.ArtifactGroupNamer;
 import com.google.devtools.build.lib.buildeventstream.BuildCompletingEvent;
 import com.google.devtools.build.lib.buildeventstream.BuildEvent;
+import com.google.devtools.build.lib.buildeventstream.BuildEventArtifactUploader;
 import com.google.devtools.build.lib.buildeventstream.BuildEventContext;
 import com.google.devtools.build.lib.buildeventstream.BuildEventProtocolOptions;
 import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos;
@@ -48,12 +50,14 @@ import com.google.devtools.build.lib.util.AbruptExitException;
 import com.google.devtools.build.lib.util.ExitCode;
 import com.google.devtools.build.lib.util.JavaSleeper;
 import com.google.devtools.build.lib.util.Sleeper;
+import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.build.v1.BuildStatus.Result;
 import com.google.devtools.build.v1.PublishBuildToolEventStreamRequest;
 import com.google.devtools.build.v1.PublishBuildToolEventStreamResponse;
 import com.google.devtools.build.v1.PublishLifecycleEventRequest;
 import com.google.protobuf.Any;
 import io.grpc.Status;
+import java.io.IOException;
 import java.time.Duration;
 import java.util.Deque;
 import java.util.Set;
@@ -95,7 +99,7 @@ public class BuildEventServiceTransport implements BuildEventTransport {
   private final ModuleEnvironment moduleEnvironment;
   private final EventHandler commandLineReporter;
   private final BuildEventProtocolOptions protocolOptions;
-  private final PathConverter pathConverter;
+  private final BuildEventArtifactUploader artifactUploader;
   private final Sleeper sleeper;
   /** Contains all pendingAck events that might be retried in case of failures. */
   private ConcurrentLinkedDeque<InternalOrderedBuildEvent> pendingAck;
@@ -131,11 +135,11 @@ public class BuildEventServiceTransport implements BuildEventTransport {
       ModuleEnvironment moduleEnvironment,
       Clock clock,
       BuildEventProtocolOptions protocolOptions,
-      PathConverter pathConverter,
       EventHandler commandLineReporter,
       @Nullable String projectId,
       Set<String> keywords,
-      @Nullable String besResultsUrl) {
+      @Nullable String besResultsUrl,
+      BuildEventArtifactUploader artifactUploader) {
     this(
         besClient,
         uploadTimeout,
@@ -146,12 +150,12 @@ public class BuildEventServiceTransport implements BuildEventTransport {
         moduleEnvironment,
         clock,
         protocolOptions,
-        pathConverter,
         commandLineReporter,
         projectId,
         keywords,
-        new JavaSleeper(),
-        besResultsUrl);
+        besResultsUrl,
+        artifactUploader,
+        new JavaSleeper());
   }
 
   @VisibleForTesting
@@ -165,12 +169,12 @@ public class BuildEventServiceTransport implements BuildEventTransport {
       ModuleEnvironment moduleEnvironment,
       Clock clock,
       BuildEventProtocolOptions protocolOptions,
-      PathConverter pathConverter,
       EventHandler commandLineReporter,
       @Nullable String projectId,
       Set<String> keywords,
-      Sleeper sleeper,
-      @Nullable String besResultsUrl) {
+      @Nullable String besResultsUrl,
+      BuildEventArtifactUploader artifactUploader,
+      Sleeper sleeper) {
     this.besClient = besClient;
     this.besProtoUtil =
         new BuildEventServiceProtoUtil(
@@ -187,9 +191,9 @@ public class BuildEventServiceTransport implements BuildEventTransport {
     // TODO(buchgr): Fix it.
     this.uploaderExecutorService = listeningDecorator(Executors.newFixedThreadPool(2));
     this.protocolOptions = protocolOptions;
-    this.pathConverter = pathConverter;
     this.invocationResult = UNKNOWN_STATUS;
     this.uploadTimeout = uploadTimeout;
+    this.artifactUploader = artifactUploader;
     this.sleeper = sleeper;
     this.besResultsUrl = besResultsUrl;
   }
@@ -480,6 +484,27 @@ public class BuildEventServiceTransport implements BuildEventTransport {
       do {
         orderedBuildEvent = pendingSend.pollFirst(STREAMING_RPC_POLL_IN_SECS, TimeUnit.SECONDS);
         if (orderedBuildEvent != null) {
+          final PathConverter pathConverter;
+          try {
+            pathConverter = artifactUploader.upload(orderedBuildEvent.referencedLocalFiles());
+          } catch (IOException e) {
+            logger.log(
+                Level.WARNING,
+                String.format(
+                    "Aborting publishBuildToolEventStream RPC because of a failure to "
+                        + "upload referenced artifacts: %s",
+                    e.getMessage()),
+                e);
+            besClient.abortStream(Status.INTERNAL.augmentDescription(e.getMessage()));
+            throw e;
+          } catch (InterruptedException e) {
+            // By convention the interrupted flag should have been cleared,
+            // but just to be sure clear it.
+            Thread.interrupted();
+            String additionalDetails = "Uploading referenced artifacts";
+            besClient.abortStream(Status.CANCELLED.augmentDescription(additionalDetails));
+            throw e;
+          }
           pendingAck.add(orderedBuildEvent);
           besClient.sendOverStream(orderedBuildEvent.serialize(pathConverter));
         }
@@ -621,6 +646,8 @@ public class BuildEventServiceTransport implements BuildEventTransport {
 
     int getSequenceNumber();
 
+    Set<Path> referencedLocalFiles();
+
     PublishBuildToolEventStreamRequest serialize(PathConverter pathConverter);
   }
 
@@ -647,6 +674,11 @@ public class BuildEventServiceTransport implements BuildEventTransport {
     }
 
     @Override
+    public Set<Path> referencedLocalFiles() {
+      return event.referencedLocalFiles();
+    }
+
+    @Override
     public PublishBuildToolEventStreamRequest serialize(PathConverter pathConverter) {
       BuildEventStreamProtos.BuildEvent eventProto =
           event.asStreamProto(
@@ -666,7 +698,7 @@ public class BuildEventServiceTransport implements BuildEventTransport {
                   return protocolOptions;
                 }
               });
-      return besProtoUtil.bazelEvent(sequenceNumber, Any.pack(eventProto));
+      return besProtoUtil.bazelEvent(Any.pack(eventProto), sequenceNumber);
     }
   }
 
@@ -688,6 +720,11 @@ public class BuildEventServiceTransport implements BuildEventTransport {
     }
 
     @Override
+    public Set<Path> referencedLocalFiles() {
+      return ImmutableSet.of();
+    }
+
+    @Override
     public PublishBuildToolEventStreamRequest serialize(PathConverter pathConverter) {
       return besProtoUtil.streamFinished(sequenceNumber);
     }