Remote execution: post CAS links to the BEP instead of local file URIs

Also use DigestUtils for performance, which has a static cache of the digests.
This isn't ideal, but necessary for now to avoid computing the digests multiple
times. We'll need to clean this up afterwards by including the digests in the
posted events, and making them available to the PathConverter. I have some
changes towards that (as well as some submitted already), but it requires
quite some refactoring.

RELNOTES: Bazel posts links to the CAS to the BEP if remote caching / execution is enabled
PiperOrigin-RevId: 158513165

3 files changed, 63 insertions, 5 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/remote/BUILD b/src/main/java/com/google/devtools/build/lib/remote/BUILD
index d2e82efa60..9951598650 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/remote/BUILD
@@ -13,6 +13,7 @@ java_library(
     deps = [
         "//src/main/java/com/google/devtools/build/lib:auth_and_tls_options",
         "//src/main/java/com/google/devtools/build/lib:build-base",
+        "//src/main/java/com/google/devtools/build/lib:buildeventstream",
         "//src/main/java/com/google/devtools/build/lib:concurrent",
         "//src/main/java/com/google/devtools/build/lib:events",
         "//src/main/java/com/google/devtools/build/lib:io",
diff --git a/src/main/java/com/google/devtools/build/lib/remote/Digests.java b/src/main/java/com/google/devtools/build/lib/remote/Digests.java
index 9121bc2782..1c70db306e 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/Digests.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/Digests.java
@@ -20,6 +20,7 @@ import com.google.common.hash.HashCode;
 import com.google.common.hash.Hashing;
 import com.google.devtools.build.lib.actions.ActionInput;
 import com.google.devtools.build.lib.actions.ActionInputFileCache;
+import com.google.devtools.build.lib.actions.cache.DigestUtils;
 import com.google.devtools.build.lib.actions.cache.VirtualActionInput;
 import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
 import com.google.devtools.build.lib.vfs.Path;
@@ -39,7 +40,9 @@ public final class Digests {
   }
 
   public static Digest computeDigest(Path file) throws IOException {
-    return buildDigest(file.getSHA1Digest(), file.getFileSize());
+    long fileSize = file.getFileSize();
+    byte[] digest = DigestUtils.getDigestOrFail(file, fileSize);
+    return buildDigest(digest, fileSize);
   }
 
   public static Digest computeDigest(VirtualActionInput input) throws IOException {
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
index ae0299721e..9312cb2e55 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
@@ -17,29 +17,79 @@ package com.google.devtools.build.lib.remote;
 import com.google.common.collect.ImmutableList;
 import com.google.common.eventbus.Subscribe;
 import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions;
+import com.google.devtools.build.lib.buildeventstream.PathConverter;
 import com.google.devtools.build.lib.buildtool.BuildRequest;
 import com.google.devtools.build.lib.buildtool.buildevent.BuildStartingEvent;
 import com.google.devtools.build.lib.exec.ExecutorBuilder;
 import com.google.devtools.build.lib.runtime.BlazeModule;
 import com.google.devtools.build.lib.runtime.Command;
 import com.google.devtools.build.lib.runtime.CommandEnvironment;
+import com.google.devtools.build.lib.runtime.ServerBuilder;
 import com.google.devtools.build.lib.util.AbruptExitException;
 import com.google.devtools.build.lib.util.ExitCode;
 import com.google.devtools.build.lib.vfs.FileSystem;
 import com.google.devtools.build.lib.vfs.FileSystem.HashFunction;
+import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.common.options.OptionsBase;
+import com.google.devtools.common.options.OptionsProvider;
+import com.google.devtools.remoteexecution.v1test.Digest;
+import java.io.IOException;
 
 /** RemoteModule provides distributed cache and remote execution for Bazel. */
 public final class RemoteModule extends BlazeModule {
+  private final class CasPathConverter implements PathConverter {
+    private CasPathConverter() {
+    }
+
+    @Override
+    public String apply(Path path) {
+      if (options == null || !remoteEnabled(options)) {
+        return null;
+      }
+      String server = options.remoteCache;
+      String remoteInstanceName = options.remoteInstanceName;
+      try {
+        Digest digest = Digests.computeDigest(path);
+        return remoteInstanceName.isEmpty()
+            ? String.format(
+                "//%s/blobs/%s/%d",
+                server,
+                digest.getHash(),
+                digest.getSizeBytes())
+            : String.format(
+                "//%s/projects/%s/blobs/%s/%d",
+                server,
+                remoteInstanceName,
+                digest.getHash(),
+                digest.getSizeBytes());
+      } catch (IOException e) {
+        // TODO(ulfjack): Don't fail silently!
+        return null;
+      }
+    }
+  }
+
+  private RemoteOptions options;
   private CommandEnvironment env;
 
   @Override
+  public void serverInit(OptionsProvider startupOptions, ServerBuilder builder)
+      throws AbruptExitException {
+    builder.addPathToUriConverter(new CasPathConverter());
+  }
+
+  @Override
   public void beforeCommand(Command command, CommandEnvironment env) {
     this.env = env;
     env.getEventBus().register(this);
   }
 
   @Override
+  public void handleOptions(OptionsProvider optionsProvider) {
+    this.options = optionsProvider.getOptions(RemoteOptions.class);
+  }
+
+  @Override
   public void afterCommand() {
     this.env = null;
   }
@@ -53,8 +103,7 @@ public final class RemoteModule extends BlazeModule {
   public void buildStarting(BuildStartingEvent event) {
     RemoteOptions options = event.getRequest().getOptions(RemoteOptions.class);
 
-    if (SimpleBlobStoreFactory.isRemoteCacheOptions(options)
-        || GrpcActionCache.isRemoteCacheOptions(options)) {
+    if (remoteEnabled(options)) {
       HashFunction hf = FileSystem.getDigestFunction();
       if (hf != HashFunction.SHA1) {
         env.getBlazeModuleEnvironment().exit(new AbruptExitException(
@@ -68,8 +117,13 @@ public final class RemoteModule extends BlazeModule {
   @Override
   public Iterable<Class<? extends OptionsBase>> getCommandOptions(Command command) {
     return "build".equals(command.name())
-        ? ImmutableList.<Class<? extends OptionsBase>>of(RemoteOptions.class,
-        AuthAndTLSOptions.class)
+        ? ImmutableList.<Class<? extends OptionsBase>>of(
+            RemoteOptions.class, AuthAndTLSOptions.class)
         : ImmutableList.<Class<? extends OptionsBase>>of();
   }
+
+  public static boolean remoteEnabled(RemoteOptions options) {
+    return SimpleBlobStoreFactory.isRemoteCacheOptions(options)
+        || GrpcActionCache.isRemoteCacheOptions(options);
+  }
 }