Using an ActionFileInputCache for SHA1 digests used with remote execution.

--
MOS_MIGRATED_REVID=139613925

6 files changed, 66 insertions, 13 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/remote/ContentDigests.java b/src/main/java/com/google/devtools/build/lib/remote/ContentDigests.java
index 92941891f4..e7772b23e8 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/ContentDigests.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/ContentDigests.java
@@ -82,9 +82,7 @@ public final class ContentDigests {
   }
 
   public static String toHexString(ContentDigest digest) {
-    return digest.getSizeBytes() > 0
-        ? HashCode.fromBytes(digest.getDigest().toByteArray()).toString()
-        : "";
+    return HashCode.fromBytes(digest.getDigest().toByteArray()).toString();
   }
 
   public static String toString(ContentDigest digest) {
diff --git a/src/main/java/com/google/devtools/build/lib/remote/GrpcActionCache.java b/src/main/java/com/google/devtools/build/lib/remote/GrpcActionCache.java
index 5004748b00..0563c6848b 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/GrpcActionCache.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/GrpcActionCache.java
@@ -629,7 +629,6 @@ public final class GrpcActionCache implements RemoteActionCache {
     ExecutionCacheReply reply = stub.getCachedResult(request);
     ExecutionCacheStatus status = reply.getStatus();
     if (!status.getSucceeded()
-        && status.getError() != ExecutionCacheStatus.ErrorCode.UNSUPPORTED
         && status.getError() != ExecutionCacheStatus.ErrorCode.MISSING_RESULT) {
       throw new RuntimeException(status.getErrorDetail());
     }
@@ -650,7 +649,8 @@ public final class GrpcActionCache implements RemoteActionCache {
             .build();
     ExecutionCacheSetReply reply = stub.setCachedResult(request);
     ExecutionCacheStatus status = reply.getStatus();
-    if (!status.getSucceeded()) {
+    if (!status.getSucceeded()
+        && status.getError() != ExecutionCacheStatus.ErrorCode.UNSUPPORTED) {
       throw new RuntimeException(status.getErrorDetail());
     }
   }
diff --git a/src/main/java/com/google/devtools/build/lib/remote/README.md b/src/main/java/com/google/devtools/build/lib/remote/README.md
index f778edee2a..b0598acb21 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/README.md
+++ b/src/main/java/com/google/devtools/build/lib/remote/README.md
@@ -9,7 +9,8 @@ this step.
 
 - Then you run Bazel pointing to the Hazelcast server.
 
-    bazel build --hazelcast_node=localhost:5701 --spawn_strategy=remote \
+    bazel --host_jvm_args=-Dbazel.DigestFunction=SHA1 build \
+        --hazelcast_node=localhost:5701 --spawn_strategy=remote \
         src/tools/generate_workspace:all
 
 Above command will build generate_workspace with remote spawn strategy that uses
@@ -25,6 +26,33 @@ with default configuration.
 
 - Then run Bazel pointing to the Hazelcast server and remote worker.
 
-        bazel build --hazelcast_node=localhost:5701 \
+        bazel --host_jvm_args=-Dbazel.DigestFunction=SHA1 build \
+            --hazelcast_node=localhost:5701 \
             --remote_worker=localhost:8080 \
             --spawn_strategy=remote src/tools/generate_workspace:all
+
+# How to run a remote worker with a remote cache server.
+
+- First you need to run a standalone Hazelcast server with default
+configuration. If you already have a separate Hazelcast cluster you can skip
+this step.
+
+    java -cp third_party/hazelcast/hazelcast-3.6.4.jar \
+        com.hazelcast.core.server.StartServer
+
+- Then run the remote cache server:
+
+    bazel-bin/src/tools/remote_worker/remote_cache --listen_port 8081
+
+- The run the remote worker:
+
+    bazel-bin/src/tools/remote_worker/remote_worker \
+        --work_path=/tmp/remote --listen_port 8080
+
+- Then run Bazel pointing to the cache server and remote worker.
+
+        bazel --host_jvm_args=-Dbazel.DigestFunction=SHA1 build \
+            --hazelcast_node=localhost:5701 \
+            --remote_worker=localhost:8080 \
+            --remote_cache=localhost:8081 \
+            --spawn_strategy=remote src/tools/generate_workspace:all
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionCache.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionCache.java
index 62790e8a76..cc0720b57f 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionCache.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionCache.java
@@ -61,9 +61,8 @@ interface RemoteActionCache {
       throws IOException, InterruptedException;
 
   /**
-   * Put the file contents cache if it is not already in it. No-op if the file is already stored in
-   * cache. The given path must be a full absolute path. Note: this is horribly inefficient, need to
-   * patch through an overload that uses an ActionInputFile cache to compute the digests!
+   * Put the file contents in cache if it is not already in it. No-op if the file is already stored
+   * in cache. The given path must be a full absolute path.
    *
    * @return The key for fetching the file contents blob from cache.
    */
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
index 52343b0799..7d2845c6ff 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java
@@ -24,6 +24,10 @@ import com.google.devtools.build.lib.events.Event;
 import com.google.devtools.build.lib.runtime.BlazeModule;
 import com.google.devtools.build.lib.runtime.Command;
 import com.google.devtools.build.lib.runtime.CommandEnvironment;
+import com.google.devtools.build.lib.util.AbruptExitException;
+import com.google.devtools.build.lib.util.ExitCode;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.FileSystem.HashFunction;
 import com.google.devtools.common.options.OptionsBase;
 
 /** RemoteModule provides distributed cache and remote execution for Bazel. */
@@ -66,8 +70,17 @@ public final class RemoteModule extends BlazeModule {
       }
       // Otherwise actionCache remains null and remote caching/execution are disabled.
 
-      if (actionCache != null && RemoteWorkExecutor.isRemoteExecutionOptions(options)) {
-        workExecutor = new RemoteWorkExecutor(options);
+      if (actionCache != null) {
+        HashFunction hf = FileSystem.getDigestFunction();
+        if (hf != HashFunction.SHA1) {
+          env.getBlazeModuleEnvironment().exit(new AbruptExitException(
+              "Remote cache/execution requires SHA1 digests, got " + hf
+              + ", run with --host_jvm_args=-Dbazel.DigestFunction=SHA1",
+              ExitCode.COMMAND_LINE_ERROR));
+        }
+        if (RemoteWorkExecutor.isRemoteExecutionOptions(options)) {
+          workExecutor = new RemoteWorkExecutor(options);
+        }
       }
     } catch (InvalidConfigurationException e) {
       env.getReporter().handle(Event.warn(e.toString()));
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnStrategy.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnStrategy.java
index d2b30cef97..a4dcf80829 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnStrategy.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnStrategy.java
@@ -21,8 +21,10 @@ import com.google.common.collect.ImmutableMap;
 import com.google.devtools.build.lib.actions.ActionExecutionContext;
 import com.google.devtools.build.lib.actions.ActionInput;
 import com.google.devtools.build.lib.actions.ActionInputHelper;
+import com.google.devtools.build.lib.actions.ActionStatusMessage;
 import com.google.devtools.build.lib.actions.ExecException;
 import com.google.devtools.build.lib.actions.ExecutionStrategy;
+import com.google.devtools.build.lib.actions.Executor;
 import com.google.devtools.build.lib.actions.Spawn;
 import com.google.devtools.build.lib.actions.SpawnActionContext;
 import com.google.devtools.build.lib.actions.Spawns;
@@ -128,6 +130,11 @@ final class RemoteSpawnStrategy implements SpawnActionContext {
             .handle(
                 Event.warn(
                     spawn.getMnemonic() + " unsupported operation for action cache (" + e + ")"));
+      } catch (StatusRuntimeException e) {
+        actionExecutionContext
+            .getExecutor()
+            .getEventHandler()
+            .handle(Event.warn(spawn.getMnemonic() + " failed uploading results (" + e + ")"));
       }
     }
   }
@@ -147,6 +154,11 @@ final class RemoteSpawnStrategy implements SpawnActionContext {
     }
   }
 
+  @Override
+  public String toString() {
+    return "remote";
+  }
+
   /** Executes the given {@code spawn}. */
   @Override
   public void exec(Spawn spawn, ActionExecutionContext actionExecutionContext)
@@ -158,7 +170,10 @@ final class RemoteSpawnStrategy implements SpawnActionContext {
 
     ActionKey actionKey = null;
     String mnemonic = spawn.getMnemonic();
-    EventHandler eventHandler = actionExecutionContext.getExecutor().getEventHandler();
+    Executor executor = actionExecutionContext.getExecutor();
+    EventHandler eventHandler = executor.getEventHandler();
+    executor.getEventBus().post(
+        ActionStatusMessage.runningStrategy(spawn.getResourceOwner(), "remote"));
 
     try {
       // Temporary hack: the TreeNodeRepository should be created and maintained upstream!