diff options
9 files changed, 72 insertions, 16 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/remote/ContentDigests.java b/src/main/java/com/google/devtools/build/lib/remote/ContentDigests.java index 92941891f4..e7772b23e8 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/ContentDigests.java +++ b/src/main/java/com/google/devtools/build/lib/remote/ContentDigests.java @@ -82,9 +82,7 @@ public final class ContentDigests { } public static String toHexString(ContentDigest digest) { - return digest.getSizeBytes() > 0 - ? HashCode.fromBytes(digest.getDigest().toByteArray()).toString() - : ""; + return HashCode.fromBytes(digest.getDigest().toByteArray()).toString(); } public static String toString(ContentDigest digest) { diff --git a/src/main/java/com/google/devtools/build/lib/remote/GrpcActionCache.java b/src/main/java/com/google/devtools/build/lib/remote/GrpcActionCache.java index 5004748b00..0563c6848b 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/GrpcActionCache.java +++ b/src/main/java/com/google/devtools/build/lib/remote/GrpcActionCache.java @@ -629,7 +629,6 @@ public final class GrpcActionCache implements RemoteActionCache { ExecutionCacheReply reply = stub.getCachedResult(request); ExecutionCacheStatus status = reply.getStatus(); if (!status.getSucceeded() - && status.getError() != ExecutionCacheStatus.ErrorCode.UNSUPPORTED && status.getError() != ExecutionCacheStatus.ErrorCode.MISSING_RESULT) { throw new RuntimeException(status.getErrorDetail()); } @@ -650,7 +649,8 @@ public final class GrpcActionCache implements RemoteActionCache { .build(); ExecutionCacheSetReply reply = stub.setCachedResult(request); ExecutionCacheStatus status = reply.getStatus(); - if (!status.getSucceeded()) { + if (!status.getSucceeded() + && status.getError() != ExecutionCacheStatus.ErrorCode.UNSUPPORTED) { throw new RuntimeException(status.getErrorDetail()); } } diff --git a/src/main/java/com/google/devtools/build/lib/remote/README.md b/src/main/java/com/google/devtools/build/lib/remote/README.md index f778edee2a..b0598acb21 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/README.md +++ b/src/main/java/com/google/devtools/build/lib/remote/README.md @@ -9,7 +9,8 @@ this step. - Then you run Bazel pointing to the Hazelcast server. - bazel build --hazelcast_node=localhost:5701 --spawn_strategy=remote \ + bazel --host_jvm_args=-Dbazel.DigestFunction=SHA1 build \ + --hazelcast_node=localhost:5701 --spawn_strategy=remote \ src/tools/generate_workspace:all Above command will build generate_workspace with remote spawn strategy that uses @@ -25,6 +26,33 @@ with default configuration. - Then run Bazel pointing to the Hazelcast server and remote worker. - bazel build --hazelcast_node=localhost:5701 \ + bazel --host_jvm_args=-Dbazel.DigestFunction=SHA1 build \ + --hazelcast_node=localhost:5701 \ --remote_worker=localhost:8080 \ --spawn_strategy=remote src/tools/generate_workspace:all + +# How to run a remote worker with a remote cache server. + +- First you need to run a standalone Hazelcast server with default +configuration. If you already have a separate Hazelcast cluster you can skip +this step. + + java -cp third_party/hazelcast/hazelcast-3.6.4.jar \ + com.hazelcast.core.server.StartServer + +- Then run the remote cache server: + + bazel-bin/src/tools/remote_worker/remote_cache --listen_port 8081 + +- The run the remote worker: + + bazel-bin/src/tools/remote_worker/remote_worker \ + --work_path=/tmp/remote --listen_port 8080 + +- Then run Bazel pointing to the cache server and remote worker. + + bazel --host_jvm_args=-Dbazel.DigestFunction=SHA1 build \ + --hazelcast_node=localhost:5701 \ + --remote_worker=localhost:8080 \ + --remote_cache=localhost:8081 \ + --spawn_strategy=remote src/tools/generate_workspace:all diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionCache.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionCache.java index 62790e8a76..cc0720b57f 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionCache.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionCache.java @@ -61,9 +61,8 @@ interface RemoteActionCache { throws IOException, InterruptedException; /** - * Put the file contents cache if it is not already in it. No-op if the file is already stored in - * cache. The given path must be a full absolute path. Note: this is horribly inefficient, need to - * patch through an overload that uses an ActionInputFile cache to compute the digests! + * Put the file contents in cache if it is not already in it. No-op if the file is already stored + * in cache. The given path must be a full absolute path. * * @return The key for fetching the file contents blob from cache. */ diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java index 52343b0799..7d2845c6ff 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java @@ -24,6 +24,10 @@ import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.runtime.BlazeModule; import com.google.devtools.build.lib.runtime.Command; import com.google.devtools.build.lib.runtime.CommandEnvironment; +import com.google.devtools.build.lib.util.AbruptExitException; +import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.FileSystem.HashFunction; import com.google.devtools.common.options.OptionsBase; /** RemoteModule provides distributed cache and remote execution for Bazel. */ @@ -66,8 +70,17 @@ public final class RemoteModule extends BlazeModule { } // Otherwise actionCache remains null and remote caching/execution are disabled. - if (actionCache != null && RemoteWorkExecutor.isRemoteExecutionOptions(options)) { - workExecutor = new RemoteWorkExecutor(options); + if (actionCache != null) { + HashFunction hf = FileSystem.getDigestFunction(); + if (hf != HashFunction.SHA1) { + env.getBlazeModuleEnvironment().exit(new AbruptExitException( + "Remote cache/execution requires SHA1 digests, got " + hf + + ", run with --host_jvm_args=-Dbazel.DigestFunction=SHA1", + ExitCode.COMMAND_LINE_ERROR)); + } + if (RemoteWorkExecutor.isRemoteExecutionOptions(options)) { + workExecutor = new RemoteWorkExecutor(options); + } } } catch (InvalidConfigurationException e) { env.getReporter().handle(Event.warn(e.toString())); diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnStrategy.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnStrategy.java index d2b30cef97..a4dcf80829 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnStrategy.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnStrategy.java @@ -21,8 +21,10 @@ import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.actions.ActionExecutionContext; import com.google.devtools.build.lib.actions.ActionInput; import com.google.devtools.build.lib.actions.ActionInputHelper; +import com.google.devtools.build.lib.actions.ActionStatusMessage; import com.google.devtools.build.lib.actions.ExecException; import com.google.devtools.build.lib.actions.ExecutionStrategy; +import com.google.devtools.build.lib.actions.Executor; import com.google.devtools.build.lib.actions.Spawn; import com.google.devtools.build.lib.actions.SpawnActionContext; import com.google.devtools.build.lib.actions.Spawns; @@ -128,6 +130,11 @@ final class RemoteSpawnStrategy implements SpawnActionContext { .handle( Event.warn( spawn.getMnemonic() + " unsupported operation for action cache (" + e + ")")); + } catch (StatusRuntimeException e) { + actionExecutionContext + .getExecutor() + .getEventHandler() + .handle(Event.warn(spawn.getMnemonic() + " failed uploading results (" + e + ")")); } } } @@ -147,6 +154,11 @@ final class RemoteSpawnStrategy implements SpawnActionContext { } } + @Override + public String toString() { + return "remote"; + } + /** Executes the given {@code spawn}. */ @Override public void exec(Spawn spawn, ActionExecutionContext actionExecutionContext) @@ -158,7 +170,10 @@ final class RemoteSpawnStrategy implements SpawnActionContext { ActionKey actionKey = null; String mnemonic = spawn.getMnemonic(); - EventHandler eventHandler = actionExecutionContext.getExecutor().getEventHandler(); + Executor executor = actionExecutionContext.getExecutor(); + EventHandler eventHandler = executor.getEventHandler(); + executor.getEventBus().post( + ActionStatusMessage.runningStrategy(spawn.getResourceOwner(), "remote")); try { // Temporary hack: the TreeNodeRepository should be created and maintained upstream! diff --git a/src/test/java/com/google/devtools/build/lib/BUILD b/src/test/java/com/google/devtools/build/lib/BUILD index 9db93c01ac..47e35eaf96 100644 --- a/src/test/java/com/google/devtools/build/lib/BUILD +++ b/src/test/java/com/google/devtools/build/lib/BUILD @@ -1041,6 +1041,7 @@ java_test( ":foundations_testutil", ":test_runner", ":testutil", + "//src/main/java/com/google/devtools/build/lib:build-base", "//src/main/java/com/google/devtools/build/lib:preconditions", "//src/main/java/com/google/devtools/build/lib:vfs", "//src/main/java/com/google/devtools/build/lib/actions", diff --git a/src/test/shell/bazel/remote_execution_test.sh b/src/test/shell/bazel/remote_execution_test.sh index 75bef52e9a..897bdbb0e6 100755 --- a/src/test/shell/bazel/remote_execution_test.sh +++ b/src/test/shell/bazel/remote_execution_test.sh @@ -69,7 +69,7 @@ function test_cc_binary() { cp bazel-bin/a/test ${TEST_TMPDIR}/test_expected bazel clean --expunge - bazel build \ + bazel --host_jvm_args=-Dbazel.DigestFunction=SHA1 build \ --spawn_strategy=remote \ --hazelcast_node=localhost:${hazelcast_port} \ --remote_worker=localhost:${worker_port} \ diff --git a/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/RemoteCache.java b/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/RemoteCache.java index 2e933f39c5..023145f230 100644 --- a/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/RemoteCache.java +++ b/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/RemoteCache.java @@ -188,8 +188,10 @@ public class RemoteCache { offset == chunk.getOffset(), "Missing input chunk for digest %s", ContentDigests.toString(digest)); - chunk.getData().copyTo(blob, (int) offset); - offset = (offset + chunk.getData().size()) % digest.getSizeBytes(); + if (digest.getSizeBytes() > 0) { + chunk.getData().copyTo(blob, (int) offset); + offset = (offset + chunk.getData().size()) % digest.getSizeBytes(); + } if (offset == 0) { ContentDigest uploadedDigest = cache.uploadBlob(blob); Preconditions.checkArgument( |