diff options
23 files changed, 709 insertions, 76 deletions
@@ -19,6 +19,25 @@ bind( actual = "//:dummy", ) +# Protobuf code generation for GRPC requires three external labels: +# //external:grpc-java_plugin +# //external:grpc-jar +# //external:guava +bind( + name = "grpc-java-plugin", + actual = "//third_party/grpc:grpc-java-plugin", +) + +bind( + name = "grpc-jar", + actual = "//third_party/grpc:grpc-jar", +) + +bind( + name = "guava", + actual = "//third_party:guava", +) + # For tools/cpp/test/... load("//tools/cpp/test:docker_repository.bzl", "docker_repository") docker_repository() diff --git a/scripts/bootstrap/compile.sh b/scripts/bootstrap/compile.sh index cba3ae3e0a..3862bdec4e 100755 --- a/scripts/bootstrap/compile.sh +++ b/scripts/bootstrap/compile.sh @@ -41,11 +41,13 @@ linux) JAVA_HOME="${JAVA_HOME:-$(readlink -f $(which javac) | sed 's_/bin/javac__')}" if [ "${MACHINE_IS_64BIT}" = 'yes' ]; then PROTOC=${PROTOC:-third_party/protobuf/protoc-linux-x86_64.exe} + GRPC_JAVA_PLUGIN=${GRPC_JAVA_PLUGIN:-third_party/grpc/protoc-gen-grpc-java-0.13.2-linux-x86_64.exe} else if [ "${MACHINE_IS_ARM}" = 'yes' ]; then PROTOC=${PROTOC:-third_party/protobuf/protoc-linux-arm32.exe} else PROTOC=${PROTOC:-third_party/protobuf/protoc-linux-x86_32.exe} + GRPC_JAVA_PLUGIN=${GRPC_JAVA_PLUGIN:-third_party/grpc/protoc-gen-grpc-java-0.13.2-linux-x86_32.exe} fi fi ;; @@ -57,6 +59,7 @@ freebsd) # We choose the 32-bit version for maximum compatiblity since 64-bit # linux binaries are only supported in FreeBSD-11. PROTOC=${PROTOC:-third_party/protobuf/protoc-linux-x86_32.exe} + GRPC_JAVA_PLUGIN=${GRPC_JAVA_PLUGIN:-third_party/grpc/protoc-gen-grpc-java-0.13.2-linux-x86_32.exe} ;; darwin) @@ -66,6 +69,7 @@ darwin) fi if [ "${MACHINE_IS_64BIT}" = 'yes' ]; then PROTOC=${PROTOC:-third_party/protobuf/protoc-osx-x86_64.exe} + GRPC_JAVA_PLUGIN=${GRPC_JAVA_PLUGIN:-third_party/grpc/protoc-gen-grpc-java-0.13.2-osx-x86_64.exe} else PROTOC=${PROTOC:-third_party/protobuf/protoc-osx-x86_32.exe} fi @@ -80,14 +84,19 @@ msys*|mingw*) # We do not use the JNI library on Windows. if [ "${MACHINE_IS_64BIT}" = 'yes' ]; then PROTOC=${PROTOC:-third_party/protobuf/protoc-windows-x86_64.exe} + GRPC_JAVA_PLUGIN=${GRPC_JAVA_PLUGIN:-third_party/grpc/protoc-gen-grpc-java-0.13.2-windows-x86_64.exe} else PROTOC=${PROTOC:-third_party/protobuf/protoc-windows-x86_32.exe} + GRPC_JAVA_PLUGIN=${GRPC_JAVA_PLUGIN:-third_party/grpc/protoc-gen-grpc-java-0.13.2-windows-x86_32.exe} fi esac [[ -x "${PROTOC-}" ]] \ || fail "Protobuf compiler not found in ${PROTOC-}" +[[ -x "${GRPC_JAVA_PLUGIN-}" ]] \ + || fail "gRPC Java plugin not found in ${GRPC_JAVA_PLUGIN-}" + # Check that javac -version returns a upper version than $JAVA_VERSION. get_java_version [ ${JAVA_VERSION#*.} -le ${JAVAC_VERSION#*.} ] || \ @@ -168,7 +177,9 @@ function create_deploy_jar() { if [ -z "${BAZEL_SKIP_JAVA_COMPILATION}" ]; then log "Compiling Java stubs for protocol buffers..." for f in $PROTO_FILES ; do - run "${PROTOC}" -Isrc/main/protobuf/ --java_out=${OUTPUT_DIR}/src "$f" + run "${PROTOC}" -Isrc/main/protobuf/ --java_out=${OUTPUT_DIR}/src \ + --plugin=protoc-gen-grpc="${GRPC_JAVA_PLUGIN-}" \ + --grpc_out=${OUTPUT_DIR}/src "$f" done java_compilation "Bazel Java" "$DIRS" "$EXCLUDE_FILES" "$LIBRARY_JARS" "${OUTPUT_DIR}" @@ -14,6 +14,7 @@ filegroup( visibility = [ "//src/test/java:__pkg__", "//src/tools/generate_workspace:__pkg__", + "//src/tools/remote_worker:__subpackages__", ], ) diff --git a/src/main/java/com/google/devtools/build/lib/remote/BUILD b/src/main/java/com/google/devtools/build/lib/remote/BUILD index 40fd1b4476..b20fa704f6 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/BUILD +++ b/src/main/java/com/google/devtools/build/lib/remote/BUILD @@ -5,6 +5,7 @@ package( java_library( name = "remote", srcs = glob(["*.java"]), + tags = ["bazel"], deps = [ "//src/main/java/com/google/devtools/build/lib:build-base", "//src/main/java/com/google/devtools/build/lib:concurrent", @@ -12,6 +13,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib:io", "//src/main/java/com/google/devtools/build/lib:packages-internal", "//src/main/java/com/google/devtools/build/lib:runtime", + "//src/main/java/com/google/devtools/build/lib:shell", "//src/main/java/com/google/devtools/build/lib:util", "//src/main/java/com/google/devtools/build/lib:vfs", "//src/main/java/com/google/devtools/build/lib/actions", @@ -23,6 +25,7 @@ java_library( "//third_party:gson", "//third_party:guava", "//third_party:hazelcast", + "//third_party/grpc:grpc-jar", "//third_party/protobuf", ], ) diff --git a/src/main/java/com/google/devtools/build/lib/remote/HazelcastCacheFactory.java b/src/main/java/com/google/devtools/build/lib/remote/HazelcastCacheFactory.java index e524770652..f4e34867e1 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/HazelcastCacheFactory.java +++ b/src/main/java/com/google/devtools/build/lib/remote/HazelcastCacheFactory.java @@ -26,14 +26,14 @@ import java.util.concurrent.ConcurrentMap; * A factory class for providing a {@link ConcurrentMap} object implemented by Hazelcast. * Hazelcast will work as a distributed memory cache. */ -final class HazelcastCacheFactory { +public final class HazelcastCacheFactory { private static final String CACHE_NAME = "hazelcast-build-cache"; - static ConcurrentMap<String, byte[]> create(RemoteOptions options) { + public static ConcurrentMap<String, byte[]> create(RemoteOptions options) { HazelcastInstance instance; if (options.hazelcastNode != null) { - // If --hazelast_node is then create a client instance. + // If --hazelcast_node is then create a client instance. ClientConfig config = new ClientConfig(); ClientNetworkConfig net = config.getNetworkConfig(); net.addAddress(options.hazelcastNode.split(",")); diff --git a/src/main/java/com/google/devtools/build/lib/remote/MemcacheActionCache.java b/src/main/java/com/google/devtools/build/lib/remote/MemcacheActionCache.java index e83ad69987..e821f1289f 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/MemcacheActionCache.java +++ b/src/main/java/com/google/devtools/build/lib/remote/MemcacheActionCache.java @@ -39,7 +39,7 @@ import java.util.concurrent.Semaphore; * The thread satefy is guaranteed by the underlying memcache client. */ @ThreadSafe -final class MemcacheActionCache implements RemoteActionCache { +public final class MemcacheActionCache implements RemoteActionCache { private final Path execRoot; private final ConcurrentMap<String, byte[]> cache; private static final int MAX_MEMORY_KBYTES = 512 * 1024; @@ -48,7 +48,7 @@ final class MemcacheActionCache implements RemoteActionCache { /** * Construct an action cache using JCache API. */ - MemcacheActionCache( + public MemcacheActionCache( Path execRoot, RemoteOptions options, ConcurrentMap<String, byte[]> cache) { this.execRoot = execRoot; this.cache = cache; diff --git a/src/main/java/com/google/devtools/build/lib/remote/MemcacheWorkExecutor.java b/src/main/java/com/google/devtools/build/lib/remote/MemcacheWorkExecutor.java new file mode 100644 index 0000000000..04c3c3f1d2 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/remote/MemcacheWorkExecutor.java @@ -0,0 +1,208 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.remote; + +import com.google.common.collect.ImmutableMap; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.ActionInputFileCache; +import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.remote.RemoteProtocol.FileEntry; +import com.google.devtools.build.lib.remote.RemoteProtocol.RemoteWorkRequest; +import com.google.devtools.build.lib.remote.RemoteProtocol.RemoteWorkResponse; +import com.google.devtools.build.lib.remote.RemoteWorkGrpc.RemoteWorkFutureStub; +import com.google.devtools.build.lib.shell.Command; +import com.google.devtools.build.lib.shell.CommandException; +import com.google.devtools.build.lib.shell.CommandResult; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; + +import io.grpc.ManagedChannel; +import io.grpc.netty.NettyChannelBuilder; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.nio.file.FileAlreadyExistsException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * Implementation of {@link RemoteWorkExecutor} that uses MemcacheActionCache and gRPC for + * communicating the work, inputs and outputs. + */ +@ThreadSafe +public class MemcacheWorkExecutor implements RemoteWorkExecutor { + /** + * A cache used to store the input and output files as well as the build status + * of the remote work. + */ + protected final MemcacheActionCache cache; + + /** Execution root for running this work locally. */ + private final Path execRoot; + + /** Channel over which to send work to run remotely. */ + private final ManagedChannel channel; + + private static final int MAX_WORK_SIZE_BYTES = 1024 * 1024 * 512; + + /** + * This constructor is used when this class is used in a client. + * It requires a host address and port to connect to a remote service. + */ + private MemcacheWorkExecutor(MemcacheActionCache cache, String host, int port) { + this.cache = cache; + this.execRoot = null; + this.channel = NettyChannelBuilder.forAddress(host, port).usePlaintext(true).build(); + } + + /** + * This constructor is used when this class is used in the remote worker. + * A path to the execution root is needed for executing work locally. + */ + private MemcacheWorkExecutor(MemcacheActionCache cache, Path execRoot) { + this.cache = cache; + this.execRoot = execRoot; + this.channel = null; + } + + /** + * Create an instance of MemcacheWorkExecutor that talks to a remote server. + * @param cache An instance of MemcacheActionCache. + * @param host Hostname of the server to connect to. + * @param port Port of the server to connect to. + * @return An instance of MemcacheWorkExecutor that talks to a remote server. + */ + public static MemcacheWorkExecutor createRemoteWorkExecutor( + MemcacheActionCache cache, String host, int port) { + return new MemcacheWorkExecutor(cache, host, port); + } + + /** + * Create an instance of MemcacheWorkExecutor that runs locally. + * @param cache An instance of MemcacheActionCache. + * @param execRoot Path of the execution root where work is executed. + * @return An instance of MemcacheWorkExecutor tthat runs locally in the execution root. + */ + public static MemcacheWorkExecutor createLocalWorkExecutor( + MemcacheActionCache cache, Path execRoot) { + return new MemcacheWorkExecutor(cache, execRoot); + } + + @Override + public ListenableFuture<RemoteWorkResponse> executeRemotely( + Path execRoot, + ActionInputFileCache actionCache, + String actionOutputKey, + Collection<String> arguments, + Collection<ActionInput> inputs, + ImmutableMap<String, String> environment, + Collection<? extends ActionInput> outputs, + int timeout) + throws IOException, WorkTooLargeException { + RemoteWorkRequest.Builder work = RemoteWorkRequest.newBuilder(); + work.setOutputKey(actionOutputKey); + + long workSize = 0; + for (ActionInput input : inputs) { + if (!(input instanceof Artifact)) { + continue; + } + if (!actionCache.isFile((Artifact) input)) { + continue; + } + workSize += actionCache.getSizeInBytes(input); + } + + if (workSize > MAX_WORK_SIZE_BYTES) { + throw new WorkTooLargeException("Work is too large: " + workSize + " bytes."); + } + + // Save all input files to cache. + for (ActionInput input : inputs) { + Path file = execRoot.getRelative(input.getExecPathString()); + + if (file.isDirectory()) { + // TODO(alpha): Handle this case better. + throw new UnsupportedOperationException( + "Does not support directory artifacts: " + file + "."); + } + + String contentKey = cache.putFileIfNotExist(actionCache, input); + work.addInputFilesBuilder() + .setPath(input.getExecPathString()) + .setContentKey(contentKey) + .setExecutable(file.isExecutable()); + } + + work.addAllArguments(arguments); + work.getMutableEnvironment().putAll(environment); + for (ActionInput output : outputs) { + work.addOutputFilesBuilder().setPath(output.getExecPathString()); + } + + RemoteWorkFutureStub stub = RemoteWorkGrpc.newFutureStub(channel); + work.setTimeout(timeout); + return stub.executeSynchronously(work.build()); + } + + /** Execute a work item locally. */ + public RemoteWorkResponse executeLocally(RemoteWorkRequest work) throws IOException { + ByteArrayOutputStream stdout = new ByteArrayOutputStream(); + ByteArrayOutputStream stderr = new ByteArrayOutputStream(); + try { + // Prepare directories and input files. + for (FileEntry input : work.getInputFilesList()) { + Path file = execRoot.getRelative(input.getPath()); + FileSystemUtils.createDirectoryAndParents(file.getParentDirectory()); + cache.writeFile(input.getContentKey(), file, input.getExecutable()); + } + + List<Path> outputs = new ArrayList<>(work.getOutputFilesList().size()); + for (FileEntry output : work.getOutputFilesList()) { + Path file = execRoot.getRelative(output.getPath()); + if (file.exists()) { + throw new FileAlreadyExistsException("Output file already exists: " + file); + } + FileSystemUtils.createDirectoryAndParents(file.getParentDirectory()); + outputs.add(file); + } + + Command cmd = + new Command( + work.getArgumentsList().toArray(new String[] {}), + work.getEnvironment(), + new File(execRoot.getPathString())); + CommandResult result = + cmd.execute(Command.NO_INPUT, Command.NO_OBSERVER, stdout, stderr, true); + cache.putActionOutput(work.getOutputKey(), execRoot, outputs); + return RemoteWorkResponse.newBuilder() + .setSuccess(result.getTerminationStatus().success()) + .setOut(stdout.toString()) + .setErr(stderr.toString()) + .build(); + } catch (CommandException e) { + return RemoteWorkResponse.newBuilder() + .setSuccess(false) + .setOut(stdout.toString()) + .setErr(stderr.toString()) + .setException(e.toString()) + .build(); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/remote/README.md b/src/main/java/com/google/devtools/build/lib/remote/README.md index c020564c96..43ba87703f 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/README.md +++ b/src/main/java/com/google/devtools/build/lib/remote/README.md @@ -1,15 +1,30 @@ -How to run a standalone Hazelcast server for testing distributed cache. +# How to run a standalone Hazelcast server for testing distributed cache. -* First you need to run a standalone Hazelcast server with JCache API in the -classpath. This will start Hazelcast with the default configuration. +- First you need to run a standalone Hazelcast server with default +configuration. If you already have a separate Hazelcast cluster you can skip +this step. -java -cp third_party/hazelcast/hazelcast-3.5.4.jar \ - com.hazelcast.core.server.StartServer + java -cp third_party/hazelcast/hazelcast-3.5.4.jar \ + com.hazelcast.core.server.StartServer -* Then you run Bazel pointing to the Hazelcast server. +- Then you run Bazel pointing to the Hazelcast server. -bazel build --hazelcast_node=127.0.0.1:5701 --spawn_strategy=remote \ - src/tools/generate_workspace:all + bazel build --hazelcast_node=127.0.0.1:5701 --spawn_strategy=remote \ + src/tools/generate_workspace:all Above command will build generate_workspace with remote spawn strategy that uses Hazelcast as the distributed caching backend. + +# How to run a remote worker for testing remote execution. + +- First run the remote worker. This will start a standalone Hazelcast server +with default configuration. + + bazel-bin/src/tools/remote_worker/remote_worker \ + --work_path=/tmp/remote --listen_port 8080 + +- Then run Bazel pointing to the Hazelcast server and remote worker. + + bazel build --hazelcast_node=127.0.0.1:5701 \ + --remote_worker=127.0.0.1:8080 \ + --spawn_strategy=remote src/tools/generate_workspace:all diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java index 3ed2968f78..88e7f86e58 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java @@ -19,11 +19,15 @@ import com.google.common.eventbus.Subscribe; import com.google.devtools.build.lib.actions.ActionContextProvider; import com.google.devtools.build.lib.buildtool.BuildRequest; import com.google.devtools.build.lib.buildtool.buildevent.BuildStartingEvent; +import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.runtime.BlazeModule; import com.google.devtools.build.lib.runtime.Command; import com.google.devtools.build.lib.runtime.CommandEnvironment; import com.google.devtools.common.options.OptionsBase; +import java.net.URI; +import java.net.URISyntaxException; + /** * RemoteModule provides distributed cache and remote execution for Bazel. */ @@ -37,11 +41,8 @@ public final class RemoteModule extends BlazeModule { @Override public Iterable<ActionContextProvider> getActionContextProviders() { - if (actionCache != null) { - return ImmutableList.<ActionContextProvider>of( - new RemoteActionContextProvider(env, buildRequest, actionCache, workExecutor)); - } - return ImmutableList.<ActionContextProvider>of(); + return ImmutableList.<ActionContextProvider>of( + new RemoteActionContextProvider(env, buildRequest, actionCache, workExecutor)); } @Override @@ -63,12 +64,25 @@ public final class RemoteModule extends BlazeModule { // Don't provide the remote spawn unless at least action cache is initialized. if (actionCache == null && options.hazelcastNode != null) { - actionCache = + MemcacheActionCache cache = new MemcacheActionCache( - env.getExecRoot(), + this.env.getDirectories().getExecRoot(), options, HazelcastCacheFactory.create(options)); - // TODO(alpha): Initialize a RemoteWorkExecutor. + actionCache = cache; + if (workExecutor == null && options.remoteWorker != null) { + try { + URI uri = new URI("dummy://" + options.remoteWorker); + if (uri.getHost() == null || uri.getPort() == -1) { + throw new URISyntaxException("Invalid host or port.", ""); + } + workExecutor = + MemcacheWorkExecutor.createRemoteWorkExecutor(cache, uri.getHost(), uri.getPort()); + } catch (URISyntaxException e) { + env.getReporter() + .handle(Event.warn("Invalid argument for the address of remote worker.")); + } + } } } diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteOptions.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteOptions.java index 933b2b1389..5ca8abf3c3 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteOptions.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteOptions.java @@ -30,10 +30,12 @@ public final class RemoteOptions extends OptionsBase { public String hazelcastNode; @Option( - name = "rest_worker_url", + name = "remote_worker", defaultValue = "null", category = "remote", - help = "URL for the REST worker." + help = + "Hostname and port number of remote worker in the form of host:port. " + + "For client mode only." ) - public String restWorkerUrl; + public String remoteWorker; } diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnStrategy.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnStrategy.java index 75ba75ebd5..1b93d5679d 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnStrategy.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnStrategy.java @@ -31,6 +31,7 @@ import com.google.devtools.build.lib.actions.SpawnActionContext; import com.google.devtools.build.lib.actions.UserExecException; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.remote.RemoteProtocol.RemoteWorkResponse; import com.google.devtools.build.lib.standalone.StandaloneSpawnStrategy; import com.google.devtools.build.lib.util.Preconditions; import com.google.devtools.build.lib.util.io.FileOutErr; @@ -88,6 +89,14 @@ final class RemoteSpawnStrategy implements SpawnActionContext { ActionInputFileCache inputFileCache = actionExecutionContext.getActionInputFileCache(); EventHandler eventHandler = executor.getEventHandler(); + if (remoteActionCache == null) { + eventHandler.handle( + Event.warn( + spawn.getMnemonic() + " Cannot instantiate remote action cache. Running locally.")); + standaloneStrategy.exec(spawn, actionExecutionContext); + return; + } + // Compute a hash code to uniquely identify the action plus the action inputs. Hasher hasher = Hashing.sha256().newHasher(); @@ -183,8 +192,8 @@ final class RemoteSpawnStrategy implements SpawnActionContext { return false; } try { - ListenableFuture<RemoteWorkExecutor.Response> future = - remoteWorkExecutor.submit( + ListenableFuture<RemoteWorkResponse> future = + remoteWorkExecutor.executeRemotely( execRoot, actionCache, actionOutputKey, @@ -193,8 +202,8 @@ final class RemoteSpawnStrategy implements SpawnActionContext { environment, outputs, timeout); - RemoteWorkExecutor.Response response = future.get(timeout, TimeUnit.SECONDS); - if (!response.success()) { + RemoteWorkResponse response = future.get(timeout, TimeUnit.SECONDS); + if (!response.getSuccess()) { String exception = ""; if (!response.getException().isEmpty()) { exception = " (" + response.getException() + ")"; @@ -204,12 +213,8 @@ final class RemoteSpawnStrategy implements SpawnActionContext { mnemonic + " failed to execute work remotely" + exception + ", running locally")); return false; } - if (response.getOut() != null) { - outErr.printOut(response.getOut()); - } - if (response.getErr() != null) { - outErr.printErr(response.getErr()); - } + outErr.printOut(response.getOut()); + outErr.printErr(response.getErr()); } catch (ExecutionException e) { eventHandler.handle( Event.warn(mnemonic + " failed to execute work remotely (" + e + "), running locally")); diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteWorkExecutor.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteWorkExecutor.java index 89e12cde55..22def42a07 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteWorkExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteWorkExecutor.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.devtools.build.lib.actions.ActionInput; import com.google.devtools.build.lib.actions.ActionInputFileCache; import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import com.google.devtools.build.lib.remote.RemoteProtocol.RemoteWorkResponse; import com.google.devtools.build.lib.vfs.Path; import java.io.IOException; @@ -28,41 +29,7 @@ import java.util.Collection; * Interface for exeucting work remotely. */ @ThreadCompatible -interface RemoteWorkExecutor { - - /** - * The response of running a remote work. - */ - class Response { - private final boolean success; - private final String out; - private final String err; - private final String exception; - - boolean success() { - return success; - } - - String getOut() { - return out; - } - - String getErr() { - return err; - } - - String getException() { - return exception; - } - - Response(boolean success, String out, String err, String exception) { - this.success = success; - this.out = out; - this.err = err; - this.exception = exception; - } - } - +public interface RemoteWorkExecutor { /** * Submit the work to this work executor. * The output of running this action should be written to {@link RemoteActionCache} indexed @@ -70,7 +37,7 @@ interface RemoteWorkExecutor { * * Returns a future for the response of this work request. */ - ListenableFuture<Response> submit( + ListenableFuture<RemoteWorkResponse> executeRemotely( Path execRoot, ActionInputFileCache cache, String actionOutputKey, diff --git a/src/main/java/com/google/devtools/build/lib/remote/WorkTooLargeException.java b/src/main/java/com/google/devtools/build/lib/remote/WorkTooLargeException.java index d112ebe199..d1498d73df 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/WorkTooLargeException.java +++ b/src/main/java/com/google/devtools/build/lib/remote/WorkTooLargeException.java @@ -14,9 +14,7 @@ package com.google.devtools.build.lib.remote; -/** - * An exception that indicates the work is too large to run remotely. - */ +/** An exception that indicates the work is too large to run remotely. */ final class WorkTooLargeException extends RuntimeException { WorkTooLargeException() { super(); diff --git a/src/main/protobuf/BUILD b/src/main/protobuf/BUILD index 1e71e025bb..63e0f67579 100644 --- a/src/main/protobuf/BUILD +++ b/src/main/protobuf/BUILD @@ -18,7 +18,6 @@ FILES = [ "xcodegen", "worker_protocol", "invocation_policy", - "remote_protocol", "android_deploy_info", "apk_manifest", ] @@ -39,6 +38,12 @@ cc_grpc_library( src = "command_server.proto", ) +java_proto_library( + name = "remote_protocol_java_proto", + src = "remote_protocol.proto", + use_grpc_plugin = True, +) + py_proto_library( name = "build_pb_py", srcs = ["build.proto"], diff --git a/src/main/protobuf/remote_protocol.proto b/src/main/protobuf/remote_protocol.proto index a9d476d320..341061a120 100644 --- a/src/main/protobuf/remote_protocol.proto +++ b/src/main/protobuf/remote_protocol.proto @@ -85,4 +85,10 @@ message RemoteWorkResponse { // String for the exception when running this work. string exception = 4; -}
\ No newline at end of file +} + +service RemoteWork { + // Perform work synchronously. + rpc ExecuteSynchronously(RemoteWorkRequest) returns (RemoteWorkResponse) { + } +} diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD index 9c7b5562ec..7f3937e71e 100644 --- a/src/test/shell/bazel/BUILD +++ b/src/test/shell/bazel/BUILD @@ -326,6 +326,16 @@ sh_test( ], ) +sh_test( + name = "remote_execution_test", + size = "large", + srcs = ["remote_execution_test.sh"], + data = [ + ":test-deps", + "//src/tools/remote_worker", + ], +) + test_suite( name = "all_tests", visibility = ["//visibility:public"], diff --git a/src/test/shell/bazel/remote_execution_test.sh b/src/test/shell/bazel/remote_execution_test.sh new file mode 100755 index 0000000000..0083ba7357 --- /dev/null +++ b/src/test/shell/bazel/remote_execution_test.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# +# Copyright 2016 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Tests remote execution and caching. +# + +# Load test environment +src_dir=$(cd "$(dirname ${BASH_SOURCE[0]})" && pwd) +source $src_dir/test-setup.sh \ + || { echo "test-setup.sh not found!" >&2; exit 1; } + +function set_up() { + mkdir -p a + cat > a/BUILD <<EOF +package(default_visibility = ["//visibility:public"]) +cc_binary( +name = 'test', +srcs = [ 'test.cc' ], +) +EOF + cat > a/test.cc <<EOF +#include <iostream> +int main() { std::cout << "Hello world!" << std::endl; return 0; } +EOF + work_path=$(mktemp -d ${TEST_TMPDIR}/remote.XXXXXXXX) + pid_file=$(mktemp -u ${TEST_TMPDIR}/remote.XXXXXXXX) + ${bazel_data}/src/tools/remote_worker/remote_worker \ + --work_path=${work_path} \ + --listen_port 8080 \ + --pid_file=${pid_file} >& $TEST_log & + local wait_seconds=0 + until [ -s "${pid_file}" ] || [ $wait_seconds -eq 30 ]; do + sleep 1 + ((wait_seconds++)) + done + if [ ! -s "${pid_file}" ]; then + fail "Timed out waiting for remote worker to start." + fi +} + +function tear_down() { + if [ -s ${pid_file} ]; then + local pid=$(cat ${pid_file}) + kill ${pid} || true + fi + rm -rf ${pid_file} + rm -rf ${work_path} +} + +function test_cc_binary() { + bazel build //a:test >& $TEST_log \ + || fail "Failed to build //a:test without remote execution" + cp bazel-bin/a/test ${TEST_TMPDIR}/test_expected + bazel clean --expunge + + bazel build \ + --spawn_strategy=remote \ + --hazelcast_node=localhost:5701 \ + --remote_worker=localhost:8080 \ + //a:test >& $TEST_log \ + || fail "Failed to build //a:test with remote execution" + diff bazel-bin/a/test ${TEST_TMPDIR}/test_expected \ + || fail "Remote execution generated different result" +} + +# TODO(alpha): Add a test that fails remote execution when remote worker +# supports sandbox. + +run_suite "Remote execution tests" diff --git a/src/tools/remote_worker/BUILD b/src/tools/remote_worker/BUILD new file mode 100644 index 0000000000..7f44a71e83 --- /dev/null +++ b/src/tools/remote_worker/BUILD @@ -0,0 +1,6 @@ +java_binary( + name = "remote_worker", + main_class = "com.google.devtools.build.remote.RemoteWorker", + visibility = ["//visibility:public"], + runtime_deps = ["//src/tools/remote_worker/src/main/java/com/google/devtools/build/remote"], +) diff --git a/src/tools/remote_worker/README.md b/src/tools/remote_worker/README.md new file mode 100644 index 0000000000..7664d3345a --- /dev/null +++ b/src/tools/remote_worker/README.md @@ -0,0 +1,17 @@ +This program implements a remote execution worker that uses gRPC to accept work +requests. It also serves as a Hazelcast server for distributed caching. + +- First build remote_worker and run it. + + bazel build src/tools/remote_worker:all + bazel-bin/src/tools/remote_worker/remote_worker --work_path=/tmp/test \ + --listen_port=8080 + +- Then you run Bazel pointing to the remote_worker instance. + + bazel build --hazelcast_node=127.0.0.1:5701 --spawn_strategy=remote \ + --remote_worker=127.0.0.1:8080 src/tools/generate_workspace:all + +The above command will build generate_workspace with remote spawn strategy that +uses Hazelcast as the distributed caching backend and executes work remotely on +the localhost remote_worker. diff --git a/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/BUILD b/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/BUILD new file mode 100644 index 0000000000..495d640ba8 --- /dev/null +++ b/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/BUILD @@ -0,0 +1,26 @@ +java_library( + name = "remote", + srcs = glob(["*.java"]), + data = ["//src:libunix"], + visibility = ["//src/tools/remote_worker:__subpackages__"], + deps = [ + "//src/main/java/com/google/devtools/build/lib:build-base", + "//src/main/java/com/google/devtools/build/lib:concurrent", + "//src/main/java/com/google/devtools/build/lib:events", + "//src/main/java/com/google/devtools/build/lib:io", + "//src/main/java/com/google/devtools/build/lib:os_util", + "//src/main/java/com/google/devtools/build/lib:packages-internal", + "//src/main/java/com/google/devtools/build/lib:runtime", + "//src/main/java/com/google/devtools/build/lib:shell", + "//src/main/java/com/google/devtools/build/lib:util", + "//src/main/java/com/google/devtools/build/lib:vfs", + "//src/main/java/com/google/devtools/build/lib/remote", + "//src/main/java/com/google/devtools/common/options", + "//src/main/protobuf:remote_protocol_java_proto", + "//third_party:guava", + "//third_party:hazelcast", + "//third_party:netty", + "//third_party/grpc:grpc-jar", + "//third_party/protobuf", + ], +) diff --git a/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/RemoteWorker.java b/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/RemoteWorker.java new file mode 100644 index 0000000000..9bb4d2bb12 --- /dev/null +++ b/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/RemoteWorker.java @@ -0,0 +1,177 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.remote; + +import com.google.devtools.build.lib.remote.HazelcastCacheFactory; +import com.google.devtools.build.lib.remote.MemcacheActionCache; +import com.google.devtools.build.lib.remote.MemcacheWorkExecutor; +import com.google.devtools.build.lib.remote.RemoteOptions; +import com.google.devtools.build.lib.remote.RemoteProtocol.RemoteWorkRequest; +import com.google.devtools.build.lib.remote.RemoteProtocol.RemoteWorkResponse; +import com.google.devtools.build.lib.remote.RemoteWorkGrpc; +import com.google.devtools.build.lib.util.OS; +import com.google.devtools.build.lib.util.ProcessUtils; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.JavaIoFileSystem; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.UnixFileSystem; +import com.google.devtools.common.options.OptionsParser; + +import io.grpc.Server; +import io.grpc.ServerBuilder; +import io.grpc.stub.StreamObserver; + +import java.io.IOException; +import java.io.PrintWriter; +import java.util.Collections; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Implements a remote worker that accepts work items as protobufs. + * The server implementation is based on grpc. + */ +public class RemoteWorker implements RemoteWorkGrpc.RemoteWork { + private static final Logger LOG = Logger.getLogger(RemoteWorker.class.getName()); + private static final boolean LOG_FINER = LOG.isLoggable(Level.FINER); + private final Path workPath; + private final RemoteOptions remoteOptions; + private final RemoteWorkerOptions options; + private final ConcurrentMap<String, byte[]> cache; + + public RemoteWorker( + Path workPath, + RemoteOptions remoteOptions, + RemoteWorkerOptions options, + ConcurrentMap<String, byte[]> cache) { + this.workPath = workPath; + this.remoteOptions = remoteOptions; + this.options = options; + this.cache = cache; + } + + @Override + public void executeSynchronously( + RemoteWorkRequest request, StreamObserver<RemoteWorkResponse> responseObserver) { + Path tempRoot = workPath.getRelative("build-" + UUID.randomUUID().toString()); + try { + FileSystemUtils.createDirectoryAndParents(tempRoot); + final MemcacheActionCache actionCache = + new MemcacheActionCache(tempRoot, remoteOptions, cache); + final MemcacheWorkExecutor workExecutor = + MemcacheWorkExecutor.createLocalWorkExecutor(actionCache, tempRoot); + if (LOG_FINER) { + LOG.fine( + "Work received has " + + request.getInputFilesCount() + + " input files and " + + request.getOutputFilesCount() + + " output files."); + } + RemoteWorkResponse response = workExecutor.executeLocally(request); + responseObserver.onNext(response); + if (options.debug) { + if (!response.getSuccess()) { + LOG.warning("Work failed. Request: " + request.toString() + "."); + + } else if (LOG_FINER) { + LOG.fine("Work completed."); + } + } + if (!options.debug || response.getSuccess()) { + FileSystemUtils.deleteTree(tempRoot); + } else { + LOG.warning("Preserving work directory " + tempRoot.toString() + "."); + } + } catch (IOException e) { + RemoteWorkResponse.Builder response = RemoteWorkResponse.newBuilder(); + response.setSuccess(false).setOut("").setErr("").setException(e.toString()); + responseObserver.onNext(response.build()); + } finally { + responseObserver.onCompleted(); + } + } + + public static void main(String[] args) throws Exception { + OptionsParser parser = + OptionsParser.newOptionsParser(RemoteOptions.class, RemoteWorkerOptions.class); + parser.parseAndExitUponError(args); + RemoteOptions remoteOptions = parser.getOptions(RemoteOptions.class); + RemoteWorkerOptions remoteWorkerOptions = parser.getOptions(RemoteWorkerOptions.class); + + if (remoteWorkerOptions.workPath == null) { + printUsage(parser); + return; + } + + System.out.println("*** Starting Hazelcast server."); + ConcurrentMap<String, byte[]> cache = new HazelcastCacheFactory().create(remoteOptions); + + System.out.println( + "*** Starting grpc server at 0.0.0.0:" + remoteWorkerOptions.listenPort + "."); + Path workPath = getFileSystem().getPath(remoteWorkerOptions.workPath); + FileSystemUtils.createDirectoryAndParents(workPath); + RemoteWorker worker = new RemoteWorker(workPath, remoteOptions, remoteWorkerOptions, cache); + Server server = + ServerBuilder.forPort(remoteWorkerOptions.listenPort) + .addService(RemoteWorkGrpc.bindService(worker)) + .build(); + server.start(); + + final Path pidFile; + if (remoteWorkerOptions.pidFile != null) { + pidFile = getFileSystem().getPath(remoteWorkerOptions.pidFile); + PrintWriter writer = new PrintWriter(pidFile.getOutputStream()); + writer.append(Integer.toString(ProcessUtils.getpid())); + writer.append("\n"); + writer.close(); + } else { + pidFile = null; + } + + Runtime.getRuntime() + .addShutdownHook( + new Thread() { + @Override + public void run() { + System.err.println("*** Shutting down grpc server."); + server.shutdown(); + if (pidFile != null) { + try { + pidFile.delete(); + } catch (IOException e) { + System.err.println("Cannot remove pid file: " + pidFile.toString()); + } + } + System.err.println("*** Server shut down."); + } + }); + server.awaitTermination(); + } + + public static void printUsage(OptionsParser parser) { + System.out.println("Usage: remote_worker \n\n" + "Starts a worker that runs a RPC service."); + System.out.println( + parser.describeOptions( + Collections.<String, String>emptyMap(), OptionsParser.HelpVerbosity.LONG)); + } + + static FileSystem getFileSystem() { + return OS.getCurrent() == OS.WINDOWS ? new JavaIoFileSystem() : new UnixFileSystem(); + } +} diff --git a/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/RemoteWorkerOptions.java b/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/RemoteWorkerOptions.java new file mode 100644 index 0000000000..f5004268eb --- /dev/null +++ b/src/tools/remote_worker/src/main/java/com/google/devtools/build/remote/RemoteWorkerOptions.java @@ -0,0 +1,55 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.remote; + +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; + +/** Options for remote worker. */ +public class RemoteWorkerOptions extends OptionsBase { + @Option( + name = "listen_port", + defaultValue = "8080", + category = "build_worker", + help = "Listening port for the netty server." + ) + public int listenPort; + + @Option( + name = "work_path", + defaultValue = "null", + category = "build_worker", + help = "A directory for the build worker to do work." + ) + public String workPath; + + @Option( + name = "debug", + defaultValue = "false", + category = "build_worker", + help = + "Turn this on for debugging remote job failures. There will be extra messages and the " + + "work directory will be preserved in the case of failure." + ) + public boolean debug; + + @Option( + name = "pid_file", + defaultValue = "null", + category = "build_worker", + help = "File for writing the process id for this worker when it is fully started." + ) + public String pidFile; +} diff --git a/third_party/README.md b/third_party/README.md index c171296a36..ed89e5af04 100644 --- a/third_party/README.md +++ b/third_party/README.md @@ -295,3 +295,9 @@ a minimal set of extra dependencies. * Version: 1.5 * License: Public Domain + +## [netty](http://netty.io/) + +* Version: 4.1.0.CR4 +* License: Apache License 2.0 + |