diff options
Diffstat (limited to 'src/main/java/com/google/devtools/build/lib/sandbox/DockerSandboxedSpawnRunner.java')
-rw-r--r-- | src/main/java/com/google/devtools/build/lib/sandbox/DockerSandboxedSpawnRunner.java | 427 |
1 files changed, 427 insertions, 0 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/DockerSandboxedSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/sandbox/DockerSandboxedSpawnRunner.java new file mode 100644 index 0000000000..6d4fda2876 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/sandbox/DockerSandboxedSpawnRunner.java @@ -0,0 +1,427 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.sandbox; + +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.MoreCollectors; +import com.google.common.eventbus.Subscribe; +import com.google.common.io.ByteStreams; +import com.google.devtools.build.lib.actions.ExecException; +import com.google.devtools.build.lib.actions.Spawn; +import com.google.devtools.build.lib.actions.SpawnResult; +import com.google.devtools.build.lib.actions.Spawns; +import com.google.devtools.build.lib.actions.UserExecException; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.exec.local.LocalEnvProvider; +import com.google.devtools.build.lib.exec.local.PosixLocalEnvProvider; +import com.google.devtools.build.lib.runtime.CommandCompleteEvent; +import com.google.devtools.build.lib.runtime.CommandEnvironment; +import com.google.devtools.build.lib.runtime.ProcessWrapperUtil; +import com.google.devtools.build.lib.shell.Command; +import com.google.devtools.build.lib.shell.CommandException; +import com.google.devtools.build.lib.util.OS; +import com.google.devtools.build.lib.util.ProcessUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.remoteexecution.v1test.Platform; +import com.google.protobuf.TextFormat; +import com.google.protobuf.TextFormat.ParseException; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +/** Spawn runner that uses Docker to execute a local subprocess. */ +final class DockerSandboxedSpawnRunner extends AbstractSandboxSpawnRunner { + + // The name of the container image entry in the Platform proto + // (see third_party/googleapis/devtools/remoteexecution/*/remote_execution.proto and + // experimental_remote_platform_override in + // src/main/java/com/google/devtools/build/lib/remote/RemoteOptions.java) + private static final String CONTAINER_IMAGE_ENTRY_NAME = "container-image"; + private static final String DOCKER_IMAGE_PREFIX = "docker://"; + + public static boolean isSupported(CommandEnvironment cmdEnv, Path dockerClient) { + boolean verbose = cmdEnv.getOptions().getOptions(SandboxOptions.class).dockerVerbose; + + if (!ProcessWrapperUtil.isSupported(cmdEnv)) { + if (verbose) { + cmdEnv + .getReporter() + .handle( + Event.error( + "Docker sandboxing is disabled, because ProcessWrapperUtil.isSupported " + + "returned false. This should never happen - is your Bazel binary " + + "corrupted?")); + } + return false; + } + + // On Linux we need to know the UID and GID that we're running as, because otherwise Docker will + // create files as 'root' and we can't move them to the execRoot. + if (OS.getCurrent() == OS.LINUX) { + try { + ProcessUtils.getuid(); + ProcessUtils.getgid(); + } catch (UnsatisfiedLinkError e) { + if (verbose) { + cmdEnv + .getReporter() + .handle( + Event.error( + "Docker sandboxing is disabled, because ProcessUtils.getuid/getgid threw an " + + "UnsatisfiedLinkError. This means that you're running a Bazel version " + + "that doesn't have JNI libraries - did you build it correctly?\n" + + Throwables.getStackTraceAsString(e))); + } + return false; + } + } + + Command cmd = + new Command( + new String[] {dockerClient.getPathString(), "info"}, + ImmutableMap.of(), + cmdEnv.getExecRoot().getPathFile()); + try { + cmd.execute(ByteStreams.nullOutputStream(), ByteStreams.nullOutputStream()); + } catch (CommandException e) { + if (verbose) { + cmdEnv + .getReporter() + .handle( + Event.error( + "Docker sandboxing is disabled, because running 'docker info' failed: " + + Throwables.getStackTraceAsString(e))); + } + return false; + } + + if (verbose) { + cmdEnv.getReporter().handle(Event.info("Docker sandboxing is supported")); + } + + return true; + } + + private static final ConcurrentHashMap<String, String> imageMap = new ConcurrentHashMap<>(); + + private final Path execRoot; + private final boolean allowNetwork; + private final Path dockerClient; + private final Path processWrapper; + private final Path sandboxBase; + private final String defaultImage; + private final LocalEnvProvider localEnvProvider; + private final Duration timeoutKillDelay; + private final String commandId; + private final Reporter reporter; + private final boolean useCustomizedImages; + private final int uid; + private final int gid; + private final List<UUID> containersToCleanup; + + /** + * Creates a sandboxed spawn runner that uses the {@code linux-sandbox} tool. + * + * @param cmdEnv the command environment to use + * @param dockerClient path to the `docker` executable + * @param sandboxBase path to the sandbox base directory + * @param defaultImage the Docker image to use if the platform doesn't specify one + * @param timeoutKillDelay an additional grace period before killing timing out commands + * @param useCustomizedImages whether to use customized images for execution + */ + DockerSandboxedSpawnRunner( + CommandEnvironment cmdEnv, + Path dockerClient, + Path sandboxBase, + String defaultImage, + Duration timeoutKillDelay, + boolean useCustomizedImages) { + super(cmdEnv); + this.execRoot = cmdEnv.getExecRoot(); + this.allowNetwork = SandboxHelpers.shouldAllowNetwork(cmdEnv.getOptions()); + this.dockerClient = dockerClient; + this.processWrapper = ProcessWrapperUtil.getProcessWrapper(cmdEnv); + this.sandboxBase = sandboxBase; + this.defaultImage = defaultImage; + this.localEnvProvider = new PosixLocalEnvProvider(cmdEnv.getClientEnv()); + this.timeoutKillDelay = timeoutKillDelay; + this.commandId = cmdEnv.getCommandId().toString(); + this.reporter = cmdEnv.getReporter(); + this.useCustomizedImages = useCustomizedImages; + if (OS.getCurrent() == OS.LINUX) { + this.uid = ProcessUtils.getuid(); + this.gid = ProcessUtils.getgid(); + } else { + this.uid = -1; + this.gid = -1; + } + this.containersToCleanup = Collections.synchronizedList(new ArrayList<>()); + + cmdEnv.getEventBus().register(this); + } + + @Override + protected SpawnResult actuallyExec(Spawn spawn, SpawnExecutionContext context) + throws IOException, ExecException, InterruptedException { + // Each invocation of "exec" gets its own sandbox base, execroot and temporary directory. + Path sandboxPath = + sandboxBase.getRelative(getName()).getRelative(Integer.toString(context.getId())); + sandboxPath.getParentDirectory().createDirectory(); + sandboxPath.createDirectory(); + + // b/64689608: The execroot of the sandboxed process must end with the workspace name, just like + // the normal execroot does. + Path sandboxExecRoot = sandboxPath.getRelative("execroot").getRelative(execRoot.getBaseName()); + sandboxExecRoot.getParentDirectory().createDirectory(); + sandboxExecRoot.createDirectory(); + + Map<String, String> environment = + localEnvProvider.rewriteLocalEnv(spawn.getEnvironment(), execRoot, "/tmp"); + + ImmutableSet<PathFragment> outputs = SandboxHelpers.getOutputFiles(spawn); + Duration timeout = context.getTimeout(); + + UUID uuid = UUID.randomUUID(); + + String baseImageName = dockerContainerFromSpawn(spawn).orElse(this.defaultImage); + if (baseImageName.isEmpty()) { + throw new UserExecException( + String.format( + "Cannot execute %s mnemonic with Docker, because no " + + "image could be found in the remote_execution_properties of the platform and " + + "no default image was set via --experimental_docker_image", + spawn.getMnemonic())); + } + + String customizedImageName = getOrCreateCustomizedImage(baseImageName); + if (customizedImageName == null) { + throw new UserExecException("Could not prepare Docker image for execution"); + } + + DockerCommandLineBuilder cmdLine = new DockerCommandLineBuilder(); + cmdLine + .setProcessWrapper(processWrapper) + .setDockerClient(dockerClient) + .setImageName(customizedImageName) + .setCommandArguments(spawn.getArguments()) + .setSandboxExecRoot(sandboxExecRoot) + .setPrivileged(getSandboxOptions().dockerPrivileged) + .setEnvironmentVariables(environment) + .setKillDelay(timeoutKillDelay) + .setCreateNetworkNamespace(!(allowNetwork || Spawns.requiresNetwork(spawn))) + .setCommandId(commandId) + .setUuid(uuid); + // If uid / gid are -1, we are on an operating system that doesn't require us to set them on the + // Docker invocation. If they're 0, it means we are running as root and don't need to set them. + if (uid > 0) { + cmdLine.setUid(uid); + } + if (gid > 0) { + cmdLine.setGid(gid); + } + if (!timeout.isZero()) { + cmdLine.setTimeout(timeout); + } + + SandboxedSpawn sandbox = + new CopyingSandboxedSpawn( + sandboxPath, + sandboxExecRoot, + cmdLine.build(), + environment, + SandboxHelpers.getInputFiles(spawn, context, execRoot), + outputs, + ImmutableSet.of()); + + try { + return runSpawn(spawn, sandbox, context, execRoot, timeout, null); + } catch (InterruptedException e) { + // If we were interrupted, it is possible that "docker run" gets killed in exactly the moment + // between the create and the start call, leaving behind a container that is created but never + // ran. This means that Docker won't automatically clean it up (as --rm only affects the start + // phase and has no effect on the create phase of "docker run"). + // We add the container UUID to a list and clean them up after the execution is over. + containersToCleanup.add(uuid); + throw e; + } + } + + private String getOrCreateCustomizedImage(String baseImage) { + // TODO(philwo) docker run implicitly does a docker pull if the image does not exist locally. + // Pulling an image can take a long time and a user might not be aware of that. We could check + // if the image exists locally (docker images -q name:tag) and if not, do a docker pull and + // notify the user in a similar way as when we download a http_archive. + // + // This is mostly relevant for the case where we don't build a customized image, as that prints + // a message when it runs. + + if (!useCustomizedImages) { + return baseImage; + } + + // If we're running as root, we can skip this step, as it's safe to assume that every image + // already has a built-in root user and group. + if (uid == 0 && gid == 0) { + return baseImage; + } + + return imageMap.computeIfAbsent( + baseImage, + (image) -> { + reporter.handle(Event.info("Preparing Docker image " + image + " for use...")); + String workDir = + PathFragment.create("/execroot").getRelative(execRoot.getBaseName()).getPathString(); + StringBuilder dockerfile = new StringBuilder(); + dockerfile.append(String.format("FROM %s\n", image)); + dockerfile.append(String.format("RUN [\"mkdir\", \"-p\", \"%s\"]\n", workDir)); + // TODO(philwo) this will fail if a user / group with the given uid / gid already exists + // in the container. For now this seems reasonably unlikely, but we'll have to come up + // with a better way. + if (gid > 0) { + dockerfile.append( + String.format("RUN [\"groupadd\", \"-g\", \"%d\", \"bazelbuild\"]\n", gid)); + } + if (uid > 0) { + dockerfile.append( + String.format( + "RUN [\"useradd\", \"-m\", \"-g\", \"%d\", \"-d\", \"%s\", \"-N\", \"-u\", " + + "\"%d\", \"bazelbuild\"]\n", + gid, workDir, uid)); + } + dockerfile.append( + String.format("RUN [\"chown\", \"-R\", \"%d:%d\", \"%s\"]\n", uid, gid, workDir)); + dockerfile.append(String.format("USER %d:%d\n", uid, gid)); + dockerfile.append(String.format("ENV HOME %s\n", workDir)); + if (uid > 0) { + dockerfile.append(String.format("ENV USER bazelbuild\n")); + } + dockerfile.append(String.format("WORKDIR %s\n", workDir)); + try { + return executeCommand( + ImmutableList.of(dockerClient.getPathString(), "build", "-q", "-"), + new ByteArrayInputStream(dockerfile.toString().getBytes(Charset.defaultCharset()))); + } catch (UserExecException e) { + reporter.handle(Event.error(e.getMessage())); + return null; + } + }); + } + + private String executeCommand(List<String> cmdLine, InputStream stdIn) throws UserExecException { + ByteArrayOutputStream stdOut = new ByteArrayOutputStream(); + ByteArrayOutputStream stdErr = new ByteArrayOutputStream(); + Command cmd = + new Command(cmdLine.toArray(new String[0]), ImmutableMap.of(), execRoot.getPathFile()); + try { + cmd.executeAsync(stdIn, stdOut, stdErr, Command.KILL_SUBPROCESS_ON_INTERRUPT).get(); + } catch (CommandException e) { + throw new UserExecException("Running command " + cmd.toDebugString() + " failed: " + stdErr); + } + return stdOut.toString().trim(); + } + + private Optional<String> dockerContainerFromSpawn(Spawn spawn) { + Platform platform = null; + // TODO(philwo) Figure out if this is the right mechanism to specify a Docker image per action. + String platformDescription = spawn.getExecutionPlatform().remoteExecutionProperties(); + if (platformDescription != null) { + try { + Platform.Builder platformBuilder = Platform.newBuilder(); + TextFormat.getParser().merge(platformDescription, platformBuilder); + platform = platformBuilder.build(); + } catch (ParseException e) { + throw new IllegalArgumentException( + String.format( + "Failed to parse remote_execution_properties from platform %s", + spawn.getExecutionPlatform().label()), + e); + } + } + + if (platform != null) { + try { + return platform + .getPropertiesList() + .stream() + .filter(p -> p.getName().equals(CONTAINER_IMAGE_ENTRY_NAME)) + .map(p -> p.getValue()) + .filter(r -> r.startsWith(DOCKER_IMAGE_PREFIX)) + .map(r -> r.substring(DOCKER_IMAGE_PREFIX.length())) + .collect(MoreCollectors.toOptional()); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + String.format( + "Platform %s contained multiple container-image entries, but only one is allowed.", + spawn.getExecutionPlatform().label())); + } + } else { + return Optional.empty(); + } + } + + // Remove all Docker containers that might be stuck in "Created" state and weren't automatically + // cleaned up by Docker itself. + public void cleanup() { + if (containersToCleanup == null || containersToCleanup.isEmpty()) { + return; + } + + ArrayList<String> cmdLine = new ArrayList<>(); + cmdLine.add(dockerClient.getPathString()); + cmdLine.add("rm"); + cmdLine.add("-fv"); + for (UUID uuid : containersToCleanup) { + cmdLine.add(uuid.toString()); + } + + Command cmd = + new Command(cmdLine.toArray(new String[0]), ImmutableMap.of(), execRoot.getPathFile()); + + try { + cmd.execute(); + } catch (CommandException e) { + // This is to be expected, as not all UUIDs that we pass to "docker rm" will still be alive + // when this method is called. However, it will successfully remove all the containers that + // *are* still there, even when the command exits with an error. + } + + containersToCleanup.clear(); + } + + @Subscribe + public void commandComplete(CommandCompleteEvent event) { + cleanup(); + } + + @Override + public String getName() { + return "docker"; + } +} |