Add an interface to interact with sandboxfs.

The new SandboxfsProcess interface allows interacting with sandboxfs.
There are two implementations: RealSandboxfsProcess, which spawns the
sandboxfs binary, and FakeSandboxfsProcess, which mimics what sandboxfs
does but using symlinks and is intended for testing purposes only.

The RealSandboxfsProcess implementation works but still carries many
TODOs.  The most "painful" one may be that the test requires manual
invocation because we do not yet have an easy way to integrate with
sandboxfs.  That will be solved later on; for now this is sufficient
for initial testing.

RELNOTES: None.
PiperOrigin-RevId: 188347393

5 files changed, 445 insertions, 5 deletions

diff --git a/src/test/java/com/google/devtools/build/lib/BUILD b/src/test/java/com/google/devtools/build/lib/BUILD
index 9e871c9298..a11f48ac1a 100644
--- a/src/test/java/com/google/devtools/build/lib/BUILD
+++ b/src/test/java/com/google/devtools/build/lib/BUILD
@@ -1226,34 +1226,76 @@ java_test(
     ],
 )
 
+java_library(
+    name = "sandboxfs-base-tests",
+    testonly = 1,
+    srcs = ["sandbox/BaseSandboxfsProcessTest.java"],
+    deps = [
+        "//src/main/java/com/google/devtools/build/lib/sandbox",
+        "//src/main/java/com/google/devtools/build/lib/vfs",
+        "//src/test/java/com/google/devtools/build/lib:testutil",
+        "//third_party:guava",
+        "//third_party:junit4",
+        "//third_party:truth",
+    ],
+)
+
 java_test(
     name = "sandbox-tests",
-    srcs = glob(["sandbox/*.java"]),
+    srcs = glob(
+        ["sandbox/*.java"],
+        exclude = [
+            "sandbox/BaseSandboxfsProcessTest.java",
+            "sandbox/RealSandboxfsProcessTest.java",
+        ],
+    ),
     data = [":embedded_scripts"],
     local = 1,
     tags = ["no_windows"],
     test_class = "com.google.devtools.build.lib.AllTests",
     deps = [
-        ":actions_testutil",
         ":analysis_testutil",
         ":foundations_testutil",
         ":guava_junit_truth",
+        ":sandboxfs-base-tests",
         ":testutil",
         "//src/main/java/com/google/devtools/build/lib:bazel-rules",
         "//src/main/java/com/google/devtools/build/lib:build-base",
-        "//src/main/java/com/google/devtools/build/lib:events",
         "//src/main/java/com/google/devtools/build/lib:os_util",
         "//src/main/java/com/google/devtools/build/lib:util",
         "//src/main/java/com/google/devtools/build/lib/actions",
-        "//src/main/java/com/google/devtools/build/lib/clock",
         "//src/main/java/com/google/devtools/build/lib/sandbox",
-        "//src/main/java/com/google/devtools/build/lib/shell",
         "//src/main/java/com/google/devtools/build/lib/vfs",
+        "//src/main/java/com/google/devtools/build/lib/vfs/inmemoryfs",
         "//src/main/java/com/google/devtools/common/options",
     ],
 )
 
 java_test(
+    name = "sandboxfs-integration-tests",
+    srcs = ["sandbox/RealSandboxfsProcessTest.java"],
+    data = [":embedded_scripts"],
+    local = 1,
+    tags = [
+        # On macOS:
+        #   sudo sysctl -w vfs.generic.osxfuse.tunables.allow_other=1
+        # Test requires:
+        #   --test_env=SANDBOXFS=/path/to/sandboxfs
+        "manual",
+        "no-sandbox",
+        "no_windows",
+    ],
+    test_class = "com.google.devtools.build.lib.AllTests",
+    deps = [
+        ":sandboxfs-base-tests",
+        ":test_runner",
+        "//src/main/java/com/google/devtools/build/lib/sandbox",
+        "//src/main/java/com/google/devtools/build/lib/vfs",
+        "//third_party:junit4",
+    ],
+)
+
+java_test(
     name = "standalone-tests",
     srcs = glob(["standalone/*.java"]),
     data = [":embedded_scripts"],
diff --git a/src/test/java/com/google/devtools/build/lib/sandbox/BaseSandboxfsProcessTest.java b/src/test/java/com/google/devtools/build/lib/sandbox/BaseSandboxfsProcessTest.java
new file mode 100644
index 0000000000..dd827761be
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/sandbox/BaseSandboxfsProcessTest.java
@@ -0,0 +1,176 @@
+// Copyright 2018 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.sandbox;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.devtools.build.lib.testutil.MoreAsserts.assertThrows;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.sandbox.SandboxfsProcess.Mapping;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import java.io.IOException;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Common tests for all implementations of {@link SandboxfsProcess}.
+ *
+ * <p>Subclasses must define the provided hooks to configure the file system the tests run in
+ * (which can be real or virtual), and a mechanism to "mount" a sandboxfs instance.
+ *
+ * <p>Subclasses inherit and run all the tests in this class.
+ */
+abstract class BaseSandboxfsProcessTest {
+
+  /** Test-specific temporary directory and file system. */
+  protected Path tmpDir;
+
+  /** Hook to obtain the path to a test-specific temporary directory and file system. */
+  abstract Path newTmpDir() throws IOException;
+
+  /** Hook to mount a new test-specific sandboxfs instance. */
+  abstract SandboxfsProcess mount(Path mountPoint) throws IOException;
+
+  @Before
+  public void setUp() throws IOException {
+    tmpDir = newTmpDir();
+  }
+
+  @After
+  public void tearDown() throws IOException {
+    FileSystemUtils.deleteTreesBelow(tmpDir);
+    tmpDir = null;
+  }
+
+  @Test
+  public void testMount_MissingDirectory() throws IOException {
+    IOException expected = assertThrows(
+        IOException.class, () -> mount(tmpDir.getRelative("missing")));
+    assertThat(expected).hasMessageThat().matches(".*(/missing.*does not exist|failed to start).*");
+  }
+
+  @Test
+  public void testLifeCycle() throws IOException {
+    Path mountPoint = tmpDir.getRelative("mnt");
+    mountPoint.createDirectory();
+    SandboxfsProcess process = mount(mountPoint);
+    try {
+      assertThat(process.isAlive()).isTrue();
+      process.destroy();
+      assertThat(process.isAlive()).isFalse();
+      process.destroy();
+      assertThat(process.isAlive()).isFalse();
+    } finally {
+      process.destroy();
+    }
+  }
+
+  @Test
+  public void testReconfigure() throws IOException {
+    Path mountPoint = tmpDir.getRelative("mnt");
+    mountPoint.createDirectory();
+    SandboxfsProcess process = mount(mountPoint);
+    try {
+      // Start by ensuring the mount point is empty.
+      assertThat(mountPoint.getDirectoryEntries()).isEmpty();
+
+      // Create a file outside of the mount point to ensure it's not touched.
+      mountPoint.getRelative("../unrelated").createDirectory();
+
+      // Create twp mappings: one to be deleted and one to be kept around throughout the test.
+      Path keepMeFile = tmpDir.getRelative("one");
+      keepMeFile.getOutputStream().close();
+      Path oneFile = tmpDir.getRelative("one");
+      FileSystemUtils.writeContent(oneFile, UTF_8, "One test data");
+      process.map(
+          ImmutableList.of(
+              Mapping.builder()
+                  .setPath(PathFragment.create("/keep-me"))
+                  .setTarget(keepMeFile.asFragment())
+                  .setWritable(false)
+                  .build(),
+              Mapping.builder()
+                  .setPath(PathFragment.create("/foo"))
+                  .setTarget(oneFile.asFragment())
+                  .setWritable(false)
+                  .build()));
+      assertThat(
+          mountPoint.getDirectoryEntries())
+          .containsExactly(mountPoint.getRelative("foo"), mountPoint.getRelative("keep-me"));
+      assertThat(
+          FileSystemUtils.readContent(mountPoint.getRelative("foo"), UTF_8))
+          .isEqualTo("One test data");
+
+      // Replace the previous mapping and create a new one.
+      Path twoFile = tmpDir.getRelative("two");
+      FileSystemUtils.writeContent(twoFile, UTF_8, "Two test data");
+      Path bazFile = tmpDir.getRelative("baz");
+      FileSystemUtils.writeContent(bazFile, UTF_8, "Baz test data");
+      process.unmap(PathFragment.create("/foo"));
+      process.map(
+          ImmutableList.of(
+              Mapping.builder()
+                  .setPath(PathFragment.create("/foo"))
+                  .setTarget(twoFile.asFragment())
+                  .setWritable(false)
+                  .build(),
+              Mapping.builder()
+                  .setPath(PathFragment.create("/bar"))
+                  .setTarget(bazFile.asFragment())
+                  .setWritable(true)
+                  .build()));
+      assertThat(
+          mountPoint.getDirectoryEntries())
+          .containsExactly(mountPoint.getRelative("foo"), mountPoint.getRelative("bar"),
+              mountPoint.getRelative("keep-me"));
+      assertThat(
+          FileSystemUtils.readContent(mountPoint.getRelative("foo"), UTF_8))
+          .isEqualTo("Two test data");
+      assertThat(
+          FileSystemUtils.readContent(mountPoint.getRelative("bar"), UTF_8))
+          .isEqualTo("Baz test data");
+
+      // Replace all existing mappings, and try with a nested one.
+      Path longLink = tmpDir.getRelative("long/link");
+      longLink.getParentDirectory().createDirectoryAndParents();
+      longLink.createSymbolicLink(oneFile);  // The target is irrelevant but must exist.
+      process.unmap(PathFragment.create("/foo"));
+      process.unmap(PathFragment.create("/bar"));
+      process.map(
+          ImmutableList.of(
+              Mapping.builder()
+                  .setPath(PathFragment.create("/something/complex"))
+                  .setTarget(longLink.asFragment())
+                  .setWritable(false)
+                  .build()));
+      assertThat(
+          mountPoint.getDirectoryEntries())
+          .containsExactly(mountPoint.getRelative("keep-me"), mountPoint.getRelative("something"));
+      assertThat(
+          FileSystemUtils.readContent(mountPoint.getRelative("something/complex"), UTF_8))
+          .isEqualTo("One test data");
+
+      // Ensure that files that should not have been touched throughout the test are still there.
+      assertThat(mountPoint.getRelative("keep-me").exists()).isTrue();
+      assertThat(mountPoint.getRelative("../unrelated").exists()).isTrue();
+    } finally {
+      process.destroy();
+    }
+  }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/sandbox/FakeSandboxfsProcess.java b/src/test/java/com/google/devtools/build/lib/sandbox/FakeSandboxfsProcess.java
new file mode 100644
index 0000000000..8a9c63b288
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/sandbox/FakeSandboxfsProcess.java
@@ -0,0 +1,111 @@
+// Copyright 2018 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.sandbox;
+
+import static com.google.common.base.Preconditions.checkState;
+
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * A fake in-process sandboxfs implementation that uses symlinks on the Bazel file system API.
+ *
+ * <p>TODO(jmmv): It's possible that this could replace {@link SymlinkedSandboxedSpawn} altogether,
+ * simplifying all callers that need to perform a sandboxed spawn because they would all go through
+ * the sandboxfs worker interface.  Evaluate this idea once we are confident enough that we won't
+ * just remove all sandboxfs support code.
+ */
+final class FakeSandboxfsProcess implements SandboxfsProcess {
+
+  /** File system on which the fake sandboxfs instance operates. */
+  private final FileSystem fileSystem;
+
+  /** Directory on which the sandboxfs is serving. */
+  private final PathFragment mountPoint;
+
+  /**
+   * Whether this "process" is valid or not.  Used to better represent the workflow of a real
+   * sandboxfs subprocess.
+   */
+  private boolean alive = true;
+
+  /**
+   * Initializes a new sandboxfs process instance.
+   *
+   * <p>To better represent reality, this ensures that the mount point is present and valid.
+   *
+   * @param fileSystem file system on which the fake sandboxfs instance operates
+   * @param mountPoint directory on which the sandboxfs instance is serving
+   * @throws IOException if the mount point is missing
+   */
+  FakeSandboxfsProcess(FileSystem fileSystem, PathFragment mountPoint) throws IOException {
+    if (!fileSystem.getPath(mountPoint).exists()) {
+      throw new IOException("Mount point " + mountPoint + " does not exist");
+    } else if (!fileSystem.getPath(mountPoint).isDirectory()) {
+      throw new IOException("Mount point " + mountPoint + " is not a directory");
+    }
+
+    this.fileSystem = fileSystem;
+    this.mountPoint = mountPoint;
+  }
+
+  @Override
+  public Path getMountPoint() {
+    return fileSystem.getPath(mountPoint);
+  }
+
+  @Override
+  public synchronized boolean isAlive() {
+    return alive;
+  }
+
+  @Override
+  public synchronized void destroy() {
+    alive = false;
+  }
+
+  @Override
+  public synchronized void map(List<Mapping> mappings) throws IOException {
+    checkState(alive, "Cannot be called after destroy()");
+
+    for (Mapping mapping : mappings) {
+      checkState(mapping.path().isAbsolute(), "Mapping specifications are expected to be absolute"
+          + " but %s is not", mapping.path());
+      Path link = fileSystem.getPath(mountPoint).getRelative(mapping.path().toRelative());
+      link.getParentDirectory().createDirectoryAndParents();
+
+      if (!fileSystem.getPath(mapping.target()).exists()) {
+        // Not a requirement for the creation of a symbolic link but this reflects the behavior of
+        // the real sandboxfs.
+        throw new IOException("Target " + mapping.target() + " does not exist");
+      }
+
+      link.createSymbolicLink(fileSystem.getPath(mapping.target()));
+    }
+  }
+
+  @Override
+  public synchronized void unmap(PathFragment mapping) throws IOException {
+    checkState(alive, "Cannot be called after destroy()");
+
+    checkState(mapping.isAbsolute(), "Mapping specifications are expected to be absolute"
+        + " but %s is not", mapping);
+    FileSystemUtils.deleteTree(fileSystem.getPath(mountPoint).getRelative(mapping.toRelative()));
+  }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/sandbox/FakeSandboxfsProcessTest.java b/src/test/java/com/google/devtools/build/lib/sandbox/FakeSandboxfsProcessTest.java
new file mode 100644
index 0000000000..f9d8c63eb9
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/sandbox/FakeSandboxfsProcessTest.java
@@ -0,0 +1,52 @@
+// Copyright 2018 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.sandbox;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.devtools.build.lib.testutil.MoreAsserts.assertThrows;
+
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
+import java.io.IOException;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link FakeSandboxfsProcess}. */
+@RunWith(JUnit4.class)
+public class FakeSandboxfsProcessTest extends BaseSandboxfsProcessTest {
+
+  @Override
+  Path newTmpDir() throws IOException {
+    FileSystem fileSystem = new InMemoryFileSystem();
+    Path tmpDir = fileSystem.getPath("/tmp");
+    tmpDir.createDirectory();
+    return tmpDir;
+  }
+
+  @Override
+  SandboxfsProcess mount(Path mountPoint) throws IOException {
+    return new FakeSandboxfsProcess(mountPoint.getFileSystem(), mountPoint.asFragment());
+  }
+
+  @Test
+  public void testMount_NotADirectory() throws IOException {
+    tmpDir.getRelative("file").getOutputStream().close();
+    IOException expected = assertThrows(
+        IOException.class, () -> mount(tmpDir.getRelative("file")));
+    assertThat(expected).hasMessageThat().matches(".*/file.*not a directory");
+  }
+}
diff --git a/src/test/java/com/google/devtools/build/lib/sandbox/RealSandboxfsProcessTest.java b/src/test/java/com/google/devtools/build/lib/sandbox/RealSandboxfsProcessTest.java
new file mode 100644
index 0000000000..ffce1d2539
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/sandbox/RealSandboxfsProcessTest.java
@@ -0,0 +1,59 @@
+// Copyright 2018 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.sandbox;
+
+import static junit.framework.TestCase.fail;
+
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.JavaIoFileSystem;
+import com.google.devtools.build.lib.vfs.Path;
+import java.io.IOException;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests for {@link RealSandboxfsProcess}. */
+@RunWith(JUnit4.class)
+public class RealSandboxfsProcessTest extends BaseSandboxfsProcessTest {
+
+  @Override
+  Path newTmpDir() {
+    String rawTmpDir = System.getenv("TEST_TMPDIR");
+    if (rawTmpDir == null) {
+      fail("Test requires TEST_TMPDIR to be defined in the environment");
+    }
+
+    FileSystem fileSystem = new JavaIoFileSystem();
+    Path tmpDir = fileSystem.getPath(rawTmpDir);
+    if (!tmpDir.isDirectory()) {
+      fail("TEST_TMPDIR must point to a directory");
+    }
+    return tmpDir;
+  }
+
+  @Override
+  SandboxfsProcess mount(Path mountPoint) throws IOException {
+    String rawSandboxfs = System.getenv("SANDBOXFS");
+    if (rawSandboxfs == null) {
+      fail("Test requires SANDBOXFS to be defined in the environment");
+    }
+
+    FileSystem fileSystem = new JavaIoFileSystem();
+    Path sandboxfs = fileSystem.getPath(rawSandboxfs);
+    if (!sandboxfs.isExecutable()) {
+      fail("SANDBOXFS must point to an executable binary");
+    }
+    return RealSandboxfsProcess.mount(sandboxfs, mountPoint, fileSystem.getPath("/dev/stderr"));
+  }
+}