Introduce a new SpawnCache API; add a RemoteSpawnCache implementation

AbstractSpawnRunner now uses a SpawnCache if one is registered, this allows
adding caching to any spawn runner without having to be aware of the
implementations.

I will delete the old CachedLocalSpawnRunner in a follow-up CL.

PiperOrigin-RevId: 165024382

1 files changed, 203 insertions, 0 deletions

diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java
new file mode 100644
index 0000000000..930815bcf5
--- /dev/null
+++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnCacheTest.java
@@ -0,0 +1,203 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.remote;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.actions.ActionInput;
+import com.google.devtools.build.lib.actions.ActionInputFileCache;
+import com.google.devtools.build.lib.actions.ActionInputHelper;
+import com.google.devtools.build.lib.actions.Artifact;
+import com.google.devtools.build.lib.actions.Artifact.ArtifactExpander;
+import com.google.devtools.build.lib.actions.ResourceSet;
+import com.google.devtools.build.lib.actions.SimpleSpawn;
+import com.google.devtools.build.lib.exec.SpawnCache.CacheHandle;
+import com.google.devtools.build.lib.exec.SpawnInputExpander;
+import com.google.devtools.build.lib.exec.SpawnResult;
+import com.google.devtools.build.lib.exec.SpawnResult.Status;
+import com.google.devtools.build.lib.exec.SpawnRunner.ProgressStatus;
+import com.google.devtools.build.lib.exec.SpawnRunner.SpawnExecutionPolicy;
+import com.google.devtools.build.lib.exec.util.FakeOwner;
+import com.google.devtools.build.lib.remote.Digests.ActionKey;
+import com.google.devtools.build.lib.remote.TreeNodeRepository.TreeNode;
+import com.google.devtools.build.lib.util.io.FileOutErr;
+import com.google.devtools.build.lib.vfs.FileSystem;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
+import com.google.devtools.common.options.Options;
+import com.google.devtools.remoteexecution.v1test.ActionResult;
+import com.google.devtools.remoteexecution.v1test.Command;
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collection;
+import java.util.SortedMap;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+
+/** Tests for {@link RemoteSpawnCache}. */
+@RunWith(JUnit4.class)
+public class RemoteSpawnCacheTest {
+  private static final ArtifactExpander SIMPLE_ARTIFACT_EXPANDER =
+      new ArtifactExpander() {
+        @Override
+        public void expand(Artifact artifact, Collection<? super Artifact> output) {
+          output.add(artifact);
+        }
+      };
+
+  private FileSystem fs;
+  private Path execRoot;
+  private SimpleSpawn simpleSpawn;
+  private FakeActionInputFileCache fakeFileCache;
+  @Mock private RemoteActionCache remoteCache;
+  private RemoteSpawnCache cache;
+  private FileOutErr outErr;
+
+  private final SpawnExecutionPolicy simplePolicy =
+      new SpawnExecutionPolicy() {
+        @Override
+        public int getId() {
+          return 0;
+        }
+
+        @Override
+        public void prefetchInputs() {
+          // CachedLocalSpawnRunner should never prefetch itself, though the nested SpawnRunner may.
+          throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public void lockOutputFiles() throws InterruptedException {
+          throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public boolean speculating() {
+          return false;
+        }
+
+        @Override
+        public ActionInputFileCache getActionInputFileCache() {
+          return fakeFileCache;
+        }
+
+        @Override
+        public ArtifactExpander getArtifactExpander() {
+          throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public Duration getTimeout() {
+          return Duration.ZERO;
+        }
+
+        @Override
+        public FileOutErr getFileOutErr() {
+          return outErr;
+        }
+
+        @Override
+        public SortedMap<PathFragment, ActionInput> getInputMapping() throws IOException {
+          return new SpawnInputExpander(/*strict*/ false)
+              .getInputMapping(simpleSpawn, SIMPLE_ARTIFACT_EXPANDER, fakeFileCache, "workspace");
+        }
+
+        @Override
+        public void report(ProgressStatus state, String name) {
+          // TODO(ulfjack): Test that the right calls are made.
+        }
+      };
+
+  @Before
+  public final void setUp() throws Exception {
+    MockitoAnnotations.initMocks(this);
+    fs = new InMemoryFileSystem();
+    execRoot = fs.getPath("/exec/root");
+    FileSystemUtils.createDirectoryAndParents(execRoot);
+    fakeFileCache = new FakeActionInputFileCache(execRoot);
+    simpleSpawn =
+        new SimpleSpawn(
+            new FakeOwner("Mnemonic", "Progress Message"),
+            ImmutableList.of("/bin/echo", "Hi!"),
+            ImmutableMap.of("VARIABLE", "value"),
+            /*executionInfo=*/ ImmutableMap.<String, String>of(),
+            /*inputs=*/ ImmutableList.of(ActionInputHelper.fromPath("input")),
+            /*outputs=*/ ImmutableList.<ActionInput>of(),
+            ResourceSet.ZERO);
+
+    Path stdout = fs.getPath("/tmp/stdout");
+    Path stderr = fs.getPath("/tmp/stderr");
+    FileSystemUtils.createDirectoryAndParents(stdout.getParentDirectory());
+    FileSystemUtils.createDirectoryAndParents(stderr.getParentDirectory());
+    outErr = new FileOutErr(stdout, stderr);
+    RemoteOptions options = Options.getDefaults(RemoteOptions.class);
+    cache = new RemoteSpawnCache(execRoot, options, remoteCache);
+    fakeFileCache.createScratchInput(simpleSpawn.getInputFiles().get(0), "xyz");
+  }
+
+  @SuppressWarnings("unchecked")
+  @Test
+  public void cacheHit() throws Exception {
+    ActionResult actionResult = ActionResult.getDefaultInstance();
+    when(remoteCache.getCachedActionResult(any(ActionKey.class))).thenReturn(actionResult);
+
+    CacheHandle entry = cache.lookup(simpleSpawn, simplePolicy);
+    assertThat(entry.hasResult()).isTrue();
+    SpawnResult result = entry.getResult();
+    // All other methods on RemoteActionCache have side effects, so we verify all of them.
+    verify(remoteCache).download(actionResult, execRoot, outErr);
+    verify(remoteCache, never())
+        .ensureInputsPresent(
+            any(TreeNodeRepository.class),
+            any(Path.class),
+            any(TreeNode.class),
+            any(Command.class));
+    verify(remoteCache, never())
+        .upload(
+            any(ActionKey.class), any(Path.class), any(Collection.class), any(FileOutErr.class));
+    assertThat(result.setupSuccess()).isTrue();
+    assertThat(result.exitCode()).isEqualTo(0);
+    // We expect the CachedLocalSpawnRunner to _not_ write to outErr at all.
+    assertThat(outErr.hasRecordedOutput()).isFalse();
+    assertThat(outErr.hasRecordedStderr()).isFalse();
+  }
+
+  @Test
+  public void cacheMiss() throws Exception {
+    CacheHandle entry = cache.lookup(simpleSpawn, simplePolicy);
+    assertThat(entry.hasResult()).isFalse();
+    SpawnResult result = new SpawnResult.Builder().setExitCode(0).setStatus(Status.SUCCESS).build();
+    ImmutableList<Path> outputFiles = ImmutableList.of(fs.getPath("/random/file"));
+    entry.store(result, outputFiles);
+    verify(remoteCache)
+        .upload(
+            any(ActionKey.class),
+            any(Path.class),
+            eq(outputFiles),
+            eq(outErr));
+  }
+}