Support local search for http archives

With --experimental_repository_cache, bazel has means of avoiding
downloading the same archive again. However, this requires bazel
to first download it itself, as we make no guarantee about the
internal structure of that cache; this, in turn, does not play
well in situations where bazel has to cooperate with other tools,
e.g., because the bazel build is just one step in a larger package
building process. Therefore, add an experimental option allowing
to specify directories where the outer process may have placed
needed files and make bazel not download them if a file with correct
name and hash could be found in one of those directories. In this
way, cooperation is possible without patching all entries in the
WORSPACE file.

Change-Id: I43240b8b59bf8472ec0310661015899e46491236
PiperOrigin-RevId: 185115713

4 files changed, 87 insertions, 0 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
index 0ff341fa27..9012ed4da0 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
@@ -83,6 +83,7 @@ import java.lang.reflect.InvocationTargetException;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
 import javax.annotation.Nullable;
 
 /**
@@ -204,6 +205,15 @@ public class BazelRepositoryModule extends BlazeModule {
         repositoryCache.setRepositoryCachePath(null);
       }
 
+      if (repoOptions.experimentalDistdir != null) {
+        httpDownloader.setDistdir(
+            repoOptions
+                .experimentalDistdir
+                .stream()
+                .map(filesystem::getPath)
+                .collect(Collectors.toList()));
+      }
+
       if (repoOptions.repositoryOverrides != null) {
         ImmutableMap.Builder<RepositoryName, PathFragment> builder = ImmutableMap.builder();
         for (RepositoryOverride override : repoOptions.repositoryOverrides) {
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java
index 72a1699a94..61e50105f6 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java
@@ -47,6 +47,20 @@ public class RepositoryOptions extends OptionsBase {
   public PathFragment experimentalRepositoryCache;
 
   @Option(
+    name = "experimental_distdir",
+    defaultValue = "null",
+    allowMultiple = true,
+    documentationCategory = OptionDocumentationCategory.BAZEL_CLIENT_OPTIONS,
+    effectTags = {OptionEffectTag.BAZEL_INTERNAL_CONFIGURATION},
+    metadataTags = {OptionMetadataTag.EXPERIMENTAL},
+    converter = OptionsUtils.PathFragmentConverter.class,
+    help =
+        "Additional places to search for archives before accessing the network "
+            + "to download them."
+  )
+  public List<PathFragment> experimentalDistdir;
+
+  @Option(
     name = "override_repository",
     defaultValue = "null",
     allowMultiple = true,
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java
index 1856bbd5af..8bcb5eb001 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java
@@ -32,6 +32,7 @@ import com.google.devtools.build.lib.syntax.EvalException;
 import com.google.devtools.build.lib.syntax.Type;
 import com.google.devtools.build.lib.util.JavaSleeper;
 import com.google.devtools.build.lib.util.Sleeper;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
 import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.build.lib.vfs.PathFragment;
 import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
@@ -58,11 +59,16 @@ public class HttpDownloader {
   private static final Semaphore semaphore = new Semaphore(MAX_PARALLEL_DOWNLOADS, true);
 
   protected final RepositoryCache repositoryCache;
+  private List<Path> distdir = ImmutableList.of();
 
   public HttpDownloader(RepositoryCache repositoryCache) {
     this.repositoryCache = repositoryCache;
   }
 
+  public void setDistdir(List<Path> distdir) {
+    this.distdir = ImmutableList.copyOf(distdir);
+  }
+
   /** Validates native repository rule attributes and calls the other download method. */
   public Path download(
       Rule rule,
@@ -182,6 +188,19 @@ public class HttpDownloader {
           return cachedDestination;
         }
       }
+
+      for (Path dir : distdir) {
+        Path candidate = dir.getRelative(destination.getBaseName());
+        if (RepositoryCache.getChecksum(KeyType.SHA256, candidate).equals(sha256)) {
+          // Found the archive in one of the distdirs, no need to download.
+          if (isCaching) {
+            repositoryCache.put(sha256, candidate, KeyType.SHA256);
+          }
+          FileSystemUtils.createDirectoryAndParents(destination.getParentDirectory());
+          FileSystemUtils.copyFile(candidate, destination);
+          return destination;
+        }
+      }
     }
 
     Clock clock = new JavaClock();
diff --git a/src/test/shell/bazel/external_integration_test.sh b/src/test/shell/bazel/external_integration_test.sh
index b5c5f844b6..cc3b18a21b 100755
--- a/src/test/shell/bazel/external_integration_test.sh
+++ b/src/test/shell/bazel/external_integration_test.sh
@@ -1168,6 +1168,50 @@ EOF
   expect_log '@ext//:foo'
 }
 
+function test_distdir() {
+  WRKDIR=$(mktemp -d "${TEST_TMPDIR}/testXXXXXX")
+  cd "${WRKDIR}"
+  mkdir ext
+  cat > ext/BUILD <<'EOF'
+genrule(
+  name="foo",
+  outs=["foo.txt"],
+  cmd="echo Hello World > $@",
+  visibility = ["//visibility:public"],
+)
+EOF
+  zip ext.zip ext/*
+  rm -rf ext
+  sha256=$(sha256sum ext.zip | head -c 64)
+
+  mkdir distfiles
+  mv ext.zip distfiles
+
+  mkdir main
+  cd main
+  cat > WORKSPACE <<EOF
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+http_archive(
+  name="ext",
+  strip_prefix="ext",
+  urls=["http://doesnotexist.example.com/outdatedpath/ext.zip"],
+  sha256="${sha256}",
+)
+EOF
+  cat > BUILD <<'EOF'
+genrule(
+  name = "local",
+  srcs = ["@ext//:foo"],
+  outs = ["local.txt"],
+  cmd = "cp $< $@",
+)
+EOF
+
+  bazel clean --expunge
+  bazel build --experimental_distdir="${WRKDIR}/distfiles" //:local \
+    || fail "expected success"
+}
+
 function test_good_symlinks() {
   WRKDIR=$(mktemp -d "${TEST_TMPDIR}/testXXXXXX")
   cd "${WRKDIR}"