Add repository_ctx.download and repository_ctx.extract function.

Fixes #1041

RELNOTES: Add repository_ctx.download and repository_ctx.download_and_extract function.

--
MOS_MIGRATED_REVID=117698142

3 files changed, 124 insertions, 7 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java
index b0c82a17f9..1cf9e80fef 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java
@@ -81,19 +81,37 @@ public class HttpDownloader {
     }
   }
 
+  @Nullable
+  public static Path download(
+      String url, String sha256, String type, Path output, EventHandler eventHandler)
+      throws RepositoryFunctionException {
+    try {
+      return new HttpDownloader(eventHandler, url, sha256, output, type).download();
+    } catch (IOException e) {
+      throw new RepositoryFunctionException(
+          new IOException(
+              "Error downloading from " + url + " to " + output + ": " + e.getMessage()),
+          SkyFunctionException.Transience.TRANSIENT);
+    }
+  }
+
   /**
    * Attempt to download a file from the repository's URL. Returns the path to the file downloaded.
    */
   public Path download() throws IOException {
     URL url = new URL(urlString);
-    String filename = new PathFragment(url.getPath()).getBaseName();
-    if (filename.isEmpty()) {
-      filename = "temp";
-    }
-    if (!type.isEmpty()) {
-      filename += "." + type;
+    Path destination;
+    if (type == null) {
+      destination = outputDirectory;
+    } else {
+      String filename = new PathFragment(url.getPath()).getBaseName();
+      if (filename.isEmpty()) {
+        filename = "temp";
+      } else if (!type.isEmpty()) {
+        filename += "." + type;
+      }
+      destination = outputDirectory.getRelative(filename);
     }
-    Path destination = outputDirectory.getRelative(filename);
 
     if (!sha256.isEmpty()) {
       try {
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkPath.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkPath.java
index 32ec145e3a..328fa74e94 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkPath.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkPath.java
@@ -39,6 +39,10 @@ final class SkylarkPath {
     this.path = path;
   }
 
+  public Path getPath() {
+    return path;
+  }
+
   @Override
   public boolean equals(Object obj) {
     return (obj instanceof SkylarkPath) &&  path.equals(((SkylarkPath) obj).path);
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java
index 7c48f5440d..97f0d5476d 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java
@@ -17,6 +17,9 @@ package com.google.devtools.build.lib.bazel.repository.skylark;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.devtools.build.lib.bazel.repository.DecompressorDescriptor;
+import com.google.devtools.build.lib.bazel.repository.DecompressorValue;
+import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader;
 import com.google.devtools.build.lib.cmdline.Label;
 import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
 import com.google.devtools.build.lib.events.Location;
@@ -337,6 +340,98 @@ public class SkylarkRepositoryContext {
     return Runtime.NONE;
   }
 
+  @SkylarkCallable(
+    name = "download",
+    doc =
+        "Download a file to the output path for the provided url."
+            + "\nParameters:"
+            + "\nurl: a URL referencing an archive file containing a Bazel repository."
+            + " Archives of type .zip, .jar, .war, .tar.gz or .tgz are supported."
+            + " There is no support for authentication. Redirections are followed."
+            + "\noutput: "
+            + "(optional) sha256: the expected SHA-256 hash of the file downloaded."
+            + " This must match the SHA-256 hash of the file downloaded. It is a security risk to"
+            + " omit the SHA-256 as remote files can change. At best omitting this field will make"
+            + " your build non-hermetic. It is optional to make development easier but should"
+            + " be set before shipping."
+  )
+  public void download(String url, Object output, String sha256)
+      throws RepositoryFunctionException, EvalException {
+    SkylarkPath outputPath = getPath("download()", output);
+    checkInOutputDirectory(outputPath);
+    HttpDownloader.download(url, sha256, null, outputPath.getPath(), env.getListener());
+  }
+
+  @SkylarkCallable(name = "download", documented = false)
+  public void download(String url, Object output)
+      throws RepositoryFunctionException, EvalException {
+    download(url, output, "");
+  }
+
+  @SkylarkCallable(
+    name = "download_and_extract",
+    doc =
+        "Download a file to the output path for the provided url, and extract it."
+            + "\nParameters:"
+            + "\nurl: a URL referencing an archive file containing a Bazel repository."
+            + " Archives of type .zip, .jar, .war, .tar.gz or .tgz are supported."
+            + " There is no support for authentication. Redirections are followed."
+            + "\noutput: "
+            + "\n(optional) sha256: the expected SHA-256 hash of the file downloaded."
+            + " This must match the SHA-256 hash of the file downloaded. It is a security risk to"
+            + " omit the SHA-256 as remote files can change. At best omitting this field will make"
+            + " your build non-hermetic. It is optional to make development easier but should"
+            + " be set before shipping."
+            + "\n(optional) type: The archive type of the downloaded file."
+            + " By default, the archive type is determined from the file extension of the URL."
+            + " If the file has no extension, you can explicitly specify either"
+            + "\"zip\", \"jar\", \"tar.gz\", or \"tgz\" here."
+            + "(optional) stripPrefix: a directory prefix to strip from the extracted files."
+            + "\nMany archives contain a top-level directory that contains alfiles in"
+            + " archive. Instead of needing to specify this prefix over and over in the"
+            + " <code>build_file</code>, this field can be used to strip it extracted"
+            + " files."
+  )
+  public void download_and_extract(
+      String url, Object output, String sha256, String type, String stripPrefix)
+      throws RepositoryFunctionException, InterruptedException, EvalException {
+    // Download to outputDirectory and delete it after extraction
+    SkylarkPath outputPath = getPath("download_and_extract()", output);
+    checkInOutputDirectory(outputPath);
+    Path downloadedPath =
+        HttpDownloader.download(url, sha256, type, outputPath.getPath(), env.getListener());
+    DecompressorValue.decompress(
+        DecompressorDescriptor.builder()
+            .setTargetKind(rule.getTargetKind())
+            .setTargetName(rule.getName())
+            .setArchivePath(downloadedPath)
+            .setRepositoryPath(outputPath.getPath())
+            .setPrefix(stripPrefix)
+            .build());
+    try {
+      if (downloadedPath.exists()) {
+        downloadedPath.delete();
+      }
+    } catch (IOException e) {
+      throw new RepositoryFunctionException(
+          new IOException(
+              "Couldn't delete temporary file (" + downloadedPath.getPathString() + ")", e),
+          Transience.TRANSIENT);
+    }
+  }
+
+  @SkylarkCallable(name = "download_and_extract", documented = false)
+  public void download_and_extract(String url, Object output, String type)
+      throws RepositoryFunctionException, InterruptedException, EvalException {
+    download_and_extract(url, output, "", "", type);
+  }
+
+  @SkylarkCallable(name = "download_and_extract", documented = false)
+  public void download_and_extract(String url, Object output)
+      throws RepositoryFunctionException, InterruptedException, EvalException {
+    download_and_extract(url, output, "", "", "");
+  }
+
   // This is just for test to overwrite the path environment
   private static ImmutableList<String> pathEnv = null;