From 97f2c842ae706eccd822d8d264aba66c6102bade Mon Sep 17 00:00:00 2001 From: Yue Gan Date: Mon, 21 Mar 2016 10:10:59 +0000 Subject: Add repository_ctx.download and repository_ctx.extract function. Fixes #1041 RELNOTES: Add repository_ctx.download and repository_ctx.download_and_extract function. -- MOS_MIGRATED_REVID=117698142 --- .../repository/downloader/HttpDownloader.java | 32 ++++++-- .../lib/bazel/repository/skylark/SkylarkPath.java | 4 + .../skylark/SkylarkRepositoryContext.java | 95 ++++++++++++++++++++++ 3 files changed, 124 insertions(+), 7 deletions(-) (limited to 'src/main/java') diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java index b0c82a17f9..1cf9e80fef 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java @@ -81,19 +81,37 @@ public class HttpDownloader { } } + @Nullable + public static Path download( + String url, String sha256, String type, Path output, EventHandler eventHandler) + throws RepositoryFunctionException { + try { + return new HttpDownloader(eventHandler, url, sha256, output, type).download(); + } catch (IOException e) { + throw new RepositoryFunctionException( + new IOException( + "Error downloading from " + url + " to " + output + ": " + e.getMessage()), + SkyFunctionException.Transience.TRANSIENT); + } + } + /** * Attempt to download a file from the repository's URL. Returns the path to the file downloaded. */ public Path download() throws IOException { URL url = new URL(urlString); - String filename = new PathFragment(url.getPath()).getBaseName(); - if (filename.isEmpty()) { - filename = "temp"; - } - if (!type.isEmpty()) { - filename += "." + type; + Path destination; + if (type == null) { + destination = outputDirectory; + } else { + String filename = new PathFragment(url.getPath()).getBaseName(); + if (filename.isEmpty()) { + filename = "temp"; + } else if (!type.isEmpty()) { + filename += "." + type; + } + destination = outputDirectory.getRelative(filename); } - Path destination = outputDirectory.getRelative(filename); if (!sha256.isEmpty()) { try { diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkPath.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkPath.java index 32ec145e3a..328fa74e94 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkPath.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkPath.java @@ -39,6 +39,10 @@ final class SkylarkPath { this.path = path; } + public Path getPath() { + return path; + } + @Override public boolean equals(Object obj) { return (obj instanceof SkylarkPath) && path.equals(((SkylarkPath) obj).path); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java index 7c48f5440d..97f0d5476d 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java @@ -17,6 +17,9 @@ package com.google.devtools.build.lib.bazel.repository.skylark; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.bazel.repository.DecompressorDescriptor; +import com.google.devtools.build.lib.bazel.repository.DecompressorValue; +import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.cmdline.LabelSyntaxException; import com.google.devtools.build.lib.events.Location; @@ -337,6 +340,98 @@ public class SkylarkRepositoryContext { return Runtime.NONE; } + @SkylarkCallable( + name = "download", + doc = + "Download a file to the output path for the provided url." + + "\nParameters:" + + "\nurl: a URL referencing an archive file containing a Bazel repository." + + " Archives of type .zip, .jar, .war, .tar.gz or .tgz are supported." + + " There is no support for authentication. Redirections are followed." + + "\noutput: " + + "(optional) sha256: the expected SHA-256 hash of the file downloaded." + + " This must match the SHA-256 hash of the file downloaded. It is a security risk to" + + " omit the SHA-256 as remote files can change. At best omitting this field will make" + + " your build non-hermetic. It is optional to make development easier but should" + + " be set before shipping." + ) + public void download(String url, Object output, String sha256) + throws RepositoryFunctionException, EvalException { + SkylarkPath outputPath = getPath("download()", output); + checkInOutputDirectory(outputPath); + HttpDownloader.download(url, sha256, null, outputPath.getPath(), env.getListener()); + } + + @SkylarkCallable(name = "download", documented = false) + public void download(String url, Object output) + throws RepositoryFunctionException, EvalException { + download(url, output, ""); + } + + @SkylarkCallable( + name = "download_and_extract", + doc = + "Download a file to the output path for the provided url, and extract it." + + "\nParameters:" + + "\nurl: a URL referencing an archive file containing a Bazel repository." + + " Archives of type .zip, .jar, .war, .tar.gz or .tgz are supported." + + " There is no support for authentication. Redirections are followed." + + "\noutput: " + + "\n(optional) sha256: the expected SHA-256 hash of the file downloaded." + + " This must match the SHA-256 hash of the file downloaded. It is a security risk to" + + " omit the SHA-256 as remote files can change. At best omitting this field will make" + + " your build non-hermetic. It is optional to make development easier but should" + + " be set before shipping." + + "\n(optional) type: The archive type of the downloaded file." + + " By default, the archive type is determined from the file extension of the URL." + + " If the file has no extension, you can explicitly specify either" + + "\"zip\", \"jar\", \"tar.gz\", or \"tgz\" here." + + "(optional) stripPrefix: a directory prefix to strip from the extracted files." + + "\nMany archives contain a top-level directory that contains alfiles in" + + " archive. Instead of needing to specify this prefix over and over in the" + + " build_file, this field can be used to strip it extracted" + + " files." + ) + public void download_and_extract( + String url, Object output, String sha256, String type, String stripPrefix) + throws RepositoryFunctionException, InterruptedException, EvalException { + // Download to outputDirectory and delete it after extraction + SkylarkPath outputPath = getPath("download_and_extract()", output); + checkInOutputDirectory(outputPath); + Path downloadedPath = + HttpDownloader.download(url, sha256, type, outputPath.getPath(), env.getListener()); + DecompressorValue.decompress( + DecompressorDescriptor.builder() + .setTargetKind(rule.getTargetKind()) + .setTargetName(rule.getName()) + .setArchivePath(downloadedPath) + .setRepositoryPath(outputPath.getPath()) + .setPrefix(stripPrefix) + .build()); + try { + if (downloadedPath.exists()) { + downloadedPath.delete(); + } + } catch (IOException e) { + throw new RepositoryFunctionException( + new IOException( + "Couldn't delete temporary file (" + downloadedPath.getPathString() + ")", e), + Transience.TRANSIENT); + } + } + + @SkylarkCallable(name = "download_and_extract", documented = false) + public void download_and_extract(String url, Object output, String type) + throws RepositoryFunctionException, InterruptedException, EvalException { + download_and_extract(url, output, "", "", type); + } + + @SkylarkCallable(name = "download_and_extract", documented = false) + public void download_and_extract(String url, Object output) + throws RepositoryFunctionException, InterruptedException, EvalException { + download_and_extract(url, output, "", "", ""); + } + // This is just for test to overwrite the path environment private static ImmutableList pathEnv = null; -- cgit v1.2.3