diff options
4 files changed, 224 insertions, 7 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java index b0c82a17f9..1cf9e80fef 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java @@ -81,19 +81,37 @@ public class HttpDownloader { } } + @Nullable + public static Path download( + String url, String sha256, String type, Path output, EventHandler eventHandler) + throws RepositoryFunctionException { + try { + return new HttpDownloader(eventHandler, url, sha256, output, type).download(); + } catch (IOException e) { + throw new RepositoryFunctionException( + new IOException( + "Error downloading from " + url + " to " + output + ": " + e.getMessage()), + SkyFunctionException.Transience.TRANSIENT); + } + } + /** * Attempt to download a file from the repository's URL. Returns the path to the file downloaded. */ public Path download() throws IOException { URL url = new URL(urlString); - String filename = new PathFragment(url.getPath()).getBaseName(); - if (filename.isEmpty()) { - filename = "temp"; - } - if (!type.isEmpty()) { - filename += "." + type; + Path destination; + if (type == null) { + destination = outputDirectory; + } else { + String filename = new PathFragment(url.getPath()).getBaseName(); + if (filename.isEmpty()) { + filename = "temp"; + } else if (!type.isEmpty()) { + filename += "." + type; + } + destination = outputDirectory.getRelative(filename); } - Path destination = outputDirectory.getRelative(filename); if (!sha256.isEmpty()) { try { diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkPath.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkPath.java index 32ec145e3a..328fa74e94 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkPath.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkPath.java @@ -39,6 +39,10 @@ final class SkylarkPath { this.path = path; } + public Path getPath() { + return path; + } + @Override public boolean equals(Object obj) { return (obj instanceof SkylarkPath) && path.equals(((SkylarkPath) obj).path); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java index 7c48f5440d..97f0d5476d 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java @@ -17,6 +17,9 @@ package com.google.devtools.build.lib.bazel.repository.skylark; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.bazel.repository.DecompressorDescriptor; +import com.google.devtools.build.lib.bazel.repository.DecompressorValue; +import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.cmdline.LabelSyntaxException; import com.google.devtools.build.lib.events.Location; @@ -337,6 +340,98 @@ public class SkylarkRepositoryContext { return Runtime.NONE; } + @SkylarkCallable( + name = "download", + doc = + "Download a file to the output path for the provided url." + + "\nParameters:" + + "\nurl: a URL referencing an archive file containing a Bazel repository." + + " Archives of type .zip, .jar, .war, .tar.gz or .tgz are supported." + + " There is no support for authentication. Redirections are followed." + + "\noutput: " + + "(optional) sha256: the expected SHA-256 hash of the file downloaded." + + " This must match the SHA-256 hash of the file downloaded. It is a security risk to" + + " omit the SHA-256 as remote files can change. At best omitting this field will make" + + " your build non-hermetic. It is optional to make development easier but should" + + " be set before shipping." + ) + public void download(String url, Object output, String sha256) + throws RepositoryFunctionException, EvalException { + SkylarkPath outputPath = getPath("download()", output); + checkInOutputDirectory(outputPath); + HttpDownloader.download(url, sha256, null, outputPath.getPath(), env.getListener()); + } + + @SkylarkCallable(name = "download", documented = false) + public void download(String url, Object output) + throws RepositoryFunctionException, EvalException { + download(url, output, ""); + } + + @SkylarkCallable( + name = "download_and_extract", + doc = + "Download a file to the output path for the provided url, and extract it." + + "\nParameters:" + + "\nurl: a URL referencing an archive file containing a Bazel repository." + + " Archives of type .zip, .jar, .war, .tar.gz or .tgz are supported." + + " There is no support for authentication. Redirections are followed." + + "\noutput: " + + "\n(optional) sha256: the expected SHA-256 hash of the file downloaded." + + " This must match the SHA-256 hash of the file downloaded. It is a security risk to" + + " omit the SHA-256 as remote files can change. At best omitting this field will make" + + " your build non-hermetic. It is optional to make development easier but should" + + " be set before shipping." + + "\n(optional) type: The archive type of the downloaded file." + + " By default, the archive type is determined from the file extension of the URL." + + " If the file has no extension, you can explicitly specify either" + + "\"zip\", \"jar\", \"tar.gz\", or \"tgz\" here." + + "(optional) stripPrefix: a directory prefix to strip from the extracted files." + + "\nMany archives contain a top-level directory that contains alfiles in" + + " archive. Instead of needing to specify this prefix over and over in the" + + " <code>build_file</code>, this field can be used to strip it extracted" + + " files." + ) + public void download_and_extract( + String url, Object output, String sha256, String type, String stripPrefix) + throws RepositoryFunctionException, InterruptedException, EvalException { + // Download to outputDirectory and delete it after extraction + SkylarkPath outputPath = getPath("download_and_extract()", output); + checkInOutputDirectory(outputPath); + Path downloadedPath = + HttpDownloader.download(url, sha256, type, outputPath.getPath(), env.getListener()); + DecompressorValue.decompress( + DecompressorDescriptor.builder() + .setTargetKind(rule.getTargetKind()) + .setTargetName(rule.getName()) + .setArchivePath(downloadedPath) + .setRepositoryPath(outputPath.getPath()) + .setPrefix(stripPrefix) + .build()); + try { + if (downloadedPath.exists()) { + downloadedPath.delete(); + } + } catch (IOException e) { + throw new RepositoryFunctionException( + new IOException( + "Couldn't delete temporary file (" + downloadedPath.getPathString() + ")", e), + Transience.TRANSIENT); + } + } + + @SkylarkCallable(name = "download_and_extract", documented = false) + public void download_and_extract(String url, Object output, String type) + throws RepositoryFunctionException, InterruptedException, EvalException { + download_and_extract(url, output, "", "", type); + } + + @SkylarkCallable(name = "download_and_extract", documented = false) + public void download_and_extract(String url, Object output) + throws RepositoryFunctionException, InterruptedException, EvalException { + download_and_extract(url, output, "", "", ""); + } + // This is just for test to overwrite the path environment private static ImmutableList<String> pathEnv = null; diff --git a/src/test/shell/bazel/skylark_repository_test.sh b/src/test/shell/bazel/skylark_repository_test.sh index 734a56caa2..16f52abb6a 100755 --- a/src/test/shell/bazel/skylark_repository_test.sh +++ b/src/test/shell/bazel/skylark_repository_test.sh @@ -18,8 +18,11 @@ # # Load test environment +src_dir=$(cd "$(dirname ${BASH_SOURCE[0]})" && pwd) source $(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/test-setup.sh \ || { echo "test-setup.sh not found!" >&2; exit 1; } +source "$src_dir/remote_helpers.sh" \ + || { echo "remote_helpers.sh not found!" >&2; exit 1; } # Basic test. function test_macro_local_repository() { @@ -429,6 +432,102 @@ EOF test ! -x "${output_base}/external/foo/test2" || fail "test2 is executable" } +function test_skylark_repository_download() { + # Prepare HTTP server with Python + mkdir "server_dir" + echo "This is one file" > server_dir/download_no_sha256.txt + echo "This is another file" > server_dir/download_with_sha256.txt + file_sha256="$(sha256sum server_dir/download_with_sha256.txt | head -c 64)" + + # Start HTTP server with Python + startup_server "server_dir" + + setup_skylark_repository + # Our custom repository rule + cat >test.bzl <<EOF +def _impl(repository_ctx): + repository_ctx.file("BUILD") + repository_ctx.download( + "http://localhost:${fileserver_port}/download_no_sha256.txt", + "download_no_sha256.txt") + repository_ctx.download( + "http://localhost:${fileserver_port}/download_with_sha256.txt", + "download_with_sha256.txt", "${file_sha256}") +repo = repository_rule(implementation=_impl, local=False) +EOF + + bazel build @foo//:all >& $TEST_log && shutdown_server \ + || fail "Execution of @foo//:all failed" + + output_base="$(bazel info output_base)" + # Test download + test -e "${output_base}/external/foo/download_no_sha256.txt" \ + || fail "download_no_sha256.txt is not downloaded" + test -e "${output_base}/external/foo/download_with_sha256.txt" \ + || fail "download_with_sha256.txt is not downloaded" + # Test download + diff "${output_base}/external/foo/download_no_sha256.txt" server_dir/download_no_sha256.txt >/dev/null \ + || fail "download_no_sha256.txt is not downloaded successfully" + diff "${output_base}/external/foo/download_with_sha256.txt" server_dir/download_with_sha256.txt >/dev/null \ + || fail "download_with_sha256.txt is not downloaded successfully" + + rm -rf "server_dir" +} + +function test_skylark_repository_download_and_extract() { + # Prepare HTTP server with Python + mkdir "server_dir" + echo "This is one file" > server_dir/download_and_extract1.txt + echo "This is another file" > server_dir/download_and_extract2.txt + echo "This is a third file" > server_dir/download_and_extract3.txt + tar -zcvf server_dir/download_and_extract1.tar.gz server_dir/download_and_extract1.txt + zip server_dir/download_and_extract2.zip server_dir/download_and_extract2.txt + zip server_dir/download_and_extract3.zip server_dir/download_and_extract3.txt + file_sha256="$(sha256sum server_dir/download_and_extract3.zip | head -c 64)" + + # Start HTTP server with Python + startup_server "server_dir" + + setup_skylark_repository + # Our custom repository rule + cat >test.bzl <<EOF +def _impl(repository_ctx): + repository_ctx.file("BUILD") + repository_ctx.download_and_extract( + "http://localhost:${fileserver_port}/download_and_extract1.tar.gz", "") + repository_ctx.download_and_extract( + "http://localhost:${fileserver_port}/download_and_extract2.zip", "", "") + repository_ctx.download_and_extract( + "http://localhost:${fileserver_port}/download_and_extract3.zip", ".", "${file_sha256}", "", "") +repo = repository_rule(implementation=_impl, local=False) +EOF + + bazel clean --expunge_async >& $TEST_log || fail "bazel clean failed" + bazel build @foo//:all >& $TEST_log && shutdown_server \ + || fail "Execution of @foo//:all failed" + + output_base="$(bazel info output_base)" + # Test cleanup + test -e "${output_base}/external/foo/server_dir/download_and_extract1.tar.gz" \ + && fail "temp file is not deleted successfully" || true + test -e "${output_base}/external/foo/server_dir/download_and_extract2.zip" \ + && fail "temp file is not deleted successfully" || true + test -e "${output_base}/external/foo/server_dir/download_and_extract3.zip" \ + && fail "temp file is not deleted successfully" || true + # Test download_and_extract + diff "${output_base}/external/foo/server_dir/download_and_extract1.txt" \ + server_dir/download_and_extract1.txt >/dev/null \ + || fail "download_and_extract1.tar.gz is not extracted successfully" + diff "${output_base}/external/foo/server_dir/download_and_extract2.txt" \ + server_dir/download_and_extract2.txt >/dev/null \ + || fail "download_and_extract2.zip is not extracted successfully" + diff "${output_base}/external/foo/server_dir/download_and_extract3.txt" \ + server_dir/download_and_extract3.txt >/dev/null \ + || fail "download_and_extract3.zip is not extracted successfully" + + rm -rf "server_dir" +} + # Test native.bazel_version function test_bazel_version() { create_new_workspace @@ -472,6 +571,7 @@ EOF } function tear_down() { + shutdown_server true } |