Add http_file() workspace function

The http_file workspace function is a remote repository that
make available inside a Bazel workspace a single file downloaded
from the web.

--
Change-Id: Ie6e9f5170d938583eda5b985c1f6377043e2b85b
Reviewed-on: https://bazel-review.googlesource.com/1620
MOS_MIGRATED_REVID=98739459

8 files changed, 278 insertions, 25 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
index 555f0cbbfe..620f24e9d4 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
@@ -22,10 +22,12 @@ import com.google.devtools.build.lib.analysis.BlazeVersionInfo;
 import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
 import com.google.devtools.build.lib.analysis.RuleDefinition;
 import com.google.devtools.build.lib.bazel.commands.FetchCommand;
+import com.google.devtools.build.lib.bazel.repository.FileFunction;
 import com.google.devtools.build.lib.bazel.repository.GitCloneFunction;
 import com.google.devtools.build.lib.bazel.repository.GitRepositoryFunction;
 import com.google.devtools.build.lib.bazel.repository.HttpArchiveFunction;
 import com.google.devtools.build.lib.bazel.repository.HttpDownloadFunction;
+import com.google.devtools.build.lib.bazel.repository.HttpFileFunction;
 import com.google.devtools.build.lib.bazel.repository.HttpJarFunction;
 import com.google.devtools.build.lib.bazel.repository.JarFunction;
 import com.google.devtools.build.lib.bazel.repository.LocalRepositoryFunction;
@@ -47,6 +49,7 @@ import com.google.devtools.build.lib.bazel.rules.android.AndroidSdkRepositoryFun
 import com.google.devtools.build.lib.bazel.rules.android.AndroidSdkRepositoryRule;
 import com.google.devtools.build.lib.bazel.rules.workspace.GitRepositoryRule;
 import com.google.devtools.build.lib.bazel.rules.workspace.HttpArchiveRule;
+import com.google.devtools.build.lib.bazel.rules.workspace.HttpFileRule;
 import com.google.devtools.build.lib.bazel.rules.workspace.HttpJarRule;
 import com.google.devtools.build.lib.bazel.rules.workspace.LocalRepositoryRule;
 import com.google.devtools.build.lib.bazel.rules.workspace.MavenJarRule;
@@ -83,22 +86,24 @@ public class BazelRepositoryModule extends BlazeModule {
   private GitCloneFunction gitCloneFunction;
 
   public BazelRepositoryModule() {
-    repositoryHandlers = ImmutableMap.<String, RepositoryFunction>builder()
-        .put(LocalRepositoryRule.NAME, new LocalRepositoryFunction())
-        .put(HttpArchiveRule.NAME, new HttpArchiveFunction())
-        .put(GitRepositoryRule.NAME, new GitRepositoryFunction())
-        .put(HttpJarRule.NAME, new HttpJarFunction())
-        .put(MavenJarRule.NAME, new MavenJarFunction())
-        .put(NewHttpArchiveRule.NAME, new NewHttpArchiveFunction())
-        .put(NewGitRepositoryRule.NAME, new NewGitRepositoryFunction())
-        .put(NewLocalRepositoryRule.NAME, new NewLocalRepositoryFunction())
-        .put(AndroidSdkRepositoryRule.NAME, new AndroidSdkRepositoryFunction())
-        .put(AndroidNdkRepositoryRule.NAME, new AndroidNdkRepositoryFunction())
-        .put(AndroidRepositoryRules.AndroidLocalRepositoryRule.NAME,
-            new AndroidLocalToolsRepositoryFunction())
-        .put(AndroidHttpToolsRepositoryRule.NAME,
-            new AndroidHttpToolsRepositoryFunction())
-        .build();
+    repositoryHandlers =
+        ImmutableMap.<String, RepositoryFunction>builder()
+            .put(LocalRepositoryRule.NAME, new LocalRepositoryFunction())
+            .put(HttpArchiveRule.NAME, new HttpArchiveFunction())
+            .put(GitRepositoryRule.NAME, new GitRepositoryFunction())
+            .put(HttpJarRule.NAME, new HttpJarFunction())
+            .put(HttpFileRule.NAME, new HttpFileFunction())
+            .put(MavenJarRule.NAME, new MavenJarFunction())
+            .put(NewHttpArchiveRule.NAME, new NewHttpArchiveFunction())
+            .put(NewGitRepositoryRule.NAME, new NewGitRepositoryFunction())
+            .put(NewLocalRepositoryRule.NAME, new NewLocalRepositoryFunction())
+            .put(AndroidSdkRepositoryRule.NAME, new AndroidSdkRepositoryFunction())
+            .put(AndroidNdkRepositoryRule.NAME, new AndroidNdkRepositoryFunction())
+            .put(
+                AndroidRepositoryRules.AndroidLocalRepositoryRule.NAME,
+                new AndroidLocalToolsRepositoryFunction())
+            .put(AndroidHttpToolsRepositoryRule.NAME, new AndroidHttpToolsRepositoryFunction())
+            .build();
   }
 
   @Override
@@ -168,6 +173,7 @@ public class BazelRepositoryModule extends BlazeModule {
     builder.put(JarFunction.NAME, new JarFunction());
     builder.put(ZipFunction.NAME, new ZipFunction());
     builder.put(TarGzFunction.NAME, new TarGzFunction());
+    builder.put(FileFunction.NAME, new FileFunction());
     return builder.build();
   }
 }
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorValue.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorValue.java
index bff9096f01..4d67554ad8 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorValue.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorValue.java
@@ -54,6 +54,13 @@ public class DecompressorValue implements SkyValue {
     return directory.hashCode();
   }
 
+  public static SkyKey fileKey(
+      String targetKind, String targetName, Path archivePath, Path repositoryPath) {
+    return new SkyKey(
+        FileFunction.NAME,
+        new DecompressorDescriptor(targetKind, targetName, archivePath, repositoryPath));
+  }
+
   public static SkyKey jarKey(
       String targetKind, String targetName, Path archivePath, Path repositoryPath) {
     return new SkyKey(JarFunction.NAME,
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/FileFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/FileFunction.java
new file mode 100644
index 0000000000..77759e0e24
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/FileFunction.java
@@ -0,0 +1,42 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository;
+
+import com.google.common.base.Joiner;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+
+/**
+ * Creates a repository for a random file.
+ */
+public class FileFunction extends JarFunction {
+
+  public static final SkyFunctionName NAME = SkyFunctionName.create("DOWNLOAD_FILE_FUNCTION");
+
+  @Override
+  protected String getPackageName() {
+    return "file";
+  }
+
+  @Override
+  protected String createBuildFile(String baseName) {
+    return Joiner.on("\n")
+        .join(
+            "filegroup(",
+            "    name = 'file',",
+            "    srcs = ['" + baseName + "'],",
+            "    visibility = ['//visibility:public']",
+            ")");
+  }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpFileFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpFileFunction.java
new file mode 100644
index 0000000000..5801368c83
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpFileFunction.java
@@ -0,0 +1,59 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository;
+
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.bazel.rules.workspace.HttpFileRule;
+import com.google.devtools.build.lib.packages.PackageIdentifier.RepositoryName;
+import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.skyframe.SkyFunctionException;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+
+import java.io.IOException;
+
+/**
+ * Downloads a jar file from a URL.
+ */
+public class HttpFileFunction extends HttpArchiveFunction {
+
+  @Override
+  public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException {
+    RepositoryName repositoryName = (RepositoryName) skyKey.argument();
+    Rule rule = RepositoryFunction.getRule(repositoryName, HttpFileRule.NAME, env);
+    if (rule == null) {
+      return null;
+    }
+    return compute(env, rule);
+  }
+
+  protected SkyKey decompressorValueKey(Rule rule, Path downloadPath, Path outputDirectory)
+      throws IOException {
+    return DecompressorValue.fileKey(
+        rule.getTargetKind(), rule.getName(), downloadPath, outputDirectory);
+  }
+
+  @Override
+  public SkyFunctionName getSkyFunctionName() {
+    return SkyFunctionName.create(HttpFileRule.NAME.toUpperCase());
+  }
+
+  @Override
+  public Class<? extends RuleDefinition> getRuleDefinition() {
+    return HttpFileRule.class;
+  }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/JarFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/JarFunction.java
index db11150f00..5ce88a8529 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/JarFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/JarFunction.java
@@ -14,6 +14,7 @@
 
 package com.google.devtools.build.lib.bazel.repository;
 
+import com.google.common.base.Joiner;
 import com.google.devtools.build.lib.bazel.repository.DecompressorValue.DecompressorDescriptor;
 import com.google.devtools.build.lib.bazel.repository.RepositoryFunction.RepositoryFunctionException;
 import com.google.devtools.build.lib.vfs.FileSystemUtils;
@@ -57,7 +58,7 @@ public class JarFunction implements SkyFunction {
           "# DO NOT EDIT: automatically generated WORKSPACE file for %s rule %s\n",
           descriptor.targetKind(), descriptor.targetName()));
       // external/some-name/jar.
-      Path jarDirectory = descriptor.repositoryPath().getRelative("jar");
+      Path jarDirectory = descriptor.repositoryPath().getRelative(getPackageName());
       FileSystemUtils.createDirectoryAndParents(jarDirectory);
       // external/some-name/repository/jar/foo.jar is a symbolic link to the jar in
       // external/some-name.
@@ -67,14 +68,14 @@ public class JarFunction implements SkyFunction {
       }
       // external/some-name/repository/jar/BUILD defines the //jar target.
       Path buildFile = jarDirectory.getRelative("BUILD");
-      FileSystemUtils.writeLinesAs(buildFile, Charset.forName("UTF-8"),
-          "# DO NOT EDIT: automatically generated BUILD file for " + descriptor.targetKind()
-              + " rule " + descriptor.targetName(),
-          "java_import(",
-          "    name = 'jar',",
-          "    jars = ['" + baseName + "'],",
-          "    visibility = ['//visibility:public']",
-          ")");
+      FileSystemUtils.writeLinesAs(
+          buildFile,
+          Charset.forName("UTF-8"),
+          "# DO NOT EDIT: automatically generated BUILD file for "
+              + descriptor.targetKind()
+              + " rule "
+              + descriptor.targetName(),
+          createBuildFile(baseName));
     } catch (IOException e) {
       throw new RepositoryFunctionException(new IOException(
           "Error auto-creating jar repo structure: " + e.getMessage()), Transience.TRANSIENT);
@@ -82,6 +83,20 @@ public class JarFunction implements SkyFunction {
     return new DecompressorValue(descriptor.repositoryPath());
   }
 
+  protected String getPackageName() {
+    return "jar";
+  }
+
+  protected String createBuildFile(String baseName) {
+    return Joiner.on("\n")
+        .join(
+            "java_import(",
+            "    name = 'jar',",
+            "    jars = ['" + baseName + "'],",
+            "    visibility = ['//visibility:public']",
+            ")");
+  }
+
   @Override
   @Nullable
   public String extractTag(SkyKey skyKey) {
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java
index d7ec9ff977..4b04b5a2cb 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java
@@ -62,6 +62,7 @@ import com.google.devtools.build.lib.bazel.rules.sh.BazelShRuleClasses;
 import com.google.devtools.build.lib.bazel.rules.sh.BazelShTestRule;
 import com.google.devtools.build.lib.bazel.rules.workspace.GitRepositoryRule;
 import com.google.devtools.build.lib.bazel.rules.workspace.HttpArchiveRule;
+import com.google.devtools.build.lib.bazel.rules.workspace.HttpFileRule;
 import com.google.devtools.build.lib.bazel.rules.workspace.HttpJarRule;
 import com.google.devtools.build.lib.bazel.rules.workspace.LocalRepositoryRule;
 import com.google.devtools.build.lib.bazel.rules.workspace.MavenJarRule;
@@ -343,6 +344,7 @@ public class BazelRuleClassProvider {
     builder.addRuleDefinition(new GitRepositoryRule());
     builder.addRuleDefinition(new HttpArchiveRule());
     builder.addRuleDefinition(new HttpJarRule());
+    builder.addRuleDefinition(new HttpFileRule());
     builder.addRuleDefinition(new LocalRepositoryRule());
     builder.addRuleDefinition(new MavenJarRule());
     builder.addRuleDefinition(new NewHttpArchiveRule());
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/HttpFileRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/HttpFileRule.java
new file mode 100644
index 0000000000..4af272c262
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/HttpFileRule.java
@@ -0,0 +1,88 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.rules.workspace;
+
+import static com.google.devtools.build.lib.packages.Attribute.attr;
+import static com.google.devtools.build.lib.packages.Type.STRING;
+
+import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment;
+import com.google.devtools.build.lib.packages.RuleClass;
+import com.google.devtools.build.lib.packages.RuleClass.Builder;
+import com.google.devtools.build.lib.packages.RuleClass.Builder.RuleClassType;
+
+/**
+ * Rule definition for the http_file rule.
+ */
+public class HttpFileRule implements RuleDefinition {
+
+  public static final String NAME = "http_file";
+
+  @Override
+  public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) {
+    return builder
+        /* <!-- #BLAZE_RULE(http_file).ATTRIBUTE(url) -->
+         A URL to a file that will be made available to Bazel.
+         ${SYNOPSIS}
+
+         <p>This must be an http or https URL. Authentication is not support and
+         redirects are not followed.</p>
+         <!-- #END_BLAZE_RULE.ATTRIBUTE --> */
+        .add(attr("url", STRING).mandatory())
+        /* <!-- #BLAZE_RULE(http_file).ATTRIBUTE(sha256) -->
+         The expected SHA-256 of the file downloaded.
+         ${SYNOPSIS}
+
+         <p>This must match the SHA-256 of the file downloaded.</p>
+         <!-- #END_BLAZE_RULE.ATTRIBUTE --> */
+        .add(attr("sha256", STRING).mandatory())
+        .setWorkspaceOnly()
+        .build();
+  }
+
+  @Override
+  public Metadata getMetadata() {
+    return RuleDefinition.Metadata.builder()
+        .name(HttpFileRule.NAME)
+        .type(RuleClassType.WORKSPACE)
+        .ancestors(WorkspaceBaseRule.class)
+        .factoryClass(WorkspaceConfiguredTargetFactory.class)
+        .build();
+  }
+}
+/*<!-- #BLAZE_RULE (NAME = http_file, TYPE = OTHER, FAMILY = Workspace)[GENERIC_RULE] -->
+
+ ${ATTRIBUTE_SIGNATURE}
+
+ <p>Downloads a file from a URL and makes it available to be used as a file group.</p>
+
+ ${ATTRIBUTE_DEFINITION}
+
+ <h4 id="http_file_examples">Examples</h4>
+
+ <p>Suppose you need to have a debian package for your custom rules. This package is available from
+ <i>http://example.com/package.deb</i>. Then you can add to your WORKSPACE file:</p>
+
+ <pre class="code">
+ http_file(
+ name = "my-deb",
+ url = "http://example.com/package.deb",
+ sha256 = "03a58ac630e59778f328af4bcc4acb4f80208ed4",
+ )
+ </pre>
+
+ <p>Targets would specify <code>@my-deb//file</code> as a dependency to depend on this file.</p>
+
+ <!-- #END_BLAZE_RULE -->*/
diff --git a/src/test/shell/bazel/external_integration_test.sh b/src/test/shell/bazel/external_integration_test.sh
index 3eb4a72e69..d5c1bb76fe 100755
--- a/src/test/shell/bazel/external_integration_test.sh
+++ b/src/test/shell/bazel/external_integration_test.sh
@@ -344,6 +344,40 @@ EOF
   expect_log "Tra-la!"
 }
 
+# Tests downloading a file and using it as a dependency.
+function test_http_download() {
+  local test_file=$TEST_TMPDIR/toto
+  echo "Tra-la!" >$test_file
+  local sha256=$(sha256sum $test_file | cut -f 1 -d ' ')
+  serve_file $test_file
+  cd ${WORKSPACE_DIR}
+
+  cat > WORKSPACE <<EOF
+http_file(name = 'toto', url = 'http://localhost:$nc_port/toto',
+    sha256 = '$sha256')
+EOF
+
+  mkdir -p test
+  cat > test/BUILD <<EOF
+sh_binary(
+    name = "test",
+    srcs = ["test.sh"],
+    data = ["@toto//file"],
+)
+EOF
+
+  cat > test/test.sh <<EOF
+#!/bin/bash
+cat external/toto/file/toto
+EOF
+
+  chmod +x test/test.sh
+  bazel fetch //test || fail "Fetch failed"
+  bazel run //test >& $TEST_log || echo "Expected run to succeed"
+  kill_nc
+  expect_log "Tra-la!"
+}
+
 function test_invalid_rule() {
   # http_jar with missing URL field.
   cat > WORKSPACE <<EOF