Separate download library so it can be reused by generate_workspace

--
MOS_MIGRATED_REVID=117268295

9 files changed, 172 insertions, 118 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/BUILD b/src/main/java/com/google/devtools/build/lib/BUILD
index 6ef6a8486e..3473d4764f 100644
--- a/src/main/java/com/google/devtools/build/lib/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/BUILD
@@ -29,6 +29,7 @@ filegroup(
         "//src/main/java/com/google/devtools/build/docgen:srcs",
         "//src/main/java/com/google/devtools/build/lib/actions:srcs",
         "//src/main/java/com/google/devtools/build/lib/bazel/dash:srcs",
+        "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader:srcs",
         "//src/main/java/com/google/devtools/build/lib/query2:srcs",
         "//src/main/java/com/google/devtools/build/lib/remote:srcs",
         "//src/main/java/com/google/devtools/build/lib/rules/apple:srcs",
@@ -614,6 +615,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib:shell",
         "//src/main/java/com/google/devtools/build/lib:skylarkinterface",
         "//src/main/java/com/google/devtools/build/lib:vfs",
+        "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader",
         "//src/main/java/com/google/devtools/build/skyframe",
         "//third_party:aether",
         "//third_party:apache_commons_compress",
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/GitCloner.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/GitCloner.java
index b862cf5de8..fc7555708e 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/GitCloner.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/GitCloner.java
@@ -14,6 +14,7 @@
 
 package com.google.devtools.build.lib.bazel.repository;
 
+import com.google.devtools.build.lib.bazel.repository.downloader.ProxyHelper;
 import com.google.devtools.build.lib.events.EventHandler;
 import com.google.devtools.build.lib.packages.AggregatingAttributeMapper;
 import com.google.devtools.build.lib.packages.Rule;
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpArchiveFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpArchiveFunction.java
index 3952360e1e..8761558931 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpArchiveFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpArchiveFunction.java
@@ -15,6 +15,7 @@
 package com.google.devtools.build.lib.bazel.repository;
 
 import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader;
 import com.google.devtools.build.lib.bazel.rules.workspace.HttpArchiveRule;
 import com.google.devtools.build.lib.packages.AggregatingAttributeMapper;
 import com.google.devtools.build.lib.packages.Rule;
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java
index 44c6ca33a9..30c2704f3d 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java
@@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.hash.Hasher;
 import com.google.common.hash.Hashing;
 import com.google.devtools.build.lib.analysis.RuleDefinition;
+import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader;
 import com.google.devtools.build.lib.bazel.rules.workspace.MavenJarRule;
 import com.google.devtools.build.lib.cmdline.RepositoryName;
 import com.google.devtools.build.lib.packages.AggregatingAttributeMapper;
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/NewHttpArchiveFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/NewHttpArchiveFunction.java
index 9d7af3ebcd..ecdbb5c165 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/NewHttpArchiveFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/NewHttpArchiveFunction.java
@@ -14,6 +14,7 @@
 
 package com.google.devtools.build.lib.bazel.repository;
 
+import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader;
 import com.google.devtools.build.lib.packages.AggregatingAttributeMapper;
 import com.google.devtools.build.lib.packages.Rule;
 import com.google.devtools.build.lib.rules.repository.NewRepositoryBuildFileHandler;
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/BUILD
new file mode 100644
index 0000000000..83818ca33e
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/BUILD
@@ -0,0 +1,23 @@
+package(
+    default_visibility = ["//src:__subpackages__"],
+)
+
+filegroup(
+    name = "srcs",
+    srcs = glob(["*"]),
+)
+
+java_library(
+    name = "downloader",
+    srcs = glob(["*.java"]),
+    deps = [
+        "//src/main/java/com/google/devtools/build/lib:build-base",
+        "//src/main/java/com/google/devtools/build/lib:events",
+        "//src/main/java/com/google/devtools/build/lib:packages-internal",
+        "//src/main/java/com/google/devtools/build/lib:syntax",
+        "//src/main/java/com/google/devtools/build/lib:vfs",
+        "//src/main/java/com/google/devtools/build/skyframe",
+        "//third_party:guava",
+        "//third_party:jsr305",
+    ],
+)
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnection.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnection.java
new file mode 100644
index 0000000000..0ec3265e99
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnection.java
@@ -0,0 +1,141 @@
+// Copyright 2016 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository.downloader;
+
+import com.google.common.io.ByteStreams;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.Proxy;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+
+/**
+ * Represents a connection over HTTP.
+ */
+class HttpConnection implements Closeable {
+  private static final int MAX_REDIRECTS = 20;
+  private final InputStream inputStream;
+  private final int contentLength;
+
+  private HttpConnection(InputStream inputStream, int contentLength) {
+    this.inputStream = inputStream;
+    this.contentLength = contentLength;
+  }
+
+  public InputStream getInputStream() {
+    return inputStream;
+  }
+
+  /**
+   * @return The length of the response, or -1 if unknown.
+   */
+  public int getContentLength() {
+    return contentLength;
+  }
+
+  @Override
+  public void close() throws IOException {
+    inputStream.close();
+  }
+
+  private static int parseContentLength(HttpURLConnection connection) {
+    String length;
+    try {
+      length = connection.getHeaderField("Content-Length");
+      if (length == null) {
+        return -1;
+      }
+      return Integer.parseInt(length);
+    } catch (NumberFormatException e) {
+      return -1;
+    }
+  }
+
+  public static HttpConnection createAndConnect(URL url) throws IOException {
+    int retries = MAX_REDIRECTS;
+    Proxy proxy = ProxyHelper.createProxyIfNeeded(url.toString());
+    do {
+      HttpURLConnection connection = (HttpURLConnection) url.openConnection(proxy);
+      try {
+        connection.connect();
+      } catch (IllegalArgumentException e) {
+        throw new IOException("Failed to connect to " + url + " : " + e.getMessage(), e);
+      }
+
+      int statusCode = connection.getResponseCode();
+      switch (statusCode) {
+        case HttpURLConnection.HTTP_OK:
+          return new HttpConnection(connection.getInputStream(), parseContentLength(connection));
+        case HttpURLConnection.HTTP_MOVED_PERM:
+        case HttpURLConnection.HTTP_MOVED_TEMP:
+          url = tryGetLocation(statusCode, connection);
+          connection.disconnect();
+          break;
+        case -1:
+          throw new IOException("An HTTP error occured");
+        default:
+          throw new IOException(String.format("%s %s: %s",
+              connection.getResponseCode(),
+              connection.getResponseMessage(),
+              readBody(connection)));
+      }
+    } while (retries-- > 0);
+    throw new IOException("Maximum redirects (" + MAX_REDIRECTS + ") exceeded");
+  }
+
+  private static URL tryGetLocation(int statusCode, HttpURLConnection connection)
+      throws IOException {
+    String newLocation = connection.getHeaderField("Location");
+    if (newLocation == null) {
+      throw new IOException(
+          "Remote returned " + statusCode + " but did not return location header.");
+    }
+
+    URL newUrl;
+    try {
+      newUrl = new URL(newLocation);
+    } catch (MalformedURLException e) {
+      throw new IOException("Remote returned invalid location header: " + newLocation);
+    }
+
+    String newProtocol = newUrl.getProtocol();
+    if (!("http".equals(newProtocol) || "https".equals(newProtocol))) {
+      throw new IOException(
+          "Remote returned invalid location header: " + newLocation);
+    }
+
+    return newUrl;
+  }
+
+  private static String readBody(HttpURLConnection connection) throws IOException {
+    InputStream errorStream = connection.getErrorStream();
+    if (errorStream != null) {
+      // TODO(kchodorow): detect encoding.
+      return new String(ByteStreams.toByteArray(errorStream), StandardCharsets.UTF_8);
+    }
+
+    InputStream responseStream = connection.getInputStream();
+    if (responseStream != null) {
+      // TODO(kchodorow): detect encoding.
+      return new String(ByteStreams.toByteArray(responseStream), StandardCharsets.UTF_8);
+    }
+
+    return null;
+  }
+}
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpDownloader.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java
index 147bb45ab2..b0c82a17f9 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpDownloader.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java
@@ -12,11 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package com.google.devtools.build.lib.bazel.repository;
+package com.google.devtools.build.lib.bazel.repository.downloader;
 
 import com.google.common.hash.Hasher;
 import com.google.common.hash.Hashing;
-import com.google.common.io.ByteStreams;
 import com.google.devtools.build.lib.events.Event;
 import com.google.devtools.build.lib.events.EventHandler;
 import com.google.devtools.build.lib.packages.AggregatingAttributeMapper;
@@ -27,15 +26,10 @@ import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.build.lib.vfs.PathFragment;
 import com.google.devtools.build.skyframe.SkyFunctionException;
 
-import java.io.Closeable;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
-import java.net.HttpURLConnection;
-import java.net.MalformedURLException;
-import java.net.Proxy;
 import java.net.URL;
-import java.nio.charset.StandardCharsets;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
@@ -48,7 +42,6 @@ import javax.annotation.Nullable;
  * Helper class for downloading a file from a URL.
  */
 public class HttpDownloader {
-  private static final int MAX_REDIRECTS = 20;
   private static final int BUFFER_SIZE = 32 * 1024;
   private static final int KB = 1024;
   private static final String UNITS = " KMGTPEY";
@@ -194,115 +187,6 @@ public class HttpDownloader {
         + (UNITS.charAt(logBaseUnitOfBytes) + "B");
   }
 
-  private static class HttpConnection implements Closeable {
-    private final InputStream inputStream;
-    private final int contentLength;
-
-    private HttpConnection(InputStream inputStream, int contentLength) {
-      this.inputStream = inputStream;
-      this.contentLength = contentLength;
-    }
-
-    public InputStream getInputStream() {
-      return inputStream;
-    }
-
-    /**
-     * @return The length of the response, or -1 if unknown.
-     */
-    public int getContentLength() {
-      return contentLength;
-    }
-
-    @Override
-    public void close() throws IOException {
-      inputStream.close();
-    }
-
-    private static int parseContentLength(HttpURLConnection connection) {
-      String length;
-      try {
-        length = connection.getHeaderField("Content-Length");
-        if (length == null) {
-          return -1;
-        }
-        return Integer.parseInt(length);
-      } catch (NumberFormatException e) {
-        return -1;
-      }
-    }
-
-    public static HttpConnection createAndConnect(URL url) throws IOException {
-      int retries = MAX_REDIRECTS;
-      Proxy proxy = ProxyHelper.createProxyIfNeeded(url.toString());
-      do {
-        HttpURLConnection connection = (HttpURLConnection) url.openConnection(proxy);
-        try {
-          connection.connect();
-        } catch (IllegalArgumentException e) {
-          throw new IOException("Failed to connect to " + url + " : " + e.getMessage(), e);
-        }
-
-        int statusCode = connection.getResponseCode();
-        switch (statusCode) {
-          case HttpURLConnection.HTTP_OK:
-            return new HttpConnection(connection.getInputStream(), parseContentLength(connection));
-          case HttpURLConnection.HTTP_MOVED_PERM:
-          case HttpURLConnection.HTTP_MOVED_TEMP:
-            url = tryGetLocation(statusCode, connection);
-            connection.disconnect();
-            break;
-          case -1:
-            throw new IOException("An HTTP error occured");
-          default:
-            throw new IOException(String.format("%s %s: %s",
-                connection.getResponseCode(),
-                connection.getResponseMessage(),
-                readBody(connection)));
-          }
-      } while (retries-- > 0);
-      throw new IOException("Maximum redirects (" + MAX_REDIRECTS + ") exceeded");
-    }
-
-    private static URL tryGetLocation(int statusCode, HttpURLConnection connection)
-        throws IOException {
-      String newLocation = connection.getHeaderField("Location");
-      if (newLocation == null) {
-        throw new IOException(
-            "Remote returned " + statusCode + " but did not return location header.");
-      }
-
-      URL newUrl;
-      try {
-        newUrl = new URL(newLocation);
-      } catch (MalformedURLException e) {
-        throw new IOException("Remote returned invalid location header: " + newLocation);
-      }
-
-      String newProtocol = newUrl.getProtocol();
-      if (!("http".equals(newProtocol) || "https".equals(newProtocol))) {
-        throw new IOException(
-            "Remote returned invalid location header: " + newLocation);
-      }
-
-      return newUrl;
-    }
-
-    private static String readBody(HttpURLConnection connection) throws IOException {
-      InputStream errorStream = connection.getErrorStream();
-      if (errorStream != null) {
-        return new String(ByteStreams.toByteArray(errorStream), StandardCharsets.UTF_8);
-      }
-
-      InputStream responseStream = connection.getInputStream();
-      if (responseStream != null) {
-        return new String(ByteStreams.toByteArray(responseStream), StandardCharsets.UTF_8);
-      }
-
-      return null;
-    }
-  }
-
   public static String getHash(Hasher hasher, Path path) throws IOException {
     byte byteBuffer[] = new byte[BUFFER_SIZE];
     try (InputStream stream = path.getInputStream()) {
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/ProxyHelper.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/ProxyHelper.java
index 0cb37723d9..1c7f8a6a4a 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/ProxyHelper.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/ProxyHelper.java
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package com.google.devtools.build.lib.bazel.repository;
+package com.google.devtools.build.lib.bazel.repository.downloader;
 
 import com.google.common.base.Strings;