Be smarter about Content-Encoding: gzip downloads

Fixes #2136

RELNOTES: n/a

--
MOS_MIGRATED_REVID=140190547

1 files changed, 21 insertions, 2 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnector.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnector.java
index 3f8f716051..c055f6f577 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnector.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnector.java
@@ -20,6 +20,7 @@ import static com.google.common.base.Strings.nullToEmpty;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Ascii;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.math.IntMath;
 import com.google.devtools.build.lib.events.Event;
 import com.google.devtools.build.lib.events.EventHandler;
@@ -50,6 +51,8 @@ final class HttpConnector {
   private static final int CONNECT_TIMEOUT_MS = 1000;
   private static final int MAX_CONNECT_TIMEOUT_MS = 10000;
   private static final int READ_TIMEOUT_MS = 20000;
+  private static final ImmutableSet<String> COMPRESSED_EXTENSIONS =
+      ImmutableSet.of("bz2", "gz", "jar", "tgz", "war", "xz", "zip");
 
   /**
    * Connects to HTTP (or file) URL with GET request and lazily returns payload.
@@ -83,7 +86,9 @@ final class HttpConnector {
       HttpURLConnection connection = null;
       try {
         connection = (HttpURLConnection) url.openConnection(proxy);
-        connection.setRequestProperty("Accept-Encoding", "gzip");
+        if (!COMPRESSED_EXTENSIONS.contains(getExtension(url.getPath()))) {
+          connection.setRequestProperty("Accept-Encoding", "gzip");
+        }
         connection.setConnectTimeout(connectTimeout);
         connection.setReadTimeout(READ_TIMEOUT_MS);
         int code;
@@ -173,7 +178,13 @@ final class HttpConnector {
         return connection.getInputStream();
       case "gzip":
       case "x-gzip":
-        return new GZIPInputStream(connection.getInputStream());
+        // Some web servers will send Content-Encoding: gzip even when we didn't request it, iff
+        // the file is a .gz file.
+        if (connection.getURL().getPath().endsWith(".gz")) {
+          return connection.getInputStream();
+        } else {
+          return new GZIPInputStream(connection.getInputStream());
+        }
       default:
         throw new UnrecoverableHttpException(
             "Unsupported and unrequested Content-Encoding: " + connection.getContentEncoding());
@@ -239,6 +250,14 @@ final class HttpConnector {
     return Ascii.equalsIgnoreCase(protocol, url.getProtocol());
   }
 
+  private static String getExtension(String path) {
+    int index = path.lastIndexOf('.');
+    if (index == -1) {
+      return "";
+    }
+    return path.substring(index + 1);
+  }
+
   private static final class UnrecoverableHttpException extends IOException {
     UnrecoverableHttpException(String message) {
       super(message);