diff options
39 files changed, 3383 insertions, 519 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java index 0d78adcb5f..47fa3f3eb7 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/FetchCommand.java @@ -123,7 +123,9 @@ public final class FetchCommand implements BlazeCommand { // Throw away the result. } }); - } catch (QueryException | InterruptedException e) { + } catch (InterruptedException e) { + return ExitCode.COMMAND_LINE_ERROR; + } catch (QueryException e) { // Keep consistent with reportBuildFileError() env.getReporter().handle(Event.error(e.getMessage())); return ExitCode.COMMAND_LINE_ERROR; diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/GitCloner.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/GitCloner.java index 12ff3966ee..9b347ae639 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/GitCloner.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/GitCloner.java @@ -14,6 +14,7 @@ package com.google.devtools.build.lib.bazel.repository; +import com.google.common.base.Ascii; import com.google.devtools.build.lib.bazel.repository.downloader.ProxyHelper; import com.google.devtools.build.lib.events.EventHandler; import com.google.devtools.build.lib.packages.Rule; @@ -25,7 +26,11 @@ import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; import com.google.devtools.build.skyframe.SkyValue; - +import java.io.IOException; +import java.net.URL; +import java.util.Map; +import java.util.Objects; +import java.util.Set; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.Status; import org.eclipse.jgit.api.errors.GitAPIException; @@ -39,11 +44,6 @@ import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.storage.file.FileRepositoryBuilder; import org.eclipse.jgit.transport.NetRCCredentialsProvider; -import java.io.IOException; -import java.util.Map; -import java.util.Objects; -import java.util.Set; - /** * Clones a Git repository, checks out the provided branch, tag, or commit, and * clones submodules if specified. @@ -123,9 +123,9 @@ public class GitCloner { } // Setup proxy if remote is http or https - if (descriptor.remote != null && descriptor.remote.startsWith("http")) { + if (descriptor.remote != null && Ascii.toLowerCase(descriptor.remote).startsWith("http")) { try { - ProxyHelper.createProxyIfNeeded(descriptor.remote, clientEnvironment); + new ProxyHelper(clientEnvironment).createProxyIfNeeded(new URL(descriptor.remote)); } catch (IOException ie) { throw new RepositoryFunctionException(ie, Transience.TRANSIENT); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/cache/RepositoryCache.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/cache/RepositoryCache.java index 64cdfdf14a..bd743b3532 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/cache/RepositoryCache.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/cache/RepositoryCache.java @@ -15,6 +15,7 @@ package com.google.devtools.build.lib.bazel.repository.cache; import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import com.google.common.hash.HashFunction; import com.google.common.hash.Hasher; import com.google.common.hash.Hashing; @@ -48,7 +49,7 @@ public class RepositoryCache { } public boolean isValid(@Nullable String checksum) { - return checksum != null && checksum.matches(regexp); + return !Strings.isNullOrEmpty(checksum) && checksum.matches(regexp); } public Path getCachePath(Path parentDirectory) { @@ -58,7 +59,7 @@ public class RepositoryCache { public Hasher newHasher() { return hashFunction.newHasher(); } - + @Override public String toString() { return stringRepr; diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/BUILD index 82f39cc928..c4ebcf0476 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/BUILD @@ -12,9 +12,11 @@ java_library( srcs = glob(["*.java"]), deps = [ "//src/main/java/com/google/devtools/build/lib:build-base", + "//src/main/java/com/google/devtools/build/lib:concurrent", "//src/main/java/com/google/devtools/build/lib:events", "//src/main/java/com/google/devtools/build/lib:packages-internal", "//src/main/java/com/google/devtools/build/lib:syntax", + "//src/main/java/com/google/devtools/build/lib:util", "//src/main/java/com/google/devtools/build/lib:vfs", "//src/main/java/com/google/devtools/build/lib/bazel/repository/cache", "//src/main/java/com/google/devtools/build/skyframe", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HashInputStream.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HashInputStream.java new file mode 100644 index 0000000000..ee7b1bd0c4 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HashInputStream.java @@ -0,0 +1,93 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import com.google.common.hash.HashCode; +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hasher; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import java.io.IOException; +import java.io.InputStream; +import javax.annotation.Nullable; +import javax.annotation.WillCloseWhenClosed; + +/** + * Input stream that guarantees its contents matches a hash code. + * + * <p>The actual checksum is computed gradually as the input is read. If it doesn't match, then an + * {@link IOException} will be thrown when {@link #close()} is called, or when any read method is + * called that detects the end of stream. This error will be thrown multiple times if these methods + * are called again for some reason. + * + * <p>This class is not thread safe, but it is safe to message pass this object between threads. + */ +@ThreadCompatible +final class HashInputStream extends InputStream { + + private final InputStream delegate; + private final Hasher hasher; + private final HashCode code; + @Nullable private volatile HashCode actual; + + HashInputStream( + @WillCloseWhenClosed InputStream delegate, HashFunction function, HashCode code) { + this.delegate = delegate; + this.hasher = function.newHasher(); + this.code = code; + } + + @Override + public int read() throws IOException { + int result = delegate.read(); + if (result == -1) { + check(); + } else { + hasher.putByte((byte) result); + } + return result; + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + int amount = delegate.read(buffer, offset, length); + if (amount == -1) { + check(); + } else { + hasher.putBytes(buffer, offset, amount); + } + return amount; + } + + @Override + public int available() throws IOException { + return delegate.available(); + } + + @Override + public void close() throws IOException { + delegate.close(); + check(); + } + + private void check() throws IOException { + if (actual == null) { + actual = hasher.hash(); + } + if (!code.equals(actual)) { + throw new UnrecoverableHttpException( + String.format("Checksum was %s but wanted %s", actual, code)); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnector.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnector.java index c055f6f577..ba2c64f424 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnector.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnector.java @@ -14,89 +14,109 @@ package com.google.devtools.build.lib.bazel.repository.downloader; -import static com.google.common.base.MoreObjects.firstNonNull; -import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.base.Strings.nullToEmpty; - -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Ascii; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.io.ByteStreams; import com.google.common.math.IntMath; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.Sleeper; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.InterruptedIOException; import java.net.HttpURLConnection; -import java.net.MalformedURLException; -import java.net.Proxy; import java.net.SocketTimeoutException; -import java.net.URI; -import java.net.URISyntaxException; import java.net.URL; +import java.net.URLConnection; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.zip.GZIPInputStream; +import java.util.Locale; +import java.util.Map; import javax.annotation.Nullable; - -/** Utility class for connecting to HTTP servers for downloading files. */ -final class HttpConnector { +import javax.annotation.WillClose; + +/** + * Class for establishing connections to HTTP servers for downloading files. + * + * <p>This class must be used in conjunction with {@link HttpConnectorMultiplexer}. + * + * <p>Instances are thread safe and can be reused. + */ +@ThreadSafe +class HttpConnector { private static final int MAX_RETRIES = 8; - private static final int MAX_REDIRECTS = 20; + private static final int MAX_REDIRECTS = 40; private static final int MIN_RETRY_DELAY_MS = 100; - private static final int CONNECT_TIMEOUT_MS = 1000; + private static final int MIN_CONNECT_TIMEOUT_MS = 1000; private static final int MAX_CONNECT_TIMEOUT_MS = 10000; private static final int READ_TIMEOUT_MS = 20000; private static final ImmutableSet<String> COMPRESSED_EXTENSIONS = ImmutableSet.of("bz2", "gz", "jar", "tgz", "war", "xz", "zip"); - /** - * Connects to HTTP (or file) URL with GET request and lazily returns payload. - * - * <p>This routine supports gzip, redirects, retries, and exponential backoff. It's designed to - * recover fast from transient errors. However please note that this this reliability magic only - * applies to the connection and header reading phase. - * - * @param url URL to download, which can be file, http, or https - * @param proxy HTTP proxy to use or {@link Proxy#NO_PROXY} if none is desired - * @param eventHandler Bazel event handler for reporting real-time progress on retries - * @throws IOException if response returned ≥400 after max retries or ≥300 after max redirects - * @throws InterruptedException if thread is being cast into oblivion - */ - static InputStream connect( - URL url, Proxy proxy, EventHandler eventHandler) - throws IOException, InterruptedException { - checkNotNull(proxy); - checkNotNull(eventHandler); - if (isProtocol(url, "file")) { - return url.openConnection().getInputStream(); + private final Locale locale; + private final EventHandler eventHandler; + private final ProxyHelper proxyHelper; + private final Sleeper sleeper; + + HttpConnector( + Locale locale, EventHandler eventHandler, ProxyHelper proxyHelper, Sleeper sleeper) { + this.locale = locale; + this.eventHandler = eventHandler; + this.proxyHelper = proxyHelper; + this.sleeper = sleeper; + } + + URLConnection connect( + URL originalUrl, ImmutableMap<String, String> requestHeaders) + throws IOException { + if (Thread.interrupted()) { + throw new InterruptedIOException(); } - if (!isHttp(url)) { - throw new IOException("Protocol must be http, https, or file"); + URL url = originalUrl; + if (HttpUtils.isProtocol(url, "file")) { + return url.openConnection(); } List<Throwable> suppressions = new ArrayList<>(); int retries = 0; int redirects = 0; - int connectTimeout = CONNECT_TIMEOUT_MS; + int connectTimeout = MIN_CONNECT_TIMEOUT_MS; while (true) { HttpURLConnection connection = null; try { - connection = (HttpURLConnection) url.openConnection(proxy); - if (!COMPRESSED_EXTENSIONS.contains(getExtension(url.getPath()))) { - connection.setRequestProperty("Accept-Encoding", "gzip"); + connection = (HttpURLConnection) + url.openConnection(proxyHelper.createProxyIfNeeded(url)); + boolean isAlreadyCompressed = + COMPRESSED_EXTENSIONS.contains(HttpUtils.getExtension(url.getPath())) + || COMPRESSED_EXTENSIONS.contains(HttpUtils.getExtension(originalUrl.getPath())); + for (Map.Entry<String, String> entry : requestHeaders.entrySet()) { + if (isAlreadyCompressed && Ascii.equalsIgnoreCase(entry.getKey(), "Accept-Encoding")) { + // We're not going to ask for compression if we're downloading a file that already + // appears to be compressed. + continue; + } + connection.setRequestProperty(entry.getKey(), entry.getValue()); } connection.setConnectTimeout(connectTimeout); + // The read timeout is always large because it stays in effect after this method. connection.setReadTimeout(READ_TIMEOUT_MS); + // Java tries to abstract HTTP error responses for us. We don't want that. So we're going + // to try and undo any IOException that doesn't appepar to be a legitimate I/O exception. int code; try { connection.connect(); code = connection.getResponseCode(); } catch (FileNotFoundException ignored) { code = connection.getResponseCode(); + } catch (UnknownHostException e) { + String message = "Unknown host: " + e.getMessage(); + eventHandler.handle(Event.progress(message)); + throw new UnrecoverableHttpException(message); } catch (IllegalArgumentException e) { // This will happen if the user does something like specify a port greater than 2^16-1. throw new UnrecoverableHttpException(e.getMessage()); @@ -106,163 +126,115 @@ final class HttpConnector { } code = connection.getResponseCode(); } - if (code == 200) { - return getInputStream(connection); - } else if (code == 301 || code == 302) { + // 206 means partial content and only happens if caller specified Range. See RFC7233 § 4.1. + if (code == 200 || code == 206) { + return connection; + } else if (code == 301 || code == 302 || code == 307) { readAllBytesAndClose(connection.getInputStream()); if (++redirects == MAX_REDIRECTS) { + eventHandler.handle(Event.progress("Redirect loop detected in " + originalUrl)); throw new UnrecoverableHttpException("Redirect loop detected"); } - url = getLocation(connection); - } else if (code < 500) { + url = HttpUtils.getLocation(connection); + if (code == 301) { + originalUrl = url; + } + } else if (code == 403) { + // jart@ has noticed BitBucket + Amazon AWS downloads frequently flake with this code. + throw new IOException(describeHttpResponse(connection)); + } else if (code == 408) { + // The 408 (Request Timeout) status code indicates that the server did not receive a + // complete request message within the time that it was prepared to wait. Server SHOULD + // send the "close" connection option (Section 6.1 of [RFC7230]) in the response, since + // 408 implies that the server has decided to close the connection rather than continue + // waiting. If the client has an outstanding request in transit, the client MAY repeat + // that request on a new connection. Quoth RFC7231 § 6.5.7 + throw new IOException(describeHttpResponse(connection)); + } else if (code < 500 // 4xx means client seems to have erred quoth RFC7231 § 6.5 + || code == 501 // Server doesn't support function quoth RFC7231 § 6.6.2 + || code == 502 // Host not configured on server cf. RFC7231 § 6.6.3 + || code == 505) { // Server refuses to support version quoth RFC7231 § 6.6.6 + // This is a permanent error so we're not going to retry. readAllBytesAndClose(connection.getErrorStream()); throw new UnrecoverableHttpException(describeHttpResponse(connection)); } else { + // However we will retry on some 5xx errors, particularly 500 and 503. throw new IOException(describeHttpResponse(connection)); } - } catch (InterruptedIOException e) { - throw new InterruptedException(); } catch (UnrecoverableHttpException e) { throw e; - } catch (UnknownHostException e) { - throw new IOException("Unknown host: " + e.getMessage()); } catch (IOException e) { if (connection != null) { + // If we got here, it means we might not have consumed the entire payload of the + // response, if any. So we're going to force this socket to disconnect and not be + // reused. This is particularly important if multiple threads end up establishing + // connections to multiple mirrors simultaneously for a large file. We don't want to + // download that large file twice. connection.disconnect(); } + // We don't respect the Retry-After header (RFC7231 § 7.1.3) because it's rarely used and + // tends to be too conservative when it is. We're already being good citizens by using + // exponential backoff. Furthermore RFC law didn't use the magic word "MUST". + int timeout = IntMath.pow(2, retries) * MIN_RETRY_DELAY_MS; if (e instanceof SocketTimeoutException) { + eventHandler.handle(Event.progress("Timeout connecting to " + url)); connectTimeout = Math.min(connectTimeout * 2, MAX_CONNECT_TIMEOUT_MS); + // If we got connect timeout, we're already doing exponential backoff, so no point + // in sleeping too. + timeout = 1; + } else if (e instanceof InterruptedIOException) { + // Please note that SocketTimeoutException is a subtype of InterruptedIOException. + throw e; } if (++retries == MAX_RETRIES) { + if (!(e instanceof SocketTimeoutException)) { + eventHandler + .handle(Event.progress(format("Error connecting to %s: %s", url, e.getMessage()))); + } for (Throwable suppressed : suppressions) { e.addSuppressed(suppressed); } throw e; } + // Java 7 allows us to create a tree of all errors that led to the ultimate failure. suppressions.add(e); - int timeout = IntMath.pow(2, retries) * MIN_RETRY_DELAY_MS; - eventHandler.handle(Event.progress( - String.format("Failed to connect to %s trying again in %,dms: %s", - url, timeout, e))); - TimeUnit.MILLISECONDS.sleep(timeout); + eventHandler.handle( + Event.progress(format("Failed to connect to %s trying again in %,dms", url, timeout))); + url = originalUrl; + try { + sleeper.sleepMillis(timeout); + } catch (InterruptedException translated) { + throw new InterruptedIOException(); + } } catch (RuntimeException e) { if (connection != null) { connection.disconnect(); } + eventHandler.handle(Event.progress(format("Unknown error connecting to %s: %s", url, e))); throw e; } } } - private static String describeHttpResponse(HttpURLConnection connection) throws IOException { - return String.format( - "%s returned %s %s", + private String describeHttpResponse(HttpURLConnection connection) throws IOException { + return format( + "%s returned %d %s", connection.getRequestMethod(), connection.getResponseCode(), - nullToEmpty(connection.getResponseMessage())); - } - - private static void readAllBytesAndClose(@Nullable InputStream stream) throws IOException { - if (stream != null) { - // TODO: Replace with ByteStreams#exhaust when Guava 20 comes out. - byte[] buf = new byte[8192]; - while (stream.read(buf) != -1) {} - stream.close(); - } - } - - private static InputStream getInputStream(HttpURLConnection connection) throws IOException { - // See RFC2616 § 3.5 and § 14.11 - switch (firstNonNull(connection.getContentEncoding(), "identity")) { - case "identity": - return connection.getInputStream(); - case "gzip": - case "x-gzip": - // Some web servers will send Content-Encoding: gzip even when we didn't request it, iff - // the file is a .gz file. - if (connection.getURL().getPath().endsWith(".gz")) { - return connection.getInputStream(); - } else { - return new GZIPInputStream(connection.getInputStream()); - } - default: - throw new UnrecoverableHttpException( - "Unsupported and unrequested Content-Encoding: " + connection.getContentEncoding()); - } + Strings.nullToEmpty(connection.getResponseMessage())); } - @VisibleForTesting - static URL getLocation(HttpURLConnection connection) throws IOException { - String newLocation = connection.getHeaderField("Location"); - if (newLocation == null) { - throw new IOException("Remote redirect missing Location."); - } - URL result = mergeUrls(URI.create(newLocation), connection.getURL()); - if (!isHttp(result)) { - throw new IOException("Bad Location: " + newLocation); - } - return result; - } - - private static URL mergeUrls(URI preferred, URL original) throws IOException { - // If the Location value provided in a 3xx (Redirection) response does not have a fragment - // component, a user agent MUST process the redirection as if the value inherits the fragment - // component of the URI reference used to generate the request target (i.e., the redirection - // inherits the original reference's fragment, if any). Quoth RFC7231 § 7.1.2 - String protocol = firstNonNull(preferred.getScheme(), original.getProtocol()); - String userInfo = preferred.getUserInfo(); - String host = preferred.getHost(); - int port; - if (host == null) { - host = original.getHost(); - port = original.getPort(); - userInfo = original.getUserInfo(); - } else { - port = preferred.getPort(); - if (userInfo == null - && host.equals(original.getHost()) - && port == original.getPort()) { - userInfo = original.getUserInfo(); - } - } - String path = preferred.getPath(); - String query = preferred.getQuery(); - String fragment = preferred.getFragment(); - if (fragment == null) { - fragment = original.getRef(); - } - URL result; - try { - result = new URI(protocol, userInfo, host, port, path, query, fragment).toURL(); - } catch (URISyntaxException | MalformedURLException e) { - throw new IOException("Could not merge " + preferred + " into " + original); - } - return result; + private String format(String format, Object... args) { + return String.format(locale, format, args); } - private static boolean isHttp(URL url) { - return isProtocol(url, "http") || isProtocol(url, "https"); - } - - private static boolean isProtocol(URL url, String protocol) { - // An implementation should accept uppercase letters as equivalent to lowercase in scheme names - // (e.g., allow "HTTP" as well as "http") for the sake of robustness. Quoth RFC3986 § 3.1 - return Ascii.equalsIgnoreCase(protocol, url.getProtocol()); - } - - private static String getExtension(String path) { - int index = path.lastIndexOf('.'); - if (index == -1) { - return ""; - } - return path.substring(index + 1); - } - - private static final class UnrecoverableHttpException extends IOException { - UnrecoverableHttpException(String message) { - super(message); + // Exhausts all bytes in an HTTP to make it easier for Java infrastructure to reuse sockets. + private static void readAllBytesAndClose( + @WillClose @Nullable InputStream stream) + throws IOException { + if (stream != null) { + ByteStreams.exhaust(stream); + stream.close(); } } - - private HttpConnector() {} } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnectorMultiplexer.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnectorMultiplexer.java new file mode 100644 index 0000000000..ac0c080cfd --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnectorMultiplexer.java @@ -0,0 +1,332 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicates; +import com.google.common.collect.FluentIterable; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Ordering; +import com.google.devtools.build.lib.bazel.repository.downloader.RetryingInputStream.Reconnector; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.util.Sleeper; +import java.io.IOException; +import java.io.InputStream; +import java.io.InterruptedIOException; +import java.net.URL; +import java.net.URLConnection; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Deque; +import java.util.LinkedList; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.annotation.Nullable; +import javax.annotation.concurrent.GuardedBy; + +/** + * Class for establishing HTTP connections. + * + * <p>This is the most amazing way to download files ever. It makes Bazel builds as reliable as + * Blaze builds in Google's internal hermettically sealed repository. But this class isn't just + * reliable. It's also fast. It even works on the worst Internet connections in the farthest corners + * of the Earth. You are just not going to believe how fast and reliable this design is. It’s + * incredible. Your builds are never going to break again due to downloads. You’re going to be so + * happy. Your developer community is going to be happy. Mr. Jenkins will be happy too. Everyone is + * going to have such a magnificent developer experience due to the product excellence of this + * class. + */ +@ThreadSafe +final class HttpConnectorMultiplexer { + + private static final Logger logger = Logger.getLogger(HttpConnectorMultiplexer.class.getName()); + + private static final int MAX_THREADS_PER_CONNECT = 2; + private static final long FAILOVER_DELAY_MS = 2000; + private static final ImmutableMap<String, String> REQUEST_HEADERS = + ImmutableMap.of("Accept-Encoding", "gzip"); + + private final EventHandler eventHandler; + private final HttpConnector connector; + private final HttpStream.Factory httpStreamFactory; + private final Clock clock; + private final Sleeper sleeper; + + /** + * Creates a new instance. + * + * <p>Instances are thread safe and can be reused. + */ + HttpConnectorMultiplexer( + EventHandler eventHandler, + HttpConnector connector, + HttpStream.Factory httpStreamFactory, + Clock clock, + Sleeper sleeper) { + this.eventHandler = eventHandler; + this.connector = connector; + this.httpStreamFactory = httpStreamFactory; + this.clock = clock; + this.sleeper = sleeper; + } + + /** + * Establishes reliable HTTP connection to a good mirror URL. + * + * <p>This routine supports HTTP redirects in an RFC compliant manner. It requests gzip content + * encoding when appropriate in order to minimize bandwidth consumption when downloading + * uncompressed files. It reports download progress. It enforces a SHA-256 checksum which + * continues to be enforced even after this method returns. + * + * <p>This routine spawns {@value #MAX_THREADS_PER_CONNECT} threads that initiate connections in + * parallel to {@code urls} with a {@value #FAILOVER_DELAY_MS} millisecond failover waterfall so + * earlier mirrors are preferred. Each connector thread retries automatically on transient errors + * with exponential backoff. It vets the first 32kB of any payload before selecting a mirror in + * order to evade captive portals and avoid ultra-low-bandwidth servers. Even after this method + * returns the reliability doesn't stop. Each read operation wiil intercept timeouts and errors + * and block until the connection can be renegotiated transparently right where it left off. + * + * @param urls mirrors by preference; each URL can be: file, http, or https + * @param sha256 hex checksum lazily checked on entire payload, or empty to disable + * @return an {@link InputStream} of response payload + * @throws IOException if all mirrors are down and contains suppressed exception of each attempt + * @throws InterruptedIOException if current thread is being cast into oblivion + * @throws IllegalArgumentException if {@code urls} is empty or has an unsupported protocol + */ + public HttpStream connect(List<URL> urls, String sha256) throws IOException { + Preconditions.checkNotNull(sha256); + HttpUtils.checkUrlsArgument(urls); + if (Thread.interrupted()) { + throw new InterruptedIOException(); + } + // If there's only one URL then there's no need for us to run all our fancy thread stuff. + if (urls.size() == 1) { + return establishConnection(urls.get(0), sha256); + } + MutexConditionSharedMemory context = new MutexConditionSharedMemory(); + // The parent thread always holds the lock except when released by wait(). + synchronized (context) { + // Create the jobs for workers to do. + long now = clock.currentTimeMillis(); + long startAtTime = now; + for (URL url : urls) { + context.jobs.add(new WorkItem(url, sha256, startAtTime)); + startAtTime += FAILOVER_DELAY_MS; + } + // Create the worker thread pool. + for (int i = 0; i < Math.min(urls.size(), MAX_THREADS_PER_CONNECT); i++) { + Thread thread = new Thread(new Worker(context)); + thread.setName("HttpConnector"); + // These threads will not start doing anything until we release the lock below. + thread.start(); + context.threads.add(thread); + } + // Wait for the first worker to compute a result, or for all workers to fail. + boolean interrupted = false; + while (context.result == null && !context.threads.isEmpty()) { + try { + // Please note that waiting on a conndition releases the mutex. It also throws + // InterruptedException if the thread is *already* interrupted. + context.wait(); + } catch (InterruptedException e) { + // The interrupted state of this thread is now cleared, so we can call wait() again. + interrupted = true; + // We need to terminate the workers before rethrowing InterruptedException. + break; + } + } + // Now that we have the answer or are interrupted, we need to terminate any remaining workers. + for (Thread thread : context.threads) { + thread.interrupt(); + } + // Now wait for all threads to exit. We technically don't need to do this, but it helps with + // the regression testing of this implementation. + while (!context.threads.isEmpty()) { + try { + context.wait(); + } catch (InterruptedException e) { + // We don't care right now. Leave us alone. + interrupted = true; + } + } + // Now that the workers are terminated, we can safely propagate interruptions. + if (interrupted) { + throw new InterruptedIOException(); + } + // Please do not modify this code to call join() because the way we've implemented this + // routine is much better and faster. join() is basically a sleep loop when multiple threads + // exist. By sharing our mutex condition across threads, we were able to make things go + // lightning fast. If the child threads have not terminated by now, they are guaranteed to do + // so very soon. + if (context.result != null) { + return context.result; + } else { + IOException error = + new IOException("All mirrors are down: " + describeErrors(context.errors)); + // By this point, we probably have a very complex tree of exceptions. Beware! + for (Throwable workerError : context.errors) { + error.addSuppressed(workerError); + } + throw error; + } + } + } + + private static class MutexConditionSharedMemory { + @GuardedBy("this") @Nullable HttpStream result; + @GuardedBy("this") final List<Thread> threads = new ArrayList<>(); + @GuardedBy("this") final Deque<WorkItem> jobs = new LinkedList<>(); + @GuardedBy("this") final List<Throwable> errors = new ArrayList<>(); + } + + private static class WorkItem { + final URL url; + final String sha256; + final long startAtTime; + + WorkItem(URL url, String sha256, long startAtTime) { + this.url = url; + this.sha256 = sha256; + this.startAtTime = startAtTime; + } + } + + private class Worker implements Runnable { + private final MutexConditionSharedMemory context; + + Worker(MutexConditionSharedMemory context) { + this.context = context; + } + + @Override + public void run() { + while (true) { + WorkItem work; + synchronized (context) { + // A lot could have happened while we were waiting for this lock. Let's check. + if (context.result != null + || context.jobs.isEmpty() + || Thread.currentThread().isInterrupted()) { + tellParentThreadWeAreDone(); + return; + } + // Now remove a the first job from the fifo. + work = context.jobs.pop(); + } + // Wait if necessary before starting this thread. + long now = clock.currentTimeMillis(); + // Java does not have a true monotonic clock; but since currentTimeMillis returns UTC, it's + // monotonic enough for our purposes. This routine will not be pwnd by DST or JVM freezes. + // However it may be trivially impacted by system clock skew correction that go backwards. + if (now < work.startAtTime) { + try { + sleeper.sleepMillis(work.startAtTime - now); + } catch (InterruptedException e) { + // The parent thread or JVM has asked us to terminate this thread. + synchronized (context) { + tellParentThreadWeAreDone(); + return; + } + } + } + // Now we're actually going to attempt to connect to the remote server. + HttpStream result; + try { + result = establishConnection(work.url, work.sha256); + } catch (InterruptedIOException e) { + // The parent thread got its result from another thread and killed this one. + synchronized (context) { + tellParentThreadWeAreDone(); + return; + } + } catch (Throwable e) { + // Oh no the connector failed for some reason. We won't let that interfere with our plans. + synchronized (context) { + context.errors.add(e); + continue; + } + } + // Our connection attempt succeeded! Let's inform the parent thread of this joyous occasion. + synchronized (context) { + if (context.result == null) { + context.result = result; + result = null; + } + tellParentThreadWeAreDone(); + } + // We created a connection but we lost the race. Now we need to close it outside the mutex. + // We're not going to slow the parent thread down waiting for this operation to complete. + if (result != null) { + try { + result.close(); + } catch (IOException | RuntimeException e) { + logger.log(Level.WARNING, "close() failed in loser zombie thread", e); + } + } + } + } + + @GuardedBy("context") + private void tellParentThreadWeAreDone() { + // Remove this thread from the list of threads so parent thread knows when all have exited. + context.threads.remove(Thread.currentThread()); + // Wake up parent thread so it can check if that list is empty. + context.notify(); + } + } + + private HttpStream establishConnection(final URL url, String sha256) throws IOException { + final URLConnection connection = connector.connect(url, REQUEST_HEADERS); + return httpStreamFactory.create( + connection, url, sha256, + new Reconnector() { + @Override + public URLConnection connect( + Throwable cause, ImmutableMap<String, String> extraHeaders) + throws IOException { + eventHandler.handle( + Event.progress(String.format("Lost connection for %s due to %s", url, cause))); + return connector.connect( + connection.getURL(), + new ImmutableMap.Builder<String, String>() + .putAll(REQUEST_HEADERS) + .putAll(extraHeaders) + .build()); + } + }); + } + + private static String describeErrors(Collection<Throwable> errors) { + return + FluentIterable + .from(errors) + .transform( + new Function<Throwable, String>() { + @Nullable + @Override + public String apply(Throwable workerError) { + return workerError.getMessage(); + } + }) + .filter(Predicates.notNull()) + .toSortedSet(Ordering.natural()) + .toString(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java index ea4cc81d7c..7ed6a924ba 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java @@ -1,4 +1,4 @@ -// Copyright 2014 The Bazel Authors. All rights reserved. +// Copyright 2016 The Bazel Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,98 +14,153 @@ package com.google.devtools.build.lib.bazel.repository.downloader; +import com.google.common.base.MoreObjects; +import com.google.common.base.Optional; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.io.ByteStreams; import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache; import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache.KeyType; -import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.EventHandler; -import com.google.devtools.build.lib.events.Location; import com.google.devtools.build.lib.packages.Rule; import com.google.devtools.build.lib.rules.repository.RepositoryFunction.RepositoryFunctionException; import com.google.devtools.build.lib.rules.repository.WorkspaceAttributeMapper; import com.google.devtools.build.lib.syntax.EvalException; import com.google.devtools.build.lib.syntax.Type; +import com.google.devtools.build.lib.util.Clock; +import com.google.devtools.build.lib.util.JavaClock; +import com.google.devtools.build.lib.util.JavaSleeper; +import com.google.devtools.build.lib.util.Sleeper; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; -import com.google.devtools.build.skyframe.SkyFunctionException; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; import java.io.IOException; -import java.io.InputStream; +import java.io.InterruptedIOException; import java.io.OutputStream; -import java.net.Proxy; -import java.net.URI; -import java.net.URISyntaxException; +import java.net.MalformedURLException; import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; import java.util.Map; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.Semaphore; /** - * Helper class for downloading a file from a URL. + * Bazel file downloader. + * + * <p>This class uses {@link HttpConnectorMultiplexer} to connect to HTTP mirrors and then reads the + * file to disk. */ public class HttpDownloader { - private static final int BUFFER_SIZE = 32 * 1024; - private static final int KB = 1024; - private static final String UNITS = " KMGTPEY"; - private static final double LOG_OF_KB = Math.log(1024); - private final ScheduledExecutorService scheduler; - private Location ruleUrlAttributeLocation; + private static final int MAX_PARALLEL_DOWNLOADS = 8; + private static final Semaphore semaphore = new Semaphore(MAX_PARALLEL_DOWNLOADS, true); protected final RepositoryCache repositoryCache; public HttpDownloader(RepositoryCache repositoryCache) { - this.scheduler = Executors.newScheduledThreadPool(1); - this.ruleUrlAttributeLocation = null; this.repositoryCache = repositoryCache; } + /** Validates native repository rule attributes and calls the other download method. */ public Path download( Rule rule, Path outputDirectory, EventHandler eventHandler, Map<String, String> clientEnv) - throws RepositoryFunctionException, InterruptedException { + throws RepositoryFunctionException, InterruptedException { WorkspaceAttributeMapper mapper = WorkspaceAttributeMapper.of(rule); - String url; + List<URL> urls = new ArrayList<>(); String sha256; String type; try { - ruleUrlAttributeLocation = rule.getAttributeLocation("url"); - - url = mapper.get("url", Type.STRING); - sha256 = mapper.get("sha256", Type.STRING); - type = mapper.isAttributeValueExplicitlySpecified("type") - ? mapper.get("type", Type.STRING) : ""; + String urlString = Strings.nullToEmpty(mapper.get("url", Type.STRING)); + if (!urlString.isEmpty()) { + try { + URL url = new URL(urlString); + if (!HttpUtils.isUrlSupportedByDownloader(url)) { + throw new EvalException( + rule.getAttributeLocation("url"), "Unsupported protocol: " + url.getProtocol()); + } + urls.add(url); + } catch (MalformedURLException e) { + throw new EvalException(rule.getAttributeLocation("url"), e.toString()); + } + } + List<String> urlStrings = + MoreObjects.firstNonNull( + mapper.get("urls", Type.STRING_LIST), + ImmutableList.<String>of()); + if (!urlStrings.isEmpty()) { + if (!urls.isEmpty()) { + throw new EvalException(rule.getAttributeLocation("url"), "Don't set url if urls is set"); + } + try { + for (String urlString2 : urlStrings) { + URL url = new URL(urlString2); + if (!HttpUtils.isUrlSupportedByDownloader(url)) { + throw new EvalException( + rule.getAttributeLocation("urls"), "Unsupported protocol: " + url.getProtocol()); + } + urls.add(url); + } + } catch (MalformedURLException e) { + throw new EvalException(rule.getAttributeLocation("urls"), e.toString()); + } + } + if (urls.isEmpty()) { + throw new EvalException(rule.getLocation(), "urls attribute not set"); + } + sha256 = Strings.nullToEmpty(mapper.get("sha256", Type.STRING)); + if (!sha256.isEmpty() && !RepositoryCache.KeyType.SHA256.isValid(sha256)) { + throw new EvalException(rule.getAttributeLocation("sha256"), "Invalid SHA256 checksum"); + } + type = Strings.nullToEmpty(mapper.get("type", Type.STRING)); } catch (EvalException e) { throw new RepositoryFunctionException(e, Transience.PERSISTENT); } - try { - return download(url, sha256, type, outputDirectory, eventHandler, clientEnv); + return download(urls, sha256, Optional.of(type), outputDirectory, eventHandler, clientEnv); } catch (IOException e) { - throw new RepositoryFunctionException(e, SkyFunctionException.Transience.TRANSIENT); + throw new RepositoryFunctionException(e, Transience.TRANSIENT); } } /** - * Attempt to download a file from the repository's URL. Returns the path to the file downloaded. + * Downloads file to disk and returns path. + * + * <p>If the SHA256 checksum and path to the repository cache is specified, attempt to load the + * file from the {@link RepositoryCache}. If it doesn't exist, proceed to download the file and + * load it into the cache prior to returning the value. * - * If the SHA256 checksum and path to the repository cache is specified, attempt - * to load the file from the RepositoryCache. If it doesn't exist, proceed to - * download the file and load it into the cache prior to returning the value. + * @param urls list of mirror URLs with identical content + * @param sha256 valid SHA256 hex checksum string which is checked, or empty to disable + * @param type extension, e.g. "tar.gz" to force on downloaded filename, or empty to not do this + * @param output destination filename if {@code type} is <i>absent</i>, otherwise output directory + * @param eventHandler CLI progress reporter + * @param clientEnv environment variables in shell issuing this command + * @throws IllegalArgumentException on parameter badness, which should be checked beforehand + * @throws IOException if download was attempted and ended up failing + * @throws InterruptedException if this thread is being cast into oblivion */ public Path download( - String urlString, String sha256, String type, Path outputDirectory, - EventHandler eventHandler, Map<String, String> clientEnv) - throws IOException, InterruptedException, RepositoryFunctionException { - Path destination = getDownloadDestination(urlString, type, outputDirectory); + List<URL> urls, + String sha256, + Optional<String> type, + Path output, + EventHandler eventHandler, + Map<String, String> clientEnv) + throws IOException, InterruptedException { + if (Thread.interrupted()) { + throw new InterruptedException(); + } + + Path destination = getDownloadDestination(urls.get(0), type, output); // Used to decide whether to cache the download at the end of this method. boolean isCaching = false; - if (RepositoryCache.KeyType.SHA256.isValid(sha256)) { + if (!sha256.isEmpty()) { try { - String currentSha256 = RepositoryCache.getChecksum(KeyType.SHA256, destination); + String currentSha256 = + RepositoryCache.getChecksum(KeyType.SHA256, destination); if (currentSha256.equals(sha256)) { // No need to download. return destination; @@ -125,36 +180,30 @@ public class HttpDownloader { } } - AtomicInteger totalBytes = new AtomicInteger(0); - final ScheduledFuture<?> loggerHandle = getLoggerHandle(totalBytes, eventHandler, urlString); - final URL url = new URL(urlString); - Proxy proxy = ProxyHelper.createProxyIfNeeded(url.toString(), clientEnv); - - try (OutputStream out = destination.getOutputStream(); - InputStream inputStream = HttpConnector.connect(url, proxy, eventHandler)) { - int read; - byte[] buf = new byte[BUFFER_SIZE]; - while ((read = inputStream.read(buf)) > 0) { - totalBytes.addAndGet(read); - out.write(buf, 0, read); - if (Thread.interrupted()) { - throw new InterruptedException("Download interrupted"); - } - } + // TODO: Consider using Dagger2 to automate this. + Clock clock = new JavaClock(); + Sleeper sleeper = new JavaSleeper(); + Locale locale = Locale.getDefault(); + ProxyHelper proxyHelper = new ProxyHelper(clientEnv); + HttpConnector connector = new HttpConnector(locale, eventHandler, proxyHelper, sleeper); + ProgressInputStream.Factory progressInputStreamFactory = + new ProgressInputStream.Factory(locale, clock, eventHandler); + HttpStream.Factory httpStreamFactory = new HttpStream.Factory(progressInputStreamFactory); + HttpConnectorMultiplexer multiplexer = + new HttpConnectorMultiplexer(eventHandler, connector, httpStreamFactory, clock, sleeper); + + // Connect to the best mirror and download the file, while reporting progress to the CLI. + semaphore.acquire(); + try (HttpStream payload = multiplexer.connect(urls, sha256); + OutputStream out = destination.getOutputStream()) { + ByteStreams.copy(payload, out); + } catch (InterruptedIOException e) { + throw new InterruptedException(); } catch (IOException e) { throw new IOException( - "Error downloading " + url + " to " + destination + ": " + e.getMessage()); + "Error downloading " + urls + " to " + destination + ": " + e.getMessage()); } finally { - scheduler.schedule(new Runnable() { - @Override - public void run() { - loggerHandle.cancel(true); - } - }, 0, TimeUnit.SECONDS); - } - - if (!sha256.isEmpty()) { - RepositoryCache.assertFileChecksum(sha256, destination, KeyType.SHA256); + semaphore.release(); } if (isCaching) { @@ -164,55 +213,20 @@ public class HttpDownloader { return destination; } - private Path getDownloadDestination(String urlString, String type, Path outputDirectory) - throws RepositoryFunctionException { - URI uri = null; - try { - uri = new URI(urlString); - } catch (URISyntaxException e) { - throw new RepositoryFunctionException( - new EvalException(ruleUrlAttributeLocation, e), Transience.PERSISTENT); - } - if (type == null) { - return outputDirectory; - } else { - String filename = new PathFragment(uri.getPath()).getBaseName(); - if (filename.isEmpty()) { - filename = "temp"; - } else if (!type.isEmpty()) { - filename += "." + type; - } - return outputDirectory.getRelative(filename); + private Path getDownloadDestination(URL url, Optional<String> type, Path output) { + if (!type.isPresent()) { + return output; } - } - - private ScheduledFuture<?> getLoggerHandle( - final AtomicInteger totalBytes, final EventHandler eventHandler, final String urlString) { - final Runnable logger = new Runnable() { - @Override - public void run() { - try { - eventHandler.handle(Event.progress( - "Downloading from " + urlString + ": " + formatSize(totalBytes.get()))); - } catch (Exception e) { - eventHandler.handle(Event.error( - "Error generating download progress: " + e.getMessage())); - } + String basename = + MoreObjects.firstNonNull( + Strings.emptyToNull(new PathFragment(url.getPath()).getBaseName()), + "temp"); + if (!type.get().isEmpty()) { + String suffix = "." + type.get(); + if (!basename.endsWith(suffix)) { + basename += suffix; } - }; - return scheduler.scheduleAtFixedRate(logger, 0, 1, TimeUnit.SECONDS); - } - - private String formatSize(int bytes) { - if (bytes < KB) { - return bytes + "B"; } - int logBaseUnitOfBytes = (int) (Math.log(bytes) / LOG_OF_KB); - if (logBaseUnitOfBytes < 0 || logBaseUnitOfBytes >= UNITS.length()) { - return bytes + "B"; - } - return (int) (bytes / Math.pow(KB, logBaseUnitOfBytes)) - + (UNITS.charAt(logBaseUnitOfBytes) + "B"); + return output.getRelative(basename); } - } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpStream.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpStream.java new file mode 100644 index 0000000000..4921b1504f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpStream.java @@ -0,0 +1,140 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import com.google.common.base.Splitter; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.hash.HashCode; +import com.google.common.hash.Hashing; +import com.google.common.io.ByteStreams; +import com.google.devtools.build.lib.bazel.repository.downloader.RetryingInputStream.Reconnector; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import java.io.ByteArrayInputStream; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.SequenceInputStream; +import java.net.URL; +import java.net.URLConnection; +import java.util.zip.GZIPInputStream; +import javax.annotation.WillCloseWhenClosed; + +/** + * Input stream that validates checksum resumes downloads on error. + * + * <p>This class is not thread safe, but it is safe to message pass its objects between threads. + */ +@ThreadCompatible +final class HttpStream extends FilterInputStream { + + static final int PRECHECK_BYTES = 32 * 1024; + private static final int GZIP_BUFFER_BYTES = 8192; // same as ByteStreams#copy + private static final ImmutableSet<String> GZIPPED_EXTENSIONS = ImmutableSet.of("gz", "tgz"); + private static final ImmutableSet<String> GZIP_CONTENT_ENCODING = + ImmutableSet.of("gzip", "x-gzip"); + + /** Factory for {@link HttpStream}. */ + @ThreadSafe + static class Factory { + + private final ProgressInputStream.Factory progressInputStreamFactory; + + Factory(ProgressInputStream.Factory progressInputStreamFactory) { + this.progressInputStreamFactory = progressInputStreamFactory; + } + + @SuppressWarnings("resource") + HttpStream create( + @WillCloseWhenClosed URLConnection connection, + URL originalUrl, + String sha256, + Reconnector reconnector) + throws IOException { + InputStream stream = new InterruptibleInputStream(connection.getInputStream()); + try { + // If server supports range requests, we can retry on read errors. See RFC7233 § 2.3. + RetryingInputStream retrier = null; + if (Iterables.contains( + Splitter.on(',') + .trimResults() + .split(Strings.nullToEmpty(connection.getHeaderField("Accept-Ranges"))), + "bytes")) { + retrier = new RetryingInputStream(stream, reconnector); + stream = retrier; + } + + stream = progressInputStreamFactory.create(stream, connection.getURL(), originalUrl); + + // Determine if we need to transparently gunzip. See RFC2616 § 3.5 and § 14.11. Please note + // that some web servers will send Content-Encoding: gzip even when we didn't request it if + // the file is a .gz file. + if (GZIP_CONTENT_ENCODING.contains(Strings.nullToEmpty(connection.getContentEncoding())) + && !GZIPPED_EXTENSIONS.contains(HttpUtils.getExtension(connection.getURL().getPath())) + && !GZIPPED_EXTENSIONS.contains(HttpUtils.getExtension(originalUrl.getPath()))) { + stream = new GZIPInputStream(stream, GZIP_BUFFER_BYTES); + } + + if (!sha256.isEmpty()) { + stream = new HashInputStream(stream, Hashing.sha256(), HashCode.fromString(sha256)); + if (retrier != null) { + retrier.disabled = true; + } + byte[] buffer = new byte[PRECHECK_BYTES]; + int read = 0; + while (read < PRECHECK_BYTES) { + int amount; + amount = stream.read(buffer, read, PRECHECK_BYTES - read); + if (amount == -1) { + break; + } + read += amount; + } + if (read < PRECHECK_BYTES) { + stream.close(); + stream = ByteStreams.limit(new ByteArrayInputStream(buffer), read); + } else { + stream = new SequenceInputStream(new ByteArrayInputStream(buffer), stream); + if (retrier != null) { + retrier.disabled = false; + } + } + } + } catch (Exception e) { + try { + stream.close(); + } catch (IOException e2) { + e.addSuppressed(e2); + } + throw e; + } + return new HttpStream(stream, connection.getURL()); + } + } + + private final URL url; + + HttpStream(@WillCloseWhenClosed InputStream delegate, URL url) { + super(delegate); + this.url = url; + } + + /** Returns final redirected URL. */ + URL getUrl() { + return url; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpUtils.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpUtils.java new file mode 100644 index 0000000000..5c558d6f18 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpUtils.java @@ -0,0 +1,110 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import com.google.common.base.Ascii; +import com.google.common.base.MoreObjects; +import com.google.common.base.Preconditions; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.Collection; + +/** HTTP utilities. */ +public final class HttpUtils { + + /** Returns {@code true} if {@code url} is supported by {@link HttpDownloader}. */ + public static boolean isUrlSupportedByDownloader(URL url) { + return isHttp(url) || isProtocol(url, "file"); + } + + static boolean isHttp(URL url) { + return isProtocol(url, "http") || isProtocol(url, "https"); + } + + static boolean isProtocol(URL url, String protocol) { + // An implementation should accept uppercase letters as equivalent to lowercase in scheme names + // (e.g., allow "HTTP" as well as "http") for the sake of robustness. Quoth RFC3986 § 3.1 + return Ascii.equalsIgnoreCase(protocol, url.getProtocol()); + } + + static void checkUrlsArgument(Collection<URL> urls) { + Preconditions.checkArgument(!urls.isEmpty(), "urls list empty"); + for (URL url : urls) { + Preconditions.checkArgument(isUrlSupportedByDownloader(url), "unsupported protocol: %s", url); + } + } + + static String getExtension(String path) { + int index = path.lastIndexOf('.'); + if (index == -1) { + return ""; + } + return Ascii.toLowerCase(path.substring(index + 1)); + } + + static URL getLocation(HttpURLConnection connection) throws IOException { + String newLocation = connection.getHeaderField("Location"); + if (newLocation == null) { + throw new IOException("Remote redirect missing Location."); + } + URL result = mergeUrls(URI.create(newLocation), connection.getURL()); + if (!isHttp(result)) { + throw new IOException("Bad Location: " + newLocation); + } + return result; + } + + private static URL mergeUrls(URI preferred, URL original) throws IOException { + // If the Location value provided in a 3xx (Redirection) response does not have a fragment + // component, a user agent MUST process the redirection as if the value inherits the fragment + // component of the URI reference used to generate the request target (i.e., the redirection + // inherits the original reference's fragment, if any). Quoth RFC7231 § 7.1.2 + String protocol = MoreObjects.firstNonNull(preferred.getScheme(), original.getProtocol()); + String userInfo = preferred.getUserInfo(); + String host = preferred.getHost(); + int port; + if (host == null) { + host = original.getHost(); + port = original.getPort(); + userInfo = original.getUserInfo(); + } else { + port = preferred.getPort(); + if (userInfo == null + && host.equals(original.getHost()) + && port == original.getPort()) { + userInfo = original.getUserInfo(); + } + } + String path = preferred.getPath(); + String query = preferred.getQuery(); + String fragment = preferred.getFragment(); + if (fragment == null) { + fragment = original.getRef(); + } + URL result; + try { + result = new URI(protocol, userInfo, host, port, path, query, fragment).toURL(); + } catch (URISyntaxException | MalformedURLException e) { + throw new IOException("Could not merge " + preferred + " into " + original); + } + return result; + } + + private HttpUtils() {} +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/InterruptibleInputStream.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/InterruptibleInputStream.java new file mode 100644 index 0000000000..81158e3a47 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/InterruptibleInputStream.java @@ -0,0 +1,88 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import com.google.devtools.build.lib.concurrent.ThreadSafety.ConditionallyThreadSafe; +import java.io.IOException; +import java.io.InputStream; +import java.io.InterruptedIOException; +import javax.annotation.WillCloseWhenClosed; + +/** + * Input stream that guarantees {@link InterruptedIOException}. + * + * <p>This class exists to hedge against the possibility that the JVM might not implement this + * functionality. See <a href="http://bugs.java.com/view_bug.do?bug_id=4385444">bug 4385444</a>. + */ +@ConditionallyThreadSafe +final class InterruptibleInputStream extends InputStream { + + private final InputStream delegate; + + InterruptibleInputStream(@WillCloseWhenClosed InputStream delegate) { + this.delegate = delegate; + } + + @Override + public int read() throws IOException { + check(); + return delegate.read(); + } + + @Override + public int read(byte[] buffer) throws IOException { + check(); + return delegate.read(buffer); + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + check(); + return delegate.read(buffer, offset, length); + } + + @Override + public int available() throws IOException { + return delegate.available(); + } + + @Override + public boolean markSupported() { + return delegate.markSupported(); + } + + @Override + @SuppressWarnings("sync-override") + public void mark(int readlimit) { + delegate.mark(readlimit); + } + + @Override + @SuppressWarnings("sync-override") + public void reset() throws IOException { + delegate.reset(); + } + + @Override + public void close() throws IOException { + delegate.close(); + } + + private static void check() throws InterruptedIOException { + if (Thread.interrupted()) { + throw new InterruptedIOException(); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/ProgressInputStream.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/ProgressInputStream.java new file mode 100644 index 0000000000..acaf3e0697 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/ProgressInputStream.java @@ -0,0 +1,130 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import com.google.common.base.Preconditions; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.util.Clock; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.util.Locale; +import java.util.concurrent.atomic.AtomicLong; +import javax.annotation.WillCloseWhenClosed; + +/** + * Input stream that reports progress on total bytes read as the download progresses. + * + * <p>This class is not thread safe, but it is safe to message pass its objects between threads. + */ +@ThreadCompatible +final class ProgressInputStream extends InputStream { + + private static final long PROGRESS_INTERVAL_MS = 200; + + /** Factory for {@link ProgressInputStream}. */ + @ThreadSafe + static class Factory { + private final Locale locale; + private final Clock clock; + private final EventHandler eventHandler; + + Factory(Locale locale, Clock clock, EventHandler eventHandler) { + this.locale = locale; + this.clock = clock; + this.eventHandler = eventHandler; + } + + InputStream create(@WillCloseWhenClosed InputStream delegate, URL url, URL originalUrl) { + return new ProgressInputStream( + locale, clock, eventHandler, PROGRESS_INTERVAL_MS, delegate, url, originalUrl); + } + } + + private final Locale locale; + private final Clock clock; + private final EventHandler eventHandler; + private final InputStream delegate; + private final long intervalMs; + private final URL url; + private final URL originalUrl; + private final AtomicLong toto = new AtomicLong(); + private final AtomicLong nextEvent; + + ProgressInputStream( + Locale locale, + Clock clock, + EventHandler eventHandler, + long intervalMs, + InputStream delegate, + URL url, + URL originalUrl) { + Preconditions.checkArgument(intervalMs >= 0); + this.locale = locale; + this.clock = clock; + this.eventHandler = eventHandler; + this.intervalMs = intervalMs; + this.delegate = delegate; + this.url = url; + this.originalUrl = originalUrl; + this.nextEvent = new AtomicLong(clock.currentTimeMillis() + intervalMs); + } + + @Override + public int read() throws IOException { + int result = delegate.read(); + if (result != -1) { + reportProgress(toto.incrementAndGet()); + } + return result; + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + int amount = delegate.read(buffer, offset, length); + if (amount > 0) { + reportProgress(toto.addAndGet(amount)); + } + return amount; + } + + @Override + public int available() throws IOException { + return delegate.available(); + } + + @Override + public void close() throws IOException { + delegate.close(); + } + + private void reportProgress(long bytesRead) { + long now = clock.currentTimeMillis(); + if (now < nextEvent.get()) { + return; + } + String via = ""; + if (!url.getHost().equals(originalUrl.getHost())) { + via = " via " + url.getHost(); + } + eventHandler.handle( + Event.progress( + String.format(locale, "Downloading %s%s: %,d bytes", originalUrl, via, bytesRead))); + nextEvent.set(now + intervalMs); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/ProxyHelper.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/ProxyHelper.java index fd0936989c..1ae265b89c 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/ProxyHelper.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/ProxyHelper.java @@ -20,36 +20,43 @@ import java.net.Authenticator; import java.net.InetSocketAddress; import java.net.PasswordAuthentication; import java.net.Proxy; +import java.net.URL; import java.net.URLDecoder; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import javax.annotation.Nullable; /** * Helper class for setting up a proxy server for network communication - * */ public class ProxyHelper { + private final Map<String, String> env; + + /** + * Creates new instance. + * + * @param env client environment to check for proxy settings + */ + public ProxyHelper(Map<String, String> env) { + this.env = env; + } + /** * This method takes a String for the resource being requested and sets up a proxy to make * the request if HTTP_PROXY and/or HTTPS_PROXY environment variables are set. - * @param requestedUrl The url for the remote resource that may need to be retrieved through a - * proxy - * @param env The client environment to check for proxy settings. - * @return Proxy - * @throws IOException + * + * @param requestedUrl remote resource that may need to be retrieved through a proxy */ - public static Proxy createProxyIfNeeded(String requestedUrl, Map<String, String> env) - throws IOException { - String lcUrl = requestedUrl.toLowerCase(); + public Proxy createProxyIfNeeded(URL requestedUrl) throws IOException { String proxyAddress = null; - if (lcUrl.startsWith("https")) { + if (HttpUtils.isProtocol(requestedUrl, "https")) { proxyAddress = env.get("https_proxy"); if (Strings.isNullOrEmpty(proxyAddress)) { proxyAddress = env.get("HTTPS_PROXY"); } - } else if (lcUrl.startsWith("http")) { + } else if (HttpUtils.isProtocol(requestedUrl, "http")) { proxyAddress = env.get("http_proxy"); if (Strings.isNullOrEmpty(proxyAddress)) { proxyAddress = env.get("HTTP_PROXY"); @@ -67,7 +74,7 @@ public class ProxyHelper { * @return Proxy * @throws IOException */ - public static Proxy createProxy(String proxyAddress) throws IOException { + public static Proxy createProxy(@Nullable String proxyAddress) throws IOException { if (Strings.isNullOrEmpty(proxyAddress)) { return Proxy.NO_PROXY; } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/RetryingInputStream.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/RetryingInputStream.java new file mode 100644 index 0000000000..4263cc257f --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/RetryingInputStream.java @@ -0,0 +1,149 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; +import java.io.IOException; +import java.io.InputStream; +import java.io.InterruptedIOException; +import java.net.SocketTimeoutException; +import java.net.URLConnection; +import java.util.Vector; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Input stream that reconnects on read timeouts and errors. + * + * <p>This class is not thread safe, but it is safe to message pass between threads. + */ +@ThreadCompatible +class RetryingInputStream extends InputStream { + + private static final int MAX_RESUMES = 3; + + /** Lambda for establishing a connection. */ + interface Reconnector { + + /** Establishes a connection with the same parameters as what was passed to us initially. */ + URLConnection connect( + Throwable cause, ImmutableMap<String, String> extraHeaders) + throws IOException; + } + + volatile boolean disabled; + private volatile InputStream delegate; + private final Reconnector reconnector; + private final AtomicLong toto = new AtomicLong(); + private final AtomicInteger resumes = new AtomicInteger(); + private final Vector<Throwable> suppressed = new Vector<>(); + + RetryingInputStream(InputStream delegate, Reconnector reconnector) { + this.delegate = delegate; + this.reconnector = reconnector; + } + + @Override + public int read() throws IOException { + while (true) { + try { + int result = delegate.read(); + if (result != -1) { + toto.incrementAndGet(); + } + return result; + } catch (IOException e) { + tryAgainIfPossible(e); + } + } + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException { + while (true) { + try { + int amount = delegate.read(buffer, offset, length); + if (amount != -1) { + toto.addAndGet(amount); + } + return amount; + } catch (IOException e) { + tryAgainIfPossible(e); + } + } + } + + @Override + public int available() throws IOException { + return delegate.available(); + } + + @Override + public void close() throws IOException { + delegate.close(); + } + + private void tryAgainIfPossible(IOException cause) throws IOException { + if (disabled) { + throw cause; + } + if (cause instanceof InterruptedIOException && !(cause instanceof SocketTimeoutException)) { + throw cause; + } + if (resumes.incrementAndGet() > MAX_RESUMES) { + propagate(cause); + } + try { + delegate.close(); + } catch (Exception ignored) { + // We know this connection failed so if it reminds us we're going to ignore it. + } + suppressed.add(cause); + reconnectWhereWeLeftOff(cause); + } + + private void reconnectWhereWeLeftOff(IOException cause) throws IOException { + try { + URLConnection connection; + long amountRead = toto.get(); + if (amountRead == 0) { + connection = reconnector.connect(cause, ImmutableMap.<String, String>of()); + } else { + connection = + reconnector.connect( + cause, ImmutableMap.of("Range", String.format("bytes %d-", amountRead))); + if (!Strings.nullToEmpty(connection.getHeaderField("Content-Range")) + .startsWith(String.format("bytes %d-", amountRead))) { + throw new IOException(String.format( + "Tried to reconnect at offset %,d but server didn't support it", amountRead)); + } + } + delegate = new InterruptibleInputStream(connection.getInputStream()); + } catch (InterruptedIOException e) { + throw e; + } catch (IOException e) { + propagate(e); + } + } + + private <T extends Throwable> void propagate(T error) throws T { + for (Throwable e : suppressed) { + error.addSuppressed(e); + } + throw error; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/UnrecoverableHttpException.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/UnrecoverableHttpException.java new file mode 100644 index 0000000000..3ccd2f4a2c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/UnrecoverableHttpException.java @@ -0,0 +1,23 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import java.io.IOException; + +final class UnrecoverableHttpException extends IOException { + UnrecoverableHttpException(String message) { + super(message); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java index 97b295f2c4..95e8532429 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java @@ -15,11 +15,14 @@ package com.google.devtools.build.lib.bazel.repository.skylark; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.bazel.repository.DecompressorDescriptor; import com.google.devtools.build.lib.bazel.repository.DecompressorValue; +import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache.KeyType; import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader; +import com.google.devtools.build.lib.bazel.repository.downloader.HttpUtils; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.cmdline.LabelSyntaxException; import com.google.devtools.build.lib.events.Location; @@ -53,7 +56,11 @@ import com.google.devtools.build.skyframe.SkyKey; import java.io.File; import java.io.IOException; import java.io.OutputStream; +import java.net.MalformedURLException; +import java.net.URL; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; import java.util.Map; /** Skylark API for the repository_rule's context. */ @@ -447,10 +454,11 @@ public class SkylarkRepositoryContext { parameters = { @Param( name = "url", - type = String.class, - doc = - "URL to the file to download. There is no authentication." - + " Redirection are followed." + allowedTypes = { + @ParamType(type = String.class), + @ParamType(type = SkylarkList.class, generic1 = String.class), + }, + doc = "List of mirror URLs referencing the same file." ), @Param( name = "output", @@ -482,18 +490,28 @@ public class SkylarkRepositoryContext { ), } ) - public void download(String url, Object output, String sha256, Boolean executable) - throws RepositoryFunctionException, EvalException, InterruptedException { + public void download( + Object url, Object output, String sha256, Boolean executable) + throws RepositoryFunctionException, EvalException, InterruptedException { + validateSha256(sha256); + List<URL> urls = getUrls(url); SkylarkPath outputPath = getPath("download()", output); try { checkInOutputDirectory(outputPath); makeDirectories(outputPath.getPath()); - - httpDownloader.download(url, sha256, null, outputPath.getPath(), env.getListener(), + httpDownloader.download( + urls, + sha256, + Optional.<String>absent(), + outputPath.getPath(), + env.getListener(), osObject.getEnvironmentVariables()); if (executable) { outputPath.getPath().setExecutable(true); } + } catch (InterruptedException e) { + throw new RepositoryFunctionException( + new IOException("thread interrupted"), Transience.TRANSIENT); } catch (IOException e) { throw new RepositoryFunctionException(e, Transience.TRANSIENT); } @@ -505,11 +523,11 @@ public class SkylarkRepositoryContext { parameters = { @Param( name = "url", - type = String.class, - doc = - "a URL referencing an archive file containing a Bazel repository." - + " Archives of type .zip, .jar, .war, .tar.gz or .tgz are supported." - + " There is no support for authentication. Redirections are followed." + allowedTypes = { + @ParamType(type = String.class), + @ParamType(type = SkylarkList.class, generic1 = String.class), + }, + doc = "List of mirror URLs referencing the same file." ), @Param( name = "output", @@ -542,8 +560,8 @@ public class SkylarkRepositoryContext { doc = "the archive type of the downloaded file." + " By default, the archive type is determined from the file extension of the URL." - + " If the file has no extension, you can explicitly specify either" - + "\"zip\", \"jar\", \"tar.gz\", or \"tgz\" here." + + " If the file has no extension, you can explicitly specify either \"zip\"," + + " \"jar\", \"war\", \"tar.gz\", \"tgz\", \"tar.bz2\", or \"tar.xz\" here." ), @Param( name = "stripPrefix", @@ -560,8 +578,11 @@ public class SkylarkRepositoryContext { } ) public void downloadAndExtract( - String url, Object output, String sha256, String type, String stripPrefix) - throws RepositoryFunctionException, InterruptedException, EvalException { + Object url, Object output, String sha256, String type, String stripPrefix) + throws RepositoryFunctionException, InterruptedException, EvalException { + validateSha256(sha256); + List<URL> urls = getUrls(url); + // Download to outputDirectory and delete it after extraction SkylarkPath outputPath = getPath("download_and_extract()", output); checkInOutputDirectory(outputPath); @@ -569,8 +590,14 @@ public class SkylarkRepositoryContext { Path downloadedPath; try { - downloadedPath = httpDownloader.download(url, sha256, type, outputPath.getPath(), - env.getListener(), osObject.getEnvironmentVariables()); + downloadedPath = + httpDownloader.download( + urls, + sha256, + Optional.of(type), + outputPath.getPath(), + env.getListener(), + osObject.getEnvironmentVariables()); } catch (IOException e) { throw new RepositoryFunctionException(e, Transience.TRANSIENT); } @@ -594,6 +621,43 @@ public class SkylarkRepositoryContext { } } + private static void validateSha256(String sha256) throws RepositoryFunctionException { + if (!sha256.isEmpty() && !KeyType.SHA256.isValid(sha256)) { + throw new RepositoryFunctionException( + new IOException("Invalid SHA256 checksum"), Transience.TRANSIENT); + } + } + + private static List<URL> getUrls(Object urlOrList) throws RepositoryFunctionException { + List<String> urlStrings; + if (urlOrList instanceof String) { + urlStrings = ImmutableList.of((String) urlOrList); + } else { + @SuppressWarnings("unchecked") + List<String> list = (List<String>) urlOrList; + urlStrings = list; + } + if (urlStrings.isEmpty()) { + throw new RepositoryFunctionException(new IOException("urls not set"), Transience.PERSISTENT); + } + List<URL> urls = new ArrayList<>(); + for (String urlString : urlStrings) { + URL url; + try { + url = new URL(urlString); + } catch (MalformedURLException e) { + throw new RepositoryFunctionException( + new IOException("Bad URL: " + urlString), Transience.PERSISTENT); + } + if (!HttpUtils.isUrlSupportedByDownloader(url)) { + throw new RepositoryFunctionException( + new IOException("Unsupported protocol: " + url.getProtocol()), Transience.PERSISTENT); + } + urls.add(url); + } + return urls; + } + // This is just for test to overwrite the path environment private static ImmutableList<String> pathEnv = null; diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/HttpArchiveRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/HttpArchiveRule.java index 0406fabf8e..3734725a08 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/HttpArchiveRule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/HttpArchiveRule.java @@ -16,6 +16,7 @@ package com.google.devtools.build.lib.bazel.rules.workspace; import static com.google.devtools.build.lib.packages.Attribute.attr; import static com.google.devtools.build.lib.syntax.Type.STRING; +import static com.google.devtools.build.lib.syntax.Type.STRING_LIST; import com.google.devtools.build.lib.analysis.RuleDefinition; import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; @@ -36,12 +37,19 @@ public class HttpArchiveRule implements RuleDefinition { public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) { return builder /* <!-- #BLAZE_RULE(http_archive).ATTRIBUTE(url) --> - A URL referencing an archive file containing a Bazel repository. + (Deprecated) A URL referencing an archive file containing a Bazel repository. - <p>Archives of type .zip, .jar, .war, .tar.gz or .tgz are supported. There is no support - for authentication.</p> + <p>This value has the same meaning as a <code>urls</code> list with a single item. This + must not be specified if <code>urls</code> is also specified.</p> <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ - .add(attr("url", STRING).mandatory()) + .add(attr("url", STRING)) + /* <!-- #BLAZE_RULE(http_archive).ATTRIBUTE(urls) --> + List of mirror URLs referencing the same archive file containing a Bazel repository. + + <p>This must be an http, https, or file URL. Archives of type .zip, .jar, .war, .tar.gz, + .tgz, tar.bz2, or tar.xz are supported. There is no support for authentication.</p> + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("urls", STRING_LIST)) /* <!-- #BLAZE_RULE(http_archive).ATTRIBUTE(sha256) --> The expected SHA-256 hash of the file downloaded. diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/HttpFileRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/HttpFileRule.java index 7c96c4cc32..9ef7b0d3fe 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/HttpFileRule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/HttpFileRule.java @@ -17,6 +17,7 @@ package com.google.devtools.build.lib.bazel.rules.workspace; import static com.google.devtools.build.lib.packages.Attribute.attr; import static com.google.devtools.build.lib.syntax.Type.BOOLEAN; import static com.google.devtools.build.lib.syntax.Type.STRING; +import static com.google.devtools.build.lib.syntax.Type.STRING_LIST; import com.google.devtools.build.lib.analysis.RuleDefinition; import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; @@ -37,24 +38,31 @@ public class HttpFileRule implements RuleDefinition { public RuleClass build(Builder builder, RuleDefinitionEnvironment environment) { return builder /* <!-- #BLAZE_RULE(http_file).ATTRIBUTE(url) --> - A URL to a file that will be made available to Bazel. - - <p>This must be an http or https URL. Authentication is not supported.</p> - <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ - .add(attr("url", STRING).mandatory()) + (Deprecated) A URL to a file that will be made available to Bazel. + + <p>This value has the same meaning as a <code>urls</code> list with a single item. This + must not be specified if <code>urls</code> is also specified.</p> + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("url", STRING)) + /* <!-- #BLAZE_RULE(http_file).ATTRIBUTE(urls) --> + List of mirror URLs referencing the same file that will be made available to Bazel. + + <p>This must be an http, https, or file URL. Authentication is not supported.</p> + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("urls", STRING_LIST)) /* <!-- #BLAZE_RULE(http_file).ATTRIBUTE(sha256) --> - The expected SHA-256 of the file downloaded. + The expected SHA-256 of the file downloaded. - <p>This must match the SHA-256 of the file downloaded. <em>It is a security risk to - omit the SHA-256 as remote files can change.</em> At best omitting this field will make - your build non-hermetic. It is optional to make development easier but should be set - before shipping.</p> - <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + <p>This must match the SHA-256 of the file downloaded. <em>It is a security risk to + omit the SHA-256 as remote files can change.</em> At best omitting this field will make + your build non-hermetic. It is optional to make development easier but should be set + before shipping.</p> + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ .add(attr("sha256", STRING)) /* <!-- #BLAZE_RULE(http_file).ATTRIBUTE(executable) --> - If the downloaded file should be made executable. Defaults to False. + If the downloaded file should be made executable. Defaults to False. - <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ .add(attr("executable", BOOLEAN)) .setWorkspaceOnly() .build(); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/NewHttpArchiveRule.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/NewHttpArchiveRule.java index 30a5cd8b41..9f29fd6156 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/NewHttpArchiveRule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/workspace/NewHttpArchiveRule.java @@ -16,6 +16,7 @@ package com.google.devtools.build.lib.bazel.rules.workspace; import static com.google.devtools.build.lib.packages.Attribute.attr; import static com.google.devtools.build.lib.syntax.Type.STRING; +import static com.google.devtools.build.lib.syntax.Type.STRING_LIST; import com.google.devtools.build.lib.analysis.RuleDefinition; import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; @@ -33,12 +34,19 @@ public class NewHttpArchiveRule implements RuleDefinition { public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment environment) { return builder /* <!-- #BLAZE_RULE(new_http_archive).ATTRIBUTE(url) --> - A URL referencing an archive file containing a Bazel repository. + (Deprecated) A URL referencing an archive file. - <p>Archives of type .zip, .jar, .war, .tar.gz or .tgz are supported. There is no support - for authentication.</p> - <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ - .add(attr("url", STRING).mandatory()) + <p>This value has the same meaning as a <code>urls</code> list with a single item. This + must not be specified if <code>urls</code> is also specified.</p> + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("url", STRING)) + /* <!-- #BLAZE_RULE(new_http_archive).ATTRIBUTE(urls) --> + List of mirror URLs referencing the same archive file containing a Bazel repository. + + <p>This must be an http, https, or file URL. Archives of type .zip, .jar, .war, .tar.gz, + .tgz, tar.bz2, or tar.xz are supported. There is no support for authentication.</p> + <!-- #END_BLAZE_RULE.ATTRIBUTE --> */ + .add(attr("urls", STRING_LIST)) /* <!-- #BLAZE_RULE(new_http_archive).ATTRIBUTE(sha256) --> The expected SHA-256 hash of the file downloaded. diff --git a/src/main/java/com/google/devtools/build/lib/packages/AttributeContainer.java b/src/main/java/com/google/devtools/build/lib/packages/AttributeContainer.java index 65b890d45a..d7856e8cf7 100644 --- a/src/main/java/com/google/devtools/build/lib/packages/AttributeContainer.java +++ b/src/main/java/com/google/devtools/build/lib/packages/AttributeContainer.java @@ -17,6 +17,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Function; import com.google.devtools.build.lib.events.Location; import java.util.Arrays; +import javax.annotation.Nullable; /** * Provides attribute setting and retrieval for a Rule. Encapsulating attribute access @@ -77,6 +78,7 @@ public class AttributeContainer { /** * Returns an attribute value by name, or null on no match. */ + @Nullable public Object getAttr(String attrName) { Integer idx = ruleClass.getAttributeIndex(attrName); return idx != null ? attributeValues[idx] : null; diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/WorkspaceAttributeMapper.java b/src/main/java/com/google/devtools/build/lib/rules/repository/WorkspaceAttributeMapper.java index b2edf3d96c..99cc363e9c 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/repository/WorkspaceAttributeMapper.java +++ b/src/main/java/com/google/devtools/build/lib/rules/repository/WorkspaceAttributeMapper.java @@ -14,12 +14,16 @@ package com.google.devtools.build.lib.rules.repository; +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.common.base.Preconditions; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.packages.AggregatingAttributeMapper; import com.google.devtools.build.lib.packages.BuildType.SelectorList; import com.google.devtools.build.lib.packages.Rule; import com.google.devtools.build.lib.syntax.EvalException; import com.google.devtools.build.lib.syntax.Type; +import javax.annotation.Nullable; /** * An attribute mapper for workspace rules. Similar to NonconfigurableAttributeWrapper, but throws @@ -37,7 +41,12 @@ public class WorkspaceAttributeMapper { this.rule = rule; } + /** + * Returns typecasted value for attribute or {@code null} on no match. + */ + @Nullable public <T> T get(String attributeName, Type<T> type) throws EvalException { + Preconditions.checkNotNull(type); Object value = getObject(attributeName); try { return type.cast(value); @@ -48,10 +57,11 @@ public class WorkspaceAttributeMapper { } /** - * Returns the value for an attribute without casting it to any particular type. + * Returns value for attribute without casting it to any particular type, or null on no match. */ + @Nullable public Object getObject(String attributeName) throws EvalException { - Object value = rule.getAttributeContainer().getAttr(attributeName); + Object value = rule.getAttributeContainer().getAttr(checkNotNull(attributeName)); if (value instanceof SelectorList) { String message; if (rule.getLocation().getPath().getBaseName().equals( diff --git a/src/main/java/com/google/devtools/build/lib/util/JavaSleeper.java b/src/main/java/com/google/devtools/build/lib/util/JavaSleeper.java new file mode 100644 index 0000000000..d31c617412 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/JavaSleeper.java @@ -0,0 +1,27 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util; + +import java.util.concurrent.TimeUnit; + +/** Production implementation of {@link Sleeper} */ +public final class JavaSleeper implements Sleeper { + + @Override + public void sleepMillis(long milliseconds) throws InterruptedException { + Preconditions.checkArgument(milliseconds >= 0, "sleeper can't time travel"); + TimeUnit.MILLISECONDS.sleep(milliseconds); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/util/Sleeper.java b/src/main/java/com/google/devtools/build/lib/util/Sleeper.java new file mode 100644 index 0000000000..53b29d20e8 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/util/Sleeper.java @@ -0,0 +1,32 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.util; + +/** + * Interface accepting requests to put current thread to sleep. + * + * <p>The only implementation of this interface intended for production use is {@link JavaSleeper}. + * Use {@link com.google.devtools.build.lib.testutil.ManualSleeper ManualSleeper} for testing. + */ +public interface Sleeper { + + /** + * Puts current thread to sleep for given duration. + * + * @throws InterruptedException if current thread is being cast into oblivion + * @throws IllegalArgumentException if {@code milliseconds} is negative + */ + void sleepMillis(long milliseconds) throws InterruptedException; +} diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/BUILD b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/BUILD index 6d9253a262..ceaaf305bf 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/BUILD +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/BUILD @@ -13,9 +13,11 @@ java_test( ], deps = [ "//src/main/java/com/google/devtools/build/lib:events", + "//src/main/java/com/google/devtools/build/lib:util", "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader", "//src/test/java/com/google/devtools/build/lib:foundations_testutil", "//src/test/java/com/google/devtools/build/lib:test_runner", + "//src/test/java/com/google/devtools/build/lib:testutil", "//third_party:guava", "//third_party:junit4", "//third_party:mockito", diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloaderTestSuite.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloaderTestSuite.java index 1a48a1ceb5..1c2477e64b 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloaderTestSuite.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloaderTestSuite.java @@ -21,7 +21,14 @@ import org.junit.runners.Suite.SuiteClasses; /** Test suite for downloader package. */ @RunWith(Suite.class) @SuiteClasses({ + HashInputStreamTest.class, + HttpConnectorMultiplexerIntegrationTest.class, + HttpConnectorMultiplexerTest.class, HttpConnectorTest.class, + HttpStreamTest.class, + HttpUtilsTest.class, + ProgressInputStreamTest.class, ProxyHelperTest.class, + RetryingInputStreamTest.class, }) class DownloaderTestSuite {} diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloaderTestUtils.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloaderTestUtils.java new file mode 100644 index 0000000000..1f55667efa --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloaderTestUtils.java @@ -0,0 +1,45 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import static java.nio.charset.StandardCharsets.ISO_8859_1; + +import com.google.common.base.Joiner; +import com.google.common.io.ByteStreams; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.Socket; +import java.net.URL; +import javax.annotation.WillNotClose; + +final class DownloaderTestUtils { + + static URL makeUrl(String url) { + try { + return new URL(url); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + } + + static void sendLines(@WillNotClose Socket socket, String... data) throws IOException { + ByteStreams.copy( + new ByteArrayInputStream(Joiner.on("\r\n").join(data).getBytes(ISO_8859_1)), + socket.getOutputStream()); + } + + private DownloaderTestUtils() {} +} diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HashInputStreamTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HashInputStreamTest.java new file mode 100644 index 0000000000..c1e14df907 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HashInputStreamTest.java @@ -0,0 +1,67 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.hash.HashCode; +import com.google.common.hash.Hashing; +import com.google.common.io.CharStreams; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link HashInputStream}. */ +@RunWith(JUnit4.class) +@SuppressWarnings("resource") +public class HashInputStreamTest { + + @Rule + public final ExpectedException thrown = ExpectedException.none(); + + @Test + public void validChecksum_readsOk() throws Exception { + assertThat( + CharStreams.toString( + new InputStreamReader( + new HashInputStream( + new ByteArrayInputStream("hello".getBytes(UTF_8)), + Hashing.sha1(), + HashCode.fromString("aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d")), + UTF_8))) + .isEqualTo("hello"); + } + + @Test + public void badChecksum_throwsIOException() throws Exception { + thrown.expect(IOException.class); + thrown.expectMessage("Checksum"); + assertThat( + CharStreams.toString( + new InputStreamReader( + new HashInputStream( + new ByteArrayInputStream("hello".getBytes(UTF_8)), + Hashing.sha1(), + HashCode.fromString("0000000000000000000000000000000000000000")), + UTF_8))) + .isNull(); // Only here to make @CheckReturnValue happy. + } +} diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnectorMultiplexerIntegrationTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnectorMultiplexerIntegrationTest.java new file mode 100644 index 0000000000..b5b77a28cf --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnectorMultiplexerIntegrationTest.java @@ -0,0 +1,227 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import static com.google.common.io.ByteStreams.toByteArray; +import static com.google.common.truth.Truth.assertThat; +import static com.google.devtools.build.lib.bazel.repository.downloader.DownloaderTestUtils.sendLines; +import static com.google.devtools.build.lib.bazel.repository.downloader.HttpParser.readHttpRequest; +import static java.nio.charset.StandardCharsets.US_ASCII; +import static java.util.Arrays.asList; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableList; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.testutil.ManualClock; +import com.google.devtools.build.lib.util.Sleeper; +import java.io.IOException; +import java.net.InetAddress; +import java.net.Proxy; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.URL; +import java.util.Locale; +import java.util.concurrent.Callable; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Phaser; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.rules.Timeout; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +/** Black box integration tests for {@link HttpConnectorMultiplexer}. */ +@RunWith(JUnit4.class) +public class HttpConnectorMultiplexerIntegrationTest { + + @Rule + public final ExpectedException thrown = ExpectedException.none(); + + @Rule + public final Timeout globalTimeout = new Timeout(10000); + + private final ExecutorService executor = Executors.newFixedThreadPool(3); + private final ProxyHelper proxyHelper = mock(ProxyHelper.class); + private final EventHandler eventHandler = mock(EventHandler.class); + private final ManualClock clock = new ManualClock(); + private final Sleeper sleeper = mock(Sleeper.class); + private final Locale locale = Locale.US; + private final HttpConnector connector = + new HttpConnector(locale, eventHandler, proxyHelper, sleeper); + private final ProgressInputStream.Factory progressInputStreamFactory = + new ProgressInputStream.Factory(locale, clock, eventHandler); + private final HttpStream.Factory httpStreamFactory = + new HttpStream.Factory(progressInputStreamFactory); + private final HttpConnectorMultiplexer multiplexer = + new HttpConnectorMultiplexer(eventHandler, connector, httpStreamFactory, clock, sleeper); + + @Before + public void before() throws Exception { + when(proxyHelper.createProxyIfNeeded(any(URL.class))).thenReturn(Proxy.NO_PROXY); + } + + @After + public void after() throws Exception { + executor.shutdown(); + } + + @Test + public void normalRequest() throws Exception { + final Phaser phaser = new Phaser(3); + try (ServerSocket server1 = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1")); + ServerSocket server2 = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) { + for (final ServerSocket server : asList(server1, server2)) { + executor.submit( + new Callable<Object>() { + @Override + public Object call() throws Exception { + for (String status : asList("503 MELTDOWN", "500 ERROR", "200 OK")) { + phaser.arriveAndAwaitAdvance(); + try (Socket socket = server.accept()) { + readHttpRequest(socket.getInputStream()); + sendLines(socket, + "HTTP/1.1 " + status, + "Date: Fri, 31 Dec 1999 23:59:59 GMT", + "Connection: close", + "", + "hello"); + } + } + return null; + } + }); + } + phaser.arriveAndAwaitAdvance(); + phaser.arriveAndDeregister(); + try (HttpStream stream = + multiplexer.connect( + ImmutableList.of( + new URL(String.format("http://127.0.0.1:%d", server1.getLocalPort())), + new URL(String.format("http://127.0.0.1:%d", server2.getLocalPort()))), + "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824")) { + assertThat(toByteArray(stream)).isEqualTo("hello".getBytes(US_ASCII)); + } + } + } + + @Test + public void captivePortal_isAvoided() throws Exception { + final CyclicBarrier barrier = new CyclicBarrier(2); + doAnswer( + new Answer<Void>() { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + barrier.await(); + return null; + } + }).when(sleeper).sleepMillis(anyLong()); + try (final ServerSocket server1 = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1")); + final ServerSocket server2 = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) { + executor.submit( + new Callable<Object>() { + @Override + public Object call() throws Exception { + try (Socket socket = server1.accept()) { + readHttpRequest(socket.getInputStream()); + sendLines(socket, + "HTTP/1.1 200 OK", + "Date: Fri, 31 Dec 1999 23:59:59 GMT", + "Warning: https://youtu.be/rJ6O5sTPn1k", + "Connection: close", + "", + "Und wird die Welt auch in Flammen stehen", + "Wir werden wieder auferstehen"); + } + barrier.await(); + return null; + } + }); + executor.submit( + new Callable<Object>() { + @Override + public Object call() throws Exception { + try (Socket socket = server2.accept()) { + readHttpRequest(socket.getInputStream()); + sendLines(socket, + "HTTP/1.1 200 OK", + "Date: Fri, 31 Dec 1999 23:59:59 GMT", + "Connection: close", + "", + "hello"); + } + return null; + } + }); + try (HttpStream stream = + multiplexer.connect( + ImmutableList.of( + new URL(String.format("http://127.0.0.1:%d", server1.getLocalPort())), + new URL(String.format("http://127.0.0.1:%d", server2.getLocalPort()))), + "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824")) { + assertThat(toByteArray(stream)).isEqualTo("hello".getBytes(US_ASCII)); + } + } + } + + @Test + public void allMirrorsDown_throwsIOException() throws Exception { + final CyclicBarrier barrier = new CyclicBarrier(4); + try (ServerSocket server1 = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1")); + ServerSocket server2 = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1")); + ServerSocket server3 = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) { + for (final ServerSocket server : asList(server1, server2, server3)) { + executor.submit( + new Callable<Object>() { + @Override + public Object call() throws Exception { + barrier.await(); + while (true) { + try (Socket socket = server.accept()) { + readHttpRequest(socket.getInputStream()); + sendLines(socket, + "HTTP/1.1 503 MELTDOWN", + "Date: Fri, 31 Dec 1999 23:59:59 GMT", + "Warning: https://youtu.be/6M6samPEMpM", + "Connection: close", + "", + ""); + } + } + } + }); + } + barrier.await(); + thrown.expect(IOException.class); + thrown.expectMessage("All mirrors are down: [GET returned 503 MELTDOWN]"); + multiplexer.connect( + ImmutableList.of( + new URL(String.format("http://127.0.0.1:%d", server1.getLocalPort())), + new URL(String.format("http://127.0.0.1:%d", server2.getLocalPort())), + new URL(String.format("http://127.0.0.1:%d", server3.getLocalPort()))), + "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9825"); + } + } +} diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnectorMultiplexerTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnectorMultiplexerTest.java new file mode 100644 index 0000000000..dedf316868 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnectorMultiplexerTest.java @@ -0,0 +1,266 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import static com.google.common.io.ByteStreams.toByteArray; +import static com.google.common.truth.Truth.assertThat; +import static com.google.devtools.build.lib.bazel.repository.downloader.DownloaderTestUtils.makeUrl; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Arrays.asList; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.eq; +import static org.mockito.Matchers.same; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.bazel.repository.downloader.RetryingInputStream.Reconnector; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.testutil.ManualClock; +import com.google.devtools.build.lib.util.Sleeper; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.net.URL; +import java.net.URLConnection; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.rules.Timeout; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +/** Unit tests for {@link HttpConnectorMultiplexer}. */ +@RunWith(JUnit4.class) +@SuppressWarnings("unchecked") +public class HttpConnectorMultiplexerTest { + + private static final URL URL1 = makeUrl("http://first.example"); + private static final URL URL2 = makeUrl("http://second.example"); + private static final URL URL3 = makeUrl("http://third.example"); + private static final byte[] data1 = "first".getBytes(UTF_8); + private static final byte[] data2 = "second".getBytes(UTF_8); + private static final byte[] data3 = "third".getBytes(UTF_8); + + @Rule + public final ExpectedException thrown = ExpectedException.none(); + + @Rule + public final Timeout globalTimeout = new Timeout(10000); + + private final HttpStream stream1 = fakeStream(URL1, data1); + private final HttpStream stream2 = fakeStream(URL2, data2); + private final HttpStream stream3 = fakeStream(URL3, data3); + private final ManualClock clock = new ManualClock(); + private final Sleeper sleeper = mock(Sleeper.class); + private final HttpConnector connector = mock(HttpConnector.class); + private final URLConnection connection1 = mock(URLConnection.class); + private final URLConnection connection2 = mock(URLConnection.class); + private final URLConnection connection3 = mock(URLConnection.class); + private final EventHandler eventHandler = mock(EventHandler.class); + private final HttpStream.Factory streamFactory = mock(HttpStream.Factory.class); + private final HttpConnectorMultiplexer multiplexer = + new HttpConnectorMultiplexer(eventHandler, connector, streamFactory, clock, sleeper); + + @Before + public void before() throws Exception { + when(connector.connect(eq(URL1), any(ImmutableMap.class))).thenReturn(connection1); + when(connector.connect(eq(URL2), any(ImmutableMap.class))).thenReturn(connection2); + when(connector.connect(eq(URL3), any(ImmutableMap.class))).thenReturn(connection3); + when(streamFactory + .create(same(connection1), any(URL.class), anyString(), any(Reconnector.class))) + .thenReturn(stream1); + when(streamFactory + .create(same(connection2), any(URL.class), anyString(), any(Reconnector.class))) + .thenReturn(stream2); + when(streamFactory + .create(same(connection3), any(URL.class), anyString(), any(Reconnector.class))) + .thenReturn(stream3); + } + + @Test + public void emptyList_throwsIae() throws Exception { + thrown.expect(IllegalArgumentException.class); + multiplexer.connect(ImmutableList.<URL>of(), ""); + } + + @Test + public void ftpUrl_throwsIae() throws Exception { + thrown.expect(IllegalArgumentException.class); + multiplexer.connect(asList(new URL("ftp://lol.example")), ""); + } + + @Test + public void threadIsInterrupted_throwsIeProntoAndDoesNothingElse() throws Exception { + final AtomicBoolean wasInterrupted = new AtomicBoolean(true); + Thread task = new Thread( + new Runnable() { + @Override + public void run() { + Thread.currentThread().interrupt(); + try { + multiplexer.connect(asList(new URL("http://lol.example")), ""); + } catch (InterruptedIOException ignored) { + return; + } catch (Exception ignored) { + // ignored + } + wasInterrupted.set(false); + } + }); + task.start(); + task.join(); + assertThat(wasInterrupted.get()).isTrue(); + verifyZeroInteractions(connector); + } + + @Test + public void singleUrl_justCallsConnector() throws Exception { + assertThat(toByteArray(multiplexer.connect(asList(URL1), "abc"))).isEqualTo(data1); + verify(connector).connect(eq(URL1), any(ImmutableMap.class)); + verify(streamFactory) + .create(any(URLConnection.class), any(URL.class), eq("abc"), any(Reconnector.class)); + verifyNoMoreInteractions(sleeper, connector, streamFactory); + } + + @Test + public void multipleUrlsFail_throwsIOException() throws Exception { + when(connector.connect(any(URL.class), any(ImmutableMap.class))).thenThrow(new IOException()); + try { + multiplexer.connect(asList(URL1, URL2, URL3), ""); + fail("Expected IOException"); + } catch (IOException e) { + assertThat(e.getMessage()).contains("All mirrors are down"); + } + verify(connector, times(3)).connect(any(URL.class), any(ImmutableMap.class)); + verify(sleeper, times(2)).sleepMillis(anyLong()); + verifyNoMoreInteractions(sleeper, connector, streamFactory); + } + + @Test + public void firstUrlFails_returnsSecond() throws Exception { + doAnswer( + new Answer<Void>() { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + clock.advanceMillis(1000); + return null; + } + }).when(sleeper).sleepMillis(anyLong()); + when(connector.connect(eq(URL1), any(ImmutableMap.class))).thenThrow(new IOException()); + assertThat(toByteArray(multiplexer.connect(asList(URL1, URL2), "abc"))).isEqualTo(data2); + assertThat(clock.currentTimeMillis()).isEqualTo(1000L); + verify(connector).connect(eq(URL1), any(ImmutableMap.class)); + verify(connector).connect(eq(URL2), any(ImmutableMap.class)); + verify(streamFactory) + .create(any(URLConnection.class), any(URL.class), eq("abc"), any(Reconnector.class)); + verify(sleeper).sleepMillis(anyLong()); + verifyNoMoreInteractions(sleeper, connector, streamFactory); + } + + @Test + public void twoSuccessfulUrlsAndFirstWins_returnsFirstAndInterruptsSecond() throws Exception { + final CyclicBarrier barrier = new CyclicBarrier(2); + final AtomicBoolean wasInterrupted = new AtomicBoolean(true); + when(connector.connect(eq(URL1), any(ImmutableMap.class))).thenAnswer( + new Answer<URLConnection>() { + @Override + public URLConnection answer(InvocationOnMock invocation) throws Throwable { + barrier.await(); + return connection1; + } + }); + doAnswer( + new Answer<Void>() { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + barrier.await(); + TimeUnit.MILLISECONDS.sleep(10000); + wasInterrupted.set(false); + return null; + } + }).when(sleeper).sleepMillis(anyLong()); + assertThat(toByteArray(multiplexer.connect(asList(URL1, URL2), "abc"))).isEqualTo(data1); + assertThat(wasInterrupted.get()).isTrue(); + } + + @Test + public void parentThreadGetsInterrupted_interruptsChildrenThenThrowsIe() throws Exception { + final CyclicBarrier barrier = new CyclicBarrier(3); + final AtomicBoolean wasInterrupted1 = new AtomicBoolean(true); + final AtomicBoolean wasInterrupted2 = new AtomicBoolean(true); + final AtomicBoolean wasInterrupted3 = new AtomicBoolean(true); + when(connector.connect(eq(URL1), any(ImmutableMap.class))).thenAnswer( + new Answer<URLConnection>() { + @Override + public URLConnection answer(InvocationOnMock invocation) throws Throwable { + barrier.await(); + TimeUnit.MILLISECONDS.sleep(10000); + wasInterrupted1.set(false); + throw new RuntimeException(); + } + }); + when(connector.connect(eq(URL2), any(ImmutableMap.class))).thenAnswer( + new Answer<URLConnection>() { + @Override + public URLConnection answer(InvocationOnMock invocation) throws Throwable { + barrier.await(); + TimeUnit.MILLISECONDS.sleep(10000); + wasInterrupted2.set(false); + throw new RuntimeException(); + } + }); + Thread task = new Thread( + new Runnable() { + @Override + public void run() { + try { + multiplexer.connect(asList(URL1, URL2), ""); + } catch (InterruptedIOException ignored) { + return; + } catch (Exception ignored) { + // ignored + } + wasInterrupted3.set(false); + } + }); + task.start(); + barrier.await(); + task.interrupt(); + task.join(); + assertThat(wasInterrupted1.get()).isTrue(); + assertThat(wasInterrupted2.get()).isTrue(); + assertThat(wasInterrupted3.get()).isTrue(); + } + + private static HttpStream fakeStream(URL url, byte[] data) { + return new HttpStream(new ByteArrayInputStream(data), url); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnectorTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnectorTest.java index fa5a1b6bb6..ccc50196d7 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnectorTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpConnectorTest.java @@ -15,41 +15,54 @@ package com.google.devtools.build.lib.bazel.repository.downloader; import static com.google.common.truth.Truth.assertThat; +import static com.google.devtools.build.lib.bazel.repository.downloader.DownloaderTestUtils.sendLines; +import static com.google.devtools.build.lib.bazel.repository.downloader.HttpParser.readHttpRequest; import static java.nio.charset.StandardCharsets.ISO_8859_1; +import static java.nio.charset.StandardCharsets.US_ASCII; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import com.google.common.io.ByteSource; +import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteStreams; import com.google.common.io.CharStreams; import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.testutil.ManualClock; +import com.google.devtools.build.lib.testutil.ManualSleeper; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; -import java.net.HttpURLConnection; import java.net.InetAddress; import java.net.Proxy; import java.net.ServerSocket; import java.net.Socket; import java.net.URL; +import java.net.URLConnection; +import java.util.Locale; +import java.util.Map; import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import org.junit.After; +import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; +import org.junit.rules.Timeout; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -/** - * Unit tests for {@link HttpConnector}. - */ +/** Unit tests for {@link HttpConnector}. */ @RunWith(JUnit4.class) public class HttpConnectorTest { @@ -57,191 +70,368 @@ public class HttpConnectorTest { public final ExpectedException thrown = ExpectedException.none(); @Rule - public TemporaryFolder testFolder = new TemporaryFolder(); + public final TemporaryFolder testFolder = new TemporaryFolder(); - private final ExecutorService executor = Executors.newSingleThreadExecutor(); - private final HttpURLConnection connection = mock(HttpURLConnection.class); + @Rule + public final Timeout globalTimeout = new Timeout(10000); + + private final ExecutorService executor = Executors.newFixedThreadPool(2); + private final ManualClock clock = new ManualClock(); + private final ManualSleeper sleeper = new ManualSleeper(clock); private final EventHandler eventHandler = mock(EventHandler.class); + private final ProxyHelper proxyHelper = mock(ProxyHelper.class); + private final HttpConnector connector = + new HttpConnector(Locale.US, eventHandler, proxyHelper, sleeper); + + @Before + public void before() throws Exception { + when(proxyHelper.createProxyIfNeeded(any(URL.class))).thenReturn(Proxy.NO_PROXY); + } @After public void after() throws Exception { - executor.shutdownNow(); + executor.shutdown(); } @Test - public void testLocalFileDownload() throws Exception { + public void localFileDownload() throws Exception { byte[] fileContents = "this is a test".getBytes(UTF_8); assertThat( ByteStreams.toByteArray( - HttpConnector.connect( - createTempFile(fileContents).toURI().toURL(), - Proxy.NO_PROXY, - eventHandler))) + connector.connect( + createTempFile(fileContents).toURI().toURL(), + ImmutableMap.<String, String>of()) + .getInputStream())) .isEqualTo(fileContents); } @Test - public void missingLocationInRedirect_throwsIOException() throws Exception { - thrown.expect(IOException.class); - when(connection.getURL()).thenReturn(new URL("http://lol.example")); - HttpConnector.getLocation(connection); - } - - @Test - public void absoluteLocationInRedirect_returnsNewUrl() throws Exception { - when(connection.getURL()).thenReturn(new URL("http://lol.example")); - when(connection.getHeaderField("Location")).thenReturn("http://new.example/hi"); - assertThat(HttpConnector.getLocation(connection)).isEqualTo(new URL("http://new.example/hi")); - } - - @Test - public void redirectOnlyHasPath_mergesHostFromOriginalUrl() throws Exception { - when(connection.getURL()).thenReturn(new URL("http://lol.example")); - when(connection.getHeaderField("Location")).thenReturn("/hi"); - assertThat(HttpConnector.getLocation(connection)).isEqualTo(new URL("http://lol.example/hi")); - } - - @Test - public void locationOnlyHasPathWithoutSlash_failsToMerge() throws Exception { + public void badHost_throwsIOException() throws Exception { thrown.expect(IOException.class); - thrown.expectMessage("Could not merge"); - when(connection.getURL()).thenReturn(new URL("http://lol.example")); - when(connection.getHeaderField("Location")).thenReturn("omg"); - HttpConnector.getLocation(connection); - } - - @Test - public void locationHasFragment_prefersNewFragment() throws Exception { - when(connection.getURL()).thenReturn(new URL("http://lol.example#a")); - when(connection.getHeaderField("Location")).thenReturn("http://new.example/hi#b"); - assertThat(HttpConnector.getLocation(connection)).isEqualTo(new URL("http://new.example/hi#b")); + thrown.expectMessage("Unknown host: bad.example"); + connector.connect(new URL("http://bad.example"), ImmutableMap.<String, String>of()); } @Test - public void locationHasNoFragmentButOriginalDoes_mergesOldFragment() throws Exception { - when(connection.getURL()).thenReturn(new URL("http://lol.example#a")); - when(connection.getHeaderField("Location")).thenReturn("http://new.example/hi"); - assertThat(HttpConnector.getLocation(connection)).isEqualTo(new URL("http://new.example/hi#a")); + public void normalRequest() throws Exception { + final Map<String, String> headers = new ConcurrentHashMap<>(); + try (ServerSocket server = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) { + executor.submit( + new Callable<Object>() { + @Override + public Object call() throws Exception { + try (Socket socket = server.accept()) { + readHttpRequest(socket.getInputStream(), headers); + sendLines(socket, + "HTTP/1.1 200 OK", + "Date: Fri, 31 Dec 1999 23:59:59 GMT", + "Connection: close", + "Content-Type: text/plain", + "Content-Length: 5", + "", + "hello"); + } + return null; + } + }); + try (Reader payload = + new InputStreamReader( + connector.connect( + new URL(String.format("http://127.0.0.1:%d/boo", server.getLocalPort())), + ImmutableMap.of("Content-Encoding", "gzip")) + .getInputStream(), + ISO_8859_1)) { + assertThat(CharStreams.toString(payload)).isEqualTo("hello"); + } + } + assertThat(headers).containsEntry("x-method", "GET"); + assertThat(headers).containsEntry("x-request-uri", "/boo"); + assertThat(headers).containsEntry("content-encoding", "gzip"); } @Test - public void oldUrlHasPasswordRedirectingToSameDomain_mergesPassword() throws Exception { - when(connection.getURL()).thenReturn(new URL("http://a:b@lol.example")); - when(connection.getHeaderField("Location")).thenReturn("http://lol.example/hi"); - assertThat(HttpConnector.getLocation(connection)) - .isEqualTo(new URL("http://a:b@lol.example/hi")); - when(connection.getURL()).thenReturn(new URL("http://a:b@lol.example")); - when(connection.getHeaderField("Location")).thenReturn("/hi"); - assertThat(HttpConnector.getLocation(connection)) - .isEqualTo(new URL("http://a:b@lol.example/hi")); + public void serverError_retriesConnect() throws Exception { + try (ServerSocket server = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) { + executor.submit( + new Callable<Object>() { + @Override + public Object call() throws Exception { + try (Socket socket = server.accept()) { + readHttpRequest(socket.getInputStream()); + sendLines(socket, + "HTTP/1.1 500 Incredible Catastrophe", + "Date: Fri, 31 Dec 1999 23:59:59 GMT", + "Connection: close", + "Content-Type: text/plain", + "Content-Length: 8", + "", + "nononono"); + } + try (Socket socket = server.accept()) { + readHttpRequest(socket.getInputStream()); + sendLines(socket, + "HTTP/1.1 200 OK", + "Date: Fri, 31 Dec 1999 23:59:59 GMT", + "Connection: close", + "Content-Type: text/plain", + "Content-Length: 5", + "", + "hello"); + } + return null; + } + }); + try (Reader payload = + new InputStreamReader( + connector.connect( + new URL(String.format("http://127.0.0.1:%d", server.getLocalPort())), + ImmutableMap.<String, String>of()) + .getInputStream(), + ISO_8859_1)) { + assertThat(CharStreams.toString(payload)).isEqualTo("hello"); + assertThat(clock.currentTimeMillis()).isEqualTo(100L); + } + } } @Test - public void oldUrlHasPasswordRedirectingToNewServer_doesntMergePassword() throws Exception { - when(connection.getURL()).thenReturn(new URL("http://a:b@lol.example")); - when(connection.getHeaderField("Location")).thenReturn("http://new.example/hi"); - assertThat(HttpConnector.getLocation(connection)).isEqualTo(new URL("http://new.example/hi")); - when(connection.getURL()).thenReturn(new URL("http://a:b@lol.example")); - when(connection.getHeaderField("Location")).thenReturn("http://lol.example:81/hi"); - assertThat(HttpConnector.getLocation(connection)) - .isEqualTo(new URL("http://lol.example:81/hi")); + public void permanentError_doesNotRetryAndThrowsIOException() throws Exception { + try (ServerSocket server = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) { + executor.submit( + new Callable<Object>() { + @Override + public Object call() throws Exception { + try (Socket socket = server.accept()) { + readHttpRequest(socket.getInputStream()); + sendLines(socket, + "HTTP/1.1 404 Not Here", + "Date: Fri, 31 Dec 1999 23:59:59 GMT", + "Connection: close", + "Content-Type: text/plain", + "Content-Length: 0", + "", + ""); + } + return null; + } + }); + thrown.expect(IOException.class); + thrown.expectMessage("404 Not Here"); + connector.connect( + new URL(String.format("http://127.0.0.1:%d", server.getLocalPort())), + ImmutableMap.<String, String>of()); + } } @Test - public void redirectToFtp_throwsIOException() throws Exception { - thrown.expect(IOException.class); - thrown.expectMessage("Bad Location"); - when(connection.getURL()).thenReturn(new URL("http://lol.example")); - when(connection.getHeaderField("Location")).thenReturn("ftp://lol.example"); - HttpConnector.getLocation(connection); + public void permanentError_consumesPayloadBeforeReturningn() throws Exception { + final CyclicBarrier barrier = new CyclicBarrier(2); + final AtomicBoolean consumed = new AtomicBoolean(); + try (ServerSocket server = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) { + executor.submit( + new Callable<Object>() { + @Override + public Object call() throws Exception { + try (Socket socket = server.accept()) { + readHttpRequest(socket.getInputStream()); + sendLines(socket, + "HTTP/1.1 501 Oh No", + "Date: Fri, 31 Dec 1999 23:59:59 GMT", + "Connection: close", + "Content-Type: text/plain", + "Content-Length: 1", + "", + "b"); + consumed.set(true); + } finally { + barrier.await(); + } + return null; + } + }); + connector.connect( + new URL(String.format("http://127.0.0.1:%d", server.getLocalPort())), + ImmutableMap.<String, String>of()); + fail(); + } catch (IOException ignored) { + // ignored + } finally { + barrier.await(); + } + assertThat(consumed.get()).isTrue(); + assertThat(clock.currentTimeMillis()).isEqualTo(0L); } @Test - public void redirectToHttps_works() throws Exception { - when(connection.getURL()).thenReturn(new URL("http://lol.example")); - when(connection.getHeaderField("Location")).thenReturn("https://lol.example"); - assertThat(HttpConnector.getLocation(connection)).isEqualTo(new URL("https://lol.example")); + public void always500_givesUpEventually() throws Exception { + final AtomicInteger tries = new AtomicInteger(); + try (ServerSocket server = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) { + executor.submit(new Callable<Object>() { + @Override + public Object call() throws Exception { + while (true) { + try (Socket socket = server.accept()) { + readHttpRequest(socket.getInputStream()); + sendLines(socket, + "HTTP/1.1 500 Oh My", + "Date: Fri, 31 Dec 1999 23:59:59 GMT", + "Connection: close", + "Content-Type: text/plain", + "Content-Length: 0", + "", + ""); + tries.incrementAndGet(); + } + } + } + }); + thrown.expect(IOException.class); + thrown.expectMessage("500 Oh My"); + try { + connector.connect( + new URL(String.format("http://127.0.0.1:%d", server.getLocalPort())), + ImmutableMap.<String, String>of()); + } finally { + assertThat(tries.get()).isGreaterThan(2); + } + } } @Test - public void testNormalRequest() throws Exception { - try (final ServerSocket server = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) { - Future<Void> thread = - executor.submit( - new Callable<Void>() { - @Override - public Void call() throws Exception { - try (Socket socket = server.accept()) { - send(socket, - "HTTP/1.1 200 OK\r\n" - + "Date: Fri, 31 Dec 1999 23:59:59 GMT\r\n" - + "Content-Type: text/plain\r\n" - + "Content-Length: 5\r\n" - + "\r\n" - + "hello"); - } - return null; + public void serverSays403_clientRetriesAnyway() throws Exception { + final AtomicInteger tries = new AtomicInteger(); + try (ServerSocket server = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) { + executor.submit( + new Callable<Object>() { + @Override + public Object call() throws Exception { + while (true) { + try (Socket socket = server.accept()) { + readHttpRequest(socket.getInputStream()); + sendLines(socket, + "HTTP/1.1 403 Forbidden", + "Date: Fri, 31 Dec 1999 23:59:59 GMT", + "Connection: close", + "Content-Type: text/plain", + "Content-Length: 0", + "", + ""); + tries.incrementAndGet(); } - }); - try (Reader payload = - new InputStreamReader( - HttpConnector.connect( - new URL(String.format("http://127.0.0.1:%d", server.getLocalPort())), - Proxy.NO_PROXY, - eventHandler), - ISO_8859_1)) { - assertThat(CharStreams.toString(payload)).isEqualTo("hello"); + } + } + }); + thrown.expect(IOException.class); + thrown.expectMessage("403 Forbidden"); + try { + connector.connect( + new URL(String.format("http://127.0.0.1:%d", server.getLocalPort())), + ImmutableMap.<String, String>of()); + } finally { + assertThat(tries.get()).isGreaterThan(2); } - thread.get(); } } @Test - public void testRetry() throws Exception { - try (final ServerSocket server = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) { - Future<Void> thread = - executor.submit( - new Callable<Void>() { - @Override - public Void call() throws Exception { - try (Socket socket = server.accept()) { - send(socket, - "HTTP/1.1 500 Incredible Catastrophe\r\n" - + "Date: Fri, 31 Dec 1999 23:59:59 GMT\r\n" - + "Content-Type: text/plain\r\n" - + "Content-Length: 8\r\n" - + "\r\n" - + "nononono"); - } - try (Socket socket = server.accept()) { - send(socket, - "HTTP/1.1 200 OK\r\n" - + "Date: Fri, 31 Dec 1999 23:59:59 GMT\r\n" - + "Content-Type: text/plain\r\n" - + "Content-Length: 5\r\n" - + "\r\n" - + "hello"); - } - return null; - } - }); - try (Reader payload = - new InputStreamReader( - HttpConnector.connect( - new URL(String.format("http://127.0.0.1:%d", server.getLocalPort())), - Proxy.NO_PROXY, - eventHandler), - ISO_8859_1)) { - assertThat(CharStreams.toString(payload)).isEqualTo("hello"); + public void redirectToDifferentPath_works() throws Exception { + final Map<String, String> headers1 = new ConcurrentHashMap<>(); + final Map<String, String> headers2 = new ConcurrentHashMap<>(); + try (ServerSocket server = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) { + executor.submit( + new Callable<Object>() { + @Override + public Object call() throws Exception { + try (Socket socket = server.accept()) { + readHttpRequest(socket.getInputStream(), headers1); + sendLines(socket, + "HTTP/1.1 301 Redirect", + "Date: Fri, 31 Dec 1999 23:59:59 GMT", + "Connection: close", + "Location: /doodle.tar.gz", + "Content-Length: 0", + "", + ""); + } + try (Socket socket = server.accept()) { + readHttpRequest(socket.getInputStream(), headers2); + sendLines(socket, + "HTTP/1.1 200 OK", + "Date: Fri, 31 Dec 1999 23:59:59 GMT", + "Connection: close", + "Content-Type: text/plain", + "Content-Length: 0", + "", + ""); + } + return null; + } + }); + URLConnection connection = + connector.connect( + new URL(String.format("http://127.0.0.1:%d", server.getLocalPort())), + ImmutableMap.<String, String>of()); + assertThat(connection.getURL()).isEqualTo( + new URL(String.format("http://127.0.0.1:%d/doodle.tar.gz", server.getLocalPort()))); + try (InputStream input = connection.getInputStream()) { + assertThat(ByteStreams.toByteArray(input)).isEmpty(); } - thread.get(); } + assertThat(headers1).containsEntry("x-request-uri", "/"); + assertThat(headers2).containsEntry("x-request-uri", "/doodle.tar.gz"); } - private static void send(Socket socket, String data) throws IOException { - ByteStreams.copy( - ByteSource.wrap(data.getBytes(ISO_8859_1)).openStream(), - socket.getOutputStream()); + @Test + public void redirectToDifferentServer_works() throws Exception { + try (ServerSocket server1 = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1")); + ServerSocket server2 = new ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) { + executor.submit( + new Callable<Object>() { + @Override + public Object call() throws Exception { + try (Socket socket = server1.accept()) { + readHttpRequest(socket.getInputStream()); + sendLines(socket, + "HTTP/1.1 301 Redirect", + "Date: Fri, 31 Dec 1999 23:59:59 GMT", + "Connection: close", + String.format("Location: http://127.0.0.1:%d/doodle.tar.gz", + server2.getLocalPort()), + "Content-Length: 0", + "", + ""); + } + return null; + } + }); + executor.submit( + new Callable<Object>() { + @Override + public Object call() throws Exception { + try (Socket socket = server2.accept()) { + readHttpRequest(socket.getInputStream()); + sendLines(socket, + "HTTP/1.1 200 OK", + "Date: Fri, 31 Dec 1999 23:59:59 GMT", + "Connection: close", + "Content-Type: text/plain", + "Content-Length: 5", + "", + "hello"); + } + return null; + } + }); + URLConnection connection = + connector.connect( + new URL(String.format("http://127.0.0.1:%d", server1.getLocalPort())), + ImmutableMap.<String, String>of()); + assertThat(connection.getURL()).isEqualTo( + new URL(String.format("http://127.0.0.1:%d/doodle.tar.gz", server2.getLocalPort()))); + try (InputStream input = connection.getInputStream()) { + assertThat(ByteStreams.toByteArray(input)).isEqualTo("hello".getBytes(US_ASCII)); + } + } } private File createTempFile(byte[] fileContents) throws IOException { diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpParser.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpParser.java new file mode 100644 index 0000000000..c901367aca --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpParser.java @@ -0,0 +1,154 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import com.google.common.base.Ascii; +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; + +/** Utility class for parsing HTTP messages. */ +final class HttpParser { + + /** Exhausts request line and headers of HTTP request. */ + static void readHttpRequest(InputStream stream) throws IOException { + readHttpRequest(stream, new HashMap<String, String>()); + } + + /** + * Parses request line and headers of HTTP request. + * + * <p>This parser is correct and extremely lax. This implementation is Θ(n) and the stream should + * be buffered. All decoding is ISO-8859-1. A 1mB upper bound on memory is enforced. + * + * @throws IOException if reading failed or premature end of stream encountered + * @throws HttpParserError if 400 error should be sent to client and connection must be closed + */ + static void readHttpRequest(InputStream stream, Map<String, String> output) throws IOException { + StringBuilder builder = new StringBuilder(256); + State state = State.METHOD; + String key = ""; + int toto = 0; + while (true) { + int c = stream.read(); + if (c == -1) { + throw new IOException(); // RFC7230 § 3.4 + } + if (++toto == 1024 * 1024) { + throw new HttpParserError(); // RFC7230 § 3.2.5 + } + switch (state) { + case METHOD: + if (c == ' ') { + if (builder.length() == 0) { + throw new HttpParserError(); + } + output.put("x-method", builder.toString()); + builder.setLength(0); + state = State.URI; + } else if (c == '\r' || c == '\n') { + break; // RFC7230 § 3.5 + } else { + builder.append(Ascii.toUpperCase((char) c)); + } + break; + case URI: + if (c == ' ') { + if (builder.length() == 0) { + throw new HttpParserError(); + } + output.put("x-request-uri", builder.toString()); + builder.setLength(0); + state = State.VERSION; + } else { + builder.append((char) c); + } + break; + case VERSION: + if (c == '\r' || c == '\n') { + output.put("x-version", builder.toString()); + builder.setLength(0); + state = c == '\r' ? State.CR1 : State.LF1; + } else { + builder.append(Ascii.toUpperCase((char) c)); + } + break; + case CR1: + if (c == '\n') { + state = State.LF1; + break; + } + throw new HttpParserError(); + case LF1: + if (c == '\r') { + state = State.LF2; + break; + } else if (c == '\n') { + return; + } else if (c == ' ' || c == '\t') { + throw new HttpParserError("Line folding unacceptable"); // RFC7230 § 3.2.4 + } + state = State.HKEY; + // epsilon transition + case HKEY: + if (c == ':') { + key = builder.toString(); + builder.setLength(0); + state = State.HSEP; + } else { + builder.append(Ascii.toLowerCase((char) c)); + } + break; + case HSEP: + if (c == ' ' || c == '\t') { + break; + } + state = State.HVAL; + // epsilon transition + case HVAL: + if (c == '\r' || c == '\n') { + output.put(key, builder.toString()); + builder.setLength(0); + state = c == '\r' ? State.CR1 : State.LF1; + } else { + builder.append((char) c); + } + break; + case LF2: + if (c == '\n') { + return; + } + throw new HttpParserError(); + default: + throw new AssertionError(); + } + } + } + + static final class HttpParserError extends IOException { + HttpParserError() { + this("Malformed Request"); + } + + HttpParserError(String messageForClient) { + super(messageForClient); + } + } + + private enum State { METHOD, URI, VERSION, HKEY, HSEP, HVAL, CR1, LF1, LF2 } + + private HttpParser() {} +} diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpStreamTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpStreamTest.java new file mode 100644 index 0000000000..ac0351ed18 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpStreamTest.java @@ -0,0 +1,195 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import static com.google.common.io.ByteStreams.toByteArray; +import static com.google.common.truth.Truth.assertThat; +import static com.google.devtools.build.lib.bazel.repository.downloader.DownloaderTestUtils.makeUrl; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.google.common.hash.Hashing; +import com.google.common.io.ByteStreams; +import com.google.devtools.build.lib.bazel.repository.downloader.RetryingInputStream.Reconnector; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InterruptedIOException; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.Random; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.zip.GZIPOutputStream; +import java.util.zip.ZipException; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.rules.Timeout; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +/** Integration tests for {@link HttpStream.Factory} and friends. */ +@RunWith(JUnit4.class) +public class HttpStreamTest { + + private static final Random randoCalrissian = new Random(); + private static final byte[] data = "hello".getBytes(UTF_8); + private static final String GOOD_CHECKSUM = + "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824"; + private static final String BAD_CHECKSUM = + "0000000000000000000000000000000000000000000000000000000000000000"; + private static final URL AURL = makeUrl("http://doodle.example"); + + @Rule + public final ExpectedException thrown = ExpectedException.none(); + + @Rule + public final Timeout globalTimeout = new Timeout(10000); + + private final HttpURLConnection connection = mock(HttpURLConnection.class); + private final Reconnector reconnector = mock(Reconnector.class); + private final ProgressInputStream.Factory progress = mock(ProgressInputStream.Factory.class); + private final HttpStream.Factory streamFactory = new HttpStream.Factory(progress); + + @Before + public void before() throws Exception { + when(connection.getInputStream()).thenReturn(new ByteArrayInputStream(data)); + when(progress.create(any(InputStream.class), any(URL.class), any(URL.class))).thenAnswer( + new Answer<InputStream>() { + @Override + public InputStream answer(InvocationOnMock invocation) throws Throwable { + return (InputStream) invocation.getArguments()[0]; + } + }); + } + + @Test + public void noChecksum_readsOk() throws Exception { + try (HttpStream stream = streamFactory.create(connection, AURL, "", reconnector)) { + assertThat(toByteArray(stream)).isEqualTo(data); + } + } + + @Test + public void smallDataWithValidChecksum_readsOk() throws Exception { + try (HttpStream stream = streamFactory.create(connection, AURL, GOOD_CHECKSUM, reconnector)) { + assertThat(toByteArray(stream)).isEqualTo(data); + } + } + + @Test + public void smallDataWithInvalidChecksum_throwsIOExceptionInCreatePhase() throws Exception { + thrown.expect(IOException.class); + thrown.expectMessage("Checksum"); + streamFactory.create(connection, AURL, BAD_CHECKSUM, reconnector); + } + + @Test + public void bigDataWithValidChecksum_readsOk() throws Exception { + // at google, we know big data + byte[] bigData = new byte[HttpStream.PRECHECK_BYTES + 70001]; + randoCalrissian.nextBytes(bigData); + when(connection.getInputStream()).thenReturn(new ByteArrayInputStream(bigData)); + try (HttpStream stream = + streamFactory.create( + connection, AURL, Hashing.sha256().hashBytes(bigData).toString(), reconnector)) { + assertThat(toByteArray(stream)).isEqualTo(bigData); + } + } + + @Test + public void bigDataWithInvalidChecksum_throwsIOExceptionAfterCreateOnEof() throws Exception { + // the probability of this test flaking is 8.6361686e-78 + byte[] bigData = new byte[HttpStream.PRECHECK_BYTES + 70001]; + randoCalrissian.nextBytes(bigData); + when(connection.getInputStream()).thenReturn(new ByteArrayInputStream(bigData)); + try (HttpStream stream = streamFactory.create(connection, AURL, BAD_CHECKSUM, reconnector)) { + thrown.expect(IOException.class); + thrown.expectMessage("Checksum"); + toByteArray(stream); + fail("Should have thrown error before close()"); + } + } + + @Test + public void httpServerSaidGzippedButNotGzipped_throwsZipExceptionInCreate() throws Exception { + when(connection.getURL()).thenReturn(AURL); + when(connection.getContentEncoding()).thenReturn("gzip"); + thrown.expect(ZipException.class); + streamFactory.create(connection, AURL, "", reconnector); + } + + @Test + public void javascriptGzippedInTransit_automaticallyGunzips() throws Exception { + when(connection.getURL()).thenReturn(AURL); + when(connection.getContentEncoding()).thenReturn("x-gzip"); + when(connection.getInputStream()).thenReturn(new ByteArrayInputStream(gzipData(data))); + try (HttpStream stream = streamFactory.create(connection, AURL, "", reconnector)) { + assertThat(toByteArray(stream)).isEqualTo(data); + } + } + + @Test + public void serverSaysTarballPathIsGzipped_doesntAutomaticallyGunzip() throws Exception { + byte[] gzData = gzipData(data); + when(connection.getURL()).thenReturn(new URL("http://doodle.example/foo.tar.gz")); + when(connection.getContentEncoding()).thenReturn("gzip"); + when(connection.getInputStream()).thenReturn(new ByteArrayInputStream(gzData)); + try (HttpStream stream = streamFactory.create(connection, AURL, "", reconnector)) { + assertThat(toByteArray(stream)).isEqualTo(gzData); + } + } + + @Test + public void threadInterrupted_haltsReadingAndThrowsInterrupt() throws Exception { + final AtomicBoolean wasInterrupted = new AtomicBoolean(); + Thread thread = new Thread( + new Runnable() { + @Override + public void run() { + try (HttpStream stream = streamFactory.create(connection, AURL, "", reconnector)) { + stream.read(); + Thread.currentThread().interrupt(); + stream.read(); + fail(); + } catch (InterruptedIOException expected) { + wasInterrupted.set(true); + } catch (IOException ignored) { + // ignored + } + } + }); + thread.start(); + thread.join(); + assertThat(wasInterrupted.get()).isTrue(); + } + + private static byte[] gzipData(byte[] bytes) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (InputStream input = new ByteArrayInputStream(bytes); + OutputStream output = new GZIPOutputStream(baos)) { + ByteStreams.copy(input, output); + } + return baos.toByteArray(); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpUtilsTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpUtilsTest.java new file mode 100644 index 0000000000..b79cbc5244 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpUtilsTest.java @@ -0,0 +1,131 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link HttpUtils}. */ +@RunWith(JUnit4.class) +public class HttpUtilsTest { + + @Rule + public final ExpectedException thrown = ExpectedException.none(); + + private final HttpURLConnection connection = mock(HttpURLConnection.class); + + @Test + public void getExtension_twoExtensions_returnsLast() throws Exception { + assertThat(HttpUtils.getExtension("doodle.tar.gz")).isEqualTo("gz"); + } + + @Test + public void getExtension_isUppercase_returnsLowered() throws Exception { + assertThat(HttpUtils.getExtension("DOODLE.TXT")).isEqualTo("txt"); + } + + @Test + public void getLocation_missingInRedirect_throwsIOException() throws Exception { + thrown.expect(IOException.class); + when(connection.getURL()).thenReturn(new URL("http://lol.example")); + HttpUtils.getLocation(connection); + } + + @Test + public void getLocation_absoluteInRedirect_returnsNewUrl() throws Exception { + when(connection.getURL()).thenReturn(new URL("http://lol.example")); + when(connection.getHeaderField("Location")).thenReturn("http://new.example/hi"); + assertThat(HttpUtils.getLocation(connection)).isEqualTo(new URL("http://new.example/hi")); + } + + @Test + public void getLocation_redirectOnlyHasPath_mergesHostFromOriginalUrl() throws Exception { + when(connection.getURL()).thenReturn(new URL("http://lol.example")); + when(connection.getHeaderField("Location")).thenReturn("/hi"); + assertThat(HttpUtils.getLocation(connection)).isEqualTo(new URL("http://lol.example/hi")); + } + + @Test + public void getLocation_onlyHasPathWithoutSlash_failsToMerge() throws Exception { + thrown.expect(IOException.class); + thrown.expectMessage("Could not merge"); + when(connection.getURL()).thenReturn(new URL("http://lol.example")); + when(connection.getHeaderField("Location")).thenReturn("omg"); + HttpUtils.getLocation(connection); + } + + @Test + public void getLocation_hasFragment_prefersNewFragment() throws Exception { + when(connection.getURL()).thenReturn(new URL("http://lol.example#a")); + when(connection.getHeaderField("Location")).thenReturn("http://new.example/hi#b"); + assertThat(HttpUtils.getLocation(connection)).isEqualTo(new URL("http://new.example/hi#b")); + } + + @Test + public void getLocation_hasNoFragmentButOriginalDoes_mergesOldFragment() throws Exception { + when(connection.getURL()).thenReturn(new URL("http://lol.example#a")); + when(connection.getHeaderField("Location")).thenReturn("http://new.example/hi"); + assertThat(HttpUtils.getLocation(connection)).isEqualTo(new URL("http://new.example/hi#a")); + } + + @Test + public void getLocation_oldUrlHasPassRedirectingToSameDomain_mergesPassword() throws Exception { + when(connection.getURL()).thenReturn(new URL("http://a:b@lol.example")); + when(connection.getHeaderField("Location")).thenReturn("http://lol.example/hi"); + assertThat(HttpUtils.getLocation(connection)) + .isEqualTo(new URL("http://a:b@lol.example/hi")); + when(connection.getURL()).thenReturn(new URL("http://a:b@lol.example")); + when(connection.getHeaderField("Location")).thenReturn("/hi"); + assertThat(HttpUtils.getLocation(connection)) + .isEqualTo(new URL("http://a:b@lol.example/hi")); + } + + @Test + public void getLocation_oldUrlHasPasswordRedirectingToNewServer_doesntMerge() throws Exception { + when(connection.getURL()).thenReturn(new URL("http://a:b@lol.example")); + when(connection.getHeaderField("Location")).thenReturn("http://new.example/hi"); + assertThat(HttpUtils.getLocation(connection)).isEqualTo(new URL("http://new.example/hi")); + when(connection.getURL()).thenReturn(new URL("http://a:b@lol.example")); + when(connection.getHeaderField("Location")).thenReturn("http://lol.example:81/hi"); + assertThat(HttpUtils.getLocation(connection)) + .isEqualTo(new URL("http://lol.example:81/hi")); + } + + @Test + public void getLocation_redirectToFtp_throwsIOException() throws Exception { + thrown.expect(IOException.class); + thrown.expectMessage("Bad Location"); + when(connection.getURL()).thenReturn(new URL("http://lol.example")); + when(connection.getHeaderField("Location")).thenReturn("ftp://lol.example"); + HttpUtils.getLocation(connection); + } + + @Test + public void getLocation_redirectToHttps_works() throws Exception { + when(connection.getURL()).thenReturn(new URL("http://lol.example")); + when(connection.getHeaderField("Location")).thenReturn("https://lol.example"); + assertThat(HttpUtils.getLocation(connection)).isEqualTo(new URL("https://lol.example")); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/ProgressInputStreamTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/ProgressInputStreamTest.java new file mode 100644 index 0000000000..2e047f773f --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/ProgressInputStreamTest.java @@ -0,0 +1,149 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.devtools.build.lib.bazel.repository.downloader.DownloaderTestUtils.makeUrl; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Matchers.eq; +import static org.mockito.Matchers.same; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.testutil.ManualClock; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.util.Locale; +import org.junit.After; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link ProgressInputStream}. */ +@RunWith(JUnit4.class) +public class ProgressInputStreamTest { + + private final ManualClock clock = new ManualClock(); + private final EventHandler eventHandler = mock(EventHandler.class); + private final InputStream delegate = mock(InputStream.class); + private final URL url = makeUrl("http://lol.example"); + private ProgressInputStream stream = + new ProgressInputStream(Locale.US, clock, eventHandler, 1, delegate, url, url); + + @After + public void after() throws Exception { + verifyNoMoreInteractions(eventHandler, delegate); + } + + @Test + public void close_callsDelegate() throws Exception { + stream.close(); + verify(delegate).close(); + } + + @Test + public void available_callsDelegate() throws Exception { + stream.available(); + verify(delegate).available(); + } + + @Test + public void read_callsdelegate() throws Exception { + stream.read(); + verify(delegate).read(); + } + + @Test + public void readThrowsException_passesThrough() throws Exception { + when(delegate.read()).thenThrow(new IOException()); + try { + stream.read(); + fail("Expected IOException"); + } catch (IOException expected) { + verify(delegate).read(); + } + } + + @Test + public void readsAfterInterval_emitsProgressOnce() throws Exception { + when(delegate.read()).thenReturn(42); + assertThat(stream.read()).isEqualTo(42); + clock.advanceMillis(1); + assertThat(stream.read()).isEqualTo(42); + assertThat(stream.read()).isEqualTo(42); + verify(delegate, times(3)).read(); + verify(eventHandler).handle(Event.progress("Downloading http://lol.example: 2 bytes")); + } + + @Test + public void multipleIntervalsElapsed_showsMultipleProgress() throws Exception { + stream.read(); + stream.read(); + clock.advanceMillis(1); + stream.read(); + stream.read(); + clock.advanceMillis(1); + stream.read(); + stream.read(); + verify(delegate, times(6)).read(); + verify(eventHandler).handle(Event.progress("Downloading http://lol.example: 3 bytes")); + verify(eventHandler).handle(Event.progress("Downloading http://lol.example: 5 bytes")); + } + + @Test + public void bufferReadsAfterInterval_emitsProgressOnce() throws Exception { + byte[] buffer = new byte[1024]; + when(delegate.read(any(byte[].class), anyInt(), anyInt())).thenReturn(1024); + assertThat(stream.read(buffer)).isEqualTo(1024); + clock.advanceMillis(1); + assertThat(stream.read(buffer)).isEqualTo(1024); + assertThat(stream.read(buffer)).isEqualTo(1024); + verify(delegate, times(3)).read(same(buffer), eq(0), eq(1024)); + verify(eventHandler).handle(Event.progress("Downloading http://lol.example: 2,048 bytes")); + } + + @Test + public void bufferReadsAfterIntervalInGermany_usesPeriodAsSeparator() throws Exception { + stream = new ProgressInputStream(Locale.GERMANY, clock, eventHandler, 1, delegate, url, url); + byte[] buffer = new byte[1024]; + when(delegate.read(any(byte[].class), anyInt(), anyInt())).thenReturn(1024); + clock.advanceMillis(1); + stream.read(buffer); + verify(delegate).read(same(buffer), eq(0), eq(1024)); + verify(eventHandler).handle(Event.progress("Downloading http://lol.example: 1.024 bytes")); + } + + @Test + public void redirectedToDifferentServer_showsOriginalUrlWithVia() throws Exception { + stream = new ProgressInputStream( + Locale.US, clock, eventHandler, 1, delegate, new URL("http://cdn.example/foo"), url); + when(delegate.read()).thenReturn(42); + assertThat(stream.read()).isEqualTo(42); + clock.advanceMillis(1); + assertThat(stream.read()).isEqualTo(42); + assertThat(stream.read()).isEqualTo(42); + verify(delegate, times(3)).read(); + verify(eventHandler).handle( + Event.progress("Downloading http://lol.example via cdn.example: 2 bytes")); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/ProxyHelperTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/ProxyHelperTest.java index 2a725c195a..6063ae5054 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/ProxyHelperTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/ProxyHelperTest.java @@ -21,46 +21,43 @@ import static org.junit.Assert.fail; import com.google.common.collect.ImmutableMap; import java.io.IOException; import java.net.Proxy; +import java.net.URL; import java.util.Map; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; /** - * Tests for @{link ProxyHelper}. + * Tests for {@link ProxyHelper}. */ @RunWith(JUnit4.class) public class ProxyHelperTest { @Test public void testCreateIfNeededHttpLowerCase() throws Exception { - Map<String, String> env = ImmutableMap.<String, String>builder() - .put("http_proxy", "http://my.example.com").build(); - Proxy proxy = ProxyHelper.createProxyIfNeeded("http://www.something.com", env); + ProxyHelper helper = new ProxyHelper(ImmutableMap.of("http_proxy", "http://my.example.com")); + Proxy proxy = helper.createProxyIfNeeded(new URL("http://www.something.com")); assertThat(proxy.toString()).endsWith("my.example.com:80"); } @Test public void testCreateIfNeededHttpUpperCase() throws Exception { - Map<String, String> env = ImmutableMap.<String, String>builder() - .put("HTTP_PROXY", "http://my.example.com").build(); - Proxy proxy = ProxyHelper.createProxyIfNeeded("http://www.something.com", env); + ProxyHelper helper = new ProxyHelper(ImmutableMap.of("HTTP_PROXY", "http://my.example.com")); + Proxy proxy = helper.createProxyIfNeeded(new URL("http://www.something.com")); assertThat(proxy.toString()).endsWith("my.example.com:80"); } @Test public void testCreateIfNeededHttpsLowerCase() throws Exception { - Map<String, String> env = ImmutableMap.<String, String>builder() - .put("https_proxy", "https://my.example.com").build(); - Proxy proxy = ProxyHelper.createProxyIfNeeded("https://www.something.com", env); + ProxyHelper helper = new ProxyHelper(ImmutableMap.of("https_proxy", "https://my.example.com")); + Proxy proxy = helper.createProxyIfNeeded(new URL("https://www.something.com")); assertThat(proxy.toString()).endsWith("my.example.com:443"); } @Test public void testCreateIfNeededHttpsUpperCase() throws Exception { - Map<String, String> env = ImmutableMap.<String, String>builder() - .put("HTTPS_PROXY", "https://my.example.com").build(); - Proxy proxy = ProxyHelper.createProxyIfNeeded("https://www.something.com", env); + ProxyHelper helper = new ProxyHelper(ImmutableMap.of("HTTPS_PROXY", "https://my.example.com")); + Proxy proxy = helper.createProxyIfNeeded(new URL("https://www.something.com")); assertThat(proxy.toString()).endsWith("my.example.com:443"); } @@ -72,7 +69,8 @@ public class ProxyHelperTest { proxy = ProxyHelper.createProxy(""); assertEquals(Proxy.NO_PROXY, proxy); Map<String, String> env = ImmutableMap.of(); - proxy = ProxyHelper.createProxyIfNeeded("https://www.something.com", env); + ProxyHelper helper = new ProxyHelper(env); + proxy = helper.createProxyIfNeeded(new URL("https://www.something.com")); assertEquals(Proxy.NO_PROXY, proxy); } diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/RetryingInputStreamTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/RetryingInputStreamTest.java new file mode 100644 index 0000000000..10f0923dcb --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/downloader/RetryingInputStreamTest.java @@ -0,0 +1,173 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.downloader; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Matchers.same; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.bazel.repository.downloader.RetryingInputStream.Reconnector; +import java.io.IOException; +import java.io.InputStream; +import java.io.InterruptedIOException; +import java.net.SocketTimeoutException; +import java.net.URLConnection; +import org.junit.After; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for {@link RetryingInputStream}. */ +@RunWith(JUnit4.class) +public class RetryingInputStreamTest { + + private final InputStream delegate = mock(InputStream.class); + private final InputStream newDelegate = mock(InputStream.class); + private final Reconnector reconnector = mock(Reconnector.class); + private final URLConnection connection = mock(URLConnection.class); + private final RetryingInputStream stream = new RetryingInputStream(delegate, reconnector); + + @After + public void after() throws Exception { + verifyNoMoreInteractions(delegate, newDelegate, reconnector); + } + + @Test + public void close_callsDelegate() throws Exception { + stream.close(); + verify(delegate).close(); + } + + @Test + public void available_callsDelegate() throws Exception { + stream.available(); + verify(delegate).available(); + } + + @Test + public void read_callsdelegate() throws Exception { + stream.read(); + verify(delegate).read(); + } + + @Test + public void bufferRead_callsdelegate() throws Exception { + byte[] buffer = new byte[1024]; + stream.read(buffer); + verify(delegate).read(same(buffer), eq(0), eq(1024)); + } + + @Test + public void readThrowsExceptionWhenDisabled_passesThrough() throws Exception { + stream.disabled = true; + when(delegate.read()).thenThrow(new IOException()); + try { + stream.read(); + fail("Expected IOException"); + } catch (IOException expected) { + verify(delegate).read(); + } + } + + @Test + public void readInterrupted_alwaysPassesThrough() throws Exception { + when(delegate.read()).thenThrow(new InterruptedIOException()); + try { + stream.read(); + fail("Expected InterruptedIOException"); + } catch (InterruptedIOException expected) { + verify(delegate).read(); + } + } + + @Test + @SuppressWarnings("unchecked") + public void readTimesOut_retries() throws Exception { + when(delegate.read()).thenReturn(1).thenThrow(new SocketTimeoutException()); + when(reconnector.connect(any(Throwable.class), any(ImmutableMap.class))).thenReturn(connection); + when(connection.getInputStream()).thenReturn(newDelegate); + when(newDelegate.read()).thenReturn(2); + when(connection.getHeaderField("Content-Range")).thenReturn("bytes 1-42/42"); + assertThat(stream.read()).isEqualTo(1); + assertThat(stream.read()).isEqualTo(2); + verify(reconnector).connect(any(Throwable.class), eq(ImmutableMap.of("Range", "bytes 1-"))); + verify(delegate, times(2)).read(); + verify(delegate).close(); + verify(newDelegate).read(); + } + + @Test + @SuppressWarnings("unchecked") + public void failureWhenNoBytesAreRead_doesntUseRange() throws Exception { + when(delegate.read()).thenThrow(new SocketTimeoutException()); + when(newDelegate.read()).thenReturn(1); + when(reconnector.connect(any(Throwable.class), any(ImmutableMap.class))).thenReturn(connection); + when(connection.getInputStream()).thenReturn(newDelegate); + assertThat(stream.read()).isEqualTo(1); + verify(reconnector).connect(any(Throwable.class), eq(ImmutableMap.<String, String>of())); + verify(delegate).read(); + verify(delegate).close(); + verify(newDelegate).read(); + } + + @Test + @SuppressWarnings("unchecked") + public void reconnectFails_alwaysPassesThrough() throws Exception { + when(delegate.read()).thenThrow(new IOException()); + when(reconnector.connect(any(Throwable.class), any(ImmutableMap.class))) + .thenThrow(new IOException()); + try { + stream.read(); + fail("Expected IOException"); + } catch (IOException expected) { + verify(delegate).read(); + verify(delegate).close(); + verify(reconnector).connect(any(Throwable.class), any(ImmutableMap.class)); + } + } + + @Test + @SuppressWarnings("unchecked") + public void maxRetries_givesUp() throws Exception { + when(delegate.read()) + .thenReturn(1) + .thenThrow(new IOException()) + .thenThrow(new IOException()) + .thenThrow(new IOException()) + .thenThrow(new SocketTimeoutException()); + when(reconnector.connect(any(Throwable.class), any(ImmutableMap.class))).thenReturn(connection); + when(connection.getInputStream()).thenReturn(delegate); + when(connection.getHeaderField("Content-Range")).thenReturn("bytes 1-42/42"); + stream.read(); + try { + stream.read(); + fail("Expected SocketTimeoutException"); + } catch (SocketTimeoutException e) { + assertThat(e.getSuppressed()).hasLength(3); + verify(reconnector, times(3)) + .connect(any(Throwable.class), eq(ImmutableMap.of("Range", "bytes 1-"))); + verify(delegate, times(5)).read(); + verify(delegate, times(3)).close(); + } + } +} diff --git a/src/test/java/com/google/devtools/build/lib/testutil/ManualClock.java b/src/test/java/com/google/devtools/build/lib/testutil/ManualClock.java index 15752cc2ef..fe500d0464 100644 --- a/src/test/java/com/google/devtools/build/lib/testutil/ManualClock.java +++ b/src/test/java/com/google/devtools/build/lib/testutil/ManualClock.java @@ -15,18 +15,18 @@ package com.google.devtools.build.lib.testutil; import com.google.devtools.build.lib.util.Clock; - import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; /** * A fake clock for testing. */ public final class ManualClock implements Clock { - private long currentTimeMillis = 0L; + private final AtomicLong currentTimeMillis = new AtomicLong(); @Override public long currentTimeMillis() { - return currentTimeMillis; + return currentTimeMillis.get(); } /** @@ -36,11 +36,11 @@ public final class ManualClock implements Clock { */ @Override public long nanoTime() { - return TimeUnit.MILLISECONDS.toNanos(currentTimeMillis) + return TimeUnit.MILLISECONDS.toNanos(currentTimeMillis.get()) + TimeUnit.SECONDS.toNanos(1000); } - public void advanceMillis(long time) { - currentTimeMillis += time; + public long advanceMillis(long time) { + return currentTimeMillis.addAndGet(time); } } diff --git a/src/test/java/com/google/devtools/build/lib/testutil/ManualSleeper.java b/src/test/java/com/google/devtools/build/lib/testutil/ManualSleeper.java new file mode 100644 index 0000000000..6e16253499 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/testutil/ManualSleeper.java @@ -0,0 +1,36 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.testutil; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.devtools.build.lib.util.Sleeper; + +/** Fake sleeper for testing. */ +public final class ManualSleeper implements Sleeper { + + private final ManualClock clock; + + public ManualSleeper(ManualClock clock) { + this.clock = checkNotNull(clock); + } + + @Override + public void sleepMillis(long milliseconds) throws InterruptedException { + checkArgument(milliseconds >= 0, "sleeper can't time travel"); + clock.advanceMillis(milliseconds); + } +} diff --git a/src/test/shell/bazel/external_integration_test.sh b/src/test/shell/bazel/external_integration_test.sh index 0f610e050c..52b01a1fb2 100755 --- a/src/test/shell/bazel/external_integration_test.sh +++ b/src/test/shell/bazel/external_integration_test.sh @@ -191,10 +191,9 @@ function test_http_archive_tar_xz() { } function test_http_archive_no_server() { - nc_port=$(pick_random_unused_tcp_port) || exit 1 cat > WORKSPACE <<EOF -http_archive(name = 'endangered', url = 'http://localhost:$nc_port/repo.zip', - sha256 = 'dummy') +http_archive(name = 'endangered', url = 'http://bad.example/repo.zip', + sha256 = '2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9826') EOF cat > zoo/BUILD <<EOF @@ -212,7 +211,7 @@ EOF chmod +x zoo/female.sh bazel fetch //zoo:breeding-program >& $TEST_log && fail "Expected fetch to fail" - expect_log "Connection refused" + expect_log "Unknown host: bad.example" } function test_http_archive_mismatched_sha256() { @@ -229,8 +228,11 @@ function test_http_archive_mismatched_sha256() { cd ${WORKSPACE_DIR} cat > WORKSPACE <<EOF -http_archive(name = 'endangered', url = 'http://localhost:$nc_port/repo.zip', - sha256 = '$wrong_sha256') +http_archive( + name = 'endangered', + url = 'http://localhost:$nc_port/repo.zip', + sha256 = '2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9826', +) EOF cat > zoo/BUILD <<EOF @@ -249,7 +251,7 @@ EOF bazel fetch //zoo:breeding-program >& $TEST_log && echo "Expected fetch to fail" kill_nc - expect_log "does not match expected SHA-256" + expect_log "Checksum" } # Bazel should not re-download the .zip unless the user requests it or the @@ -338,7 +340,7 @@ EOF http_file( name = 'toto', url = 'http://localhost:$nc_port/toto', - sha256 = 'whatever' + sha256 = '2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9826' ) EOF bazel build @toto//file &> $TEST_log && fail "Expected run to fail" @@ -356,7 +358,7 @@ function test_http_404() { http_file( name = 'toto', url = 'http://localhost:$nc_port/toto', - sha256 = 'whatever' + sha256 = '2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9826' ) EOF bazel build @toto//file &> $TEST_log && fail "Expected run to fail" @@ -476,7 +478,7 @@ EOF function test_invalid_rule() { # http_jar with missing URL field. cat > WORKSPACE <<EOF -http_jar(name = 'endangered', sha256 = 'dummy') +http_jar(name = 'endangered', sha256 = '2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9826') EOF bazel fetch //external:endangered >& $TEST_log && fail "Expected fetch to fail" |