From c8dfabbe9c4dc5fc45b19ceacef4582a8a0e0f47 Mon Sep 17 00:00:00 2001 From: George Gensure Date: Mon, 23 Jul 2018 02:57:03 -0700 Subject: Suppress RepositoryCache IOException on download With invalid contents in the repository cache, silence the IOException on RepositoryCache::get and re-download an artifact when attempting to short-circuit that operation. The repository cache can easily get into this state when a build is interrupted while downloading into the non- atomic repository cache destination. Possible solution to #5390 Closes #5392. PiperOrigin-RevId: 205634761 --- .../lib/bazel/repository/MavenDownloader.java | 31 +++++++++++++++------- .../lib/bazel/repository/MavenJarFunction.java | 13 ++++++--- .../repository/downloader/HttpDownloader.java | 12 ++++++--- 3 files changed, 39 insertions(+), 17 deletions(-) (limited to 'src/main/java/com') diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenDownloader.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenDownloader.java index 06eccccb0e..574b44afab 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenDownloader.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenDownloader.java @@ -21,6 +21,8 @@ import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache; import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache.KeyType; import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader; +import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.events.ExtendedEventHandler; import com.google.devtools.build.lib.rules.repository.WorkspaceAttributeMapper; import com.google.devtools.build.lib.syntax.EvalException; import com.google.devtools.build.lib.syntax.Type; @@ -56,8 +58,13 @@ public class MavenDownloader extends HttpDownloader { * Download the Maven artifact to the output directory. Returns the path to the jar (and the * srcjar if available). */ - public JarPaths download(String name, WorkspaceAttributeMapper mapper, Path outputDirectory, - MavenServerValue serverValue) throws IOException, EvalException { + public JarPaths download( + String name, + WorkspaceAttributeMapper mapper, + Path outputDirectory, + MavenServerValue serverValue, + ExtendedEventHandler eventHandler) + throws IOException, EvalException { String url = serverValue.getUrl(); Server server = serverValue.getServer(); @@ -80,14 +87,20 @@ public class MavenDownloader extends HttpDownloader { if (isCaching) { Path downloadPath = getDownloadDestination(outputDirectory, artifact); - Path cachedDestination = repositoryCache.get(sha1, downloadPath, KeyType.SHA1); - if (cachedDestination != null) { - Path cachedDestinationSrc = null; - if (sha1Src != null) { - Path downloadPathSrc = getDownloadDestination(outputDirectory, artifactWithSrcs); - cachedDestinationSrc = repositoryCache.get(sha1Src, downloadPathSrc, KeyType.SHA1); + try { + Path cachedDestination = repositoryCache.get(sha1, downloadPath, KeyType.SHA1); + if (cachedDestination != null) { + Path cachedDestinationSrc = null; + if (sha1Src != null) { + Path downloadPathSrc = getDownloadDestination(outputDirectory, artifactWithSrcs); + cachedDestinationSrc = repositoryCache.get(sha1Src, downloadPathSrc, KeyType.SHA1); + } + return new JarPaths(cachedDestination, Optional.fromNullable(cachedDestinationSrc)); } - return new JarPaths(cachedDestination, Optional.fromNullable(cachedDestinationSrc)); + } catch (IOException e) { + eventHandler.handle( + Event.debug("RepositoryCache entry " + sha1 + " is invalid, replacing it...")); + // Ignore error trying to get. We'll just download again. } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java index 903ac93ac6..bb15e606e5 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java @@ -20,6 +20,7 @@ import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.analysis.RuleDefinition; import com.google.devtools.build.lib.bazel.repository.MavenDownloader.JarPaths; import com.google.devtools.build.lib.bazel.rules.workspace.MavenJarRule; +import com.google.devtools.build.lib.events.ExtendedEventHandler; import com.google.devtools.build.lib.packages.Rule; import com.google.devtools.build.lib.rules.repository.RepositoryDirectoryValue; import com.google.devtools.build.lib.rules.repository.WorkspaceAttributeMapper; @@ -101,11 +102,15 @@ public class MavenJarFunction extends HttpArchiveFunction { } Path outputDir = getExternalRepositoryDirectory(directories).getRelative(rule.getName()); - return createOutputTree(rule, outputDir, serverValue); + return createOutputTree(rule, outputDir, serverValue, env.getListener()); } - private RepositoryDirectoryValue.Builder createOutputTree(Rule rule, Path outputDirectory, - MavenServerValue serverValue) throws RepositoryFunctionException { + private RepositoryDirectoryValue.Builder createOutputTree( + Rule rule, + Path outputDirectory, + MavenServerValue serverValue, + ExtendedEventHandler eventHandler) + throws RepositoryFunctionException { Preconditions.checkState(downloader instanceof MavenDownloader); MavenDownloader mavenDownloader = (MavenDownloader) downloader; @@ -115,7 +120,7 @@ public class MavenJarFunction extends HttpArchiveFunction { try { repositoryJars = mavenDownloader.download( - name, WorkspaceAttributeMapper.of(rule), outputDirectory, serverValue); + name, WorkspaceAttributeMapper.of(rule), outputDirectory, serverValue, eventHandler); } catch (IOException e) { throw new RepositoryFunctionException(e, Transience.TRANSIENT); } catch (EvalException e) { diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java index d2d3798bce..4e7ccb3cca 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java @@ -183,10 +183,14 @@ public class HttpDownloader { if (repositoryCache.isEnabled()) { isCachingByProvidedSha256 = true; - Path cachedDestination = repositoryCache.get(sha256, destination, KeyType.SHA256); - if (cachedDestination != null) { - // Cache hit! - return cachedDestination; + try { + Path cachedDestination = repositoryCache.get(sha256, destination, KeyType.SHA256); + if (cachedDestination != null) { + // Cache hit! + return cachedDestination; + } + } catch (IOException e) { + // Ignore error trying to get. We'll just download again. } } -- cgit v1.2.3