diff options
8 files changed, 483 insertions, 73 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index da9b9b9b25..162f790bc5 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -30,7 +30,6 @@ import com.google.devtools.build.lib.bazel.repository.MavenServerRepositoryFunct import com.google.devtools.build.lib.bazel.repository.NewGitRepositoryFunction; import com.google.devtools.build.lib.bazel.repository.NewHttpArchiveFunction; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions; -import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache; import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader; import com.google.devtools.build.lib.bazel.repository.skylark.SkylarkRepositoryFunction; import com.google.devtools.build.lib.bazel.repository.skylark.SkylarkRepositoryModule; @@ -64,6 +63,8 @@ import com.google.devtools.build.lib.skyframe.SkyFunctions; import com.google.devtools.build.lib.skyframe.SkyValueDirtinessChecker; import com.google.devtools.build.lib.util.AbruptExitException; import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor; +import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; import com.google.devtools.common.options.OptionsBase; @@ -86,6 +87,7 @@ public class BazelRepositoryModule extends BlazeModule { private final RepositoryDelegatorFunction delegator; private final AtomicReference<HttpDownloader> httpDownloader = new AtomicReference<>(new HttpDownloader()); + private FileSystem filesystem; public BazelRepositoryModule() { this.skylarkRepositoryFunction = new SkylarkRepositoryFunction(httpDownloader); @@ -149,6 +151,8 @@ public class BazelRepositoryModule extends BlazeModule { builder.addSkyFunction(SkyFunctions.REPOSITORY, new RepositoryLoaderFunction()); builder.addSkyFunction(SkyFunctions.REPOSITORY_DIRECTORY, delegator); builder.addSkyFunction(MavenServerFunction.NAME, new MavenServerFunction()); + + filesystem = directories.getFileSystem(); } @Override @@ -173,8 +177,8 @@ public class BazelRepositoryModule extends BlazeModule { RepositoryOptions repoOptions = optionsProvider.getOptions(RepositoryOptions.class); if (repoOptions != null && repoOptions.experimentalRepositoryCache != null) { - httpDownloader.get().setRepositoryCache( - new RepositoryCache(repoOptions.experimentalRepositoryCache)); + Path repositoryCachePath = filesystem.getPath(repoOptions.experimentalRepositoryCache); + httpDownloader.get().setRepositoryCachePath(repositoryCachePath); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java index 60ed5558f3..64abf7dcfe 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java @@ -17,10 +17,10 @@ package com.google.devtools.build.lib.bazel.repository; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.hash.Hasher; -import com.google.common.hash.Hashing; import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.analysis.RuleDefinition; +import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache; +import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache.KeyType; import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader; import com.google.devtools.build.lib.bazel.rules.workspace.MavenJarRule; import com.google.devtools.build.lib.packages.Rule; @@ -243,12 +243,7 @@ public class MavenJarFunction extends HttpArchiveFunction { Path downloadPath = outputDirectory.getRelative(artifact.getFile().getAbsolutePath()); // Verify checksum. if (!Strings.isNullOrEmpty(sha1)) { - Hasher hasher = Hashing.sha1().newHasher(); - String downloadSha1 = HttpDownloader.getHash(hasher, downloadPath); - if (!sha1.equals(downloadSha1)) { - throw new IOException("Downloaded file at " + downloadPath + " has SHA-1 of " - + downloadSha1 + ", does not match expected SHA-1 (" + sha1 + ")"); - } + RepositoryCache.assertFileChecksum(sha1, downloadPath, KeyType.SHA1); } return downloadPath; } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/cache/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/repository/cache/BUILD index 8a973cbb31..6a2c861a35 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/cache/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/cache/BUILD @@ -10,5 +10,9 @@ filegroup( java_library( name = "cache", srcs = ["RepositoryCache.java"], - deps = ["//src/main/java/com/google/devtools/build/lib:vfs"], + deps = [ + "//src/main/java/com/google/devtools/build/lib:vfs", + "//third_party:guava", + "//third_party:jsr305", + ], ) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/cache/RepositoryCache.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/cache/RepositoryCache.java index 0032b80e1b..1f898164c5 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/cache/RepositoryCache.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/cache/RepositoryCache.java @@ -14,20 +14,212 @@ package com.google.devtools.build.lib.bazel.repository.cache; -import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.common.base.Preconditions; +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hasher; +import com.google.common.hash.Hashing; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import java.io.IOException; +import java.io.InputStream; +import javax.annotation.Nullable; -/** - * The cache implementation of HttpDownloadValues. +/** The cache implementation to store download artifacts from external repositories. + * TODO(jingwen): Implement file locking for concurrent cache accesses. */ public class RepositoryCache { + /** The types of cache keys used. */ + public static enum KeyType { + SHA1("SHA-1", "\\p{XDigit}{40}", "sha1", Hashing.sha1()), + SHA256("SHA-256", "\\p{XDigit}{64}", "sha256", Hashing.sha256()); + + private final String stringRepr; + private final String regexp; + private final String cacheDirName; + private final HashFunction hashFunction; + + KeyType(String stringRepr, String regexp, String cacheDirName, HashFunction hashFunction) { + this.stringRepr = stringRepr; + this.regexp = regexp; + this.cacheDirName = cacheDirName; + this.hashFunction = hashFunction; + } + + public boolean isValid(String checksum) { + return checksum.matches(regexp); + } + + public String getDirectoryName() { + return cacheDirName; + } + + public Path getCachePath(Path parentDirectory) { + return parentDirectory.getChild(cacheDirName); + } + + @Override + public String toString() { + return stringRepr; + } + } + + private static final int BUFFER_SIZE = 32 * 1024; + + // Repository cache subdirectories + private static final String CAS_DIR = "content_addressable"; + + // Rename cached files to this value to simplify lookup. + public static final String DEFAULT_CACHE_FILENAME = "file"; + + private final Path repositoryCachePath; + private final Path contentAddressablePath; + /** * @param repositoryCachePath The base location of the repository cache. + * @throws IOException */ - @SuppressWarnings("unused") - public RepositoryCache(PathFragment repositoryCachePath) { - // TODO(jingwen): Preload in-memory set of cached values here. - // Create any cache subdirectories here as well. + public RepositoryCache(Path repositoryCachePath) throws IOException { + this.repositoryCachePath = repositoryCachePath; + this.contentAddressablePath = repositoryCachePath.getRelative(CAS_DIR); + } + + /** + * Determine if a cache entry exist, given a cache key. + * + * @param cacheKey The string key to cache the value by. + * @param keyType The type of key used. See: KeyType + * @return true if the cache entry exist, false otherwise. + */ + public boolean exists(String cacheKey, KeyType keyType) { + return keyType.getCachePath(contentAddressablePath).getChild(cacheKey).exists(); + } + + /** + * Copies a cached value to a specified directory, if it exists. + * + * We're using copying instead of symlinking because symlinking require weird checks to verify + * that the symlink still points to an existing artifact. e.g. cleaning up the central cache but + * not the workspace cache. + * + * @param cacheKey The string key to cache the value by. + * @param targetPath The path where the cache value should be copied to. + * @param keyType The type of key used. See: KeyType + * @return The Path value where the cache value has been copied to. If cache value does not exist, + * return null. + * @throws IOException + */ + @Nullable + public Path get(String cacheKey, Path targetPath, KeyType keyType) throws IOException { + ensureValidKey(cacheKey, keyType); + if (!exists(cacheKey, keyType)) { + return null; + } + + Path cacheEntry = keyType.getCachePath(contentAddressablePath).getRelative(cacheKey); + Path cacheValue = cacheEntry.getRelative(DEFAULT_CACHE_FILENAME); + + try { + assertFileChecksum(cacheKey, cacheValue, keyType); + } catch (IOException e) { + // New lines because this error message gets large printing multiple absolute filepaths. + throw new IOException(e.getMessage() + "\n\n" + + "Please delete the directory " + cacheEntry + " and try again."); + } + + FileSystemUtils.copyFile(cacheValue, targetPath); + + return targetPath; + } + + /** + * Copies a value from a specified path into the cache. + * + * @param cacheKey The string key to cache the value by. + * @param sourcePath The path of the value to be cached. + * @param keyType The type of key used. See: KeyType + * @throws IOException + */ + public void put(String cacheKey, Path sourcePath, KeyType keyType) throws IOException { + ensureValidKey(cacheKey, keyType); + ensureCacheDirectoryExists(keyType); + + Path cacheEntry = keyType.getCachePath(contentAddressablePath).getRelative(cacheKey); + Path cacheValue = cacheEntry.getRelative(DEFAULT_CACHE_FILENAME); + FileSystemUtils.createDirectoryAndParents(cacheEntry); + FileSystemUtils.copyFile(sourcePath, cacheValue); + } + + private void ensureCacheDirectoryExists(KeyType keyType) throws IOException { + Path directoryPath = keyType.getCachePath(contentAddressablePath); + if (!directoryPath.exists()) { + FileSystemUtils.createDirectoryAndParents(directoryPath); + } + } + + /** + * Assert that a file has an expected checksum. + * + * @param expectedChecksum The expected checksum of the file. + * @param filePath The path to the file. + * @param keyType The type of hash function. e.g. SHA-1, SHA-256 + * @throws IOException If the checksum does not match or the file cannot be hashed, an + * exception is thrown. + */ + public static void assertFileChecksum(String expectedChecksum, Path filePath, KeyType keyType) + throws IOException { + Preconditions.checkArgument(!expectedChecksum.isEmpty()); + + String actualChecksum; + try { + actualChecksum = getChecksum(keyType, filePath); + } catch (IOException e) { + throw new IOException( + "Could not hash file " + filePath + ": " + e.getMessage() + ", expected " + keyType + + " of " + expectedChecksum + ". "); + } + if (!actualChecksum.equalsIgnoreCase(expectedChecksum)) { + throw new IOException( + "Downloaded file at " + filePath + " has " + keyType + " of " + actualChecksum + + ", does not match expected " + keyType + " (" + expectedChecksum + ")"); + } + } + + /** + * Obtain the checksum of a file. + * + * @param keyType The type of hash function. e.g. SHA-1, SHA-256. + * @param path The path to the file. + * @throws IOException + */ + public static String getChecksum(KeyType keyType, Path path) throws IOException { + Hasher hasher = keyType.hashFunction.newHasher(); + byte[] byteBuffer = new byte[BUFFER_SIZE]; + try (InputStream stream = path.getInputStream()) { + int numBytesRead = stream.read(byteBuffer); + while (numBytesRead != -1) { + if (numBytesRead != 0) { + // If more than 0 bytes were read, add them to the hash. + hasher.putBytes(byteBuffer, 0, numBytesRead); + } + numBytesRead = stream.read(byteBuffer); + } + } + return hasher.hash().toString(); + } + + private void ensureValidKey(String key, KeyType keyType) throws IOException { + if (!keyType.isValid(key)) { + throw new IOException("Invalid key \"" + key + "\" of type " + keyType + ". "); + } + } + + public Path getRootPath() { + return repositoryCachePath; + } + + public Path getContentAddressableCachePath() { + return contentAddressablePath; } -} +}
\ No newline at end of file diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java index 7afd9b2adb..198c2d1da6 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/HttpDownloader.java @@ -14,11 +14,11 @@ package com.google.devtools.build.lib.bazel.repository.downloader; -import com.google.common.hash.Hasher; -import com.google.common.hash.Hashing; import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache; +import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache.KeyType; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.EventHandler; +import com.google.devtools.build.lib.events.Location; import com.google.devtools.build.lib.packages.Rule; import com.google.devtools.build.lib.rules.repository.RepositoryFunction.RepositoryFunctionException; import com.google.devtools.build.lib.rules.repository.WorkspaceAttributeMapper; @@ -31,6 +31,8 @@ import com.google.devtools.build.skyframe.SkyFunctionException.Transience; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.util.Map; import java.util.concurrent.Executors; @@ -49,9 +51,16 @@ public class HttpDownloader { private static final double LOG_OF_KB = Math.log(1024); private final ScheduledExecutorService scheduler; + private Path repositoryCachePath; + private Location ruleUrlAttributeLocation; public HttpDownloader() { this.scheduler = Executors.newScheduledThreadPool(1); + this.ruleUrlAttributeLocation = null; + } + + public void setRepositoryCachePath(Path repositoryCachePath) { + this.repositoryCachePath = repositoryCachePath; } public Path download( @@ -62,6 +71,8 @@ public class HttpDownloader { String sha256; String type; try { + ruleUrlAttributeLocation = rule.getAttributeLocation("url"); + url = mapper.get("url", Type.STRING); sha256 = mapper.get("sha256", Type.STRING); type = mapper.isAttributeValueExplicitlySpecified("type") @@ -82,39 +93,42 @@ public class HttpDownloader { /** * Attempt to download a file from the repository's URL. Returns the path to the file downloaded. + * + * If the SHA256 checksum and path to the repository cache is specified, attempt + * to load the file from the RepositoryCache. If it doesn't exist, proceed to + * download the file and load it into the cache prior to returning the value. */ public Path download( String urlString, String sha256, String type, Path outputDirectory, EventHandler eventHandler, Map<String, String> clientEnv) - throws IOException, InterruptedException { - URL url = new URL(urlString); - Path destination; - if (type == null) { - destination = outputDirectory; - } else { - String filename = new PathFragment(url.getPath()).getBaseName(); - if (filename.isEmpty()) { - filename = "temp"; - } else if (!type.isEmpty()) { - filename += "." + type; - } - destination = outputDirectory.getRelative(filename); - } + throws IOException, InterruptedException, RepositoryFunctionException { + Path destination = getDownloadDestination(urlString, type, outputDirectory); + RepositoryCache repositoryCache = null; if (!sha256.isEmpty()) { try { - String currentSha256 = getHash(Hashing.sha256().newHasher(), destination); + String currentSha256 = RepositoryCache.getChecksum(KeyType.SHA256, destination); if (currentSha256.equals(sha256)) { // No need to download. return destination; } } catch (IOException e) { - // Ignore error trying to hash. We'll just download again. + // Ignore error trying to hash. We'll attempt to retrieve from cache or just download again. + } + + if (RepositoryCache.KeyType.SHA256.isValid(sha256) && repositoryCachePath != null) { + repositoryCache = new RepositoryCache(repositoryCachePath); + Path cachedDestination = repositoryCache.get(sha256, destination, KeyType.SHA256); + if (cachedDestination != null) { + // Cache hit! + return cachedDestination; + } } } AtomicInteger totalBytes = new AtomicInteger(0); final ScheduledFuture<?> loggerHandle = getLoggerHandle(totalBytes, eventHandler, urlString); + final URL url = new URL(urlString); try (OutputStream out = destination.getOutputStream(); HttpConnection connection = HttpConnection.createAndConnect(url, clientEnv)) { @@ -145,26 +159,36 @@ public class HttpDownloader { }, 0, TimeUnit.SECONDS); } - compareHashes(destination, sha256); + if (!sha256.isEmpty()) { + RepositoryCache.assertFileChecksum(sha256, destination, KeyType.SHA256); + } + + if (repositoryCache != null) { + repositoryCache.put(sha256, destination, KeyType.SHA256); + } + return destination; } - private void compareHashes(Path destination, String sha256) throws IOException { - if (sha256.isEmpty()) { - return; - } - String downloadedSha256; + private Path getDownloadDestination(String urlString, String type, Path outputDirectory) + throws RepositoryFunctionException { + URI uri = null; try { - downloadedSha256 = getHash(Hashing.sha256().newHasher(), destination); - } catch (IOException e) { - throw new IOException( - "Could not hash file " + destination + ": " + e.getMessage() + ", expected SHA-256 of " - + sha256 + ")"); + uri = new URI(urlString); + } catch (URISyntaxException e) { + throw new RepositoryFunctionException( + new EvalException(ruleUrlAttributeLocation, e), Transience.PERSISTENT); } - if (!downloadedSha256.equals(sha256)) { - throw new IOException( - "Downloaded file at " + destination + " has SHA-256 of " + downloadedSha256 - + ", does not match expected SHA-256 (" + sha256 + ")"); + if (type == null) { + return outputDirectory; + } else { + String filename = new PathFragment(uri.getPath()).getBaseName(); + if (filename.isEmpty()) { + filename = "temp"; + } else if (!type.isEmpty()) { + filename += "." + type; + } + return outputDirectory.getRelative(filename); } } @@ -185,7 +209,7 @@ public class HttpDownloader { return scheduler.scheduleAtFixedRate(logger, 0, 1, TimeUnit.SECONDS); } - private static String formatSize(int bytes) { + private String formatSize(int bytes) { if (bytes < KB) { return bytes + "B"; } @@ -197,22 +221,4 @@ public class HttpDownloader { + (UNITS.charAt(logBaseUnitOfBytes) + "B"); } - public static String getHash(Hasher hasher, Path path) throws IOException { - byte byteBuffer[] = new byte[BUFFER_SIZE]; - try (InputStream stream = path.getInputStream()) { - int numBytesRead = stream.read(byteBuffer); - while (numBytesRead != -1) { - if (numBytesRead != 0) { - // If more than 0 bytes were read, add them to the hash. - hasher.putBytes(byteBuffer, 0, numBytesRead); - } - numBytesRead = stream.read(byteBuffer); - } - } - return hasher.hash().toString(); - } - - public void setRepositoryCache(@SuppressWarnings("unused") RepositoryCache repositoryCache) { - // TODO(jingwen): Implement repository cache bridge - } } diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/BUILD b/src/test/java/com/google/devtools/build/lib/bazel/repository/BUILD index 3068d0b0ce..96053933e5 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/repository/BUILD +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/BUILD @@ -1,6 +1,9 @@ filegroup( name = "srcs", - srcs = glob(["**"]) + ["//src/test/java/com/google/devtools/build/lib/bazel/repository/downloader:srcs"], + srcs = glob(["**"]) + [ + "//src/test/java/com/google/devtools/build/lib/bazel/repository/cache:srcs", + "//src/test/java/com/google/devtools/build/lib/bazel/repository/downloader:srcs", + ], visibility = ["//src/test/java/com/google/devtools/build/lib:__pkg__"], ) @@ -22,6 +25,7 @@ java_test( "//src/main/java/com/google/devtools/build/lib:packages-internal", "//src/main/java/com/google/devtools/build/lib:syntax", "//src/main/java/com/google/devtools/build/lib:vfs", + "//src/main/java/com/google/devtools/build/lib/bazel/repository/cache", "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader", "//src/main/java/com/google/devtools/build/lib/rules/cpp", "//src/main/java/com/google/devtools/build/skyframe", diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/cache/BUILD b/src/test/java/com/google/devtools/build/lib/bazel/repository/cache/BUILD new file mode 100644 index 0000000000..884e125055 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/cache/BUILD @@ -0,0 +1,23 @@ +filegroup( + name = "srcs", + srcs = glob(["**"]), + visibility = ["//src/test/java/com/google/devtools/build/lib/bazel/repository:__pkg__"], +) + +java_test( + name = "RepositoryCacheTests", + srcs = glob(["*.java"]), + tags = ["rules"], + test_class = "com.google.devtools.build.lib.AllTests", + deps = [ + "//src/main/java/com/google/devtools/build/lib:vfs", + "//src/main/java/com/google/devtools/build/lib/bazel/repository/cache", + "//src/test/java/com/google/devtools/build/lib:foundations_testutil", + "//src/test/java/com/google/devtools/build/lib:test_runner", + "//src/test/java/com/google/devtools/build/lib:testutil", + "//third_party:guava", + "//third_party:junit4", + "//third_party:mockito", + "//third_party:truth", + ], +) diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/cache/RepositoryCacheTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/cache/RepositoryCacheTest.java new file mode 100644 index 0000000000..d986de3e65 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/cache/RepositoryCacheTest.java @@ -0,0 +1,182 @@ +// Copyright 2016 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository.cache; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import com.google.common.base.Strings; +import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache.KeyType; +import com.google.devtools.build.lib.testutil.Scratch; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import java.io.IOException; +import java.nio.charset.Charset; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Tests for {@link RepositoryCache}. + */ +@RunWith(JUnit4.class) +public class RepositoryCacheTest { + + @Rule public ExpectedException thrown = ExpectedException.none(); + + private Scratch scratch; + private RepositoryCache repositoryCache; + private Path repositoryCachePath; + private Path contentAddressableCachePath; + private Path downloadedFile; + private String downloadedFileSha256; + + @Before + public void setUp() throws Exception { + scratch = new Scratch("/"); + repositoryCachePath = scratch.dir("/repository_cache"); + repositoryCache = new RepositoryCache(repositoryCachePath); + contentAddressableCachePath = repositoryCache.getContentAddressableCachePath(); + + downloadedFile = scratch.file("file.tmp", Charset.defaultCharset(), "contents"); + downloadedFileSha256 = "bfe5ed57e6e323555b379c660aa8d35b70c2f8f07cf03ad6747266495ac13be0"; + } + + @After + public void tearDown() throws IOException { + FileSystemUtils.deleteTree(repositoryCachePath); + } + + @Test + public void testNonExistentCacheValue() { + String fakeSha256 = Strings.repeat("a", 64); + assertFalse(repositoryCache.exists(fakeSha256, KeyType.SHA256)); + } + + /** + * Test that the put method correctly stores the downloaded file into the cache. + */ + @Test + public void testPutCacheValue() throws IOException { + repositoryCache.put(downloadedFileSha256, downloadedFile, KeyType.SHA256); + + Path cacheEntry = KeyType.SHA256.getCachePath(contentAddressableCachePath).getChild(downloadedFileSha256); + Path cacheValue = cacheEntry.getChild(RepositoryCache.DEFAULT_CACHE_FILENAME); + + assertEquals( + FileSystemUtils.readContent(cacheValue, Charset.defaultCharset()), + FileSystemUtils.readContent(downloadedFile, Charset.defaultCharset())); + } + + /** + * Test that the put method is idempotent, i.e. two successive put calls + * should not affect the final state in the cache. + */ + @Test + public void testPutCacheValueIdempotent() throws IOException { + repositoryCache.put(downloadedFileSha256, downloadedFile, KeyType.SHA256); + repositoryCache.put(downloadedFileSha256, downloadedFile, KeyType.SHA256); + + Path cacheEntry = KeyType.SHA256.getCachePath(contentAddressableCachePath).getChild(downloadedFileSha256); + Path cacheValue = cacheEntry.getChild(RepositoryCache.DEFAULT_CACHE_FILENAME); + + assertEquals( + FileSystemUtils.readContent(cacheValue, Charset.defaultCharset()), + FileSystemUtils.readContent(downloadedFile, Charset.defaultCharset())); + } + + /** + * Test that the get method correctly retrieves the cached file from the cache. + */ + @Test + public void testGetCacheValue() throws IOException { + // Inject file into cache + repositoryCache.put(downloadedFileSha256, downloadedFile, KeyType.SHA256); + + Path targetDirectory = scratch.dir("/external"); + Path targetPath = targetDirectory.getChild(downloadedFile.getBaseName()); + Path actualTargetPath = repositoryCache.get(downloadedFileSha256, targetPath, KeyType.SHA256); + + // Check that the contents are the same. + assertEquals( + FileSystemUtils.readContent(actualTargetPath, Charset.defaultCharset()), + FileSystemUtils.readContent(downloadedFile, Charset.defaultCharset())); + + // Check that the returned value is stored under outputBaseExternal. + assertEquals(targetPath, actualTargetPath); + } + + /** + * Test that the get method retrieves a null if the value is not cached. + */ + @Test + public void testGetNullCacheValue() throws IOException { + Path targetDirectory = scratch.dir("/external"); + Path targetPath = targetDirectory.getChild(downloadedFile.getBaseName()); + Path actualTargetPath = repositoryCache.get(downloadedFileSha256, targetPath, KeyType.SHA256); + + assertEquals(actualTargetPath, null); + } + + @Test + public void testInvalidSha256Throws() throws IOException { + String invalidSha = "foo"; + thrown.expect(IOException.class); + thrown.expectMessage("Invalid key \"foo\" of type SHA-256"); + repositoryCache.put(invalidSha, downloadedFile, KeyType.SHA256); + } + + @Test + public void testPoisonedCache() throws IOException { + Path poisonedEntry = KeyType.SHA256 + .getCachePath(contentAddressableCachePath).getChild(downloadedFileSha256); + Path poisonedValue = poisonedEntry.getChild(RepositoryCache.DEFAULT_CACHE_FILENAME); + scratch.file(poisonedValue.getPathString(), Charset.defaultCharset(), "poisoned"); + + Path targetDirectory = scratch.dir("/external"); + Path targetPath = targetDirectory.getChild(downloadedFile.getBaseName()); + + thrown.expect(IOException.class); + thrown.expectMessage("does not match expected"); + thrown.expectMessage("Please delete the directory"); + + repositoryCache.get(downloadedFileSha256, targetPath, KeyType.SHA256); + } + + @Test + public void testGetChecksum() throws IOException { + String actualChecksum = RepositoryCache.getChecksum(KeyType.SHA256, downloadedFile); + assertEquals(downloadedFileSha256, actualChecksum); + } + + @Test + public void testAssertFileChecksumPass() throws IOException { + RepositoryCache.assertFileChecksum(downloadedFileSha256, downloadedFile, KeyType.SHA256); + } + + @Test + public void testAssertFileChecksumFail() throws IOException { + thrown.expect(IOException.class); + thrown.expectMessage("does not match expected"); + RepositoryCache.assertFileChecksum( + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + downloadedFile, + KeyType.SHA256); + } + +} |