diff options
13 files changed, 204 insertions, 112 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/BUILD b/src/main/java/com/google/devtools/build/lib/BUILD index cd602c5c63..a50c22e476 100644 --- a/src/main/java/com/google/devtools/build/lib/BUILD +++ b/src/main/java/com/google/devtools/build/lib/BUILD @@ -141,6 +141,7 @@ java_library( ":preconditions", ":unix", ":windows", + "//src/main/java/com/google/devtools/common/options", "//third_party:guava", "//third_party:jsr305", ], diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/DigestUtils.java b/src/main/java/com/google/devtools/build/lib/actions/cache/DigestUtils.java index a35eed6a95..3dcfe54937 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/cache/DigestUtils.java +++ b/src/main/java/com/google/devtools/build/lib/actions/cache/DigestUtils.java @@ -19,17 +19,14 @@ import com.google.devtools.build.lib.profiler.ProfilerTask; import com.google.devtools.build.lib.util.BlazeClock; import com.google.devtools.build.lib.util.Fingerprint; import com.google.devtools.build.lib.util.LoggingUtil; -import com.google.devtools.build.lib.util.Preconditions; import com.google.devtools.build.lib.util.VarInt; import com.google.devtools.build.lib.vfs.Path; import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.Map; -import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Level; -import javax.annotation.Nullable; /** * Utility class for getting md5 digests of files. @@ -45,7 +42,7 @@ import javax.annotation.Nullable; public class DigestUtils { // Object to synchronize on when serializing large file reads. - private static final Object MD5_LOCK = new Object(); + private static final Object DIGEST_LOCK = new Object(); private static final AtomicBoolean MULTI_THREADED_DIGEST = new AtomicBoolean(false); /** Private constructor to prevent instantiation of utility class. */ @@ -57,9 +54,10 @@ public class DigestUtils { * calculations and underlying file system cannot provide it via extended * attribute. */ - private static byte[] getDigestInExclusiveMode(Path path) throws IOException { + private static byte[] getDigestInExclusiveMode(Path path) + throws IOException { long startTime = BlazeClock.nanoTime(); - synchronized (MD5_LOCK) { + synchronized (DIGEST_LOCK) { Profiler.instance().logSimpleTask(startTime, ProfilerTask.WAIT, path.getPathString()); return getDigestInternal(path); } @@ -67,29 +65,14 @@ public class DigestUtils { private static byte[] getDigestInternal(Path path) throws IOException { long startTime = BlazeClock.nanoTime(); - byte[] md5bin = path.getMD5Digest(); + byte[] digest = path.getDigest(); long millis = (BlazeClock.nanoTime() - startTime) / 1000000; if (millis > 5000L) { System.err.println("Slow read: a " + path.getFileSize() + "-byte read from " + path + " took " + millis + "ms."); } - return md5bin; - } - - private static boolean binaryDigestWellFormed(byte[] digest) { - Preconditions.checkNotNull(digest); - return digest.length == 16; - } - - /** - * Returns the the fast md5 digest of the file, or null if not available. - */ - @Nullable - private static byte[] getFastDigest(Path path) throws IOException { - // TODO(bazel-team): the action cache currently only works with md5 digests but it ought to - // work with any opaque digest. - return Objects.equals(path.getFastDigestFunctionType(), "MD5") ? path.getFastDigest() : null; + return digest; } /** @@ -100,7 +83,7 @@ public class DigestUtils { } /** - * Get the md5 digest of {@code path}, using a constant-time xattr call if the filesystem supports + * Get the digest of {@code path}, using a constant-time xattr call if the filesystem supports * it, and calculating the digest manually otherwise. * * @param path Path of the file. @@ -108,20 +91,21 @@ public class DigestUtils { * serially or in parallel. Files larger than a certain threshold will be read serially, in order * to avoid excessive disk seeks. */ - public static byte[] getDigestOrFail(Path path, long fileSize) throws IOException { - byte[] md5bin = getFastDigest(path); + public static byte[] getDigestOrFail(Path path, long fileSize) + throws IOException { + byte[] digest = path.getFastDigest(); - if (md5bin != null && !binaryDigestWellFormed(md5bin)) { + if (digest != null && !path.isValidDigest(digest)) { // Fail-soft in cases where md5bin is non-null, but not a valid digest. String msg = String.format("Malformed digest '%s' for file %s", - BaseEncoding.base16().lowerCase().encode(md5bin), + BaseEncoding.base16().lowerCase().encode(digest), path); LoggingUtil.logToRemote(Level.SEVERE, msg, new IllegalStateException(msg)); - md5bin = null; + digest = null; } - if (md5bin != null) { - return md5bin; + if (digest != null) { + return digest; } else if (fileSize > 4096 && !MULTI_THREADED_DIGEST.get()) { // We'll have to read file content in order to calculate the digest. In that case // it would be beneficial to serialize those calculations since there is a high diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenServerFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenServerFunction.java index 2111aedc82..a25bc2cf89 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenServerFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenServerFunction.java @@ -132,7 +132,7 @@ public class MavenServerFunction implements SkyFunction { Path path = entry.getValue().realRootedPath().asPath(); if (path.exists()) { fingerprint.addBoolean(true); - fingerprint.addBytes(path.getMD5Digest()); + fingerprint.addBytes(path.getDigest()); } else { fingerprint.addBoolean(false); } diff --git a/src/main/java/com/google/devtools/build/lib/exec/SingleBuildFileCache.java b/src/main/java/com/google/devtools/build/lib/exec/SingleBuildFileCache.java index 48e6f95e17..0d154f14c5 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/SingleBuildFileCache.java +++ b/src/main/java/com/google/devtools/build/lib/exec/SingleBuildFileCache.java @@ -70,7 +70,7 @@ public class SingleBuildFileCache implements ActionInputFileCache { Path path = null; try { path = fs.getPath(fullPath(input)); - byte[] digest = path.getMD5Digest(); + byte[] digest = path.getDigest(); BaseEncoding hex = BaseEncoding.base16().lowerCase(); ByteString hexDigest = ByteString.copyFrom(hex.encode(digest).getBytes(US_ASCII)); // Inject reverse mapping. Doing this unconditionally in getDigest() showed up diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationLoader.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationLoader.java index 7201797025..29ace2f135 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationLoader.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CrosstoolConfigurationLoader.java @@ -220,7 +220,7 @@ public class CrosstoolConfigurationLoader { return null; } - return new CrosstoolProto(path.getMD5Digest(), "CROSSTOOL file " + path.getPathString()) { + return new CrosstoolProto(path.getDigest(), "CROSSTOOL file " + path.getPathString()) { @Override public String getContents() throws IOException { try (InputStream inputStream = path.getInputStream()) { diff --git a/src/main/java/com/google/devtools/build/lib/rules/test/TestStrategy.java b/src/main/java/com/google/devtools/build/lib/rules/test/TestStrategy.java index 27597967ad..7590ef1d2b 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/test/TestStrategy.java +++ b/src/main/java/com/google/devtools/build/lib/rules/test/TestStrategy.java @@ -438,8 +438,8 @@ public abstract class TestStrategy implements TestActionContext { try { // Avoid rebuilding the runfiles directory if the manifest in it matches the input manifest, // implying the symlinks exist and are already up to date. - if (Arrays.equals(runfilesDir.getRelative("MANIFEST").getMD5Digest(), - execSettings.getInputManifest().getPath().getMD5Digest())) { + if (Arrays.equals(runfilesDir.getRelative("MANIFEST").getDigest(), + execSettings.getInputManifest().getPath().getDigest())) { return; } } catch (IOException e1) { diff --git a/src/main/java/com/google/devtools/build/lib/syntax/BuildFileAST.java b/src/main/java/com/google/devtools/build/lib/syntax/BuildFileAST.java index bf8276e593..ad58b48349 100644 --- a/src/main/java/com/google/devtools/build/lib/syntax/BuildFileAST.java +++ b/src/main/java/com/google/devtools/build/lib/syntax/BuildFileAST.java @@ -270,7 +270,7 @@ public class BuildFileAST extends ASTNode { Parser.ParseResult result = Parser.parseFileForSkylark(input, eventHandler); return create( ImmutableList.<Statement>of(), result, - HashCode.fromBytes(file.getMD5Digest()).toString(), eventHandler); + HashCode.fromBytes(file.getDigest()).toString(), eventHandler); } /** diff --git a/src/main/java/com/google/devtools/build/lib/vfs/FileSystem.java b/src/main/java/com/google/devtools/build/lib/vfs/FileSystem.java index 90aac0e130..4d938049e0 100644 --- a/src/main/java/com/google/devtools/build/lib/vfs/FileSystem.java +++ b/src/main/java/com/google/devtools/build/lib/vfs/FileSystem.java @@ -16,6 +16,7 @@ package com.google.devtools.build.lib.vfs; import static java.nio.charset.StandardCharsets.ISO_8859_1; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import com.google.common.hash.Hashing; import com.google.common.io.ByteSource; @@ -24,6 +25,8 @@ import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; import com.google.devtools.build.lib.vfs.Dirent.Type; import com.google.devtools.build.lib.vfs.Path.PathFactory; import com.google.devtools.build.lib.vfs.Path.PathFactory.TranslatedPath; +import com.google.devtools.common.options.EnumConverter; +import com.google.devtools.common.options.OptionsParsingException; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; @@ -39,6 +42,49 @@ import java.util.List; @ThreadSafe public abstract class FileSystem { + /** Type of hash function to use for digesting files. */ + public enum HashFunction { + MD5(16), + SHA1(20); + + private final int digestSize; + + HashFunction(int digestSize) { + this.digestSize = digestSize; + } + + /** Converts to {@link HashFunction}. */ + public static class Converter extends EnumConverter<HashFunction> { + public Converter() { + super(HashFunction.class, "hash function"); + } + } + + public boolean isValidDigest(byte[] digest) { + return digest != null && digest.length == digestSize; + } + } + + // This is effectively final, should be changed only in unit-tests! + private static HashFunction DIGEST_FUNCTION; + static { + try { + DIGEST_FUNCTION = new HashFunction.Converter().convert( + System.getProperty("bazel.DigestFunction", "MD5")); + } catch (OptionsParsingException e) { + throw new IllegalStateException(e); + } + } + + @VisibleForTesting + public static void setDigestFunctionForTesting(HashFunction value) { + DIGEST_FUNCTION = value; + } + + public static HashFunction getDigestFunction() { + return DIGEST_FUNCTION; + } + private enum UnixPathFactory implements PathFactory { INSTANCE { @Override @@ -261,10 +307,11 @@ public abstract class FileSystem { } /** - * Returns the type of digest that may be returned by {@link #getFastDigest}, or {@code null} - * if the filesystem doesn't support them. + * Gets a fast digest for the given path and hash function type, or {@code null} if there + * isn't one available or the filesystem doesn't support them. This digest should be + * suitable for detecting changes to the file. */ - protected String getFastDigestFunctionType(Path path) { + protected byte[] getFastDigest(Path path, HashFunction hashFunction) throws IOException { return null; } @@ -273,8 +320,43 @@ public abstract class FileSystem { * filesystem doesn't support them. This digest should be suitable for detecting changes to the * file. */ - protected byte[] getFastDigest(Path path) throws IOException { - return null; + protected final byte[] getFastDigest(Path path) throws IOException { + return getFastDigest(path, DIGEST_FUNCTION); + } + + /** + * Returns whether the given digest is a valid digest for the default digest function. + */ + public boolean isValidDigest(byte[] digest) { + return DIGEST_FUNCTION.isValidDigest(digest); + } + + /** + * Returns the digest of the file denoted by the path, following + * symbolic links, for the given hash digest function. + * + * @return a new byte array containing the file's digest + * @throws IOException if the digest could not be computed for any reason + */ + protected final byte[] getDigest(final Path path, HashFunction hashFunction) throws IOException { + switch(hashFunction) { + case MD5: + return getMD5Digest(path); + case SHA1: + return getSHA1Digest(path); + default: + throw new IOException("Unsupported hash function: " + hashFunction); + } + } + + /** + * Returns the digest of the file denoted by the path, following symbolic links. + * + * @return a new byte array containing the file's digest + * @throws IOException if the digest could not be computed for any reason + */ + protected byte[] getDigest(final Path path) throws IOException { + return getDigest(path, DIGEST_FUNCTION); } /** diff --git a/src/main/java/com/google/devtools/build/lib/vfs/Path.java b/src/main/java/com/google/devtools/build/lib/vfs/Path.java index 6599952541..e039efe74b 100644 --- a/src/main/java/com/google/devtools/build/lib/vfs/Path.java +++ b/src/main/java/com/google/devtools/build/lib/vfs/Path.java @@ -19,6 +19,7 @@ import com.google.common.collect.Iterables; import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; import com.google.devtools.build.lib.util.Preconditions; import com.google.devtools.build.lib.util.StringCanonicalizer; +import com.google.devtools.build.lib.vfs.FileSystem.HashFunction; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; @@ -1039,19 +1040,26 @@ public class Path implements Comparable<Path>, Serializable { } /** - * Returns the type of digest that may be returned by {@link #getFastDigest}, or {@code null} - * if the filesystem doesn't support them. + * Gets a fast digest for the given path, or {@code null} if there isn't one available. The + * digest should be suitable for detecting changes to the file. */ - public String getFastDigestFunctionType() { - return fileSystem.getFastDigestFunctionType(this); + public byte[] getFastDigest() throws IOException { + return fileSystem.getFastDigest(this); } /** * Gets a fast digest for the given path, or {@code null} if there isn't one available. The * digest should be suitable for detecting changes to the file. */ - public byte[] getFastDigest() throws IOException { - return fileSystem.getFastDigest(this); + public byte[] getFastDigest(HashFunction hashFunction) throws IOException { + return fileSystem.getFastDigest(this, hashFunction); + } + + /** + * Returns whether the given digest is a valid digest for the default system digest function. + */ + public boolean isValidDigest(byte[] digest) { + return fileSystem.isValidDigest(digest); } /** @@ -1083,6 +1091,28 @@ public class Path implements Comparable<Path>, Serializable { } /** + * Returns the digest of the file denoted by the current path, + * following symbolic links. + * + * @return a new byte array containing the file's digest + * @throws IOException if the digest could not be computed for any reason + */ + public byte[] getDigest() throws IOException { + return fileSystem.getDigest(this); + } + + /** + * Returns the digest of the file denoted by the current path and digest function, + * following symbolic links. + * + * @return a new byte array containing the file's digest + * @throws IOException if the digest could not be computed for any reason + */ + public byte[] getDigest(HashFunction hashFunction) throws IOException { + return fileSystem.getDigest(this, hashFunction); + } + + /** * Opens the file denoted by this path, following symbolic links, for reading, * and returns an input stream to it. * diff --git a/src/main/java/com/google/devtools/build/lib/vfs/UnionFileSystem.java b/src/main/java/com/google/devtools/build/lib/vfs/UnionFileSystem.java index a0a09555f2..b246aa68fd 100644 --- a/src/main/java/com/google/devtools/build/lib/vfs/UnionFileSystem.java +++ b/src/main/java/com/google/devtools/build/lib/vfs/UnionFileSystem.java @@ -18,6 +18,7 @@ import com.google.common.collect.Lists; import com.google.devtools.build.lib.concurrent.ThreadSafety; import com.google.devtools.build.lib.util.Preconditions; import com.google.devtools.build.lib.util.StringTrie; +import com.google.devtools.build.lib.vfs.FileSystem.HashFunction; import java.io.IOException; import java.io.InputStream; @@ -380,15 +381,9 @@ public class UnionFileSystem extends FileSystem { } @Override - protected String getFastDigestFunctionType(Path path) { + protected byte[] getFastDigest(Path path, HashFunction hashFunction) throws IOException { FileSystem delegate = getDelegate(path); - return delegate.getFastDigestFunctionType(adjustPath(path, delegate)); - } - - @Override - protected byte[] getFastDigest(Path path) throws IOException { - FileSystem delegate = getDelegate(path); - return delegate.getFastDigest(adjustPath(path, delegate)); + return delegate.getFastDigest(adjustPath(path, delegate), hashFunction); } @Override diff --git a/src/test/java/com/google/devtools/build/lib/actions/DigestUtilsTest.java b/src/test/java/com/google/devtools/build/lib/actions/DigestUtilsTest.java index 94d6aa7c93..70d2f40b22 100644 --- a/src/test/java/com/google/devtools/build/lib/actions/DigestUtilsTest.java +++ b/src/test/java/com/google/devtools/build/lib/actions/DigestUtilsTest.java @@ -25,6 +25,7 @@ import com.google.devtools.build.lib.testutil.TestThread; import com.google.devtools.build.lib.testutil.TestUtils; import com.google.devtools.build.lib.util.BlazeClock; import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.FileSystem.HashFunction; import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; @@ -34,6 +35,7 @@ import org.junit.runner.RunWith; import org.junit.runners.JUnit4; import java.io.IOException; +import java.util.Arrays; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -43,8 +45,9 @@ import java.util.concurrent.TimeUnit; @RunWith(JUnit4.class) public class DigestUtilsTest { - private static void assertMd5CalculationConcurrency(boolean expectConcurrent, - final boolean fastDigest, final int fileSize1, final int fileSize2) throws Exception { + private static void assertDigestCalculationConcurrency(boolean expectConcurrent, + final boolean fastDigest, final int fileSize1, final int fileSize2, + HashFunction hf) throws Exception { final CountDownLatch barrierLatch = new CountDownLatch(2); // Used to block test threads. final CountDownLatch readyLatch = new CountDownLatch(1); // Used to block main thread. @@ -64,16 +67,26 @@ public class DigestUtilsTest { } @Override - protected String getFastDigestFunctionType(Path path) { - return "MD5"; + protected byte[] getSHA1Digest(Path path) throws IOException { + try { + barrierLatch.countDown(); + readyLatch.countDown(); + // Either both threads will be inside getSHA1Digest at the same time or they + // both will be blocked. + barrierLatch.await(); + } catch (Exception e) { + throw new IOException(e); + } + return super.getSHA1Digest(path); } @Override - protected byte[] getFastDigest(Path path) throws IOException { - return fastDigest ? super.getMD5Digest(path) : null; + protected byte[] getFastDigest(Path path, HashFunction hashFunction) throws IOException { + return fastDigest ? super.getDigest(path, hashFunction) : null; } }; + FileSystem.setDigestFunctionForTesting(hf); final Path myFile1 = myfs.getPath("/f1.dat"); final Path myFile2 = myfs.getPath("/f2.dat"); FileSystemUtils.writeContentAsLatin1(myFile1, Strings.repeat("a", fileSize1)); @@ -111,13 +124,15 @@ public class DigestUtilsTest { * so machines with rotating drives don't become unusable. */ @Test - public void testMd5CalculationConcurrency() throws Exception { - assertMd5CalculationConcurrency(true, true, 4096, 4096); - assertMd5CalculationConcurrency(true, true, 4097, 4097); - assertMd5CalculationConcurrency(true, false, 4096, 4096); - assertMd5CalculationConcurrency(false, false, 4097, 4097); - assertMd5CalculationConcurrency(true, false, 1024, 4097); - assertMd5CalculationConcurrency(true, false, 1024, 1024); + public void testCalculationConcurrency() throws Exception { + for (HashFunction hf : Arrays.asList(HashFunction.MD5, HashFunction.SHA1)) { + assertDigestCalculationConcurrency(true, true, 4096, 4096, hf); + assertDigestCalculationConcurrency(true, true, 4097, 4097, hf); + assertDigestCalculationConcurrency(true, false, 4096, 4096, hf); + assertDigestCalculationConcurrency(false, false, 4097, 4097, hf); + assertDigestCalculationConcurrency(true, false, 1024, 4097, hf); + assertDigestCalculationConcurrency(true, false, 1024, 1024, hf); + } } @Test @@ -125,21 +140,19 @@ public class DigestUtilsTest { final byte[] malformed = {0, 0, 0}; FileSystem myFS = new InMemoryFileSystem(BlazeClock.instance()) { @Override - protected String getFastDigestFunctionType(Path path) { - return "MD5"; - } - - @Override - protected byte[] getFastDigest(Path path) throws IOException { - // MD5 digests are supposed to be 16 bytes. + protected byte[] getFastDigest(Path path, HashFunction hashFunction) throws IOException { + // Digest functions have more than 3 bytes, usually at least 16. return malformed; } }; Path path = myFS.getPath("/file"); FileSystemUtils.writeContentAsLatin1(path, "a"); - byte[] result = DigestUtils.getDigestOrFail(path, 1); - assertArrayEquals(path.getMD5Digest(), result); - assertNotSame(malformed, result); - assertEquals(16, result.length); + for (HashFunction hf : Arrays.asList(HashFunction.MD5, HashFunction.SHA1)) { + FileSystem.setDigestFunctionForTesting(hf); + byte[] result = DigestUtils.getDigestOrFail(path, 1); + assertArrayEquals(path.getDigest(), result); + assertNotSame(malformed, result); + assertTrue(path.isValidDigest(result)); + } } } diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/ArtifactFunctionTestCase.java b/src/test/java/com/google/devtools/build/lib/skyframe/ArtifactFunctionTestCase.java index e3c6a97a43..0ccaeaf559 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/ArtifactFunctionTestCase.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/ArtifactFunctionTestCase.java @@ -28,6 +28,7 @@ import com.google.devtools.build.lib.testutil.TestConstants; import com.google.devtools.build.lib.testutil.TestRuleClassProvider; import com.google.devtools.build.lib.testutil.TestUtils; import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor; +import com.google.devtools.build.lib.vfs.FileSystem.HashFunction; import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; @@ -158,13 +159,8 @@ abstract class ArtifactFunctionTestCase { /** InMemoryFileSystem that can pretend to do a fast digest. */ protected class CustomInMemoryFs extends InMemoryFileSystem { @Override - protected String getFastDigestFunctionType(Path path) { - return fastDigest ? "MD5" : null; - } - - @Override - protected byte[] getFastDigest(Path path) throws IOException { - return fastDigest ? getMD5Digest(path) : null; + protected byte[] getFastDigest(Path path, HashFunction hashFunction) throws IOException { + return fastDigest ? getDigest(path, hashFunction) : null; } } } diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/FileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/FileFunctionTest.java index 91dbc6c4ae..5eb13b326d 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/FileFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/FileFunctionTest.java @@ -52,6 +52,7 @@ import com.google.devtools.build.lib.util.Preconditions; import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor; import com.google.devtools.build.lib.vfs.FileStatus; import com.google.devtools.build.lib.vfs.FileSystem; +import com.google.devtools.build.lib.vfs.FileSystem.HashFunction; import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; @@ -95,13 +96,13 @@ public class FileFunctionTest { private Path pkgRoot; private Path outputBase; private PathPackageLocator pkgLocator; - private boolean fastMd5; + private boolean fastDigest; private ManualClock manualClock; private RecordingDifferencer differencer; @Before public final void createFsAndRoot() throws Exception { - fastMd5 = true; + fastDigest = true; manualClock = new ManualClock(); createFsAndRoot(new CustomInMemoryFs(manualClock)); } @@ -433,12 +434,7 @@ public class FileFunctionTest { createFsAndRoot( new CustomInMemoryFs(manualClock) { @Override - protected String getFastDigestFunctionType(Path path) { - return "magic"; - } - - @Override - protected byte[] getFastDigest(Path path) throws IOException { + protected byte[] getFastDigest(Path path, HashFunction hf) throws IOException { return digest; } }); @@ -479,7 +475,7 @@ public class FileFunctionTest { createFsAndRoot( new CustomInMemoryFs(manualClock) { @Override - protected byte[] getFastDigest(Path path) { + protected byte[] getFastDigest(Path path, HashFunction hf) { return path.getBaseName().equals("unreadable") ? expectedDigest : null; } }); @@ -494,7 +490,7 @@ public class FileFunctionTest { @Test public void testFileModificationModTime() throws Exception { - fastMd5 = false; + fastDigest = false; Path p = file("file"); FileValue a = valueForPath(p); p.setLastModifiedTime(42); @@ -504,7 +500,7 @@ public class FileFunctionTest { @Test public void testFileModificationDigest() throws Exception { - fastMd5 = true; + fastDigest = true; Path p = file("file"); FileValue a = valueForPath(p); FileSystemUtils.writeContentAsLatin1(p, "goop"); @@ -516,9 +512,9 @@ public class FileFunctionTest { public void testModTimeVsDigest() throws Exception { Path p = file("somefile", "fizzley"); - fastMd5 = true; + fastDigest = true; FileValue aMd5 = valueForPath(p); - fastMd5 = false; + fastDigest = false; FileValue aModTime = valueForPath(p); assertThat(aModTime).isNotEqualTo(aMd5); new EqualsTester().addEqualityGroup(aMd5).addEqualityGroup(aModTime).testEquals(); @@ -560,7 +556,7 @@ public class FileFunctionTest { @Test public void testSymlinkTargetContentsChangeModTime() throws Exception { - fastMd5 = false; + fastDigest = false; Path fooPath = file("foo"); FileSystemUtils.writeContentAsLatin1(fooPath, "foo"); Path p = symlink("symlink", "foo"); @@ -572,7 +568,7 @@ public class FileFunctionTest { @Test public void testSymlinkTargetContentsChangeDigest() throws Exception { - fastMd5 = true; + fastDigest = true; Path fooPath = file("foo"); FileSystemUtils.writeContentAsLatin1(fooPath, "foo"); Path p = symlink("symlink", "foo"); @@ -830,14 +826,14 @@ public class FileFunctionTest { assertArrayEquals(digest, value.getDigest()); // Digest is cached -- no filesystem access. assertEquals(expectedCalls, digestCalls.get()); - fastMd5 = false; + fastDigest = false; digestCalls.set(0); value = valueForPath(file); // No new digest calls. assertEquals(0, digestCalls.get()); assertNull(value.getDigest()); assertEquals(0, digestCalls.get()); - fastMd5 = true; + fastDigest = true; Path dir = directory("directory"); try { assertNull(valueForPath(dir).getDigest()); @@ -932,7 +928,7 @@ public class FileFunctionTest { fs.stubStat(path("a"), inconsistentParentFileStatus); // Disable fast-path md5 so that we don't try try to md5 the "a" (since it actually physically // is a directory). - fastMd5 = false; + fastDigest = false; SequentialBuildDriver driver = makeDriver(); SkyKey skyKey = skyKey("a/b"); EvaluationResult<FileValue> result = @@ -1624,21 +1620,16 @@ public class FileFunctionTest { super(manualClock); } - @Override - protected String getFastDigestFunctionType(Path path) { - return fastMd5 ? "MD5" : null; - } - public void stubFastDigestError(Path path, IOException error) { stubbedFastDigestErrors.put(path, error); } @Override - protected byte[] getFastDigest(Path path) throws IOException { + protected byte[] getFastDigest(Path path, HashFunction hashFunction) throws IOException { if (stubbedFastDigestErrors.containsKey(path)) { throw stubbedFastDigestErrors.get(path); } - return fastMd5 ? getMD5Digest(path) : null; + return fastDigest ? getDigest(path) : null; } public void stubStat(Path path, @Nullable FileStatus stubbedResult) { |