diff options
author | 2017-01-19 12:42:09 +0000 | |
---|---|---|
committer | 2017-01-19 14:46:26 +0000 | |
commit | fcf3cd483f55956a71832dd6fdab5b6248ff8d86 (patch) | |
tree | 85eea8596267aaf50b4c0a76379ab3fd86b5507b | |
parent | 03c534d3994c0300feece0582b0fc125704b75ff (diff) |
Inject the digest of the marker file in the RepositoryDirectoryValue
This digest make sure that we produce a different RepositoryDirectoryValue
for different marker data, thus invalidating the node.
--
Change-Id: Ibc6286c76de7a8d07ece2bb8125bfb75095f7784
Reviewed-on: https://cr.bazel.build/8136
PiperOrigin-RevId: 144948233
MOS_MIGRATED_REVID=144948233
15 files changed, 131 insertions, 94 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/GitRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/GitRepositoryFunction.java index 29f13489b8..a15d220fb5 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/GitRepositoryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/GitRepositoryFunction.java @@ -24,7 +24,6 @@ import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.skyframe.SkyFunction.Environment; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; -import com.google.devtools.build.skyframe.SkyValue; import java.io.IOException; import java.util.Map; @@ -39,12 +38,12 @@ public class GitRepositoryFunction extends RepositoryFunction { } @Override - public SkyValue fetch(Rule rule, Path outputDirectory, BlazeDirectories directories, - Environment env, Map<String, String> markerData) + public RepositoryDirectoryValue.Builder fetch(Rule rule, Path outputDirectory, + BlazeDirectories directories, Environment env, Map<String, String> markerData) throws InterruptedException, RepositoryFunctionException { createDirectory(outputDirectory, rule); GitCloner.clone(rule, outputDirectory, env.getListener(), clientEnvironment); - return RepositoryDirectoryValue.create(outputDirectory); + return RepositoryDirectoryValue.builder().setPath(outputDirectory); } protected static void createDirectory(Path path, Rule rule) throws RepositoryFunctionException { diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpArchiveFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpArchiveFunction.java index cbf04bda1e..6c973ea0a4 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpArchiveFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/HttpArchiveFunction.java @@ -28,7 +28,6 @@ import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.skyframe.SkyFunction.Environment; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; -import com.google.devtools.build.skyframe.SkyValue; import java.io.IOException; import java.util.Map; @@ -58,8 +57,8 @@ public class HttpArchiveFunction extends RepositoryFunction { } @Override - public SkyValue fetch(Rule rule, Path outputDirectory, BlazeDirectories directories, - Environment env, Map<String, String> markerData) + public RepositoryDirectoryValue.Builder fetch(Rule rule, Path outputDirectory, + BlazeDirectories directories, Environment env, Map<String, String> markerData) throws RepositoryFunctionException, InterruptedException { // The output directory is always under output_base/external (to stay out of the way of // artifacts from this repository) and uses the rule's name to avoid conflicts with other @@ -73,7 +72,7 @@ public class HttpArchiveFunction extends RepositoryFunction { env.getListener(), clientEnvironment); DecompressorValue.decompress(getDescriptor(rule, downloadedPath, outputDirectory)); - return RepositoryDirectoryValue.create(outputDirectory); + return RepositoryDirectoryValue.builder().setPath(outputDirectory); } protected DecompressorDescriptor getDescriptor(Rule rule, Path downloadPath, Path outputDirectory) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java index b90bcaec74..907227e959 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenJarFunction.java @@ -20,6 +20,7 @@ import com.google.devtools.build.lib.analysis.RuleDefinition; import com.google.devtools.build.lib.bazel.rules.workspace.MavenJarRule; import com.google.devtools.build.lib.packages.Rule; import com.google.devtools.build.lib.rules.repository.RepositoryDirectoryValue; +import com.google.devtools.build.lib.rules.repository.RepositoryFunction; import com.google.devtools.build.lib.rules.repository.WorkspaceAttributeMapper; import com.google.devtools.build.lib.syntax.EvalException; import com.google.devtools.build.lib.syntax.Type; @@ -27,7 +28,6 @@ import com.google.devtools.build.lib.util.Fingerprint; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.skyframe.SkyFunction.Environment; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; -import com.google.devtools.build.skyframe.SkyValue; import java.io.IOException; import java.util.Map; @@ -91,8 +91,8 @@ public class MavenJarFunction extends HttpArchiveFunction { } @Override - public SkyValue fetch(Rule rule, Path outputDirectory, BlazeDirectories directories, - Environment env, Map<String, String> markerData) + public RepositoryDirectoryValue.Builder fetch(Rule rule, Path outputDirectory, + BlazeDirectories directories, Environment env, Map<String, String> markerData) throws RepositoryFunctionException, InterruptedException { MavenServerValue serverValue = getServer(rule, env); if (env.valuesMissing()) { @@ -103,7 +103,7 @@ public class MavenJarFunction extends HttpArchiveFunction { return createOutputTree(rule, outputDir, serverValue); } - private SkyValue createOutputTree(Rule rule, Path outputDirectory, + private RepositoryDirectoryValue.Builder createOutputTree(Rule rule, Path outputDirectory, MavenServerValue serverValue) throws RepositoryFunctionException, InterruptedException { Preconditions.checkState(downloader instanceof MavenDownloader); MavenDownloader mavenDownloader = (MavenDownloader) downloader; @@ -127,7 +127,7 @@ public class MavenJarFunction extends HttpArchiveFunction { .setTargetName(name) .setArchivePath(repositoryJar) .setRepositoryPath(outputDirectory).build()); - return RepositoryDirectoryValue.create(result); + return RepositoryDirectoryValue.builder().setPath(result); } /** diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenServerRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenServerRepositoryFunction.java index 1fc98e4b0e..d104415463 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenServerRepositoryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/MavenServerRepositoryFunction.java @@ -18,13 +18,13 @@ import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.analysis.RuleDefinition; import com.google.devtools.build.lib.bazel.rules.workspace.MavenServerRule; import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.rules.repository.RepositoryDirectoryValue; import com.google.devtools.build.lib.rules.repository.RepositoryFunction; import com.google.devtools.build.lib.syntax.EvalException; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.skyframe.SkyFunction.Environment; import com.google.devtools.build.skyframe.SkyFunctionException; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; -import com.google.devtools.build.skyframe.SkyValue; import java.util.Map; import javax.annotation.Nullable; @@ -40,8 +40,8 @@ public class MavenServerRepositoryFunction extends RepositoryFunction { @Nullable @Override - public SkyValue fetch(Rule rule, Path outputDirectory, BlazeDirectories directories, - Environment env, Map<String, String> markerData) + public RepositoryDirectoryValue.Builder fetch(Rule rule, Path outputDirectory, + BlazeDirectories directories, Environment env, Map<String, String> markerData) throws SkyFunctionException, InterruptedException { throw new RepositoryFunctionException(new EvalException( rule.getLocation(), diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/NewGitRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/NewGitRepositoryFunction.java index 0e3d2ccce8..f7ae469949 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/NewGitRepositoryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/NewGitRepositoryFunction.java @@ -20,7 +20,6 @@ import com.google.devtools.build.lib.rules.repository.NewRepositoryBuildFileHand import com.google.devtools.build.lib.rules.repository.RepositoryDirectoryValue; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.skyframe.SkyFunction.Environment; -import com.google.devtools.build.skyframe.SkyValue; import java.util.Map; /** @@ -28,8 +27,8 @@ import java.util.Map; */ public class NewGitRepositoryFunction extends GitRepositoryFunction { @Override - public SkyValue fetch(Rule rule, Path outputDirectory, BlazeDirectories directories, - Environment env, Map<String, String> markerData) + public RepositoryDirectoryValue.Builder fetch(Rule rule, Path outputDirectory, + BlazeDirectories directories, Environment env, Map<String, String> markerData) throws InterruptedException, RepositoryFunctionException { NewRepositoryBuildFileHandler buildFileHandler = new NewRepositoryBuildFileHandler(directories.getWorkspace()); @@ -42,6 +41,6 @@ public class NewGitRepositoryFunction extends GitRepositoryFunction { createWorkspaceFile(outputDirectory, rule.getTargetKind(), rule.getName()); buildFileHandler.finishBuildFile(outputDirectory); - return RepositoryDirectoryValue.create(outputDirectory); + return RepositoryDirectoryValue.builder().setPath(outputDirectory); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/NewHttpArchiveFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/NewHttpArchiveFunction.java index 029c51f7b7..50e992eb1b 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/NewHttpArchiveFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/NewHttpArchiveFunction.java @@ -26,7 +26,6 @@ import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.skyframe.SkyFunction.Environment; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; -import com.google.devtools.build.skyframe.SkyValue; import java.io.IOException; import java.util.Map; import javax.annotation.Nullable; @@ -43,8 +42,8 @@ public class NewHttpArchiveFunction extends HttpArchiveFunction { @Nullable @Override - public SkyValue fetch(Rule rule, Path outputDirectory, BlazeDirectories directories, - Environment env, Map<String, String> markerData) + public RepositoryDirectoryValue.Builder fetch(Rule rule, Path outputDirectory, + BlazeDirectories directories, Environment env, Map<String, String> markerData) throws RepositoryFunctionException, InterruptedException { NewRepositoryBuildFileHandler buildFileHandler = new NewRepositoryBuildFileHandler(directories.getWorkspace()); @@ -86,6 +85,6 @@ public class NewHttpArchiveFunction extends HttpArchiveFunction { createWorkspaceFile(decompressed, rule.getTargetKind(), rule.getName()); buildFileHandler.finishBuildFile(outputDirectory); - return RepositoryDirectoryValue.create(outputDirectory); + return RepositoryDirectoryValue.builder().setPath(outputDirectory); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryFunction.java index bab97d7c0f..08074827d7 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryFunction.java @@ -32,7 +32,6 @@ import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.skyframe.SkyFunction.Environment; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; -import com.google.devtools.build.skyframe.SkyValue; import java.io.IOException; import java.util.Map; import javax.annotation.Nullable; @@ -70,8 +69,8 @@ public class SkylarkRepositoryFunction extends RepositoryFunction { @Nullable @Override - public SkyValue fetch(Rule rule, Path outputDirectory, BlazeDirectories directories, - Environment env, Map<String, String> markerData) + public RepositoryDirectoryValue.Builder fetch(Rule rule, Path outputDirectory, + BlazeDirectories directories, Environment env, Map<String, String> markerData) throws RepositoryFunctionException, InterruptedException { BaseFunction function = rule.getRuleClassObject().getConfiguredTargetFunction(); try (Mutability mutability = Mutability.create("skylark repository")) { @@ -134,7 +133,7 @@ public class SkylarkRepositoryFunction extends RepositoryFunction { createWorkspaceFile(outputDirectory, rule.getTargetKind(), rule.getName()); } - return RepositoryDirectoryValue.create(outputDirectory); + return RepositoryDirectoryValue.builder().setPath(outputDirectory); } @Override diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/android/AndroidNdkRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/android/AndroidNdkRepositoryFunction.java index 0fee5ef762..e0975bf315 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/android/AndroidNdkRepositoryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/android/AndroidNdkRepositoryFunction.java @@ -46,7 +46,6 @@ import com.google.devtools.build.lib.view.config.crosstool.CrosstoolConfig.ToolP import com.google.devtools.build.skyframe.SkyFunction.Environment; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; import com.google.devtools.build.skyframe.SkyKey; -import com.google.devtools.build.skyframe.SkyValue; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -77,8 +76,8 @@ public class AndroidNdkRepositoryFunction extends RepositoryFunction { } @Override - public SkyValue fetch(Rule rule, Path outputDirectory, BlazeDirectories directories, - Environment env, Map<String, String> markerData) + public RepositoryDirectoryValue.Builder fetch(Rule rule, Path outputDirectory, + BlazeDirectories directories, Environment env, Map<String, String> markerData) throws InterruptedException, RepositoryFunctionException { prepareLocalRepositorySymlinkTree(rule, outputDirectory); WorkspaceAttributeMapper attributes = WorkspaceAttributeMapper.of(rule); @@ -161,7 +160,7 @@ public class AndroidNdkRepositoryFunction extends RepositoryFunction { String buildFile = createBuildFile(ruleName, crosstoolsAndStls.build()); writeBuildFile(outputDirectory, buildFile); - return RepositoryDirectoryValue.create(outputDirectory); + return RepositoryDirectoryValue.builder().setPath(outputDirectory); } @Override diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/android/AndroidSdkRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/android/AndroidSdkRepositoryFunction.java index 850da7d874..179f5b51a4 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/android/AndroidSdkRepositoryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/android/AndroidSdkRepositoryFunction.java @@ -41,7 +41,6 @@ import com.google.devtools.build.skyframe.SkyFunction.Environment; import com.google.devtools.build.skyframe.SkyFunctionException; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; import com.google.devtools.build.skyframe.SkyKey; -import com.google.devtools.build.skyframe.SkyValue; import java.io.IOException; import java.util.Map; import java.util.Properties; @@ -65,8 +64,8 @@ public class AndroidSdkRepositoryFunction extends RepositoryFunction { } @Override - public SkyValue fetch(Rule rule, Path outputDirectory, BlazeDirectories directories, - Environment env, Map<String, String> markerData) + public RepositoryDirectoryValue.Builder fetch(Rule rule, Path outputDirectory, + BlazeDirectories directories, Environment env, Map<String, String> markerData) throws SkyFunctionException, InterruptedException { prepareLocalRepositorySymlinkTree(rule, outputDirectory); @@ -195,7 +194,7 @@ public class AndroidSdkRepositoryFunction extends RepositoryFunction { } writeBuildFile(outputDirectory, buildFile); - return RepositoryDirectoryValue.create(outputDirectory); + return RepositoryDirectoryValue.builder().setPath(outputDirectory); } @Override diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/LocalRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/LocalRepositoryFunction.java index 4fecf90753..ee387f735d 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/repository/LocalRepositoryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/repository/LocalRepositoryFunction.java @@ -22,7 +22,6 @@ import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.skyframe.SkyFunction.Environment; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; -import com.google.devtools.build.skyframe.SkyValue; import java.io.IOException; import java.util.Map; @@ -37,8 +36,8 @@ public class LocalRepositoryFunction extends RepositoryFunction { } @Override - public SkyValue fetch(Rule rule, Path outputDirectory, BlazeDirectories directories, - Environment env, Map<String, String> markerData) + public RepositoryDirectoryValue.Builder fetch(Rule rule, Path outputDirectory, + BlazeDirectories directories, Environment env, Map<String, String> markerData) throws InterruptedException, RepositoryFunctionException { PathFragment pathFragment = RepositoryFunction.getTargetPath(rule, directories.getWorkspace()); try { @@ -60,7 +59,7 @@ public class LocalRepositoryFunction extends RepositoryFunction { new IOException(rule + " must specify an existing directory"), Transience.TRANSIENT); } - return RepositoryDirectoryValue.create(outputDirectory); + return RepositoryDirectoryValue.builder().setPath(outputDirectory); } @Override diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/NewLocalRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/NewLocalRepositoryFunction.java index 16b311c5e6..91aadb7364 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/repository/NewLocalRepositoryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/repository/NewLocalRepositoryFunction.java @@ -29,7 +29,6 @@ import com.google.devtools.build.skyframe.SkyFunction.Environment; import com.google.devtools.build.skyframe.SkyFunctionException; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; import com.google.devtools.build.skyframe.SkyKey; -import com.google.devtools.build.skyframe.SkyValue; import java.io.IOException; import java.util.Map; @@ -44,8 +43,8 @@ public class NewLocalRepositoryFunction extends RepositoryFunction { } @Override - public SkyValue fetch(Rule rule, Path outputDirectory, BlazeDirectories directories, - Environment env, Map<String, String> markerData) + public RepositoryDirectoryValue.Builder fetch(Rule rule, Path outputDirectory, + BlazeDirectories directories, Environment env, Map<String, String> markerData) throws SkyFunctionException, InterruptedException { NewRepositoryBuildFileHandler buildFileHandler = @@ -133,7 +132,7 @@ public class NewLocalRepositoryFunction extends RepositoryFunction { } createWorkspaceFile(outputDirectory, rule.getTargetKind(), rule.getName()); - return RepositoryDirectoryValue.createWithSourceDirectory(outputDirectory, directoryValue); + return RepositoryDirectoryValue.builder().setPath(outputDirectory).setSourceDir(directoryValue); } @Override diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java index 166f9fd28d..771231b72b 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java @@ -132,22 +132,27 @@ public final class RepositoryDelegatorFunction implements SkyFunction { // not depend on non-local data, so it does not make much sense to try to cache from across // server instances. setupRepositoryRoot(repoRoot); - SkyValue localRepo = handler.fetch(rule, repoRoot, directories, env, markerData); - if (localRepo != null) { - writeMarkerFile(markerPath, markerData, ruleKey); + RepositoryDirectoryValue.Builder localRepo = + handler.fetch(rule, repoRoot, directories, env, markerData); + if (localRepo == null) { + return null; + } else { + // We write the marker file for local repository essentially for getting the digest and + // injecting it in the RepositoryDirectoryValue. + byte[] digest = writeMarkerFile(markerPath, markerData, ruleKey); + return localRepo.setDigest(digest).build(); } - return localRepo; } // We check the repository root for existence here, but we can't depend on the FileValue, // because it's possible that we eventually create that directory in which case the FileValue // and the state of the file system would be inconsistent. - Boolean markerUpToDate = isFilesystemUpToDate(markerPath, rule, ruleKey, handler, env); - if (markerUpToDate == null) { + byte[] markerHash = isFilesystemUpToDate(markerPath, rule, ruleKey, handler, env); + if (env.valuesMissing()) { return null; } - if (markerUpToDate && repoRoot.exists()) { + if (markerHash != null && repoRoot.exists()) { // Now that we know that it exists, we can declare a Skyframe dependency on the repository // root. RepositoryFunction.getRepositoryDirectory(repoRoot, env); @@ -155,13 +160,14 @@ public final class RepositoryDelegatorFunction implements SkyFunction { return null; } - return RepositoryDirectoryValue.create(repoRoot); + return RepositoryDirectoryValue.builder().setPath(repoRoot).setDigest(markerHash).build(); } if (isFetch.get()) { // Fetching enabled, go ahead. setupRepositoryRoot(repoRoot); - SkyValue result = handler.fetch(rule, repoRoot, directories, env, markerData); + RepositoryDirectoryValue.Builder result = + handler.fetch(rule, repoRoot, directories, env, markerData); if (env.valuesMissing()) { return null; } @@ -170,8 +176,8 @@ public final class RepositoryDelegatorFunction implements SkyFunction { // and writing the marker file because if they aren't computed, it would cause a Skyframe // restart thus calling the possibly very slow (networking, decompression...) fetch() // operation again. So we write the marker file here immediately. - writeMarkerFile(markerPath, markerData, ruleKey); - return result; + byte[] digest = writeMarkerFile(markerPath, markerData, ruleKey); + return result.setDigest(digest).build(); } if (!repoRoot.exists()) { @@ -197,7 +203,8 @@ public final class RepositoryDelegatorFunction implements SkyFunction { + "run the build without the '--nofetch' command line option.", rule.getName()))); - return RepositoryDirectoryValue.fetchingDelayed(repoRootValue.realRootedPath().asPath()); + return RepositoryDirectoryValue.builder().setPath(repoRootValue.realRootedPath().asPath()) + .setFetchingDelayed().build(); } private final String computeRuleKey(Rule rule, byte[] ruleSpecificData) { @@ -215,18 +222,21 @@ public final class RepositoryDelegatorFunction implements SkyFunction { * system stays consistent. * * <p> - * Returns null if some Skyframe values need to be computed before giving a full answer. + * Returns null if the file system is not up to date and a hash of the marker file if the file + * system is up to date. */ @Nullable - private final Boolean isFilesystemUpToDate(Path markerPath, Rule rule, String ruleKey, + private final byte[] isFilesystemUpToDate(Path markerPath, Rule rule, String ruleKey, RepositoryFunction handler, Environment env) throws RepositoryFunctionException, InterruptedException { try { if (!markerPath.exists()) { - return false; + return null; } - String[] lines = FileSystemUtils.readContent(markerPath, StandardCharsets.UTF_8).split("\n"); + String content = FileSystemUtils.readContent(markerPath, StandardCharsets.UTF_8); + + String[] lines = content.split("\n"); Map<String, String> markerData = new TreeMap<>(); String markerRuleKey = ""; boolean firstLine = true; @@ -252,12 +262,15 @@ public final class RepositoryDelegatorFunction implements SkyFunction { return null; } } - if (!result) { + + if (result) { + return new Fingerprint().addString(content).digestAndReset(); + } else { // So that we are in a consistent state if something happens while fetching the repository markerPath.delete(); + return null; } - return result; } catch (IOException e) { throw new RepositoryFunctionException(e, Transience.TRANSIENT); } @@ -297,7 +310,7 @@ public final class RepositoryDelegatorFunction implements SkyFunction { return result.toString(); } - private final void writeMarkerFile( + private final byte[] writeMarkerFile( Path markerPath, Map<String, String> markerData, String ruleKey) throws RepositoryFunctionException { try { @@ -308,7 +321,9 @@ public final class RepositoryDelegatorFunction implements SkyFunction { String value = data.getValue(); builder.append(escape(key)).append(" ").append(escape(value)).append("\n"); } - FileSystemUtils.writeContent(markerPath, StandardCharsets.UTF_8, builder.toString()); + String content = builder.toString(); + FileSystemUtils.writeContent(markerPath, StandardCharsets.UTF_8, content); + return new Fingerprint().addString(content).digestAndReset(); } catch (IOException e) { throw new RepositoryFunctionException(e, Transience.TRANSIENT); } diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDirectoryValue.java b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDirectoryValue.java index 9ada38de87..cb0d0ab48a 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDirectoryValue.java +++ b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDirectoryValue.java @@ -15,13 +15,14 @@ package com.google.devtools.build.lib.rules.repository; import com.google.common.base.Objects; +import com.google.common.base.Preconditions; import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.skyframe.DirectoryListingValue; import com.google.devtools.build.lib.skyframe.SkyFunctions; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; - +import java.util.Arrays; import javax.annotation.Nullable; /** @@ -31,36 +32,67 @@ public class RepositoryDirectoryValue implements SkyValue { private final Path path; private final boolean fetchingDelayed; @Nullable + private final byte[] digest; + @Nullable private final DirectoryListingValue sourceDir; private RepositoryDirectoryValue( - Path path, boolean fetchingDelayed, DirectoryListingValue sourceDir) { + Path path, boolean fetchingDelayed, DirectoryListingValue sourceDir, byte[] digest) { this.path = path; this.fetchingDelayed = fetchingDelayed; this.sourceDir = sourceDir; + this.digest = digest; } - /** - * Creates an immutable external repository. - */ - public static RepositoryDirectoryValue create(Path repositoryDirectory) { - return new RepositoryDirectoryValue(repositoryDirectory, false, null); - } + /** A builder to create a {@link RepositoryDirectoryValue}. */ + public static class Builder { + private Path path = null; + private boolean fetchingDelayed = false; + private byte[] digest = null; + private DirectoryListingValue sourceDir = null; - /** - * new_local_repositories - */ - public static RepositoryDirectoryValue createWithSourceDirectory( - Path repositoryDirectory, DirectoryListingValue sourceDir) { - return new RepositoryDirectoryValue(repositoryDirectory, false, sourceDir); + private Builder() {} + + public Builder copy(RepositoryDirectoryValue value) { + this.digest = value.digest; + this.fetchingDelayed = value.fetchingDelayed; + this.path = value.path; + this.sourceDir = value.sourceDir; + return this; + } + + public Builder setPath(Path path) { + this.path = path; + return this; + } + + public Builder setFetchingDelayed() { + this.fetchingDelayed = true; + return this; + } + + public Builder setDigest(byte[] digest) { + this.digest = digest; + return this; + } + + public Builder setSourceDir(DirectoryListingValue sourceDir) { + this.sourceDir = sourceDir; + return this; + } + + public RepositoryDirectoryValue build() { + Preconditions.checkNotNull(path, "Repository path must be specified!"); + // Only if fetching is delayed then we are allowed to have a null digest. + if (!this.fetchingDelayed) { + Preconditions.checkNotNull(digest, "Repository marker digest must be specified!"); + } + return new RepositoryDirectoryValue(path, fetchingDelayed, sourceDir, digest); + } } - /** - * Creates a value that represents a repository whose fetching has been delayed by a - * {@code --nofetch} command line option. - */ - public static RepositoryDirectoryValue fetchingDelayed(Path repositoryDirectory) { - return new RepositoryDirectoryValue(repositoryDirectory, true, null); + public static Builder builder() { + return new Builder(); } /** @@ -86,14 +118,15 @@ public class RepositoryDirectoryValue implements SkyValue { if (other instanceof RepositoryDirectoryValue) { RepositoryDirectoryValue otherValue = (RepositoryDirectoryValue) other; return Objects.equal(path, otherValue.path) - && Objects.equal(sourceDir, otherValue.sourceDir); + && Objects.equal(sourceDir, otherValue.sourceDir) + && Arrays.equals(digest, otherValue.digest); } return false; } @Override public int hashCode() { - return Objects.hashCode(path, sourceDir); + return Objects.hashCode(path, sourceDir, Arrays.hashCode(digest)); } @Override diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java index 6f0459ed6c..bf02eb11ff 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java @@ -46,7 +46,6 @@ import com.google.devtools.build.skyframe.SkyFunction.Environment; import com.google.devtools.build.skyframe.SkyFunctionException; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; import com.google.devtools.build.skyframe.SkyKey; -import com.google.devtools.build.skyframe.SkyValue; import java.io.IOException; import java.nio.charset.Charset; import java.util.Map; @@ -151,8 +150,8 @@ public abstract class RepositoryFunction { */ @ThreadSafe @Nullable - public abstract SkyValue fetch(Rule rule, Path outputDirectory, BlazeDirectories directories, - Environment env, Map<String, String> markerData) + public abstract RepositoryDirectoryValue.Builder fetch(Rule rule, Path outputDirectory, + BlazeDirectories directories, Environment env, Map<String, String> markerData) throws SkyFunctionException, InterruptedException; /** @@ -212,7 +211,7 @@ public abstract class RepositoryFunction { } } - protected static RepositoryDirectoryValue writeBuildFile( + protected static RepositoryDirectoryValue.Builder writeBuildFile( Path repositoryDirectory, String contents) throws RepositoryFunctionException { Path buildFilePath = repositoryDirectory.getRelative("BUILD.bazel"); try { @@ -227,7 +226,7 @@ public abstract class RepositoryFunction { throw new RepositoryFunctionException(e, Transience.TRANSIENT); } - return RepositoryDirectoryValue.create(repositoryDirectory); + return RepositoryDirectoryValue.builder().setPath(repositoryDirectory); } @VisibleForTesting diff --git a/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryFunctionTest.java b/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryFunctionTest.java index 177f2db017..629a418e2c 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryFunctionTest.java @@ -27,7 +27,6 @@ import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.skyframe.SkyFunction; import com.google.devtools.build.skyframe.SkyFunctionException; -import com.google.devtools.build.skyframe.SkyValue; import java.util.Map; import javax.annotation.Nullable; import org.junit.Test; @@ -47,8 +46,8 @@ public class RepositoryFunctionTest extends BuildViewTestCase { static class TestingRepositoryFunction extends RepositoryFunction { @Nullable @Override - public SkyValue fetch(Rule rule, Path outputDirectory, BlazeDirectories directories, - SkyFunction.Environment env, Map<String, String> markerData) + public RepositoryDirectoryValue.Builder fetch(Rule rule, Path outputDirectory, + BlazeDirectories directories, SkyFunction.Environment env, Map<String, String> markerData) throws SkyFunctionException, InterruptedException { return null; } |