Make external repository implementations not re-fetch things on server restart.

This is accomplished by saving a proto of the repository rule in the output tree, then comparing it to that of the previous version. This makes HTTP_DOWNLOAD_CHECKER somewhat superfluous because it only matters if the external repository directory is modified manually.

Local repository implementations are not included, mainly because the symlinking is cheap (maybe they should be for reasons of symmetry?)

--
MOS_MIGRATED_REVID=108706396

1 files changed, 79 insertions, 15 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java
index 2bba0bfcd5..f949e0a8b1 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java
@@ -24,6 +24,7 @@ import com.google.devtools.build.lib.packages.BuildFileContainsErrorsException;
 import com.google.devtools.build.lib.packages.BuildFileNotFoundException;
 import com.google.devtools.build.lib.packages.NoSuchPackageException;
 import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.PackageSerializer;
 import com.google.devtools.build.lib.packages.Rule;
 import com.google.devtools.build.lib.skyframe.FileSymlinkException;
 import com.google.devtools.build.lib.skyframe.FileValue;
@@ -32,6 +33,7 @@ import com.google.devtools.build.lib.skyframe.PackageValue;
 import com.google.devtools.build.lib.skyframe.RepositoryValue;
 import com.google.devtools.build.lib.syntax.EvalException;
 import com.google.devtools.build.lib.syntax.Type;
+import com.google.devtools.build.lib.util.Fingerprint;
 import com.google.devtools.build.lib.vfs.FileSystemUtils;
 import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.build.lib.vfs.PathFragment;
@@ -44,6 +46,7 @@ import com.google.devtools.build.skyframe.SkyKey;
 
 import java.io.IOException;
 import java.nio.charset.Charset;
+import java.util.Arrays;
 
 import javax.annotation.Nullable;
 
@@ -51,6 +54,8 @@ import javax.annotation.Nullable;
  * Parent class for repository-related Skyframe functions.
  */
 public abstract class RepositoryFunction implements SkyFunction {
+  protected static final byte[] NO_RULE_SPECIFIC_DATA = new byte[] {};
+
   /**
    * Exception thrown when something goes wrong accessing a remote repository.
    *
@@ -79,6 +84,58 @@ public abstract class RepositoryFunction implements SkyFunction {
 
   private BlazeDirectories directories;
 
+  private byte[] computeRuleKey(Rule rule, byte[] ruleSpecificData) {
+
+    return new Fingerprint()
+        .addBytes(new PackageSerializer().serializeRule(rule).toByteArray())
+        .addBytes(ruleSpecificData)
+        .digestAndReset();
+  }
+
+  private Path getMarkerPath(Rule rule) {
+    return getExternalRepositoryDirectory().getChild("@" + rule.getName() + ".marker");
+  }
+
+  /**
+   * Checks if the state of the repository in the file system is consistent with the rule in the
+   * WORKSPACE file.
+   *
+   * <p>Deletes the marker file if not so that no matter what happens after, the state of the file
+   * system stays consistent.
+   */
+  protected boolean isFilesystemUpToDate(Rule rule, byte[] ruleSpecificData)
+      throws RepositoryFunctionException {
+    try {
+      Path markerPath = getMarkerPath(rule);
+      if (!markerPath.exists()) {
+        return false;
+      }
+
+      boolean result = Arrays.equals(
+          computeRuleKey(rule, ruleSpecificData),
+          FileSystemUtils.readContent(markerPath));
+      if (!result) {
+        // So that we are in a consistent state if something happens while fetching the repository
+        markerPath.delete();
+      }
+
+      return result;
+
+    } catch (IOException e) {
+      throw new RepositoryFunctionException(e, Transience.TRANSIENT);
+    }
+  }
+
+  protected void writeMarkerFile(Rule rule, byte[] ruleSpecificData)
+      throws RepositoryFunctionException {
+    try {
+      FileSystemUtils.writeContent(getMarkerPath(rule), computeRuleKey(rule, ruleSpecificData));
+    } catch (IOException e) {
+      throw new RepositoryFunctionException(e, Transience.TRANSIENT);
+    }
+  }
+
+  
   protected Path prepareLocalRepositorySymlinkTree(Rule rule, Environment env)
       throws RepositoryFunctionException {
     Path repositoryDirectory = getExternalRepositoryDirectory().getRelative(rule.getName());
@@ -117,22 +174,11 @@ public abstract class RepositoryFunction implements SkyFunction {
     return RepositoryValue.create(repositoryDirectory);
   }
 
-  /**
-   * Symlinks a BUILD file from the local filesystem into the external repository's root.
-   * @param rule the rule that declares the build_file path.
-   * @param workspaceDirectory the workspace root for the build.
-   * @param outputDirectory the directory of the remote repository
-   * @param env the Skyframe environment.
-   * @return the file value of the symlink created.
-   * @throws RepositoryFunctionException if the BUILD file specified does not exist or cannot be
-   *         linked.
-   */
-  protected RepositoryValue symlinkBuildFile(
-      Rule rule, Path workspaceDirectory, Path outputDirectory, Environment env)
+  protected FileValue getBuildFileValue(Rule rule, Environment env)
       throws RepositoryFunctionException {
     AggregatingAttributeMapper mapper = AggregatingAttributeMapper.of(rule);
     PathFragment buildFile = new PathFragment(mapper.get("build_file", Type.STRING));
-    Path buildFileTarget = workspaceDirectory.getRelative(buildFile);
+    Path buildFileTarget = directories.getWorkspace().getRelative(buildFile);
     if (!buildFileTarget.exists()) {
       throw new RepositoryFunctionException(
           new EvalException(rule.getLocation(),
@@ -146,7 +192,7 @@ public abstract class RepositoryFunction implements SkyFunction {
       rootedBuild = RootedPath.toRootedPath(
           buildFileTarget.getParentDirectory(), new PathFragment(buildFileTarget.getBaseName()));
     } else {
-      rootedBuild = RootedPath.toRootedPath(workspaceDirectory, buildFile);
+      rootedBuild = RootedPath.toRootedPath(directories.getWorkspace(), buildFile);
     }
     SkyKey buildFileKey = FileValue.key(rootedBuild);
     FileValue buildFileValue;
@@ -162,8 +208,26 @@ public abstract class RepositoryFunction implements SkyFunction {
           Transience.TRANSIENT);
     }
 
+    return buildFileValue;
+  }
+
+  /**
+   * Symlinks a BUILD file from the local filesystem into the external repository's root.
+   * @param rule the rule that declares the build_file path.
+   * @param outputDirectory the directory of the remote repository
+   * @param env the Skyframe environment.
+   * @return the file value of the symlink created.
+   * @throws RepositoryFunctionException if the BUILD file specified does not exist or cannot be
+   *         linked.
+   */
+  protected RepositoryValue symlinkBuildFile(Rule rule, Path outputDirectory, Environment env)
+      throws RepositoryFunctionException {
+    FileValue buildFileValue = getBuildFileValue(rule, env);
+    if (env.valuesMissing()) {
+      return null;
+    }
     Path buildFilePath = outputDirectory.getRelative("BUILD");
-    if (createSymbolicLink(buildFilePath, buildFileTarget, env) == null) {
+    if (createSymbolicLink(buildFilePath, buildFileValue.realRootedPath().asPath(), env) == null) {
       return null;
     }
     return RepositoryValue.createNew(outputDirectory, buildFileValue);