Store content digests in repository marker files. https://www.bazel.build/designs/2016/10/18/repository-invalidation.html

Change-Id: I6cb01397a35cd32169a0e415f8d7f944e7d840df
PiperOrigin-RevId: 166200841

3 files changed, 41 insertions, 9 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java
index 72a873c0cf..de78271c3a 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java
@@ -714,13 +714,17 @@ public class SkylarkRepositoryContext {
     if (fileValue == null) {
       throw RepositoryFunction.restart();
     }
-    if (!fileValue.isFile()) {
-      throw new EvalException(Location.BUILTIN,
-          "Not a file: " + rootedPath.asPath().getPathString());
+    if (!fileValue.isFile() || fileValue.isSpecialFile()) {
+      throw new EvalException(
+          Location.BUILTIN, "Not a regular file: " + rootedPath.asPath().getPathString());
     }
 
-    // A label do not contains space so it safe to use as a key.
-    markerData.put("FILE:" + label, Integer.toString(fileValue.realFileStateValue().hashCode()));
+    // A label does not contains space so it safe to use as a key.
+    try {
+      markerData.put("FILE:" + label, RepositoryFunction.fileValueToMarkerValue(fileValue));
+    } catch (IOException e) {
+      throw new EvalException(Location.BUILTIN, e);
+    }
     return new SkylarkPath(rootedPath.asPath());
   }
 
diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/NewRepositoryFileHandler.java b/src/main/java/com/google/devtools/build/lib/rules/repository/NewRepositoryFileHandler.java
index 094f5e5ec7..6317a65dca 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/repository/NewRepositoryFileHandler.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/repository/NewRepositoryFileHandler.java
@@ -157,8 +157,11 @@ public class NewRepositoryFileHandler {
           // TODO(pcloudy): Don't add absolute path into markerData once it's not supported
           fileKey = fileValue.realRootedPath().asPath().getPathString();
         }
-        markerData.put(
-            "FILE:" + fileKey, Integer.toString(fileValue.realFileStateValue().hashCode()));
+        try {
+          markerData.put("FILE:" + fileKey, RepositoryFunction.fileValueToMarkerValue(fileValue));
+        } catch (IOException e) {
+          throw new RepositoryFunctionException(e, Transience.TRANSIENT);
+        }
       } else if (fileContent != null) {
         RepositoryFunction.writeFile(outputDirectory, filename, fileContent);
       } else {
@@ -265,6 +268,13 @@ public class NewRepositoryFileHandler {
             Transience.TRANSIENT);
       }
 
+      if (!fileValue.isFile() || fileValue.isSpecialFile()) {
+        throw new RepositoryFunctionException(
+            new EvalException(
+                rule.getLocation(), String.format("%s is not a regular file", rootedFile.asPath())),
+            Transience.PERSISTENT);
+      }
+
       return fileValue;
     }
 
diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java
index 469eb61222..e82ae7bd60 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java
@@ -17,6 +17,7 @@ package com.google.devtools.build.lib.rules.repository;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
+import com.google.common.io.BaseEncoding;
 import com.google.devtools.build.lib.analysis.BlazeDirectories;
 import com.google.devtools.build.lib.analysis.RuleDefinition;
 import com.google.devtools.build.lib.cmdline.Label;
@@ -30,6 +31,7 @@ import com.google.devtools.build.lib.packages.NoSuchPackageException;
 import com.google.devtools.build.lib.packages.Rule;
 import com.google.devtools.build.lib.rules.ExternalPackageUtil;
 import com.google.devtools.build.lib.skyframe.ActionEnvironmentFunction;
+import com.google.devtools.build.lib.skyframe.FileStateValue.RegularFileStateValue;
 import com.google.devtools.build.lib.skyframe.FileSymlinkException;
 import com.google.devtools.build.lib.skyframe.FileValue;
 import com.google.devtools.build.lib.skyframe.InconsistentFilesystemException;
@@ -222,11 +224,11 @@ public abstract class RepositoryFunction {
                   FileSymlinkException.class,
                   InconsistentFilesystemException.class);
 
-      if (fileValue == null || !fileValue.isFile()) {
+      if (fileValue == null || !fileValue.isFile() || fileValue.isSpecialFile()) {
         return false;
       }
 
-      return Objects.equals(value, Integer.toString(fileValue.realFileStateValue().hashCode()));
+      return Objects.equals(value, fileValueToMarkerValue(fileValue));
     } catch (LabelSyntaxException e) {
       throw new IllegalStateException(
           "Key " + key + " is not a correct file key (should be in form FILE:label)", e);
@@ -239,6 +241,22 @@ public abstract class RepositoryFunction {
     }
   }
 
+  /**
+   * Convert to a @{link com.google.devtools.build.lib.skyframe.FileValue} to a String appropriate
+   * for placing in a repository marker file.
+   *
+   * @param fileValue The value to convert. It must correspond to a regular file.
+   */
+  public static String fileValueToMarkerValue(FileValue fileValue) throws IOException {
+    Preconditions.checkArgument(fileValue.isFile() && !fileValue.isSpecialFile());
+    // Return the file content digest in hex. fileValue may or may not have the digest available.
+    byte[] digest = ((RegularFileStateValue) fileValue.realFileStateValue()).getDigest();
+    if (digest == null) {
+      digest = fileValue.realRootedPath().asPath().getDigest();
+    }
+    return BaseEncoding.base16().lowerCase().encode(digest);
+  }
+
   static boolean verifyMarkerDataForFiles(
       Rule rule, Map<String, String> markerData, Environment env) throws InterruptedException {
     for (Map.Entry<String, String> entry : markerData.entrySet()) {