Move verifyMarkerData into RepositoryFunction

And for new_XX_repository rules, add BUILD and WORKSPACE file into
markerData.

In markerData, key starting with FILE: can be an absolute label or an
absoulte path, but the latter one will be depracated in future.

Fixed https://github.com/bazelbuild/bazel/issues/3093

Change-Id: Ic3e16c123b3f1f781ab12c41d13f5e540b05686c
PiperOrigin-RevId: 160382024

3 files changed, 167 insertions, 30 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/NewLocalRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/NewLocalRepositoryFunction.java
index 3670dcdd44..1e2a5317ae 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/repository/NewLocalRepositoryFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/repository/NewLocalRepositoryFunction.java
@@ -117,7 +117,7 @@ public class NewLocalRepositoryFunction extends RepositoryFunction {
       return null;
     }
 
-    fileHandler.finishFile(outputDirectory);
+    fileHandler.finishFile(rule, outputDirectory, markerData);
 
     return RepositoryDirectoryValue.builder().setPath(outputDirectory).setSourceDir(directoryValue);
   }
diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/NewRepositoryFileHandler.java b/src/main/java/com/google/devtools/build/lib/rules/repository/NewRepositoryFileHandler.java
index 64c4502b71..094f5e5ec7 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/repository/NewRepositoryFileHandler.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/repository/NewRepositoryFileHandler.java
@@ -32,6 +32,7 @@ import com.google.devtools.build.skyframe.SkyFunction.Environment;
 import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
 import com.google.devtools.build.skyframe.SkyKey;
 import java.io.IOException;
+import java.util.Map;
 
 /**
  * Encapsulates the 2-step behavior of creating workspace and build files for the new_*_repository
@@ -59,9 +60,10 @@ public class NewRepositoryFileHandler {
     return true;
   }
 
-  public void finishFile(Path outputDirectory) throws RepositoryFunctionException {
-    this.workspaceFileHandler.finishFile(outputDirectory);
-    this.buildFileHandler.finishFile(outputDirectory);
+  public void finishFile(Rule rule, Path outputDirectory, Map<String, String> markerData)
+      throws RepositoryFunctionException {
+    this.workspaceFileHandler.finishFile(rule, outputDirectory, markerData);
+    this.buildFileHandler.finishFile(rule, outputDirectory, markerData);
   }
 
   /**
@@ -142,10 +144,21 @@ public class NewRepositoryFileHandler {
      * @throws IllegalStateException if {@link #prepareFile} was not called before this, or if
      *     {@link #prepareFile} failed and this was called.
      */
-    public void finishFile(Path outputDirectory) throws RepositoryFunctionException {
+    public void finishFile(Rule rule, Path outputDirectory, Map<String, String> markerData)
+        throws RepositoryFunctionException {
       if (fileValue != null) {
         // Link x/FILENAME to <build_root>/x.FILENAME.
         symlinkFile(fileValue, filename, outputDirectory);
+        String fileAttribute = getFileAttributeValue(rule);
+        String fileKey;
+        if (LabelValidator.isAbsolute(fileAttribute)) {
+          fileKey = getFileAttributeAsLabel(rule).toString();
+        } else {
+          // TODO(pcloudy): Don't add absolute path into markerData once it's not supported
+          fileKey = fileValue.realRootedPath().asPath().getPathString();
+        }
+        markerData.put(
+            "FILE:" + fileKey, Integer.toString(fileValue.realFileStateValue().hashCode()));
       } else if (fileContent != null) {
         RepositoryFunction.writeFile(outputDirectory, filename, fileContent);
       } else {
@@ -153,8 +166,7 @@ public class NewRepositoryFileHandler {
       }
     }
 
-    private FileValue getFileValue(Rule rule, Environment env)
-        throws RepositoryFunctionException, InterruptedException {
+    private String getFileAttributeValue(Rule rule) throws RepositoryFunctionException {
       WorkspaceAttributeMapper mapper = WorkspaceAttributeMapper.of(rule);
       String fileAttribute;
       try {
@@ -162,37 +174,49 @@ public class NewRepositoryFileHandler {
       } catch (EvalException e) {
         throw new RepositoryFunctionException(e, Transience.PERSISTENT);
       }
+      return fileAttribute;
+    }
+
+    private Label getFileAttributeAsLabel(Rule rule) throws RepositoryFunctionException {
+      Label label;
+      try {
+        // Parse a label
+        label = Label.parseAbsolute(getFileAttributeValue(rule));
+      } catch (LabelSyntaxException ex) {
+        throw new RepositoryFunctionException(
+            new EvalException(
+                rule.getLocation(),
+                String.format(
+                    "In %s the '%s' attribute does not specify a valid label: %s",
+                    rule, getFileAttrName(), ex.getMessage())),
+            Transience.PERSISTENT);
+      }
+      return label;
+    }
+
+    private FileValue getFileValue(Rule rule, Environment env)
+        throws RepositoryFunctionException, InterruptedException {
+      String fileAttribute = getFileAttributeValue(rule);
       RootedPath rootedFile;
 
       if (LabelValidator.isAbsolute(fileAttribute)) {
-        try {
-          // Parse a label
-          Label label = Label.parseAbsolute(fileAttribute);
-          SkyKey pkgSkyKey = PackageLookupValue.key(label.getPackageIdentifier());
-          PackageLookupValue pkgLookupValue = (PackageLookupValue) env.getValue(pkgSkyKey);
-          if (pkgLookupValue == null) {
-            return null;
-          }
-          if (!pkgLookupValue.packageExists()) {
-            throw new RepositoryFunctionException(
-                new EvalException(
-                    rule.getLocation(),
-                    "Unable to load package for " + fileAttribute + ": not found."),
-                Transience.PERSISTENT);
-          }
-
-          // And now for the file
-          Path packageRoot = pkgLookupValue.getRoot();
-          rootedFile = RootedPath.toRootedPath(packageRoot, label.toPathFragment());
-        } catch (LabelSyntaxException ex) {
+        Label label = getFileAttributeAsLabel(rule);
+        SkyKey pkgSkyKey = PackageLookupValue.key(label.getPackageIdentifier());
+        PackageLookupValue pkgLookupValue = (PackageLookupValue) env.getValue(pkgSkyKey);
+        if (pkgLookupValue == null) {
+          return null;
+        }
+        if (!pkgLookupValue.packageExists()) {
           throw new RepositoryFunctionException(
               new EvalException(
                   rule.getLocation(),
-                  String.format(
-                      "In %s the '%s' attribute does not specify a valid label: %s",
-                      rule, getFileAttrName(), ex.getMessage())),
+                  "Unable to load package for " + fileAttribute + ": not found."),
               Transience.PERSISTENT);
         }
+
+        // And now for the file
+        Path packageRoot = pkgLookupValue.getRoot();
+        rootedFile = RootedPath.toRootedPath(packageRoot, label.toPathFragment());
       } else {
         // TODO(dmarting): deprecate using a path for the workspace_file attribute.
         PathFragment file = PathFragment.create(fileAttribute);
diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java
index 16c29673ba..469eb61222 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java
@@ -15,11 +15,16 @@
 package com.google.devtools.build.lib.rules.repository;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
 import com.google.devtools.build.lib.analysis.BlazeDirectories;
 import com.google.devtools.build.lib.analysis.RuleDefinition;
 import com.google.devtools.build.lib.cmdline.Label;
+import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
+import com.google.devtools.build.lib.cmdline.LabelValidator;
 import com.google.devtools.build.lib.cmdline.RepositoryName;
 import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.events.Location;
 import com.google.devtools.build.lib.packages.BuildFileContainsErrorsException;
 import com.google.devtools.build.lib.packages.NoSuchPackageException;
 import com.google.devtools.build.lib.packages.Rule;
@@ -28,6 +33,7 @@ import com.google.devtools.build.lib.skyframe.ActionEnvironmentFunction;
 import com.google.devtools.build.lib.skyframe.FileSymlinkException;
 import com.google.devtools.build.lib.skyframe.FileValue;
 import com.google.devtools.build.lib.skyframe.InconsistentFilesystemException;
+import com.google.devtools.build.lib.skyframe.PackageLookupValue;
 import com.google.devtools.build.lib.skyframe.PackageLookupValue.BuildFileName;
 import com.google.devtools.build.lib.syntax.EvalException;
 import com.google.devtools.build.lib.syntax.Type;
@@ -118,6 +124,24 @@ public abstract class RepositoryFunction {
   }
 
   /**
+   * An exception thrown when a dependency is missing to notify the SkyFunction from an evaluation.
+   */
+  protected static class RepositoryMissingDependencyException extends EvalException {
+
+    RepositoryMissingDependencyException() {
+      super(Location.BUILTIN, "Internal exception");
+    }
+  }
+
+  /**
+   * repository functions can throw the result of this function to notify the RepositoryFunction
+   * that a dependency was missing and the evaluation of the function must be restarted.
+   */
+  public static EvalException restart() {
+    return new RepositoryMissingDependencyException();
+  }
+
+  /**
    * Fetch the remote repository represented by the given rule.
    *
    * <p>When this method is called, it has already been determined that the repository is stale and
@@ -152,6 +176,14 @@ public abstract class RepositoryFunction {
       Map<String, String> markerData)
       throws SkyFunctionException, InterruptedException;
 
+  @SuppressWarnings("unchecked")
+  private static Iterable<String> getEnviron(Rule rule) {
+    if (rule.isAttrDefined("$environ", Type.STRING_LIST)) {
+      return (Iterable<String>) rule.getAttributeContainer().getAttr("$environ");
+    }
+    return ImmutableList.of();
+  }
+
   /**
    * Verify the data provided by the marker file to check if a refetch is needed. Returns true if
    * the data is up to date and no refetch is needed and false if the data is obsolete and a refetch
@@ -159,10 +191,91 @@ public abstract class RepositoryFunction {
    */
   @Nullable
   public boolean verifyMarkerData(Rule rule, Map<String, String> markerData, Environment env)
+      throws InterruptedException, RepositoryFunctionException {
+    return verifyEnvironMarkerData(markerData, env, getEnviron(rule))
+        && verifyMarkerDataForFiles(rule, markerData, env);
+  }
+
+  private static boolean verifyLabelMarkerData(Rule rule, String key, String value, Environment env)
       throws InterruptedException {
+    Preconditions.checkArgument(key.startsWith("FILE:"));
+    try {
+      RootedPath rootedPath;
+      String fileKey = key.substring(5);
+      if (LabelValidator.isAbsolute(fileKey)) {
+        rootedPath = getRootedPathFromLabel(Label.parseAbsolute(fileKey), env);
+      } else {
+        // TODO(pcloudy): Removing checking absolute path, they should all be absolute label.
+        PathFragment filePathFragment = PathFragment.create(fileKey);
+        Path file = rule.getPackage().getPackageDirectory().getRelative(filePathFragment);
+        rootedPath =
+            RootedPath.toRootedPath(
+                file.getParentDirectory(), PathFragment.create(file.getBaseName()));
+      }
+
+      SkyKey fileSkyKey = FileValue.key(rootedPath);
+      FileValue fileValue =
+          (FileValue)
+              env.getValueOrThrow(
+                  fileSkyKey,
+                  IOException.class,
+                  FileSymlinkException.class,
+                  InconsistentFilesystemException.class);
+
+      if (fileValue == null || !fileValue.isFile()) {
+        return false;
+      }
+
+      return Objects.equals(value, Integer.toString(fileValue.realFileStateValue().hashCode()));
+    } catch (LabelSyntaxException e) {
+      throw new IllegalStateException(
+          "Key " + key + " is not a correct file key (should be in form FILE:label)", e);
+    } catch (IOException
+        | FileSymlinkException
+        | InconsistentFilesystemException
+        | EvalException e) {
+      // Consider those exception to be a cause for invalidation
+      return false;
+    }
+  }
+
+  static boolean verifyMarkerDataForFiles(
+      Rule rule, Map<String, String> markerData, Environment env) throws InterruptedException {
+    for (Map.Entry<String, String> entry : markerData.entrySet()) {
+      if (entry.getKey().startsWith("FILE:")) {
+        if (!verifyLabelMarkerData(rule, entry.getKey(), entry.getValue(), env)) {
+          return false;
+        }
+      }
+    }
     return true;
   }
 
+  public static RootedPath getRootedPathFromLabel(Label label, Environment env)
+      throws InterruptedException, EvalException {
+    // Look for package.
+    if (label.getPackageIdentifier().getRepository().isDefault()) {
+      try {
+        label = Label.create(label.getPackageIdentifier().makeAbsolute(), label.getName());
+      } catch (LabelSyntaxException e) {
+        throw new AssertionError(e); // Can't happen because the input label is valid
+      }
+    }
+    SkyKey pkgSkyKey = PackageLookupValue.key(label.getPackageIdentifier());
+    PackageLookupValue pkgLookupValue = (PackageLookupValue) env.getValue(pkgSkyKey);
+    if (pkgLookupValue == null) {
+      throw RepositoryFunction.restart();
+    }
+    if (!pkgLookupValue.packageExists()) {
+      throw new EvalException(
+          Location.BUILTIN, "Unable to load package for " + label + ": not found.");
+    }
+
+    // And now for the file
+    Path packageRoot = pkgLookupValue.getRoot();
+    return RootedPath.toRootedPath(packageRoot, label.toPathFragment());
+  }
+
   /**
    * A method that can be called from a implementation of
    * {@link #fetch(Rule, Path, BlazeDirectories, Environment, Map)} to declare a list of Skyframe