Support optional repository verification

Add an option to provide a file with a resolved value, that will be
used to verify that the repositories mentioned in this file produce
a correct directory tree.

RELNOTES: newly added options --experimental_repository_hash_file and
  --experimental_verify_repository_rules allow to verify for repositories
  the directory generated against pre-recorded hashes. See documentation
  for those options.

Work towards #5660.

Change-Id: I2d8becb188d0fa51e890fb8f6139f321cca14b7b
PiperOrigin-RevId: 206016792

4 files changed, 93 insertions, 6 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
index 2b05a506e1..e34830acf3 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java
@@ -14,6 +14,8 @@
 
 package com.google.devtools.build.lib.bazel;
 
+import com.google.common.base.Optional;
+import com.google.common.base.Strings;
 import com.google.common.base.Supplier;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -83,6 +85,8 @@ import com.google.devtools.build.lib.vfs.FileSystem;
 import com.google.devtools.build.lib.vfs.FileSystemUtils;
 import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.Root;
+import com.google.devtools.build.lib.vfs.RootedPath;
 import com.google.devtools.build.skyframe.SkyKey;
 import com.google.devtools.build.skyframe.SkyValue;
 import com.google.devtools.common.options.OptionsBase;
@@ -90,6 +94,7 @@ import com.google.devtools.common.options.OptionsProvider;
 import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.stream.Collectors;
 import javax.annotation.Nullable;
@@ -110,6 +115,8 @@ public class BazelRepositoryModule extends BlazeModule {
   private final MutableSupplier<Map<String, String>> clientEnvironmentSupplier =
       new MutableSupplier<>();
   private ImmutableMap<RepositoryName, PathFragment> overrides = ImmutableMap.of();
+  private Optional<RootedPath> resolvedFile = Optional.<RootedPath>absent();
+  private Set<String> outputVerificationRules = ImmutableSet.<String>of();
   private FileSystem filesystem;
 
   public BazelRepositoryModule() {
@@ -227,6 +234,8 @@ public class BazelRepositoryModule extends BlazeModule {
     clientEnvironmentSupplier.set(env.getActionClientEnv());
     PackageCacheOptions pkgOptions = env.getOptions().getOptions(PackageCacheOptions.class);
     isFetch.set(pkgOptions != null && pkgOptions.fetch);
+    resolvedFile = Optional.<RootedPath>absent();
+    outputVerificationRules = ImmutableSet.<String>of();
 
     RepositoryOptions repoOptions = env.getOptions().getOptions(RepositoryOptions.class);
     if (repoOptions != null) {
@@ -287,6 +296,19 @@ public class BazelRepositoryModule extends BlazeModule {
       } else {
         overrides = ImmutableMap.of();
       }
+
+      if (!Strings.isNullOrEmpty(repoOptions.repositoryHashFile)) {
+        resolvedFile =
+            Optional.of(
+                RootedPath.toRootedPath(
+                    Root.absoluteRoot(filesystem),
+                    filesystem.getPath(repoOptions.repositoryHashFile)));
+      }
+
+      if (repoOptions.experimentalVerifyRepositoryRules != null) {
+        outputVerificationRules =
+            ImmutableSet.copyOf(repoOptions.experimentalVerifyRepositoryRules);
+      }
     }
   }
 
@@ -294,6 +316,11 @@ public class BazelRepositoryModule extends BlazeModule {
   public ImmutableList<Injected> getPrecomputedValues() {
     return ImmutableList.of(
         PrecomputedValue.injected(RepositoryDelegatorFunction.REPOSITORY_OVERRIDES, overrides),
+        PrecomputedValue.injected(
+            RepositoryDelegatorFunction.RESOLVED_FILE_FOR_VERIFICATION, resolvedFile),
+        PrecomputedValue.injected(
+            RepositoryDelegatorFunction.OUTPUT_VERIFICATION_REPOSITORY_RULES,
+            outputVerificationRules),
         // That key will be reinjected by the sync command with a universally unique identifier.
         // Nevertheless, we need to provide a default value for other commands.
         PrecomputedValue.injected(
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java
index 5b06cab1a3..021c436f9e 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java
@@ -81,6 +81,28 @@ public class RepositoryOptions extends OptionsBase {
   )
   public List<RepositoryOverride> repositoryOverrides;
 
+  @Option(
+      name = "experimental_repository_hash_file",
+      defaultValue = "",
+      documentationCategory = OptionDocumentationCategory.LOGGING,
+      effectTags = {OptionEffectTag.AFFECTS_OUTPUTS},
+      help =
+          "If non-empty, specifies a file containing a resolved value, against which"
+              + " the repository directory hashes should be verified")
+  public String repositoryHashFile;
+
+  @Option(
+      name = "experimental_verify_repository_rules",
+      allowMultiple = true,
+      defaultValue = "",
+      documentationCategory = OptionDocumentationCategory.LOGGING,
+      effectTags = {OptionEffectTag.AFFECTS_OUTPUTS},
+      help =
+          "If list of repository rules for which the hash of the output directory should be"
+              + " verified, provided a file is specified by"
+              + " --experimental_respository_hash_file.")
+  public List<String> experimentalVerifyRepositoryRules;
+
   /**
    * Converts from an equals-separated pair of strings into RepositoryName->PathFragment mapping.
    */
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedEvent.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedEvent.java
index 47801f3cfb..a3218e1222 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedEvent.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedEvent.java
@@ -13,6 +13,13 @@
 // limitations under the License.
 package com.google.devtools.build.lib.bazel.repository;
 
+import static com.google.devtools.build.lib.rules.repository.ResolvedHashesFunction.ATTRIBUTES;
+import static com.google.devtools.build.lib.rules.repository.ResolvedHashesFunction.ORIGINAL_ATTRIBUTES;
+import static com.google.devtools.build.lib.rules.repository.ResolvedHashesFunction.ORIGINAL_RULE_CLASS;
+import static com.google.devtools.build.lib.rules.repository.ResolvedHashesFunction.OUTPUT_TREE_HASH;
+import static com.google.devtools.build.lib.rules.repository.ResolvedHashesFunction.REPOSITORIES;
+import static com.google.devtools.build.lib.rules.repository.ResolvedHashesFunction.RULE_CLASS;
+
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.devtools.build.lib.events.ExtendedEventHandler.ProgressLike;
@@ -29,12 +36,6 @@ import java.util.Map;
  * Event indicating that a repository rule was executed, together with the return value of the rule.
  */
 public class RepositoryResolvedEvent implements ProgressLike {
-  public static final String ORIGINAL_RULE_CLASS = "original_rule_class";
-  public static final String ORIGINAL_ATTRIBUTES = "original_attributes";
-  public static final String RULE_CLASS = "rule_class";
-  public static final String ATTRIBUTES = "attributes";
-  public static final String OUTPUT_TREE_HASH = "output_tree_hash";
-  public static final String REPOSITORIES = "repositories";
 
   /**
    * The entry for WORSPACE.resolved corresponding to that rule invocation.
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryFunction.java
index a4131bdbc2..196921d226 100644
--- a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryFunction.java
@@ -23,8 +23,10 @@ import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader;
 import com.google.devtools.build.lib.cmdline.Label;
 import com.google.devtools.build.lib.events.Event;
 import com.google.devtools.build.lib.packages.Rule;
+import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction;
 import com.google.devtools.build.lib.rules.repository.RepositoryDirectoryValue;
 import com.google.devtools.build.lib.rules.repository.RepositoryFunction;
+import com.google.devtools.build.lib.rules.repository.ResolvedHashesValue;
 import com.google.devtools.build.lib.skyframe.PrecomputedValue;
 import com.google.devtools.build.lib.syntax.BaseFunction;
 import com.google.devtools.build.lib.syntax.EvalException;
@@ -37,6 +39,7 @@ import com.google.devtools.build.skyframe.SkyFunction.Environment;
 import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
 import java.io.IOException;
 import java.util.Map;
+import java.util.Set;
 import javax.annotation.Nullable;
 
 /**
@@ -63,6 +66,19 @@ public class SkylarkRepositoryFunction extends RepositoryFunction {
     if (skylarkSemantics == null) {
       return null;
     }
+
+    Set<String> verificationRules =
+        RepositoryDelegatorFunction.OUTPUT_VERIFICATION_REPOSITORY_RULES.get(env);
+    if (verificationRules == null) {
+      return null;
+    }
+    ResolvedHashesValue resolvedHashesValue =
+        (ResolvedHashesValue) env.getValue(ResolvedHashesValue.key());
+    if (resolvedHashesValue == null) {
+      return null;
+    }
+    Map<String, String> resolvedHashes = resolvedHashesValue.getHashes();
+
     try (Mutability mutability = Mutability.create("skylark repository")) {
       com.google.devtools.build.lib.syntax.Environment buildEnv =
           com.google.devtools.build.lib.syntax.Environment.builder(mutability)
@@ -105,6 +121,27 @@ public class SkylarkRepositoryFunction extends RepositoryFunction {
         env.getListener()
             .handle(Event.info("Repository rule '" + rule.getName() + "' returned: " + retValue));
       }
+
+      String ruleClass =
+          rule.getRuleClassObject().getRuleDefinitionEnvironmentLabel() + "%" + rule.getRuleClass();
+      if (verificationRules.contains(ruleClass)) {
+        String expectedHash = resolvedHashes.get(rule.getName());
+        if (expectedHash != null) {
+          try {
+            String actualHash = outputDirectory.getDirectoryDigest();
+            if (!expectedHash.equals(actualHash)) {
+              throw new RepositoryFunctionException(
+                  new IOException(
+                      rule + " failed to create a directory with expected hash " + expectedHash),
+                  Transience.PERSISTENT);
+            }
+          } catch (IOException e) {
+            throw new RepositoryFunctionException(
+                new IOException("Rule failed to produce a directory with computable hash", e),
+                Transience.PERSISTENT);
+          }
+        }
+      }
       env.getListener()
           .post(
               new RepositoryResolvedEvent(