diff options
author | Klaus Aehlig <aehlig@google.com> | 2018-07-25 10:39:40 -0700 |
---|---|---|
committer | Copybara-Service <copybara-piper@google.com> | 2018-07-25 10:41:37 -0700 |
commit | 24d3a01b53a603059091a690e0bddb20ef5fbc98 (patch) | |
tree | b2e4d1f6a8aa30b3bed8bfbf847a8ca8e342d1f8 | |
parent | 2a8b6579c9535b649f2970307bc058895b880eb5 (diff) |
Support optional repository verification
Add an option to provide a file with a resolved value, that will be
used to verify that the repositories mentioned in this file produce
a correct directory tree.
RELNOTES: newly added options --experimental_repository_hash_file and
--experimental_verify_repository_rules allow to verify for repositories
the directory generated against pre-recorded hashes. See documentation
for those options.
Work towards #5660.
Change-Id: I2d8becb188d0fa51e890fb8f6139f321cca14b7b
PiperOrigin-RevId: 206016792
12 files changed, 372 insertions, 7 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index 2b05a506e1..e34830acf3 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -14,6 +14,8 @@ package com.google.devtools.build.lib.bazel; +import com.google.common.base.Optional; +import com.google.common.base.Strings; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -83,6 +85,8 @@ import com.google.devtools.build.lib.vfs.FileSystem; import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.Root; +import com.google.devtools.build.lib.vfs.RootedPath; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; import com.google.devtools.common.options.OptionsBase; @@ -90,6 +94,7 @@ import com.google.devtools.common.options.OptionsProvider; import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import javax.annotation.Nullable; @@ -110,6 +115,8 @@ public class BazelRepositoryModule extends BlazeModule { private final MutableSupplier<Map<String, String>> clientEnvironmentSupplier = new MutableSupplier<>(); private ImmutableMap<RepositoryName, PathFragment> overrides = ImmutableMap.of(); + private Optional<RootedPath> resolvedFile = Optional.<RootedPath>absent(); + private Set<String> outputVerificationRules = ImmutableSet.<String>of(); private FileSystem filesystem; public BazelRepositoryModule() { @@ -227,6 +234,8 @@ public class BazelRepositoryModule extends BlazeModule { clientEnvironmentSupplier.set(env.getActionClientEnv()); PackageCacheOptions pkgOptions = env.getOptions().getOptions(PackageCacheOptions.class); isFetch.set(pkgOptions != null && pkgOptions.fetch); + resolvedFile = Optional.<RootedPath>absent(); + outputVerificationRules = ImmutableSet.<String>of(); RepositoryOptions repoOptions = env.getOptions().getOptions(RepositoryOptions.class); if (repoOptions != null) { @@ -287,6 +296,19 @@ public class BazelRepositoryModule extends BlazeModule { } else { overrides = ImmutableMap.of(); } + + if (!Strings.isNullOrEmpty(repoOptions.repositoryHashFile)) { + resolvedFile = + Optional.of( + RootedPath.toRootedPath( + Root.absoluteRoot(filesystem), + filesystem.getPath(repoOptions.repositoryHashFile))); + } + + if (repoOptions.experimentalVerifyRepositoryRules != null) { + outputVerificationRules = + ImmutableSet.copyOf(repoOptions.experimentalVerifyRepositoryRules); + } } } @@ -294,6 +316,11 @@ public class BazelRepositoryModule extends BlazeModule { public ImmutableList<Injected> getPrecomputedValues() { return ImmutableList.of( PrecomputedValue.injected(RepositoryDelegatorFunction.REPOSITORY_OVERRIDES, overrides), + PrecomputedValue.injected( + RepositoryDelegatorFunction.RESOLVED_FILE_FOR_VERIFICATION, resolvedFile), + PrecomputedValue.injected( + RepositoryDelegatorFunction.OUTPUT_VERIFICATION_REPOSITORY_RULES, + outputVerificationRules), // That key will be reinjected by the sync command with a universally unique identifier. // Nevertheless, we need to provide a default value for other commands. PrecomputedValue.injected( diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java index 5b06cab1a3..021c436f9e 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java @@ -81,6 +81,28 @@ public class RepositoryOptions extends OptionsBase { ) public List<RepositoryOverride> repositoryOverrides; + @Option( + name = "experimental_repository_hash_file", + defaultValue = "", + documentationCategory = OptionDocumentationCategory.LOGGING, + effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, + help = + "If non-empty, specifies a file containing a resolved value, against which" + + " the repository directory hashes should be verified") + public String repositoryHashFile; + + @Option( + name = "experimental_verify_repository_rules", + allowMultiple = true, + defaultValue = "", + documentationCategory = OptionDocumentationCategory.LOGGING, + effectTags = {OptionEffectTag.AFFECTS_OUTPUTS}, + help = + "If list of repository rules for which the hash of the output directory should be" + + " verified, provided a file is specified by" + + " --experimental_respository_hash_file.") + public List<String> experimentalVerifyRepositoryRules; + /** * Converts from an equals-separated pair of strings into RepositoryName->PathFragment mapping. */ diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedEvent.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedEvent.java index 47801f3cfb..a3218e1222 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedEvent.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryResolvedEvent.java @@ -13,6 +13,13 @@ // limitations under the License. package com.google.devtools.build.lib.bazel.repository; +import static com.google.devtools.build.lib.rules.repository.ResolvedHashesFunction.ATTRIBUTES; +import static com.google.devtools.build.lib.rules.repository.ResolvedHashesFunction.ORIGINAL_ATTRIBUTES; +import static com.google.devtools.build.lib.rules.repository.ResolvedHashesFunction.ORIGINAL_RULE_CLASS; +import static com.google.devtools.build.lib.rules.repository.ResolvedHashesFunction.OUTPUT_TREE_HASH; +import static com.google.devtools.build.lib.rules.repository.ResolvedHashesFunction.REPOSITORIES; +import static com.google.devtools.build.lib.rules.repository.ResolvedHashesFunction.RULE_CLASS; + import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.events.ExtendedEventHandler.ProgressLike; @@ -29,12 +36,6 @@ import java.util.Map; * Event indicating that a repository rule was executed, together with the return value of the rule. */ public class RepositoryResolvedEvent implements ProgressLike { - public static final String ORIGINAL_RULE_CLASS = "original_rule_class"; - public static final String ORIGINAL_ATTRIBUTES = "original_attributes"; - public static final String RULE_CLASS = "rule_class"; - public static final String ATTRIBUTES = "attributes"; - public static final String OUTPUT_TREE_HASH = "output_tree_hash"; - public static final String REPOSITORIES = "repositories"; /** * The entry for WORSPACE.resolved corresponding to that rule invocation. diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryFunction.java index a4131bdbc2..196921d226 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryFunction.java @@ -23,8 +23,10 @@ import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.packages.Rule; +import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction; import com.google.devtools.build.lib.rules.repository.RepositoryDirectoryValue; import com.google.devtools.build.lib.rules.repository.RepositoryFunction; +import com.google.devtools.build.lib.rules.repository.ResolvedHashesValue; import com.google.devtools.build.lib.skyframe.PrecomputedValue; import com.google.devtools.build.lib.syntax.BaseFunction; import com.google.devtools.build.lib.syntax.EvalException; @@ -37,6 +39,7 @@ import com.google.devtools.build.skyframe.SkyFunction.Environment; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; import java.io.IOException; import java.util.Map; +import java.util.Set; import javax.annotation.Nullable; /** @@ -63,6 +66,19 @@ public class SkylarkRepositoryFunction extends RepositoryFunction { if (skylarkSemantics == null) { return null; } + + Set<String> verificationRules = + RepositoryDelegatorFunction.OUTPUT_VERIFICATION_REPOSITORY_RULES.get(env); + if (verificationRules == null) { + return null; + } + ResolvedHashesValue resolvedHashesValue = + (ResolvedHashesValue) env.getValue(ResolvedHashesValue.key()); + if (resolvedHashesValue == null) { + return null; + } + Map<String, String> resolvedHashes = resolvedHashesValue.getHashes(); + try (Mutability mutability = Mutability.create("skylark repository")) { com.google.devtools.build.lib.syntax.Environment buildEnv = com.google.devtools.build.lib.syntax.Environment.builder(mutability) @@ -105,6 +121,27 @@ public class SkylarkRepositoryFunction extends RepositoryFunction { env.getListener() .handle(Event.info("Repository rule '" + rule.getName() + "' returned: " + retValue)); } + + String ruleClass = + rule.getRuleClassObject().getRuleDefinitionEnvironmentLabel() + "%" + rule.getRuleClass(); + if (verificationRules.contains(ruleClass)) { + String expectedHash = resolvedHashes.get(rule.getName()); + if (expectedHash != null) { + try { + String actualHash = outputDirectory.getDirectoryDigest(); + if (!expectedHash.equals(actualHash)) { + throw new RepositoryFunctionException( + new IOException( + rule + " failed to create a directory with expected hash " + expectedHash), + Transience.PERSISTENT); + } + } catch (IOException e) { + throw new RepositoryFunctionException( + new IOException("Rule failed to produce a directory with computable hash", e), + Transience.PERSISTENT); + } + } + } env.getListener() .post( new RepositoryResolvedEvent( diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java index a590c0bfb6..19271ee179 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java @@ -15,6 +15,7 @@ package com.google.devtools.build.lib.rules.repository; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Optional; import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.actions.FileValue; import com.google.devtools.build.lib.analysis.BlazeDirectories; @@ -32,6 +33,7 @@ import com.google.devtools.build.lib.util.Fingerprint; import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.lib.vfs.RootedPath; import com.google.devtools.build.skyframe.SkyFunction; import com.google.devtools.build.skyframe.SkyFunctionException; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; @@ -40,6 +42,7 @@ import com.google.devtools.build.skyframe.SkyValue; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Map; +import java.util.Set; import java.util.TreeMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; @@ -60,6 +63,13 @@ public final class RepositoryDelegatorFunction implements SkyFunction { new Precomputed<>( PrecomputedValue.Key.create("dependency_for_unconditional_repository_fetching")); + public static final Precomputed<Optional<RootedPath>> RESOLVED_FILE_FOR_VERIFICATION = + new Precomputed<>( + PrecomputedValue.Key.create("resolved_file_for_external_repository_verification")); + + public static final Precomputed<Set<String>> OUTPUT_VERIFICATION_REPOSITORY_RULES = + new Precomputed<>(PrecomputedValue.Key.create("output_verification_repository_rules")); + public static final String DONT_FETCH_UNCONDITIONALLY = ""; // The marker file version is inject in the rule key digest so the rule key is always different diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/ResolvedHashesFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/ResolvedHashesFunction.java new file mode 100644 index 0000000000..71d086a11b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/repository/ResolvedHashesFunction.java @@ -0,0 +1,168 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.repository; + +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.FileValue; +import com.google.devtools.build.lib.cmdline.Label; +import com.google.devtools.build.lib.packages.BazelLibrary; +import com.google.devtools.build.lib.packages.BuildFileContainsErrorsException; +import com.google.devtools.build.lib.packages.NoSuchThingException; +import com.google.devtools.build.lib.skyframe.PrecomputedValue; +import com.google.devtools.build.lib.syntax.BuildFileAST; +import com.google.devtools.build.lib.syntax.Environment; +import com.google.devtools.build.lib.syntax.Mutability; +import com.google.devtools.build.lib.syntax.ParserInputSource; +import com.google.devtools.build.lib.syntax.SkylarkSemantics; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import javax.annotation.Nullable; + +/** + * Value of output hashes for the repositories specified in the resolved file designated for this + * purpose. + */ +public class ResolvedHashesFunction implements SkyFunction { + public static final String ORIGINAL_RULE_CLASS = "original_rule_class"; + public static final String ORIGINAL_ATTRIBUTES = "original_attributes"; + public static final String RULE_CLASS = "rule_class"; + public static final String ATTRIBUTES = "attributes"; + public static final String OUTPUT_TREE_HASH = "output_tree_hash"; + public static final String REPOSITORIES = "repositories"; + + @Override + @Nullable + public SkyValue compute(SkyKey skyKey, Environment env) + throws InterruptedException, SkyFunctionException { + + Optional<RootedPath> resolvedFile = + RepositoryDelegatorFunction.RESOLVED_FILE_FOR_VERIFICATION.get(env); + if (resolvedFile == null) { + return null; + } + if (!resolvedFile.isPresent()) { + return new ResolvedHashesValue(ImmutableMap.<String, String>of()); + } + SkylarkSemantics skylarkSemantics = PrecomputedValue.SKYLARK_SEMANTICS.get(env); + if (skylarkSemantics == null) { + return null; + } + FileValue resolvedFileValue = (FileValue) env.getValue(FileValue.key(resolvedFile.get())); + if (resolvedFileValue == null) { + return null; + } + try { + if (!resolvedFileValue.exists()) { + throw new ResolvedHashesFunctionException( + new NoSuchThingException( + "Specified file for resolved hashes '" + resolvedFile.get() + "' not found.")); + } else { + byte[] bytes = + FileSystemUtils.readWithKnownFileSize( + resolvedFile.get().asPath(), resolvedFile.get().asPath().getFileSize()); + BuildFileAST ast = + BuildFileAST.parseSkylarkFile( + ParserInputSource.create(bytes, resolvedFile.get().asPath().asFragment()), + env.getListener()); + if (ast.containsErrors()) { + throw new ResolvedHashesFunctionException( + new BuildFileContainsErrorsException( + Label.EXTERNAL_PACKAGE_IDENTIFIER, + "Failed to parse file resolved file for hash verification")); + } + com.google.devtools.build.lib.syntax.Environment resolvedEnvironment; + try (Mutability mutability = Mutability.create("resolved hashes %s", resolvedFile.get())) { + resolvedEnvironment = + com.google.devtools.build.lib.syntax.Environment.builder(mutability) + .setSemantics(skylarkSemantics) + .setGlobals(BazelLibrary.GLOBALS) + .build(); + if (!ast.exec(resolvedEnvironment, env.getListener())) { + throw new ResolvedHashesFunctionException( + new BuildFileContainsErrorsException( + Label.EXTERNAL_PACKAGE_IDENTIFIER, + "Failed to evaluate resolved file for hash verification")); + } + } + Object resolved = resolvedEnvironment.lookup("resolved"); + if (resolved == null) { + throw new ResolvedHashesFunctionException( + new BuildFileContainsErrorsException( + Label.EXTERNAL_PACKAGE_IDENTIFIER, + "Symbol 'resolved' not exported in file for hash verification")); + } + if (!(resolved instanceof List)) { + throw new ResolvedHashesFunctionException( + new BuildFileContainsErrorsException( + Label.EXTERNAL_PACKAGE_IDENTIFIER, + "Symbol 'resolved' not a list in file for hash verification")); + } + // Collect the hases in a mutable map, to be able to detect duplicates and + // only take the first entry, following the "maybe pattern" of external repositories, + // adding a repository only if not already present. + Map<String, String> hashes = new LinkedHashMap<String, String>(); + for (Object entry : (List) resolved) { + if (entry instanceof Map) { + Object repositories = ((Map) entry).get(REPOSITORIES); + if (repositories instanceof List) { + for (Object repo : (List) repositories) { + if (repo instanceof Map) { + Object hash = ((Map) repo).get(OUTPUT_TREE_HASH); + Object attributes = ((Map) repo).get(ATTRIBUTES); + if (attributes instanceof Map) { + Object name = ((Map) attributes).get("name"); + if ((name instanceof String) && (hash instanceof String)) { + if (!hashes.containsKey((String) name)) { + hashes.put((String) name, (String) hash); + } + } + } + } + } + } + } + } + return new ResolvedHashesValue(ImmutableMap.copyOf(hashes)); + } + } catch (IOException e) { + throw new ResolvedHashesFunctionException(e); + } + } + + @Override + @Nullable + public String extractTag(SkyKey skyKey) { + return null; + } + + private static final class ResolvedHashesFunctionException extends SkyFunctionException { + ResolvedHashesFunctionException(IOException e) { + super(e, SkyFunctionException.Transience.PERSISTENT); + } + + ResolvedHashesFunctionException(NoSuchThingException e) { + super(e, SkyFunctionException.Transience.PERSISTENT); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/ResolvedHashesValue.java b/src/main/java/com/google/devtools/build/lib/rules/repository/ResolvedHashesValue.java new file mode 100644 index 0000000000..17be36885e --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/rules/repository/ResolvedHashesValue.java @@ -0,0 +1,58 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.repository; + +import com.google.devtools.build.lib.skyframe.SkyFunctions; +import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; +import java.util.Map; + +/** The list of expected hashes of the directories */ +public class ResolvedHashesValue implements SkyValue { + @AutoCodec @AutoCodec.VisibleForSerialization + static final SkyKey KEY = () -> SkyFunctions.RESOLVED_HASH_VALUES; + + private final Map<String, String> hashes; + + ResolvedHashesValue(Map<String, String> hashes) { + this.hashes = hashes; + } + + public Map<String, String> getHashes() { + return hashes; + } + + @Override + public int hashCode() { + return hashes.hashCode(); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof ResolvedHashesValue)) { + return false; + } + return this.getHashes().equals(((ResolvedHashesValue) other).getHashes()); + } + + /** Returns the (singleton) {@link SkyKey} for {@link ResolvedHashesValue}s. */ + public static SkyKey key() { + return KEY; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java index be40e0619f..09bdcd3ba5 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java @@ -134,6 +134,8 @@ public final class SkyFunctions { SkyFunctionName.createHermetic("TOOLCHAIN_RESOLUTION"); public static final SkyFunctionName REPOSITORY_MAPPING = SkyFunctionName.createHermetic("REPOSITORY_MAPPING"); + public static final SkyFunctionName RESOLVED_HASH_VALUES = + SkyFunctionName.createHermetic("RESOLVED_HASH_VALUES"); public static Predicate<SkyKey> isSkyFunction(final SkyFunctionName functionName) { return new Predicate<SkyKey>() { diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java index 040306b589..8094c71eef 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java @@ -114,6 +114,7 @@ import com.google.devtools.build.lib.pkgcache.TransitivePackageLoader; import com.google.devtools.build.lib.profiler.AutoProfiler; import com.google.devtools.build.lib.profiler.Profiler; import com.google.devtools.build.lib.profiler.SilentCloseable; +import com.google.devtools.build.lib.rules.repository.ResolvedHashesFunction; import com.google.devtools.build.lib.skyframe.AspectValue.AspectValueKey; import com.google.devtools.build.lib.skyframe.DirtinessCheckerUtils.FileDirtinessChecker; import com.google.devtools.build.lib.skyframe.ExternalFilesHelper.ExternalFileAction; @@ -514,6 +515,7 @@ public abstract class SkyframeExecutor implements WalkableGraphFactory { map.put(SkyFunctions.REGISTERED_TOOLCHAINS, new RegisteredToolchainsFunction()); map.put(SkyFunctions.TOOLCHAIN_RESOLUTION, new ToolchainResolutionFunction()); map.put(SkyFunctions.REPOSITORY_MAPPING, new RepositoryMappingFunction()); + map.put(SkyFunctions.RESOLVED_HASH_VALUES, new ResolvedHashesFunction()); map.putAll(extraSkyFunctions); return map.build(); } diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java b/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java index 739437d231..00acc720f7 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java @@ -19,6 +19,7 @@ import static com.google.devtools.build.lib.actions.util.ActionsTestUtil.getFirs import static org.junit.Assert.fail; import com.google.common.base.Function; +import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.collect.ImmutableList; @@ -149,6 +150,7 @@ import com.google.devtools.build.lib.vfs.ModifiedFileSet; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.lib.vfs.Root; +import com.google.devtools.build.lib.vfs.RootedPath; import com.google.devtools.build.skyframe.ErrorInfo; import com.google.devtools.build.skyframe.MemoizingEvaluator; import com.google.devtools.build.skyframe.SkyFunction; @@ -405,6 +407,14 @@ public abstract class BuildViewTestCase extends FoundationTestCase { ImmutableMap.<String, String>of(), tsgm); skyframeExecutor.setDeletedPackages(ImmutableSet.copyOf(packageCacheOptions.getDeletedPackages())); + skyframeExecutor.injectExtraPrecomputedValues( + ImmutableList.of( + PrecomputedValue.injected( + RepositoryDelegatorFunction.OUTPUT_VERIFICATION_REPOSITORY_RULES, + ImmutableSet.<String>of()), + PrecomputedValue.injected( + RepositoryDelegatorFunction.RESOLVED_FILE_FOR_VERIFICATION, + Optional.<RootedPath>absent()))); } protected void setPackageCacheOptions(String... options) throws Exception { diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD index 0d977f2aa5..e8fd9c09a6 100644 --- a/src/test/shell/bazel/BUILD +++ b/src/test/shell/bazel/BUILD @@ -421,7 +421,7 @@ sh_test( size = "medium", srcs = ["workspace_resolved_test.sh"], data = [":test-deps"], - shard_count = 9, + shard_count = 10, tags = ["no_windows"], ) diff --git a/src/test/shell/bazel/workspace_resolved_test.sh b/src/test/shell/bazel/workspace_resolved_test.sh index 73fa099e38..cfcd485944 100755 --- a/src/test/shell/bazel/workspace_resolved_test.sh +++ b/src/test/shell/bazel/workspace_resolved_test.sh @@ -514,4 +514,32 @@ EOF diff hashA.txt hashB.txt || fail "Expected hash to be reproducible" } +test_non_reproducibility_detected() { + # Verify that a non-reproducible rule is detected by hash verification + mkdir repo + cd repo + touch BUILD + cat > rule.bzl <<'EOF' +def _time_rule_impl(ctx): + ctx.execute(["bash", "-c", "date +%s > timestamp"]) + +time_rule = repository_rule( + implementation = _time_rule_impl, + attrs = {}, +) +EOF + cat > WORKSPACE <<'EOF' +load("//:rule.bzl", "time_rule") + +time_rule(name="timestamprepo") +EOF + + bazel sync --experimental_repository_resolved_file=resolved.bzl + sync; sleep 10 + bazel sync --experimental_repository_hash_file=`pwd`/resolved.bzl \ + --experimental_verify_repository_rules='//:rule.bzl%time_rule' \ + > "${TEST_log}" 2>&1 && fail "expected failure" || : + expect_log "timestamprepo.*hash" +} + run_suite "workspace_resolved_test tests" |