diff options
22 files changed, 448 insertions, 27 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/BUILD b/src/main/java/com/google/devtools/build/lib/BUILD index a3707e7b65..4c4a80e983 100644 --- a/src/main/java/com/google/devtools/build/lib/BUILD +++ b/src/main/java/com/google/devtools/build/lib/BUILD @@ -752,6 +752,7 @@ java_library( "//src/main/java/com/google/devtools/common/options", "//third_party:aether", "//third_party:apache_commons_compress", + "//third_party:auto_value", "//third_party:guava", "//third_party:jgit", "//third_party:jsr305", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index 4c7388031f..91b2226c49 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -16,6 +16,8 @@ package com.google.devtools.build.lib.bazel; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; import com.google.devtools.build.lib.analysis.RuleDefinition; @@ -31,6 +33,7 @@ import com.google.devtools.build.lib.bazel.repository.MavenServerRepositoryFunct import com.google.devtools.build.lib.bazel.repository.NewGitRepositoryFunction; import com.google.devtools.build.lib.bazel.repository.NewHttpArchiveFunction; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.RepositoryOverride; import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache; import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader; import com.google.devtools.build.lib.bazel.repository.skylark.SkylarkRepositoryFunction; @@ -47,6 +50,7 @@ import com.google.devtools.build.lib.bazel.rules.workspace.MavenJarRule; import com.google.devtools.build.lib.bazel.rules.workspace.MavenServerRule; import com.google.devtools.build.lib.bazel.rules.workspace.NewGitRepositoryRule; import com.google.devtools.build.lib.bazel.rules.workspace.NewHttpArchiveRule; +import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.pkgcache.PackageCacheOptions; import com.google.devtools.build.lib.rules.repository.LocalRepositoryFunction; import com.google.devtools.build.lib.rules.repository.LocalRepositoryRule; @@ -61,12 +65,15 @@ import com.google.devtools.build.lib.runtime.Command; import com.google.devtools.build.lib.runtime.CommandEnvironment; import com.google.devtools.build.lib.runtime.ServerBuilder; import com.google.devtools.build.lib.runtime.WorkspaceBuilder; +import com.google.devtools.build.lib.skyframe.PrecomputedValue; +import com.google.devtools.build.lib.skyframe.PrecomputedValue.Injected; import com.google.devtools.build.lib.skyframe.SkyFunctions; import com.google.devtools.build.lib.skyframe.SkyValueDirtinessChecker; import com.google.devtools.build.lib.util.AbruptExitException; import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor; import com.google.devtools.build.lib.vfs.FileSystem; import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; import com.google.devtools.common.options.OptionsBase; @@ -88,6 +95,7 @@ public class BazelRepositoryModule extends BlazeModule { private final RepositoryCache repositoryCache = new RepositoryCache(); private final HttpDownloader httpDownloader = new HttpDownloader(repositoryCache); private final MavenDownloader mavenDownloader = new MavenDownloader(repositoryCache); + private ImmutableMap<RepositoryName, PathFragment> overrides = ImmutableMap.of(); private FileSystem filesystem; public BazelRepositoryModule() { @@ -152,7 +160,6 @@ public class BazelRepositoryModule extends BlazeModule { builder.addSkyFunction(SkyFunctions.REPOSITORY, new RepositoryLoaderFunction()); builder.addSkyFunction(SkyFunctions.REPOSITORY_DIRECTORY, delegator); builder.addSkyFunction(MavenServerFunction.NAME, new MavenServerFunction()); - filesystem = directories.getFileSystem(); } @@ -184,17 +191,37 @@ public class BazelRepositoryModule extends BlazeModule { } else { repositoryCache.setRepositoryCachePath(null); } + + if (repoOptions.repositoryOverrides != null) { + ImmutableMap.Builder<RepositoryName, PathFragment> builder = ImmutableMap.builder(); + for (RepositoryOverride override : repoOptions.repositoryOverrides) { + builder.put(override.repositoryName(), override.path()); + } + ImmutableMap<RepositoryName, PathFragment> newOverrides = builder.build(); + if (!Maps.difference(overrides, newOverrides).areEqual()) { + overrides = newOverrides; + } + } else { + overrides = ImmutableMap.of(); + } } } @Override + public ImmutableList<Injected> getPrecomputedValues() { + return ImmutableList.of( + PrecomputedValue.injected( + RepositoryDelegatorFunction.REPOSITORY_OVERRIDES, overrides)); + } + + @Override public void beforeCommand(Command command, CommandEnvironment env) throws AbruptExitException { delegator.setClientEnvironment(env.getActionClientEnv()); } @Override public Iterable<Class<? extends OptionsBase>> getCommandOptions(Command command) { - return "fetch".equals(command.name()) || "build".equals(command.name()) + return ImmutableSet.of("fetch", "build", "query").contains(command.name()) ? ImmutableList.<Class<? extends OptionsBase>>of(RepositoryOptions.class) : ImmutableList.<Class<? extends OptionsBase>>of(); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java index 3fdf939f4c..0072f10ecb 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java @@ -14,11 +14,17 @@ package com.google.devtools.build.lib.bazel.repository; +import com.google.auto.value.AutoValue; +import com.google.devtools.build.lib.cmdline.LabelSyntaxException; +import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.util.OptionsUtils; import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.common.options.Converter; import com.google.devtools.common.options.Option; import com.google.devtools.common.options.OptionsBase; import com.google.devtools.common.options.OptionsParser.OptionUsageRestrictions; +import com.google.devtools.common.options.OptionsParsingException; +import java.util.List; /** * Command-line options for repositories. @@ -36,4 +42,54 @@ public class RepositoryOptions extends OptionsBase { ) public PathFragment experimentalRepositoryCache; -}
\ No newline at end of file + @Option(name = "override_repository", + defaultValue = "null", + allowMultiple = true, + converter = RepositoryOverrideConverter.class, + help = "Overrides a repository with a local directory.") + public List<RepositoryOverride> repositoryOverrides; + + /** + * Converts from an equals-separated pair of strings into RepositoryName->PathFragment mapping. + */ + public static class RepositoryOverrideConverter implements Converter<RepositoryOverride> { + + @Override + public RepositoryOverride convert(String input) throws OptionsParsingException { + String[] pieces = input.split("="); + if (pieces.length != 2) { + throw new OptionsParsingException( + "Repository overrides must be of the form 'repository-name=path'", input); + } + PathFragment path = PathFragment.create(pieces[1]); + if (!path.isAbsolute()) { + throw new OptionsParsingException( + "Repository override directory must be an absolute path", input); + } + try { + return RepositoryOverride.create(RepositoryName.create("@" + pieces[0]), path); + } catch (LabelSyntaxException e) { + throw new OptionsParsingException("Invalid repository name given to override", input); + } + } + + @Override + public String getTypeDescription() { + return "an equals-separated mapping of repository name to path"; + } + } + + /** + * A repository override, represented by a name and an absolute path to a repository. + */ + @AutoValue + public abstract static class RepositoryOverride { + + private static RepositoryOverride create(RepositoryName repositoryName, PathFragment path) { + return new AutoValue_RepositoryOptions_RepositoryOverride(repositoryName, path); + } + + public abstract RepositoryName repositoryName(); + public abstract PathFragment path(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/LocalRepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/LocalRepositoryFunction.java index ee387f735d..916569c4cb 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/repository/LocalRepositoryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/repository/LocalRepositoryFunction.java @@ -40,14 +40,20 @@ public class LocalRepositoryFunction extends RepositoryFunction { BlazeDirectories directories, Environment env, Map<String, String> markerData) throws InterruptedException, RepositoryFunctionException { PathFragment pathFragment = RepositoryFunction.getTargetPath(rule, directories.getWorkspace()); + return LocalRepositoryFunction.symlink(outputDirectory, pathFragment, env); + } + + public static RepositoryDirectoryValue.Builder symlink( + Path source, PathFragment destination, Environment env) + throws RepositoryFunctionException, InterruptedException { try { - outputDirectory.createSymbolicLink(pathFragment); + source.createSymbolicLink(destination); } catch (IOException e) { throw new RepositoryFunctionException( - new IOException("Could not create symlink to repository " + pathFragment + ": " + new IOException("Could not create symlink to repository " + destination + ": " + e.getMessage(), e), Transience.TRANSIENT); } - FileValue repositoryValue = getRepositoryDirectory(outputDirectory, env); + FileValue repositoryValue = getRepositoryDirectory(source, env); if (repositoryValue == null) { // TODO(bazel-team): If this returns null, we unnecessarily recreate the symlink above on the // second execution. @@ -56,10 +62,10 @@ public class LocalRepositoryFunction extends RepositoryFunction { if (!repositoryValue.isDirectory()) { throw new RepositoryFunctionException( - new IOException(rule + " must specify an existing directory"), Transience.TRANSIENT); + new IOException(source + " must be an existing directory"), Transience.TRANSIENT); } - return RepositoryDirectoryValue.builder().setPath(outputDirectory); + return RepositoryDirectoryValue.builder().setPath(source); } @Override diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java index 771231b72b..259950beda 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java @@ -25,10 +25,13 @@ import com.google.devtools.build.lib.packages.RuleFormatter; import com.google.devtools.build.lib.rules.repository.RepositoryFunction.RepositoryFunctionException; import com.google.devtools.build.lib.skyframe.FileValue; import com.google.devtools.build.lib.skyframe.PrecomputedValue; +import com.google.devtools.build.lib.skyframe.PrecomputedValue.Precomputed; +import com.google.devtools.build.lib.skyframe.SkyFunctions; import com.google.devtools.build.lib.syntax.EvalException; import com.google.devtools.build.lib.util.Fingerprint; import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.skyframe.SkyFunction; import com.google.devtools.build.skyframe.SkyFunctionException; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; @@ -49,6 +52,8 @@ import javax.annotation.Nullable; * this function. */ public final class RepositoryDelegatorFunction implements SkyFunction { + public static final Precomputed<Map<RepositoryName, PathFragment>> REPOSITORY_OVERRIDES = + new Precomputed<>(SkyKey.create(SkyFunctions.PRECOMPUTED, "repository_overrides")); // The marker file version is inject in the rule key digest so the rule key is always different // when we decide to update the format. @@ -93,13 +98,22 @@ public final class RepositoryDelegatorFunction implements SkyFunction { public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException, InterruptedException { RepositoryName repositoryName = (RepositoryName) skyKey.argument(); - Rule rule = RepositoryFunction.getRule(repositoryName, null, env); - if (rule == null) { + BlazeDirectories directories = PrecomputedValue.BLAZE_DIRECTORIES.get(env); + Map<RepositoryName, PathFragment> overrides = REPOSITORY_OVERRIDES.get(env); + if (env.valuesMissing()) { return null; } - BlazeDirectories directories = PrecomputedValue.BLAZE_DIRECTORIES.get(env); - if (directories == null) { + Path repoRoot = RepositoryFunction.getExternalRepositoryDirectory(directories) + .getRelative(repositoryName.strippedName()); + Path markerPath = getMarkerPath(directories, repositoryName.strippedName()); + if (overrides.containsKey(repositoryName)) { + return setupOverride( + repositoryName, overrides.get(repositoryName), env, repoRoot, markerPath); + } + + Rule rule = RepositoryFunction.getRule(repositoryName, null, env); + if (rule == null) { return null; } RepositoryFunction handler; @@ -110,23 +124,20 @@ public final class RepositoryDelegatorFunction implements SkyFunction { } if (handler == null) { throw new RepositoryFunctionException( - new EvalException(Location.fromFile(directories.getWorkspace().getRelative("WORKSPACE")), + new EvalException( + Location.fromFile(directories.getWorkspace().getRelative("WORKSPACE")), "Could not find handler for " + rule), Transience.PERSISTENT); } handler.setClientEnvironment(clientEnvironment); - Path repoRoot = - RepositoryFunction.getExternalRepositoryDirectory(directories).getRelative(rule.getName()); byte[] ruleSpecificData = handler.getRuleSpecificMarkerData(rule, env); if (ruleSpecificData == null) { return null; } String ruleKey = computeRuleKey(rule, ruleSpecificData); Map<String, String> markerData = new TreeMap<>(); - Path markerPath = getMarkerPath(directories, rule); - if (handler.isLocal(rule)) { // Local repositories are always fetched because the operation is generally fast and they do // not depend on non-local data, so it does not make much sense to try to cache from across @@ -207,7 +218,7 @@ public final class RepositoryDelegatorFunction implements SkyFunction { .setFetchingDelayed().build(); } - private final String computeRuleKey(Rule rule, byte[] ruleSpecificData) { + private String computeRuleKey(Rule rule, byte[] ruleSpecificData) { return new Fingerprint().addBytes(RuleFormatter.serializeRule(rule).build().toByteArray()) .addBytes(ruleSpecificData) .addInt(MARKER_FILE_VERSION).hexDigestAndReset(); @@ -226,7 +237,7 @@ public final class RepositoryDelegatorFunction implements SkyFunction { * system is up to date. */ @Nullable - private final byte[] isFilesystemUpToDate(Path markerPath, Rule rule, String ruleKey, + private byte[] isFilesystemUpToDate(Path markerPath, Rule rule, String ruleKey, RepositoryFunction handler, Environment env) throws RepositoryFunctionException, InterruptedException { try { @@ -310,7 +321,7 @@ public final class RepositoryDelegatorFunction implements SkyFunction { return result.toString(); } - private final byte[] writeMarkerFile( + private byte[] writeMarkerFile( Path markerPath, Map<String, String> markerData, String ruleKey) throws RepositoryFunctionException { try { @@ -329,13 +340,30 @@ public final class RepositoryDelegatorFunction implements SkyFunction { } } - private static Path getMarkerPath(BlazeDirectories directories, Rule rule) { + private static Path getMarkerPath(BlazeDirectories directories, String ruleName) { return RepositoryFunction.getExternalRepositoryDirectory(directories) - .getChild("@" + rule.getName() + ".marker"); + .getChild("@" + ruleName + ".marker"); } @Override public String extractTag(SkyKey skyKey) { return null; } + + private RepositoryDirectoryValue setupOverride( + RepositoryName repositoryName, PathFragment sourcePath, Environment env, Path repoRoot, + Path markerPath) + throws RepositoryFunctionException, InterruptedException { + setupRepositoryRoot(repoRoot); + RepositoryDirectoryValue.Builder directoryValue = LocalRepositoryFunction.symlink( + repoRoot, sourcePath, env); + if (directoryValue == null) { + return null; + } + String ruleKey = new Fingerprint().addBytes(repositoryName.strippedName().getBytes()) + .addBytes(repoRoot.getFileSystem().getPath(sourcePath).getPathString().getBytes()) + .addInt(MARKER_FILE_VERSION).hexDigestAndReset(); + byte[] digest = writeMarkerFile(markerPath, new TreeMap<String, String>(), ruleKey); + return directoryValue.setDigest(digest).build(); + } } diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java index 51486eda50..8afb6eeb83 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryFunction.java @@ -470,7 +470,12 @@ public abstract class RepositoryFunction { // function so we get invalidation when the repository is fetched. // For the repository directory itself, we cannot depends on the RepositoryDirectoryValue // (cycle). - env.getValue(RepositoryDirectoryValue.key(RepositoryName.create("@" + repositoryName))); + env.getValue( + RepositoryDirectoryValue.key( + RepositoryName.createFromValidStrippedName(repositoryName))); + } else { + // Invalidate external/<repo> if the repository overrides change. + RepositoryDelegatorFunction.REPOSITORY_OVERRIDES.get(env); } } catch (RepositoryFunction.RepositoryNotFoundException ex) { // The repository we are looking for does not exist so we should depend on the whole @@ -478,7 +483,7 @@ public abstract class RepositoryFunction { // already requested all repository functions from the WORKSPACE file from Skyframe as part // of the resolution. Therefore we are safe to ignore that Exception. return; - } catch (RepositoryFunctionException | LabelSyntaxException ex) { + } catch (RepositoryFunctionException ex) { // This should never happen. throw new IllegalStateException( "Repository " + repositoryName + " cannot be resolved for path " + rootedPath, ex); diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandDispatcher.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandDispatcher.java index 3b26e1e990..26a96d00ab 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandDispatcher.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandDispatcher.java @@ -511,6 +511,10 @@ public class BlazeCommandDispatcher { env.getEventBus().post(originalCommandLine); + for (BlazeModule module : runtime.getBlazeModules()) { + env.getSkyframeExecutor().injectExtraPrecomputedValues(module.getPrecomputedValues()); + } + ExitCode outcome = command.exec(env, optionsParser); outcome = env.precompleteCommand(outcome); numericExitCode = outcome.getNumericExitCode(); diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeModule.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeModule.java index 39d6d4a9af..f05586c844 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/BlazeModule.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeModule.java @@ -27,6 +27,7 @@ import com.google.devtools.build.lib.packages.Package; import com.google.devtools.build.lib.packages.PackageFactory; import com.google.devtools.build.lib.packages.RuleClassProvider; import com.google.devtools.build.lib.rules.test.CoverageReportActionFactory; +import com.google.devtools.build.lib.skyframe.PrecomputedValue; import com.google.devtools.build.lib.util.AbruptExitException; import com.google.devtools.build.lib.util.Clock; import com.google.devtools.build.lib.vfs.FileSystem; @@ -290,4 +291,8 @@ public abstract class BlazeModule { */ void exit(AbruptExitException exception); } + + public ImmutableList<PrecomputedValue.Injected> getPrecomputedValues() { + return ImmutableList.of(); + } } diff --git a/src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java b/src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java index 4fcea318cb..7f2d2e81fb 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/CommandEnvironment.java @@ -495,6 +495,10 @@ public final class CommandEnvironment { if (!skyframeExecutor.hasIncrementalState()) { skyframeExecutor.resetEvaluator(); } + + for (BlazeModule module : runtime.getBlazeModules()) { + skyframeExecutor.injectExtraPrecomputedValues(module.getPrecomputedValues()); + } skyframeExecutor.sync( reporter, options.getOptions(PackageCacheOptions.class), diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PrecomputedValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/PrecomputedValue.java index f0d06e0be8..2619af2b39 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/PrecomputedValue.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/PrecomputedValue.java @@ -60,6 +60,11 @@ public final class PrecomputedValue implements SkyValue { void inject(Injectable injectable) { injectable.inject(precomputed.key, new PrecomputedValue(supplier.get())); } + + @Override + public String toString() { + return precomputed + ": " + supplier.get(); + } } public static <T> Injected injected(Precomputed<T> precomputed, Supplier<T> value) { diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java index fe5b1d4513..dfc2657aa7 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java @@ -701,12 +701,13 @@ public abstract class SkyframeExecutor implements WalkableGraphFactory { PrecomputedValue.BLAZE_DIRECTORIES.set(injectable(), directories); PrecomputedValue.PRODUCT_NAME.set(injectable(), productName); injectBuildInfoFactories(); - injectExtraPrecomputedValues(); + injectExtraPrecomputedValues(extraPrecomputedValues); needToInjectPrecomputedValuesForAnalysis = false; } } - private void injectExtraPrecomputedValues() { + public void injectExtraPrecomputedValues( + List<PrecomputedValue.Injected> extraPrecomputedValues) { for (PrecomputedValue.Injected injected : extraPrecomputedValues) { injected.inject(injectable()); } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceNameValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceNameValue.java index 1fccf6d4da..c752d5829d 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceNameValue.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceNameValue.java @@ -98,5 +98,10 @@ public class WorkspaceNameValue implements SkyValue { public int hashCode() { return HASHCODE; } + + @Override + public String toString() { + return "#"; + } } } diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java index 032599f4e3..1cd568596b 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java @@ -38,6 +38,7 @@ import com.google.devtools.build.lib.analysis.config.ConfigurationFactory; import com.google.devtools.build.lib.buildtool.BuildRequest.BuildRequestOptions; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.cmdline.LabelSyntaxException; +import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.exec.ExecutionOptions; import com.google.devtools.build.lib.flags.InvocationPolicyEnforcer; import com.google.devtools.build.lib.packages.PackageFactory; @@ -49,6 +50,7 @@ import com.google.devtools.build.lib.pkgcache.LoadingResult; import com.google.devtools.build.lib.pkgcache.PackageCacheOptions; import com.google.devtools.build.lib.pkgcache.PackageManager; import com.google.devtools.build.lib.pkgcache.PathPackageLocator; +import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction; import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey; import com.google.devtools.build.lib.skyframe.DiffAwareness; import com.google.devtools.build.lib.skyframe.PackageLookupFunction.CrossRepositoryLabelViolationStrategy; @@ -192,6 +194,9 @@ public abstract class AnalysisTestCase extends FoundationTestCase { ImmutableMap.<String, String>of(), ImmutableMap.<String, String>of(), new TimestampGranularityMonitor(BlazeClock.instance())); + skyframeExecutor.injectExtraPrecomputedValues(ImmutableList.of(PrecomputedValue.injected( + RepositoryDelegatorFunction.REPOSITORY_OVERRIDES, + ImmutableMap.<RepositoryName, PathFragment>of()))); packageManager = skyframeExecutor.getPackageManager(); loadingPhaseRunner = skyframeExecutor.getLoadingPhaseRunner( pkgFactory.getRuleClassNames(), defaultFlags().contains(Flag.SKYFRAME_LOADING_PHASE)); diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java b/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java index 140f99efb6..33ebdee37a 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/util/BuildViewTestCase.java @@ -118,6 +118,7 @@ import com.google.devtools.build.lib.pkgcache.PackageCacheOptions; import com.google.devtools.build.lib.pkgcache.PackageManager; import com.google.devtools.build.lib.pkgcache.PathPackageLocator; import com.google.devtools.build.lib.rules.extra.ExtraAction; +import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction; import com.google.devtools.build.lib.rules.test.BaselineCoverageAction; import com.google.devtools.build.lib.rules.test.InstrumentedFilesProvider; import com.google.devtools.build.lib.skyframe.AspectValue; @@ -232,6 +233,9 @@ public abstract class BuildViewTestCase extends FoundationTestCase { analysisMock.getProductName(), CrossRepositoryLabelViolationStrategy.ERROR, ImmutableList.of(BuildFileName.BUILD_DOT_BAZEL, BuildFileName.BUILD)); + skyframeExecutor.injectExtraPrecomputedValues(ImmutableList.of(PrecomputedValue.injected( + RepositoryDelegatorFunction.REPOSITORY_OVERRIDES, + ImmutableMap.<RepositoryName, PathFragment>of()))); packageCacheOptions.defaultVisibility = ConstantRuleVisibility.PUBLIC; packageCacheOptions.showLoadingProgress = true; packageCacheOptions.globbingThreads = 7; diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/ConfigurationTestCase.java b/src/test/java/com/google/devtools/build/lib/analysis/util/ConfigurationTestCase.java index 490b642668..fbf170269d 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/util/ConfigurationTestCase.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/util/ConfigurationTestCase.java @@ -36,6 +36,7 @@ import com.google.devtools.build.lib.packages.PackageFactory; import com.google.devtools.build.lib.packages.util.MockToolsConfig; import com.google.devtools.build.lib.pkgcache.PackageCacheOptions; import com.google.devtools.build.lib.pkgcache.PathPackageLocator; +import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction; import com.google.devtools.build.lib.skyframe.DiffAwareness; import com.google.devtools.build.lib.skyframe.PackageLookupFunction.CrossRepositoryLabelViolationStrategy; import com.google.devtools.build.lib.skyframe.PackageLookupValue.BuildFileName; @@ -116,7 +117,9 @@ public abstract class ConfigurationTestCase extends FoundationTestCase { analysisMock.getProductName(), CrossRepositoryLabelViolationStrategy.ERROR, ImmutableList.of(BuildFileName.BUILD_DOT_BAZEL, BuildFileName.BUILD)); - + skyframeExecutor.injectExtraPrecomputedValues(ImmutableList.of(PrecomputedValue.injected( + RepositoryDelegatorFunction.REPOSITORY_OVERRIDES, + ImmutableMap.<RepositoryName, PathFragment>of()))); PackageCacheOptions packageCacheOptions = Options.getDefaults(PackageCacheOptions.class); packageCacheOptions.showLoadingProgress = true; packageCacheOptions.globbingThreads = 7; diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/BUILD b/src/test/java/com/google/devtools/build/lib/bazel/repository/BUILD index 96053933e5..d3c387dbec 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/repository/BUILD +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/BUILD @@ -29,6 +29,7 @@ java_test( "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader", "//src/main/java/com/google/devtools/build/lib/rules/cpp", "//src/main/java/com/google/devtools/build/skyframe", + "//src/main/java/com/google/devtools/common/options", "//src/test/java/com/google/devtools/build/lib:analysis_testutil", "//src/test/java/com/google/devtools/build/lib:foundations_testutil", "//src/test/java/com/google/devtools/build/lib:packages_testutil", diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptionsTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptionsTest.java new file mode 100644 index 0000000000..d70b6d8100 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptionsTest.java @@ -0,0 +1,70 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.repository; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.RepositoryOverride; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.RepositoryOverrideConverter; +import com.google.devtools.build.lib.cmdline.RepositoryName; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.common.options.OptionsParsingException; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Test for {@link RepositoryOptions}. + */ +@RunWith(JUnit4.class) +public class RepositoryOptionsTest { + + private final RepositoryOverrideConverter converter = new RepositoryOverrideConverter(); + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Test + public void testOverrideConverter() throws Exception { + RepositoryOverride actual = converter.convert("foo=/bar"); + assertThat(actual.repositoryName()) + .isEqualTo(RepositoryName.createFromValidStrippedName("foo")); + assertThat(actual.path()).isEqualTo(PathFragment.create("/bar")); + } + + @Test + public void testInvalidOverride() throws Exception { + expectedException.expect(OptionsParsingException.class); + expectedException.expectMessage( + "Repository overrides must be of the form 'repository-name=path'"); + converter.convert("foo"); + } + + @Test + public void testInvalidRepoOverride() throws Exception { + expectedException.expect(OptionsParsingException.class); + expectedException.expectMessage("Invalid repository name given to override"); + converter.convert("foo/bar=/baz"); + } + + @Test + public void testInvalidPathOverride() throws Exception { + expectedException.expect(OptionsParsingException.class); + expectedException.expectMessage("Repository override directory must be an absolute path"); + converter.convert("foo=bar"); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/rules/repository/BUILD b/src/test/java/com/google/devtools/build/lib/rules/repository/BUILD index ed24a5081f..8d94ddba6c 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/repository/BUILD +++ b/src/test/java/com/google/devtools/build/lib/rules/repository/BUILD @@ -19,6 +19,8 @@ java_test( "//src/main/java/com/google/devtools/build/lib:bazel-main", "//src/main/java/com/google/devtools/build/lib:bazel-repository", "//src/main/java/com/google/devtools/build/lib:build-base", + "//src/main/java/com/google/devtools/build/lib:events", + "//src/main/java/com/google/devtools/build/lib:io", "//src/main/java/com/google/devtools/build/lib:packages-internal", "//src/main/java/com/google/devtools/build/lib:runtime", "//src/main/java/com/google/devtools/build/lib:unix", diff --git a/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java b/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java new file mode 100644 index 0000000000..f6955e5e2f --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java @@ -0,0 +1,142 @@ +// Copyright 2017 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.rules.repository; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.cmdline.RepositoryName; +import com.google.devtools.build.lib.events.StoredEventHandler; +import com.google.devtools.build.lib.pkgcache.PathPackageLocator; +import com.google.devtools.build.lib.skyframe.ExternalFilesHelper; +import com.google.devtools.build.lib.skyframe.ExternalFilesHelper.ExternalFileAction; +import com.google.devtools.build.lib.skyframe.ExternalPackageFunction; +import com.google.devtools.build.lib.skyframe.FileFunction; +import com.google.devtools.build.lib.skyframe.FileStateFunction; +import com.google.devtools.build.lib.skyframe.LocalRepositoryLookupFunction; +import com.google.devtools.build.lib.skyframe.PackageFunction; +import com.google.devtools.build.lib.skyframe.PackageLookupFunction; +import com.google.devtools.build.lib.skyframe.PackageLookupFunction.CrossRepositoryLabelViolationStrategy; +import com.google.devtools.build.lib.skyframe.PackageLookupValue.BuildFileName; +import com.google.devtools.build.lib.skyframe.PrecomputedValue; +import com.google.devtools.build.lib.skyframe.SkyFunctions; +import com.google.devtools.build.lib.skyframe.WorkspaceASTFunction; +import com.google.devtools.build.lib.skyframe.WorkspaceFileFunction; +import com.google.devtools.build.lib.testutil.FoundationTestCase; +import com.google.devtools.build.lib.testutil.TestConstants; +import com.google.devtools.build.lib.testutil.TestRuleClassProvider; +import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.PathFragment; +import com.google.devtools.build.skyframe.EvaluationResult; +import com.google.devtools.build.skyframe.InMemoryMemoizingEvaluator; +import com.google.devtools.build.skyframe.MemoizingEvaluator; +import com.google.devtools.build.skyframe.RecordingDifferencer; +import com.google.devtools.build.skyframe.SequentialBuildDriver; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Tests for {@link RepositoryDelegatorFunction} + */ +@RunWith(JUnit4.class) +public class RepositoryDelegatorTest extends FoundationTestCase { + private RepositoryDelegatorFunction delegatorFunction; + private Path overrideDirectory; + private SequentialBuildDriver driver; + + @Before + public void setupDelegator() throws Exception { + Path root = scratch.dir("/outputbase"); + delegatorFunction = new RepositoryDelegatorFunction( + ImmutableMap.<String, RepositoryFunction>of(), null, new AtomicBoolean(true)); + AtomicReference<PathPackageLocator> pkgLocator = new AtomicReference<>( + new PathPackageLocator(root, ImmutableList.of(root))); + BlazeDirectories directories = new BlazeDirectories(root, root, root, + TestConstants.PRODUCT_NAME); + ExternalFilesHelper externalFilesHelper = new ExternalFilesHelper( + pkgLocator, + ExternalFileAction.DEPEND_ON_EXTERNAL_PKG_FOR_EXTERNAL_REPO_PATHS, + directories); + RecordingDifferencer differencer = new RecordingDifferencer(); + MemoizingEvaluator evaluator = + new InMemoryMemoizingEvaluator( + ImmutableMap.<SkyFunctionName, SkyFunction>builder() + .put( + SkyFunctions.FILE_STATE, + new FileStateFunction( + new AtomicReference<TimestampGranularityMonitor>(), externalFilesHelper)) + .put(SkyFunctions.FILE, new FileFunction(pkgLocator)) + .put(SkyFunctions.REPOSITORY_DIRECTORY, delegatorFunction) + .put( + SkyFunctions.PACKAGE, + new PackageFunction(null, null, null, null, null, null, null)) + .put( + SkyFunctions.PACKAGE_LOOKUP, + new PackageLookupFunction( + null, + CrossRepositoryLabelViolationStrategy.ERROR, + ImmutableList.of(BuildFileName.BUILD_DOT_BAZEL, BuildFileName.BUILD))) + .put( + SkyFunctions.WORKSPACE_AST, + new WorkspaceASTFunction(TestRuleClassProvider.getRuleClassProvider())) + .put( + SkyFunctions.WORKSPACE_FILE, + new WorkspaceFileFunction( + TestRuleClassProvider.getRuleClassProvider(), + TestConstants.PACKAGE_FACTORY_FACTORY_FOR_TESTING.create( + TestRuleClassProvider.getRuleClassProvider(), root.getFileSystem()), + directories)) + .put(SkyFunctions.LOCAL_REPOSITORY_LOOKUP, new LocalRepositoryLookupFunction()) + .put(SkyFunctions.EXTERNAL_PACKAGE, new ExternalPackageFunction()) + .build(), + differencer); + driver = new SequentialBuildDriver(evaluator); + overrideDirectory = scratch.dir("/foo"); + RepositoryDelegatorFunction.REPOSITORY_OVERRIDES.set( + differencer, + ImmutableMap.<RepositoryName, PathFragment>builder() + .put(RepositoryName.createFromValidStrippedName("foo"), overrideDirectory.asFragment()) + .build()); + PrecomputedValue.BLAZE_DIRECTORIES.set(differencer, directories); + PrecomputedValue.PATH_PACKAGE_LOCATOR.set(differencer, pkgLocator.get()); + } + + @Test + public void testOverride() throws Exception { + StoredEventHandler eventHandler = new StoredEventHandler(); + SkyKey key = RepositoryDirectoryValue.key(RepositoryName.createFromValidStrippedName("foo")); + EvaluationResult<SkyValue> result = + driver.evaluate(ImmutableList.of(key), false, 8, eventHandler); + assertThat(result.hasError()).isFalse(); + RepositoryDirectoryValue repositoryDirectoryValue = (RepositoryDirectoryValue) result.get(key); + Path expectedPath = scratch.dir("/outputbase/external/foo"); + Path actualPath = repositoryDirectoryValue.getPath(); + assertThat(actualPath).isEqualTo(expectedPath); + assertThat(actualPath.isSymbolicLink()).isTrue(); + assertThat(actualPath.readSymbolicLink()).isEqualTo(overrideDirectory.asFragment()); + } + +} diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/FileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/FileFunctionTest.java index 68f0386bd6..f8253903d4 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/FileFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/FileFunctionTest.java @@ -37,9 +37,11 @@ import com.google.common.testing.EqualsTester; import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.cmdline.PackageIdentifier; +import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.events.NullEventHandler; import com.google.devtools.build.lib.events.StoredEventHandler; import com.google.devtools.build.lib.pkgcache.PathPackageLocator; +import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction; import com.google.devtools.build.lib.skyframe.ExternalFilesHelper.ExternalFileAction; import com.google.devtools.build.lib.skyframe.PackageLookupFunction.CrossRepositoryLabelViolationStrategy; import com.google.devtools.build.lib.skyframe.PackageLookupValue.BuildFileName; @@ -165,6 +167,8 @@ public class FileFunctionTest { differencer); PrecomputedValue.BUILD_ID.set(differencer, UUID.randomUUID()); PrecomputedValue.PATH_PACKAGE_LOCATOR.set(differencer, pkgLocator); + RepositoryDelegatorFunction.REPOSITORY_OVERRIDES.set( + differencer, ImmutableMap.<RepositoryName, PathFragment>of()); return new SequentialBuildDriver(evaluator); } diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/PackageLookupFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/PackageLookupFunctionTest.java index 6ad7725f4c..9fe08801d5 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/PackageLookupFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/PackageLookupFunctionTest.java @@ -27,6 +27,7 @@ import com.google.common.testing.EqualsTester; import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.analysis.util.AnalysisMock; import com.google.devtools.build.lib.cmdline.PackageIdentifier; +import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.events.NullEventHandler; import com.google.devtools.build.lib.packages.BuildFileNotFoundException; import com.google.devtools.build.lib.packages.PackageFactory; @@ -143,6 +144,8 @@ public abstract class PackageLookupFunctionTest extends FoundationTestCase { PrecomputedValue.BLACKLISTED_PACKAGE_PREFIXES_FILE.set( differencer, PathFragment.EMPTY_FRAGMENT); PrecomputedValue.BLAZE_DIRECTORIES.set(differencer, directories); + RepositoryDelegatorFunction.REPOSITORY_OVERRIDES.set( + differencer, ImmutableMap.<RepositoryName, PathFragment>of()); } protected PackageLookupValue lookupPackage(String packageName) throws InterruptedException { diff --git a/src/test/shell/bazel/workspace_test.sh b/src/test/shell/bazel/workspace_test.sh index a319cbb109..5b4b7de3db 100755 --- a/src/test/shell/bazel/workspace_test.sh +++ b/src/test/shell/bazel/workspace_test.sh @@ -203,4 +203,44 @@ EOF bazel build @bar//:depend-on-foo || fail "Expected build to succeed" } +function test_workspace_override() { + mkdir -p original + touch original/WORKSPACE + cat > original/BUILD <<'EOF' +genrule( + name = "gen", + cmd = "echo 'original' > $@", + outs = ["gen.out"], +) +EOF + + mkdir -p override + touch override/WORKSPACE + cat > override/BUILD <<'EOF' +genrule( + name = "gen", + cmd = "echo 'override' > $@", + outs = ["gen.out"], +) +EOF + + cat > WORKSPACE <<EOF +local_repository( + name = "o", + path = "original", +) +EOF + bazel build --override_repository="o=$PWD/override" @o//:gen &> $TEST_log \ + || fail "Expected build to succeed" + assert_contains "override" bazel-genfiles/external/o/gen.out + + bazel build @o//:gen &> $TEST_log \ + || fail "Expected build to succeed" + assert_contains "original" bazel-genfiles/external/o/gen.out + + bazel build --override_repository="o=$PWD/override" @o//:gen &> $TEST_log \ + || fail "Expected build to succeed" + assert_contains "override" bazel-genfiles/external/o/gen.out +} + run_suite "workspace tests" |