From d6a98282e229b311dd56e65b72003197120f299a Mon Sep 17 00:00:00 2001 From: carmi Date: Tue, 13 Mar 2018 19:19:16 -0700 Subject: Allow BazelPackageLoader to load external repositories. Also, disallow BazelPackageLoader from fetching missing external repos. Integration tests for BazelPackageLoader wrt external repos will be left for a follow-up CL. RELNOTES: None. PiperOrigin-RevId: 188967694 --- src/main/java/com/google/devtools/build/lib/BUILD | 53 ++++++++++-- .../build/lib/analysis/ServerDirectories.java | 6 ++ .../build/lib/bazel/BazelRepositoryModule.java | 49 +++++------ .../lib/packages/BuilderFactoryForTesting.java | 27 ++++++ .../build/lib/packages/PackageFactory.java | 9 -- .../lib/skyframe/ClientEnvironmentFunction.java | 2 +- .../build/lib/skyframe/PrecomputedValue.java | 2 +- .../skyframe/packages/AbstractPackageLoader.java | 96 ++++++++++++---------- .../devtools/build/lib/skyframe/packages/BUILD | 4 + .../lib/skyframe/packages/BazelPackageLoader.java | 80 ++++++++++++------ 10 files changed, 223 insertions(+), 105 deletions(-) create mode 100644 src/main/java/com/google/devtools/build/lib/packages/BuilderFactoryForTesting.java (limited to 'src/main') diff --git a/src/main/java/com/google/devtools/build/lib/BUILD b/src/main/java/com/google/devtools/build/lib/BUILD index af0b5edc2d..cfa53797cf 100644 --- a/src/main/java/com/google/devtools/build/lib/BUILD +++ b/src/main/java/com/google/devtools/build/lib/BUILD @@ -373,10 +373,13 @@ java_library( java_library( name = "packages-internal", - srcs = glob([ - "packages/*.java", - "pkgcache/*.java", - ]), + srcs = glob( + [ + "packages/*.java", + "pkgcache/*.java", + ], + exclude = ["packages/BuilderFactoryForTesting.java"], + ), exports = [ "//src/main/java/com/google/devtools/build/lib:syntax", "//src/main/java/com/google/devtools/build/lib/cmdline", @@ -409,6 +412,17 @@ java_library( ], ) +java_library( + name = "packages/BuilderFactoryForTesting", + srcs = ["packages/BuilderFactoryForTesting.java"], + deps = [ + ":build-base", + ":packages-internal", + "//third_party:guava", + "//third_party:jsr305", + ], +) + java_library( name = "packages", exports = [ @@ -611,9 +625,36 @@ java_library( ], ) +java_library( + name = "bazel/BazelRepositoryModule", + srcs = ["bazel/BazelRepositoryModule.java"], + deps = [ + "//src/main/java/com/google/devtools/build/lib:bazel-commands", + "//src/main/java/com/google/devtools/build/lib:bazel-repository", + "//src/main/java/com/google/devtools/build/lib:bazel-rules", + "//src/main/java/com/google/devtools/build/lib:build-base", + "//src/main/java/com/google/devtools/build/lib:events", + "//src/main/java/com/google/devtools/build/lib:io", + "//src/main/java/com/google/devtools/build/lib:packages-internal", + "//src/main/java/com/google/devtools/build/lib:runtime", + "//src/main/java/com/google/devtools/build/lib/bazel/repository/cache", + "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader", + "//src/main/java/com/google/devtools/build/lib/cmdline", + "//src/main/java/com/google/devtools/build/lib/vfs", + "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", + "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", + "//src/main/java/com/google/devtools/common/options:options_internal", + "//third_party:guava", + "//third_party:jsr305", + ], +) + java_library( name = "bazel-main", - srcs = glob(["bazel/*.java"]), + srcs = glob( + ["bazel/*.java"], + exclude = ["bazel/BazelRepositoryModule.java"], + ), resources = [ "bazel/rules/java/java_stub_template.txt", "bazel/rules/java/java_stub_template_windows.txt", @@ -621,6 +662,7 @@ java_library( "bazel/rules/sh/sh_stub_template_windows.txt", ], deps = [ + ":bazel/BazelRepositoryModule", "//src/main/java/com/google/devtools/build/lib:bazel", "//src/main/java/com/google/devtools/build/lib:bazel-commands", "//src/main/java/com/google/devtools/build/lib:bazel-repository", @@ -648,6 +690,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/worker", "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", "//src/main/java/com/google/devtools/common/options", + "//src/test/java/com/google/devtools/build/lib:testutil/BazelPackageBuilderHelperForTesting", "//third_party:guava", "//third_party:jsr305", ], diff --git a/src/main/java/com/google/devtools/build/lib/analysis/ServerDirectories.java b/src/main/java/com/google/devtools/build/lib/analysis/ServerDirectories.java index ac89158bb1..dc9d60e23f 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/ServerDirectories.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/ServerDirectories.java @@ -14,6 +14,7 @@ package com.google.devtools.build.lib.analysis; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.hash.HashCode; @@ -89,6 +90,11 @@ public final class ServerDirectories { /** Returns the installed embedded binaries directory, under the shared installBase location. */ public Path getEmbeddedBinariesRoot() { + return getEmbeddedBinariesRoot(installBase); + } + + @VisibleForTesting + public static Path getEmbeddedBinariesRoot(Path installBase) { return installBase.getChild("_embedded_binaries"); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index 8a18f3c59f..d6793da025 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -89,9 +89,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import javax.annotation.Nullable; -/** - * Adds support for fetching external code. - */ +/** Adds support for fetching external code. */ public class BazelRepositoryModule extends BlazeModule { // Default location (relative to output user root) of the repository cache. @@ -111,21 +109,25 @@ public class BazelRepositoryModule extends BlazeModule { public BazelRepositoryModule() { this.skylarkRepositoryFunction = new SkylarkRepositoryFunction(httpDownloader); - this.repositoryHandlers = - ImmutableMap.builder() - .put(LocalRepositoryRule.NAME, new LocalRepositoryFunction()) - .put(HttpArchiveRule.NAME, new HttpArchiveFunction(httpDownloader)) - .put(GitRepositoryRule.NAME, new GitRepositoryFunction(httpDownloader)) - .put(HttpJarRule.NAME, new HttpJarFunction(httpDownloader)) - .put(HttpFileRule.NAME, new HttpFileFunction(httpDownloader)) - .put(MavenJarRule.NAME, new MavenJarFunction(mavenDownloader)) - .put(NewHttpArchiveRule.NAME, new NewHttpArchiveFunction(httpDownloader)) - .put(NewGitRepositoryRule.NAME, new NewGitRepositoryFunction(httpDownloader)) - .put(NewLocalRepositoryRule.NAME, new NewLocalRepositoryFunction()) - .put(AndroidSdkRepositoryRule.NAME, new AndroidSdkRepositoryFunction()) - .put(AndroidNdkRepositoryRule.NAME, new AndroidNdkRepositoryFunction()) - .put(MavenServerRule.NAME, new MavenServerRepositoryFunction()) - .build(); + this.repositoryHandlers = repositoryRules(httpDownloader, mavenDownloader); + } + + public static ImmutableMap repositoryRules( + HttpDownloader httpDownloader, MavenDownloader mavenDownloader) { + return ImmutableMap.builder() + .put(LocalRepositoryRule.NAME, new LocalRepositoryFunction()) + .put(HttpArchiveRule.NAME, new HttpArchiveFunction(httpDownloader)) + .put(GitRepositoryRule.NAME, new GitRepositoryFunction(httpDownloader)) + .put(HttpJarRule.NAME, new HttpJarFunction(httpDownloader)) + .put(HttpFileRule.NAME, new HttpFileFunction(httpDownloader)) + .put(MavenJarRule.NAME, new MavenJarFunction(mavenDownloader)) + .put(NewHttpArchiveRule.NAME, new NewHttpArchiveFunction(httpDownloader)) + .put(NewGitRepositoryRule.NAME, new NewGitRepositoryFunction(httpDownloader)) + .put(NewLocalRepositoryRule.NAME, new NewLocalRepositoryFunction()) + .put(AndroidSdkRepositoryRule.NAME, new AndroidSdkRepositoryFunction()) + .put(AndroidNdkRepositoryRule.NAME, new AndroidNdkRepositoryFunction()) + .put(MavenServerRule.NAME, new MavenServerRepositoryFunction()) + .build(); } /** @@ -185,9 +187,11 @@ public class BazelRepositoryModule extends BlazeModule { for (Entry handler : repositoryHandlers.entrySet()) { RuleDefinition ruleDefinition; try { - ruleDefinition = handler.getValue().getRuleDefinition().getDeclaredConstructor() - .newInstance(); - } catch (IllegalAccessException | InstantiationException | NoSuchMethodException + ruleDefinition = + handler.getValue().getRuleDefinition().getDeclaredConstructor().newInstance(); + } catch (IllegalAccessException + | InstantiationException + | NoSuchMethodException | InvocationTargetException e) { throw new IllegalStateException(e); } @@ -266,8 +270,7 @@ public class BazelRepositoryModule extends BlazeModule { @Override public ImmutableList getPrecomputedValues() { return ImmutableList.of( - PrecomputedValue.injected( - RepositoryDelegatorFunction.REPOSITORY_OVERRIDES, overrides)); + PrecomputedValue.injected(RepositoryDelegatorFunction.REPOSITORY_OVERRIDES, overrides)); } @Override diff --git a/src/main/java/com/google/devtools/build/lib/packages/BuilderFactoryForTesting.java b/src/main/java/com/google/devtools/build/lib/packages/BuilderFactoryForTesting.java new file mode 100644 index 0000000000..c807714b7d --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/packages/BuilderFactoryForTesting.java @@ -0,0 +1,27 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.packages; + +import com.google.common.annotations.VisibleForTesting; +import com.google.devtools.build.lib.analysis.BlazeDirectories; + +/** + * Factory for {@link com.google.devtools.build.lib.packages.PackageFactory.BuilderForTesting} + * instances. Intended to only be used by unit tests. + */ +@VisibleForTesting +public interface BuilderFactoryForTesting { + PackageFactory.BuilderForTesting builder(BlazeDirectories directories); +} diff --git a/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java b/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java index 13e904589f..370df06e5a 100644 --- a/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java +++ b/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java @@ -372,15 +372,6 @@ public final class PackageFactory { public abstract PackageFactory build(RuleClassProvider ruleClassProvider, FileSystem fs); } - /** - * Factory for {@link PackageFactory.BuilderForTesting} instances. Intended to only be used by - * unit tests. - */ - @VisibleForTesting - public abstract static class BuilderFactoryForTesting { - public abstract BuilderForTesting builder(); - } - @VisibleForTesting public Package.Builder.Helper getPackageBuilderHelperForTesting() { return packageBuilderHelper; diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/ClientEnvironmentFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/ClientEnvironmentFunction.java index 7c4c798b4a..37f111ff43 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/ClientEnvironmentFunction.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/ClientEnvironmentFunction.java @@ -55,7 +55,7 @@ public final class ClientEnvironmentFunction implements SkyFunction { private final AtomicReference> clientEnv; - ClientEnvironmentFunction(AtomicReference> clientEnv) { + public ClientEnvironmentFunction(AtomicReference> clientEnv) { this.clientEnv = clientEnv; } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/PrecomputedValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/PrecomputedValue.java index 1d92dc2bca..ad91c60730 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/PrecomputedValue.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/PrecomputedValue.java @@ -90,7 +90,7 @@ public final class PrecomputedValue implements SkyValue { static final Precomputed BUILD_ID = new Precomputed<>(Key.create("build_id")); - static final Precomputed> ACTION_ENV = + public static final Precomputed> ACTION_ENV = new Precomputed<>(Key.create("action_env")); static final Precomputed> COVERAGE_REPORT_KEY = diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/packages/AbstractPackageLoader.java b/src/main/java/com/google/devtools/build/lib/skyframe/packages/AbstractPackageLoader.java index fa5bbd66dd..382010ee17 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/packages/AbstractPackageLoader.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/packages/AbstractPackageLoader.java @@ -107,18 +107,19 @@ public abstract class AbstractPackageLoader implements PackageLoader { private static final int MAX_DIRECTORIES_TO_EAGERLY_VISIT_IN_GLOBBING = 3000; private final ImmutableDiff preinjectedDiff; - private final Differencer preinjectedDifferencer = new Differencer() { - @Override - public Diff getDiff(WalkableGraph fromGraph, Version fromVersion, Version toVersion) - throws InterruptedException { - return preinjectedDiff; - } - }; + private final Differencer preinjectedDifferencer = + new Differencer() { + @Override + public Diff getDiff(WalkableGraph fromGraph, Version fromVersion, Version toVersion) + throws InterruptedException { + return preinjectedDiff; + } + }; private final Reporter reporter; protected final RuleClassProvider ruleClassProvider; protected SkylarkSemantics skylarkSemantics; protected final ImmutableMap extraSkyFunctions; - protected final AtomicReference pkgLocatorRef; + private final AtomicReference pkgLocatorRef; protected final ExternalFilesHelper externalFilesHelper; protected final BlazeDirectories directories; private final int legacyGlobbingThreads; @@ -128,24 +129,36 @@ public abstract class AbstractPackageLoader implements PackageLoader { public abstract static class Builder { protected final Path workspaceDir; protected final BlazeDirectories directories; + protected final PathPackageLocator pkgLocator; + final AtomicReference pkgLocatorRef; + protected final ExternalFilesHelper externalFilesHelper; protected RuleClassProvider ruleClassProvider = getDefaultRuleClassProvider(); protected SkylarkSemantics skylarkSemantics; protected Reporter reporter = new Reporter(new EventBus()); protected Map extraSkyFunctions = new HashMap<>(); - protected List extraPrecomputedValues = new ArrayList<>(); - protected String defaultsPackageContents = getDefaultDefaultPackageContents(); - protected int legacyGlobbingThreads = 1; + List extraPrecomputedValues = new ArrayList<>(); + String defaultsPackageContents = getDefaultDefaultPackageContents(); + int legacyGlobbingThreads = 1; int skyframeThreads = 1; - protected Builder(Path workspaceDir) { + protected Builder(Path workspaceDir, Path installBase, Path outputBase) { this.workspaceDir = workspaceDir; - // The 'installBase' and 'outputBase' directories won't be meaningfully used by - // WorkspaceFileFunction, so we pass in a dummy Path. - // TODO(nharmata): Refactor WorkspaceFileFunction to make this a non-issue. Path devNull = workspaceDir.getFileSystem().getPath("/dev/null"); directories = new BlazeDirectories( - new ServerDirectories(devNull, devNull, devNull), workspaceDir, "blaze"); + new ServerDirectories(installBase, outputBase, devNull), workspaceDir, "blaze"); + + this.pkgLocator = + new PathPackageLocator( + directories.getOutputBase(), + ImmutableList.of(Root.fromPath(workspaceDir)), + BazelSkyframeExecutorConstants.BUILD_FILES_BY_PRIORITY); + this.pkgLocatorRef = new AtomicReference<>(pkgLocator); + this.externalFilesHelper = + ExternalFilesHelper.create( + pkgLocatorRef, + ExternalFileAction.DEPEND_ON_EXTERNAL_PKG_FOR_EXTERNAL_REPO_PATHS, + directories); } public Builder setRuleClassProvider(RuleClassProvider ruleClassProvider) { @@ -214,30 +227,22 @@ public abstract class AbstractPackageLoader implements PackageLoader { protected abstract String getDefaultDefaultPackageContents(); } - protected AbstractPackageLoader(Builder builder) { - Path workspaceDir = builder.workspaceDir; - PathPackageLocator pkgLocator = - new PathPackageLocator( - null, - ImmutableList.of(Root.fromPath(workspaceDir)), - BazelSkyframeExecutorConstants.BUILD_FILES_BY_PRIORITY); + AbstractPackageLoader(Builder builder) { this.ruleClassProvider = builder.ruleClassProvider; this.skylarkSemantics = builder.skylarkSemantics; this.reporter = builder.reporter; this.extraSkyFunctions = ImmutableMap.copyOf(builder.extraSkyFunctions); - this.pkgLocatorRef = new AtomicReference<>(pkgLocator); + this.pkgLocatorRef = builder.pkgLocatorRef; this.legacyGlobbingThreads = builder.legacyGlobbingThreads; this.skyframeThreads = builder.skyframeThreads; this.directories = builder.directories; - this.externalFilesHelper = ExternalFilesHelper.create( - pkgLocatorRef, - ExternalFileAction.DEPEND_ON_EXTERNAL_PKG_FOR_EXTERNAL_REPO_PATHS, - directories); + this.externalFilesHelper = builder.externalFilesHelper; + this.preinjectedDiff = makePreinjectedDiff( skylarkSemantics, - pkgLocator, + builder.pkgLocator, builder.defaultsPackageContents, ImmutableList.copyOf(builder.extraPrecomputedValues)); } @@ -267,7 +272,7 @@ public abstract class AbstractPackageLoader implements PackageLoader { PrecomputedValue.DEFAULT_VISIBILITY.set(injectable, ConstantRuleVisibility.PRIVATE); PrecomputedValue.SKYLARK_SEMANTICS.set(injectable, skylarkSemantics); PrecomputedValue.DEFAULTS_PACKAGE_CONTENTS.set(injectable, defaultsPackageContents); - return new ImmutableDiff(ImmutableList.of(), valuesToInject); + return new ImmutableDiff(ImmutableList.of(), valuesToInject); } @Override @@ -333,20 +338,25 @@ public abstract class AbstractPackageLoader implements PackageLoader { } protected abstract String getName(); + protected abstract ImmutableList getEnvironmentExtensions(); + protected abstract CrossRepositoryLabelViolationStrategy getCrossRepositoryLabelViolationStrategy(); + protected abstract ImmutableList getBuildFilesByPriority(); + protected abstract ActionOnIOExceptionReadingBuildFile getActionOnIOExceptionReadingBuildFile(); - protected final ImmutableMap makeFreshSkyFunctions() { + private ImmutableMap makeFreshSkyFunctions() { AtomicReference tsgm = new AtomicReference<>(new TimestampGranularityMonitor(BlazeClock.instance())); Cache packageFunctionCache = CacheBuilder.newBuilder().build(); Cache astCache = CacheBuilder.newBuilder().build(); - AtomicReference syscallCacheRef = new AtomicReference<>( - PerBuildSyscallCache.newBuilder().setConcurrencyLevel(legacyGlobbingThreads).build()); + AtomicReference syscallCacheRef = + new AtomicReference<>( + PerBuildSyscallCache.newBuilder().setConcurrencyLevel(legacyGlobbingThreads).build()); PackageFactory pkgFactory = new PackageFactory( ruleClassProvider, @@ -359,13 +369,14 @@ public abstract class AbstractPackageLoader implements PackageLoader { pkgFactory.setSyscalls(syscallCacheRef); pkgFactory.setMaxDirectoriesToEagerlyVisitInGlobbing( MAX_DIRECTORIES_TO_EAGERLY_VISIT_IN_GLOBBING); - CachingPackageLocator cachingPackageLocator = new CachingPackageLocator() { - @Override - @Nullable - public Path getBuildFileForPackage(PackageIdentifier packageName) { - return pkgLocatorRef.get().getPackageBuildFileNullable(packageName, syscallCacheRef); - } - }; + CachingPackageLocator cachingPackageLocator = + new CachingPackageLocator() { + @Override + @Nullable + public Path getBuildFileForPackage(PackageIdentifier packageName) { + return pkgLocatorRef.get().getPackageBuildFileNullable(packageName, syscallCacheRef); + } + }; ImmutableMap.Builder builder = ImmutableMap.builder(); builder .put(SkyFunctions.PRECOMPUTED, new PrecomputedFunction()) @@ -378,10 +389,11 @@ public abstract class AbstractPackageLoader implements PackageLoader { .put( SkyFunctions.PACKAGE_LOOKUP, new PackageLookupFunction( - /* deletedPackages= */ new AtomicReference<>(ImmutableSet.of()), + /* deletedPackages= */ new AtomicReference<>(ImmutableSet.of()), getCrossRepositoryLabelViolationStrategy(), getBuildFilesByPriority())) - .put(SkyFunctions.BLACKLISTED_PACKAGE_PREFIXES, + .put( + SkyFunctions.BLACKLISTED_PACKAGE_PREFIXES, new BlacklistedPackagePrefixesFunction( /*hardcodedBlacklistedPackagePrefixes=*/ ImmutableSet.of(), /*additionalBlacklistedPackagePrefixesFile=*/ PathFragment.EMPTY_FRAGMENT)) diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/packages/BUILD b/src/main/java/com/google/devtools/build/lib/skyframe/packages/BUILD index 7ca7242ba0..7cb108a81d 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/packages/BUILD +++ b/src/main/java/com/google/devtools/build/lib/skyframe/packages/BUILD @@ -10,11 +10,15 @@ java_library( name = "packages", srcs = glob(["*.java"]), deps = [ + "//src/main/java/com/google/devtools/build/lib:bazel-repository", "//src/main/java/com/google/devtools/build/lib:bazel-rules", + "//src/main/java/com/google/devtools/build/lib:bazel/BazelRepositoryModule", "//src/main/java/com/google/devtools/build/lib:build-base", "//src/main/java/com/google/devtools/build/lib:events", "//src/main/java/com/google/devtools/build/lib:io", "//src/main/java/com/google/devtools/build/lib:packages-internal", + "//src/main/java/com/google/devtools/build/lib/bazel/repository/cache", + "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader", "//src/main/java/com/google/devtools/build/lib/clock", "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/vfs", diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoader.java b/src/main/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoader.java index dfed69265d..cdaa07c1eb 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoader.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoader.java @@ -17,17 +17,25 @@ import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; -import com.google.devtools.build.lib.bazel.rules.BazelRuleClassProvider; +import com.google.devtools.build.lib.bazel.BazelRepositoryModule; +import com.google.devtools.build.lib.bazel.repository.MavenDownloader; +import com.google.devtools.build.lib.bazel.repository.MavenServerFunction; +import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache; +import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader; +import com.google.devtools.build.lib.bazel.repository.skylark.SkylarkRepositoryFunction; +import com.google.devtools.build.lib.bazel.rules.BazelRulesModule; import com.google.devtools.build.lib.packages.BuildFileName; import com.google.devtools.build.lib.packages.PackageFactory.EnvironmentExtension; import com.google.devtools.build.lib.packages.RuleClassProvider; -import com.google.devtools.build.lib.rules.repository.LocalRepositoryFunction; -import com.google.devtools.build.lib.rules.repository.LocalRepositoryRule; import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction; import com.google.devtools.build.lib.rules.repository.RepositoryFunction; import com.google.devtools.build.lib.rules.repository.RepositoryLoaderFunction; import com.google.devtools.build.lib.runtime.proto.InvocationPolicyOuterClass.InvocationPolicy; +import com.google.devtools.build.lib.skyframe.ActionEnvironmentFunction; import com.google.devtools.build.lib.skyframe.BazelSkyframeExecutorConstants; +import com.google.devtools.build.lib.skyframe.ClientEnvironmentFunction; +import com.google.devtools.build.lib.skyframe.DirectoryListingFunction; +import com.google.devtools.build.lib.skyframe.DirectoryListingStateFunction; import com.google.devtools.build.lib.skyframe.LocalRepositoryLookupFunction; import com.google.devtools.build.lib.skyframe.PackageFunction.ActionOnIOExceptionReadingBuildFile; import com.google.devtools.build.lib.skyframe.PackageLookupFunction.CrossRepositoryLabelViolationStrategy; @@ -37,37 +45,54 @@ import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.skyframe.SkyFunction; import com.google.devtools.build.skyframe.SkyFunctionName; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; /** - * Concrete implementation of {@link PackageLoader} that uses skyframe under the covers, but with - * no caching or incrementality. + * Concrete implementation of {@link PackageLoader} that uses skyframe under the covers, but with no + * caching or incrementality. */ public class BazelPackageLoader extends AbstractPackageLoader { /** Returns a fresh {@link Builder} instance. */ - public static Builder builder(Path workspaceDir) { - Builder builder = new Builder(workspaceDir); + public static Builder builder(Path workspaceDir, Path installBase, Path outputBase) { + Builder builder = new Builder(workspaceDir, installBase, outputBase); + + RepositoryCache repositoryCache = new RepositoryCache(); + HttpDownloader httpDownloader = new HttpDownloader(repositoryCache); // Set up SkyFunctions and PrecomputedValues needed to make local repositories work correctly. ImmutableMap repositoryHandlers = - ImmutableMap.of( - LocalRepositoryRule.NAME, (RepositoryFunction) new LocalRepositoryFunction()); + BazelRepositoryModule.repositoryRules(httpDownloader, new MavenDownloader(repositoryCache)); + + // Prevent PackageLoader from fetching any remote repositories; these should only be fetched by + // Bazel before calling PackageLoader. + AtomicBoolean isFetch = new AtomicBoolean(false); builder.addExtraSkyFunctions( - ImmutableMap.of( - SkyFunctions.LOCAL_REPOSITORY_LOOKUP, - new LocalRepositoryLookupFunction(), - SkyFunctions.REPOSITORY_DIRECTORY, - new RepositoryDelegatorFunction( - repositoryHandlers, - null, - new AtomicBoolean(true), - ImmutableMap::of, - builder.directories), - SkyFunctions.REPOSITORY, - new RepositoryLoaderFunction())); + ImmutableMap.builder() + .put( + SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE, + new ClientEnvironmentFunction(new AtomicReference<>(ImmutableMap.of()))) + .put( + SkyFunctions.DIRECTORY_LISTING_STATE, + new DirectoryListingStateFunction(builder.externalFilesHelper)) + .put(SkyFunctions.ACTION_ENVIRONMENT_VARIABLE, new ActionEnvironmentFunction()) + .put(SkyFunctions.DIRECTORY_LISTING, new DirectoryListingFunction()) + .put(SkyFunctions.LOCAL_REPOSITORY_LOOKUP, new LocalRepositoryLookupFunction()) + .put( + SkyFunctions.REPOSITORY_DIRECTORY, + new RepositoryDelegatorFunction( + repositoryHandlers, + new SkylarkRepositoryFunction(httpDownloader), + isFetch, + ImmutableMap::of, + builder.directories)) + .put(SkyFunctions.REPOSITORY, new RepositoryLoaderFunction()) + .put(MavenServerFunction.NAME, new MavenServerFunction(builder.directories)) + .build()); // Set extra precomputed values. builder.addExtraPrecomputedValues( + PrecomputedValue.injected(PrecomputedValue.ACTION_ENV, ImmutableMap.of()), PrecomputedValue.injected( RepositoryDelegatorFunction.REPOSITORY_OVERRIDES, Suppliers.ofInstance(ImmutableMap.of()))); @@ -78,10 +103,17 @@ public class BazelPackageLoader extends AbstractPackageLoader { /** Builder for {@link BazelPackageLoader} instances. */ public static class Builder extends AbstractPackageLoader.Builder { private static final ConfiguredRuleClassProvider DEFAULT_RULE_CLASS_PROVIDER = - BazelRuleClassProvider.create(); + createRuleClassProvider(); + + private static ConfiguredRuleClassProvider createRuleClassProvider() { + ConfiguredRuleClassProvider.Builder classProvider = new ConfiguredRuleClassProvider.Builder(); + new BazelRepositoryModule().initializeRuleClasses(classProvider); + new BazelRulesModule().initializeRuleClasses(classProvider); + return classProvider.build(); + } - private Builder(Path workspaceDir) { - super(workspaceDir); + private Builder(Path workspaceDir, Path installBase, Path outputBase) { + super(workspaceDir, installBase, outputBase); } @Override -- cgit v1.2.3