Allow BazelPackageLoader to load external repositories.

Also, disallow BazelPackageLoader from fetching missing external repos.

Integration tests for BazelPackageLoader wrt external repos will be left for a follow-up CL.

RELNOTES: None.
PiperOrigin-RevId: 188967694

3 files changed, 114 insertions, 66 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/packages/AbstractPackageLoader.java b/src/main/java/com/google/devtools/build/lib/skyframe/packages/AbstractPackageLoader.java
index fa5bbd66dd..382010ee17 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/packages/AbstractPackageLoader.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/packages/AbstractPackageLoader.java
@@ -107,18 +107,19 @@ public abstract class AbstractPackageLoader implements PackageLoader {
   private static final int MAX_DIRECTORIES_TO_EAGERLY_VISIT_IN_GLOBBING = 3000;
 
   private final ImmutableDiff preinjectedDiff;
-  private final Differencer preinjectedDifferencer = new Differencer() {
-    @Override
-    public Diff getDiff(WalkableGraph fromGraph, Version fromVersion, Version toVersion)
-        throws InterruptedException {
-      return preinjectedDiff;
-    }
-  };
+  private final Differencer preinjectedDifferencer =
+      new Differencer() {
+        @Override
+        public Diff getDiff(WalkableGraph fromGraph, Version fromVersion, Version toVersion)
+            throws InterruptedException {
+          return preinjectedDiff;
+        }
+      };
   private final Reporter reporter;
   protected final RuleClassProvider ruleClassProvider;
   protected SkylarkSemantics skylarkSemantics;
   protected final ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions;
-  protected final AtomicReference<PathPackageLocator> pkgLocatorRef;
+  private final AtomicReference<PathPackageLocator> pkgLocatorRef;
   protected final ExternalFilesHelper externalFilesHelper;
   protected final BlazeDirectories directories;
   private final int legacyGlobbingThreads;
@@ -128,24 +129,36 @@ public abstract class AbstractPackageLoader implements PackageLoader {
   public abstract static class Builder {
     protected final Path workspaceDir;
     protected final BlazeDirectories directories;
+    protected final PathPackageLocator pkgLocator;
+    final AtomicReference<PathPackageLocator> pkgLocatorRef;
+    protected final ExternalFilesHelper externalFilesHelper;
     protected RuleClassProvider ruleClassProvider = getDefaultRuleClassProvider();
     protected SkylarkSemantics skylarkSemantics;
     protected Reporter reporter = new Reporter(new EventBus());
     protected Map<SkyFunctionName, SkyFunction> extraSkyFunctions = new HashMap<>();
-    protected List<PrecomputedValue.Injected> extraPrecomputedValues = new ArrayList<>();
-    protected String defaultsPackageContents = getDefaultDefaultPackageContents();
-    protected int legacyGlobbingThreads = 1;
+    List<PrecomputedValue.Injected> extraPrecomputedValues = new ArrayList<>();
+    String defaultsPackageContents = getDefaultDefaultPackageContents();
+    int legacyGlobbingThreads = 1;
     int skyframeThreads = 1;
 
-    protected Builder(Path workspaceDir) {
+    protected Builder(Path workspaceDir, Path installBase, Path outputBase) {
       this.workspaceDir = workspaceDir;
-      // The 'installBase' and 'outputBase' directories won't be meaningfully used by
-      // WorkspaceFileFunction, so we pass in a dummy Path.
-      // TODO(nharmata): Refactor WorkspaceFileFunction to make this a non-issue.
       Path devNull = workspaceDir.getFileSystem().getPath("/dev/null");
       directories =
           new BlazeDirectories(
-              new ServerDirectories(devNull, devNull, devNull), workspaceDir, "blaze");
+              new ServerDirectories(installBase, outputBase, devNull), workspaceDir, "blaze");
+
+      this.pkgLocator =
+          new PathPackageLocator(
+              directories.getOutputBase(),
+              ImmutableList.of(Root.fromPath(workspaceDir)),
+              BazelSkyframeExecutorConstants.BUILD_FILES_BY_PRIORITY);
+      this.pkgLocatorRef = new AtomicReference<>(pkgLocator);
+      this.externalFilesHelper =
+          ExternalFilesHelper.create(
+              pkgLocatorRef,
+              ExternalFileAction.DEPEND_ON_EXTERNAL_PKG_FOR_EXTERNAL_REPO_PATHS,
+              directories);
     }
 
     public Builder setRuleClassProvider(RuleClassProvider ruleClassProvider) {
@@ -214,30 +227,22 @@ public abstract class AbstractPackageLoader implements PackageLoader {
     protected abstract String getDefaultDefaultPackageContents();
   }
 
-  protected AbstractPackageLoader(Builder builder) {
-    Path workspaceDir = builder.workspaceDir;
-    PathPackageLocator pkgLocator =
-        new PathPackageLocator(
-            null,
-            ImmutableList.of(Root.fromPath(workspaceDir)),
-            BazelSkyframeExecutorConstants.BUILD_FILES_BY_PRIORITY);
+  AbstractPackageLoader(Builder builder) {
     this.ruleClassProvider = builder.ruleClassProvider;
     this.skylarkSemantics = builder.skylarkSemantics;
     this.reporter = builder.reporter;
     this.extraSkyFunctions = ImmutableMap.copyOf(builder.extraSkyFunctions);
-    this.pkgLocatorRef = new AtomicReference<>(pkgLocator);
+    this.pkgLocatorRef = builder.pkgLocatorRef;
     this.legacyGlobbingThreads = builder.legacyGlobbingThreads;
     this.skyframeThreads = builder.skyframeThreads;
     this.directories = builder.directories;
 
-    this.externalFilesHelper = ExternalFilesHelper.create(
-        pkgLocatorRef,
-        ExternalFileAction.DEPEND_ON_EXTERNAL_PKG_FOR_EXTERNAL_REPO_PATHS,
-        directories);
+    this.externalFilesHelper = builder.externalFilesHelper;
+
     this.preinjectedDiff =
         makePreinjectedDiff(
             skylarkSemantics,
-            pkgLocator,
+            builder.pkgLocator,
             builder.defaultsPackageContents,
             ImmutableList.copyOf(builder.extraPrecomputedValues));
   }
@@ -267,7 +272,7 @@ public abstract class AbstractPackageLoader implements PackageLoader {
     PrecomputedValue.DEFAULT_VISIBILITY.set(injectable, ConstantRuleVisibility.PRIVATE);
     PrecomputedValue.SKYLARK_SEMANTICS.set(injectable, skylarkSemantics);
     PrecomputedValue.DEFAULTS_PACKAGE_CONTENTS.set(injectable, defaultsPackageContents);
-    return new ImmutableDiff(ImmutableList.<SkyKey>of(), valuesToInject);
+    return new ImmutableDiff(ImmutableList.of(), valuesToInject);
   }
 
   @Override
@@ -333,20 +338,25 @@ public abstract class AbstractPackageLoader implements PackageLoader {
   }
 
   protected abstract String getName();
+
   protected abstract ImmutableList<EnvironmentExtension> getEnvironmentExtensions();
+
   protected abstract CrossRepositoryLabelViolationStrategy
       getCrossRepositoryLabelViolationStrategy();
+
   protected abstract ImmutableList<BuildFileName> getBuildFilesByPriority();
+
   protected abstract ActionOnIOExceptionReadingBuildFile getActionOnIOExceptionReadingBuildFile();
 
-  protected final ImmutableMap<SkyFunctionName, SkyFunction> makeFreshSkyFunctions() {
+  private ImmutableMap<SkyFunctionName, SkyFunction> makeFreshSkyFunctions() {
     AtomicReference<TimestampGranularityMonitor> tsgm =
         new AtomicReference<>(new TimestampGranularityMonitor(BlazeClock.instance()));
     Cache<PackageIdentifier, LoadedPackageCacheEntry> packageFunctionCache =
         CacheBuilder.newBuilder().build();
     Cache<PackageIdentifier, AstParseResult> astCache = CacheBuilder.newBuilder().build();
-    AtomicReference<PerBuildSyscallCache> syscallCacheRef = new AtomicReference<>(
-        PerBuildSyscallCache.newBuilder().setConcurrencyLevel(legacyGlobbingThreads).build());
+    AtomicReference<PerBuildSyscallCache> syscallCacheRef =
+        new AtomicReference<>(
+            PerBuildSyscallCache.newBuilder().setConcurrencyLevel(legacyGlobbingThreads).build());
     PackageFactory pkgFactory =
         new PackageFactory(
             ruleClassProvider,
@@ -359,13 +369,14 @@ public abstract class AbstractPackageLoader implements PackageLoader {
     pkgFactory.setSyscalls(syscallCacheRef);
     pkgFactory.setMaxDirectoriesToEagerlyVisitInGlobbing(
         MAX_DIRECTORIES_TO_EAGERLY_VISIT_IN_GLOBBING);
-    CachingPackageLocator cachingPackageLocator = new CachingPackageLocator() {
-      @Override
-      @Nullable
-      public Path getBuildFileForPackage(PackageIdentifier packageName) {
-        return pkgLocatorRef.get().getPackageBuildFileNullable(packageName, syscallCacheRef);
-      }
-    };
+    CachingPackageLocator cachingPackageLocator =
+        new CachingPackageLocator() {
+          @Override
+          @Nullable
+          public Path getBuildFileForPackage(PackageIdentifier packageName) {
+            return pkgLocatorRef.get().getPackageBuildFileNullable(packageName, syscallCacheRef);
+          }
+        };
     ImmutableMap.Builder<SkyFunctionName, SkyFunction> builder = ImmutableMap.builder();
     builder
         .put(SkyFunctions.PRECOMPUTED, new PrecomputedFunction())
@@ -378,10 +389,11 @@ public abstract class AbstractPackageLoader implements PackageLoader {
         .put(
             SkyFunctions.PACKAGE_LOOKUP,
             new PackageLookupFunction(
-                /* deletedPackages= */ new AtomicReference<>(ImmutableSet.<PackageIdentifier>of()),
+                /* deletedPackages= */ new AtomicReference<>(ImmutableSet.of()),
                 getCrossRepositoryLabelViolationStrategy(),
                 getBuildFilesByPriority()))
-        .put(SkyFunctions.BLACKLISTED_PACKAGE_PREFIXES,
+        .put(
+            SkyFunctions.BLACKLISTED_PACKAGE_PREFIXES,
             new BlacklistedPackagePrefixesFunction(
                 /*hardcodedBlacklistedPackagePrefixes=*/ ImmutableSet.of(),
                 /*additionalBlacklistedPackagePrefixesFile=*/ PathFragment.EMPTY_FRAGMENT))
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/packages/BUILD b/src/main/java/com/google/devtools/build/lib/skyframe/packages/BUILD
index 7ca7242ba0..7cb108a81d 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/packages/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/packages/BUILD
@@ -10,11 +10,15 @@ java_library(
     name = "packages",
     srcs = glob(["*.java"]),
     deps = [
+        "//src/main/java/com/google/devtools/build/lib:bazel-repository",
         "//src/main/java/com/google/devtools/build/lib:bazel-rules",
+        "//src/main/java/com/google/devtools/build/lib:bazel/BazelRepositoryModule",
         "//src/main/java/com/google/devtools/build/lib:build-base",
         "//src/main/java/com/google/devtools/build/lib:events",
         "//src/main/java/com/google/devtools/build/lib:io",
         "//src/main/java/com/google/devtools/build/lib:packages-internal",
+        "//src/main/java/com/google/devtools/build/lib/bazel/repository/cache",
+        "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader",
         "//src/main/java/com/google/devtools/build/lib/clock",
         "//src/main/java/com/google/devtools/build/lib/cmdline",
         "//src/main/java/com/google/devtools/build/lib/vfs",
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoader.java b/src/main/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoader.java
index dfed69265d..cdaa07c1eb 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoader.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoader.java
@@ -17,17 +17,25 @@ import com.google.common.base.Suppliers;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
-import com.google.devtools.build.lib.bazel.rules.BazelRuleClassProvider;
+import com.google.devtools.build.lib.bazel.BazelRepositoryModule;
+import com.google.devtools.build.lib.bazel.repository.MavenDownloader;
+import com.google.devtools.build.lib.bazel.repository.MavenServerFunction;
+import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache;
+import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader;
+import com.google.devtools.build.lib.bazel.repository.skylark.SkylarkRepositoryFunction;
+import com.google.devtools.build.lib.bazel.rules.BazelRulesModule;
 import com.google.devtools.build.lib.packages.BuildFileName;
 import com.google.devtools.build.lib.packages.PackageFactory.EnvironmentExtension;
 import com.google.devtools.build.lib.packages.RuleClassProvider;
-import com.google.devtools.build.lib.rules.repository.LocalRepositoryFunction;
-import com.google.devtools.build.lib.rules.repository.LocalRepositoryRule;
 import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction;
 import com.google.devtools.build.lib.rules.repository.RepositoryFunction;
 import com.google.devtools.build.lib.rules.repository.RepositoryLoaderFunction;
 import com.google.devtools.build.lib.runtime.proto.InvocationPolicyOuterClass.InvocationPolicy;
+import com.google.devtools.build.lib.skyframe.ActionEnvironmentFunction;
 import com.google.devtools.build.lib.skyframe.BazelSkyframeExecutorConstants;
+import com.google.devtools.build.lib.skyframe.ClientEnvironmentFunction;
+import com.google.devtools.build.lib.skyframe.DirectoryListingFunction;
+import com.google.devtools.build.lib.skyframe.DirectoryListingStateFunction;
 import com.google.devtools.build.lib.skyframe.LocalRepositoryLookupFunction;
 import com.google.devtools.build.lib.skyframe.PackageFunction.ActionOnIOExceptionReadingBuildFile;
 import com.google.devtools.build.lib.skyframe.PackageLookupFunction.CrossRepositoryLabelViolationStrategy;
@@ -37,37 +45,54 @@ import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.build.skyframe.SkyFunction;
 import com.google.devtools.build.skyframe.SkyFunctionName;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
 
 /**
- * Concrete implementation of {@link PackageLoader} that uses skyframe under the covers, but with
- * no caching or incrementality.
+ * Concrete implementation of {@link PackageLoader} that uses skyframe under the covers, but with no
+ * caching or incrementality.
  */
 public class BazelPackageLoader extends AbstractPackageLoader {
   /** Returns a fresh {@link Builder} instance. */
-  public static Builder builder(Path workspaceDir) {
-    Builder builder = new Builder(workspaceDir);
+  public static Builder builder(Path workspaceDir, Path installBase, Path outputBase) {
+    Builder builder = new Builder(workspaceDir, installBase, outputBase);
+
+    RepositoryCache repositoryCache = new RepositoryCache();
+    HttpDownloader httpDownloader = new HttpDownloader(repositoryCache);
 
     // Set up SkyFunctions and PrecomputedValues needed to make local repositories work correctly.
     ImmutableMap<String, RepositoryFunction> repositoryHandlers =
-        ImmutableMap.of(
-            LocalRepositoryRule.NAME, (RepositoryFunction) new LocalRepositoryFunction());
+        BazelRepositoryModule.repositoryRules(httpDownloader, new MavenDownloader(repositoryCache));
+
+    // Prevent PackageLoader from fetching any remote repositories; these should only be fetched by
+    // Bazel before calling PackageLoader.
+    AtomicBoolean isFetch = new AtomicBoolean(false);
 
     builder.addExtraSkyFunctions(
-        ImmutableMap.<SkyFunctionName, SkyFunction>of(
-            SkyFunctions.LOCAL_REPOSITORY_LOOKUP,
-            new LocalRepositoryLookupFunction(),
-            SkyFunctions.REPOSITORY_DIRECTORY,
-            new RepositoryDelegatorFunction(
-                repositoryHandlers,
-                null,
-                new AtomicBoolean(true),
-                ImmutableMap::of,
-                builder.directories),
-            SkyFunctions.REPOSITORY,
-            new RepositoryLoaderFunction()));
+        ImmutableMap.<SkyFunctionName, SkyFunction>builder()
+            .put(
+                SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE,
+                new ClientEnvironmentFunction(new AtomicReference<>(ImmutableMap.of())))
+            .put(
+                SkyFunctions.DIRECTORY_LISTING_STATE,
+                new DirectoryListingStateFunction(builder.externalFilesHelper))
+            .put(SkyFunctions.ACTION_ENVIRONMENT_VARIABLE, new ActionEnvironmentFunction())
+            .put(SkyFunctions.DIRECTORY_LISTING, new DirectoryListingFunction())
+            .put(SkyFunctions.LOCAL_REPOSITORY_LOOKUP, new LocalRepositoryLookupFunction())
+            .put(
+                SkyFunctions.REPOSITORY_DIRECTORY,
+                new RepositoryDelegatorFunction(
+                    repositoryHandlers,
+                    new SkylarkRepositoryFunction(httpDownloader),
+                    isFetch,
+                    ImmutableMap::of,
+                    builder.directories))
+            .put(SkyFunctions.REPOSITORY, new RepositoryLoaderFunction())
+            .put(MavenServerFunction.NAME, new MavenServerFunction(builder.directories))
+            .build());
 
     // Set extra precomputed values.
     builder.addExtraPrecomputedValues(
+        PrecomputedValue.injected(PrecomputedValue.ACTION_ENV, ImmutableMap.of()),
         PrecomputedValue.injected(
             RepositoryDelegatorFunction.REPOSITORY_OVERRIDES,
             Suppliers.ofInstance(ImmutableMap.of())));
@@ -78,10 +103,17 @@ public class BazelPackageLoader extends AbstractPackageLoader {
   /** Builder for {@link BazelPackageLoader} instances. */
   public static class Builder extends AbstractPackageLoader.Builder {
     private static final ConfiguredRuleClassProvider DEFAULT_RULE_CLASS_PROVIDER =
-        BazelRuleClassProvider.create();
+        createRuleClassProvider();
+
+    private static ConfiguredRuleClassProvider createRuleClassProvider() {
+      ConfiguredRuleClassProvider.Builder classProvider = new ConfiguredRuleClassProvider.Builder();
+      new BazelRepositoryModule().initializeRuleClasses(classProvider);
+      new BazelRulesModule().initializeRuleClasses(classProvider);
+      return classProvider.build();
+    }
 
-    private Builder(Path workspaceDir) {
-      super(workspaceDir);
+    private Builder(Path workspaceDir, Path installBase, Path outputBase) {
+      super(workspaceDir, installBase, outputBase);
     }
 
     @Override