Introduce BazelPackageLoader.

This is a standalone library for loading packages. It is intended to be used by clients outside of Bazel (for now, Kythe).

Perform BazelPackageLoader#loadPackage in the testing hook used in all of Bazel's unit tests and integration tests. This gives us very good test coverage for BazelPackageLoader.

RELNOTES: None
PiperOrigin-RevId: 155004070

1 files changed, 329 insertions, 0 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/packages/AbstractPackageLoader.java b/src/main/java/com/google/devtools/build/lib/skyframe/packages/AbstractPackageLoader.java
new file mode 100644
index 0000000000..d0b771dc31
--- /dev/null
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/packages/AbstractPackageLoader.java
@@ -0,0 +1,329 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package com.google.devtools.build.lib.skyframe.packages;
+
+import static com.google.common.base.Throwables.throwIfInstanceOf;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.eventbus.EventBus;
+import com.google.devtools.build.lib.analysis.BlazeDirectories;
+import com.google.devtools.build.lib.cmdline.PackageIdentifier;
+import com.google.devtools.build.lib.events.Reporter;
+import com.google.devtools.build.lib.packages.AttributeContainer;
+import com.google.devtools.build.lib.packages.BuildFileContainsErrorsException;
+import com.google.devtools.build.lib.packages.CachingPackageLocator;
+import com.google.devtools.build.lib.packages.ConstantRuleVisibility;
+import com.google.devtools.build.lib.packages.NoSuchPackageException;
+import com.google.devtools.build.lib.packages.Package;
+import com.google.devtools.build.lib.packages.PackageFactory;
+import com.google.devtools.build.lib.packages.PackageFactory.EnvironmentExtension;
+import com.google.devtools.build.lib.packages.Preprocessor;
+import com.google.devtools.build.lib.packages.RuleClassProvider;
+import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
+import com.google.devtools.build.lib.skyframe.ASTFileLookupFunction;
+import com.google.devtools.build.lib.skyframe.BlacklistedPackagePrefixesFunction;
+import com.google.devtools.build.lib.skyframe.ContainingPackageLookupFunction;
+import com.google.devtools.build.lib.skyframe.DirectoryListingFunction;
+import com.google.devtools.build.lib.skyframe.DirectoryListingStateFunction;
+import com.google.devtools.build.lib.skyframe.ExternalFilesHelper;
+import com.google.devtools.build.lib.skyframe.ExternalFilesHelper.ExternalFileAction;
+import com.google.devtools.build.lib.skyframe.ExternalPackageFunction;
+import com.google.devtools.build.lib.skyframe.FileFunction;
+import com.google.devtools.build.lib.skyframe.FileStateFunction;
+import com.google.devtools.build.lib.skyframe.FileSymlinkCycleUniquenessFunction;
+import com.google.devtools.build.lib.skyframe.FileSymlinkInfiniteExpansionUniquenessFunction;
+import com.google.devtools.build.lib.skyframe.GlobFunction;
+import com.google.devtools.build.lib.skyframe.PackageFunction;
+import com.google.devtools.build.lib.skyframe.PackageFunction.CacheEntryWithGlobDeps;
+import com.google.devtools.build.lib.skyframe.PackageLookupFunction;
+import com.google.devtools.build.lib.skyframe.PackageValue;
+import com.google.devtools.build.lib.skyframe.PrecomputedFunction;
+import com.google.devtools.build.lib.skyframe.PrecomputedValue;
+import com.google.devtools.build.lib.skyframe.SkyFunctions;
+import com.google.devtools.build.lib.skyframe.SkylarkImportLookupFunction;
+import com.google.devtools.build.lib.skyframe.WorkspaceASTFunction;
+import com.google.devtools.build.lib.skyframe.WorkspaceFileFunction;
+import com.google.devtools.build.lib.skyframe.WorkspaceNameFunction;
+import com.google.devtools.build.lib.syntax.SkylarkSemanticsOptions;
+import com.google.devtools.build.lib.util.BlazeClock;
+import com.google.devtools.build.lib.util.Preconditions;
+import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
+import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.vfs.UnixGlob;
+import com.google.devtools.build.skyframe.BuildDriver;
+import com.google.devtools.build.skyframe.Differencer;
+import com.google.devtools.build.skyframe.ErrorInfo;
+import com.google.devtools.build.skyframe.EvaluationResult;
+import com.google.devtools.build.skyframe.ImmutableDiff;
+import com.google.devtools.build.skyframe.InMemoryMemoizingEvaluator;
+import com.google.devtools.build.skyframe.Injectable;
+import com.google.devtools.build.skyframe.MemoizingEvaluator;
+import com.google.devtools.build.skyframe.SequentialBuildDriver;
+import com.google.devtools.build.skyframe.SkyFunction;
+import com.google.devtools.build.skyframe.SkyFunctionName;
+import com.google.devtools.build.skyframe.SkyKey;
+import com.google.devtools.build.skyframe.SkyValue;
+import com.google.devtools.build.skyframe.Version;
+import com.google.devtools.build.skyframe.WalkableGraph;
+import com.google.devtools.common.options.Options;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import javax.annotation.Nullable;
+
+/**
+ * Abstract base class of a {@link PackageLoader} implementation that has no incrementality or
+ * caching.
+ */
+public abstract class AbstractPackageLoader implements PackageLoader {
+  private final ImmutableDiff preinjectedDiff;
+  private final Differencer preinjectedDifferencer = new Differencer() {
+    @Override
+    public Diff getDiff(WalkableGraph fromGraph, Version fromVersion, Version toVersion)
+        throws InterruptedException {
+      return preinjectedDiff;
+    }
+  };
+  private final Reporter reporter;
+  protected final RuleClassProvider ruleClassProvider;
+  protected final ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions;
+  protected final AtomicReference<PathPackageLocator> pkgLocatorRef;
+  protected final ExternalFilesHelper externalFilesHelper;
+  protected final AtomicReference<ImmutableSet<PackageIdentifier>> deletedPackagesRef =
+      new AtomicReference<>(ImmutableSet.<PackageIdentifier>of());
+  protected final CachingPackageLocator packageManager;
+  protected final BlazeDirectories directories;
+
+  /** Abstract base class of a builder for {@link PackageLoader} instances. */
+  public abstract static class Builder {
+    protected final Path workspaceDir;
+    protected RuleClassProvider ruleClassProvider = getDefaultRuleClassProvider();
+    protected Reporter reporter = new Reporter(new EventBus());
+    protected ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions = ImmutableMap.of();
+    protected ImmutableList<PrecomputedValue.Injected> extraPrecomputedValues = ImmutableList.of();
+    protected String defaultsPackageContents = getDefaultDefaulsPackageContents();
+
+    protected Builder(Path workspaceDir) {
+      this.workspaceDir = workspaceDir;
+    }
+
+    public Builder setRuleClassProvider(RuleClassProvider ruleClassProvider) {
+      this.ruleClassProvider = ruleClassProvider;
+      return this;
+    }
+
+    public Builder setDefaultsPackageContents(String defaultsPackageContents) {
+      this.defaultsPackageContents = defaultsPackageContents;
+      return this;
+    }
+
+    public Builder setReporter(Reporter reporter) {
+      this.reporter = reporter;
+      return this;
+    }
+
+    public Builder setExtraSkyFunctions(
+        ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions) {
+      this.extraSkyFunctions = extraSkyFunctions;
+      return this;
+    }
+
+    public Builder setExtraPrecomputedValues(
+        ImmutableList<PrecomputedValue.Injected> extraPrecomputedValues) {
+      this.extraPrecomputedValues = extraPrecomputedValues;
+      return this;
+    }
+
+    public abstract PackageLoader build();
+
+    protected abstract RuleClassProvider getDefaultRuleClassProvider();
+
+    protected abstract String getDefaultDefaulsPackageContents();
+  }
+
+  protected AbstractPackageLoader(Builder builder) {
+    Path workspaceDir = builder.workspaceDir;
+    PathPackageLocator pkgLocator =
+        new PathPackageLocator(null, ImmutableList.of(workspaceDir));
+    this.ruleClassProvider = builder.ruleClassProvider;
+    this.reporter = builder.reporter;
+    this.extraSkyFunctions = builder.extraSkyFunctions;
+    this.pkgLocatorRef = new AtomicReference<>(pkgLocator);
+
+    // The 'installBase' and 'outputBase' directories won't be meaningfully used by
+    // WorkspaceFileFunction, so we pass in a dummy Path.
+    // TODO(nharmata): Refactor WorkspaceFileFunction to make this a non-issue.
+    Path devNull = workspaceDir.getFileSystem().getPath("/dev/null");
+    this.directories = new BlazeDirectories(/*installBase=*/devNull,
+        /*outputBase=*/devNull, /*workspace=*/workspaceDir, "blaze");
+    this.externalFilesHelper = new ExternalFilesHelper(
+        pkgLocatorRef,
+        ExternalFileAction.DEPEND_ON_EXTERNAL_PKG_FOR_EXTERNAL_REPO_PATHS,
+        directories);
+    this.packageManager = new CachingPackageLocator() {
+      @Override
+      @Nullable
+      public Path getBuildFileForPackage(PackageIdentifier packageName) {
+        return pkgLocatorRef.get().getPackageBuildFileNullable(packageName,
+            UnixGlob.DEFAULT_SYSCALLS_REF);
+      }
+    };
+    this.preinjectedDiff = makePreinjectedDiff(
+        pkgLocator,
+        builder.defaultsPackageContents,
+        builder.extraPrecomputedValues,
+        directories);
+  }
+
+  private static ImmutableDiff makePreinjectedDiff(
+      PathPackageLocator pkgLocator,
+      String defaultsPackageContents,
+      ImmutableList<PrecomputedValue.Injected> extraPrecomputedValues,
+      BlazeDirectories directories) {
+    final Map<SkyKey, SkyValue> valuesToInject = new HashMap<>();
+    Injectable injectable =
+        new Injectable() {
+          @Override
+          public void inject(Map<SkyKey, ? extends SkyValue> values) {
+            valuesToInject.putAll(values);
+          }
+
+          @Override
+          public void inject(SkyKey key, SkyValue value) {
+            valuesToInject.put(key, value);
+          }
+        };
+    for (PrecomputedValue.Injected injected : extraPrecomputedValues) {
+      injected.inject(injectable);
+    }
+    PrecomputedValue.PATH_PACKAGE_LOCATOR.set(injectable, pkgLocator);
+    PrecomputedValue.DEFAULT_VISIBILITY.set(injectable, ConstantRuleVisibility.PRIVATE);
+    PrecomputedValue.SKYLARK_SEMANTICS.set(
+        injectable,
+        Options.getDefaults(SkylarkSemanticsOptions.class));
+    PrecomputedValue.DEFAULTS_PACKAGE_CONTENTS.set(injectable, defaultsPackageContents);
+    PrecomputedValue.BLACKLISTED_PACKAGE_PREFIXES_FILE.set(injectable, PathFragment.EMPTY_FRAGMENT);
+    PrecomputedValue.BLAZE_DIRECTORIES.set(injectable, directories);
+    return new ImmutableDiff(ImmutableList.<SkyKey>of(), valuesToInject);
+  }
+
+  /**
+   * Returns a {@link Package} instance, if any, representing the Blaze package specified by
+   * {@code pkgId}. Note that the returned {@link Package} instance may be in error (see
+   * {@link Package#containsErrors}), e.g. if there was syntax error in the package's BUILD file.
+   *
+   * @throws InterruptedException if the package loading was interrupted.
+   * @throws NoSuchPackageException if there was a non-recoverable error loading the package, e.g.
+   *     an io error reading the BUILD file.
+   */
+  @Override
+  public Package loadPackage(PackageIdentifier pkgId) throws NoSuchPackageException,
+      InterruptedException {
+    SkyKey key = PackageValue.key(pkgId);
+    EvaluationResult<PackageValue> result =
+        makeFreshDriver()
+            .evaluate(ImmutableList.of(key), /*keepGoing=*/ true, /*numThreads=*/ 1, reporter);
+    if (result.hasError()) {
+      ErrorInfo error = result.getError();
+      if (!Iterables.isEmpty(error.getCycleInfo())) {
+        throw new BuildFileContainsErrorsException(
+            pkgId, "Cycle encountered while loading package " + pkgId);
+      }
+      Throwable e = Preconditions.checkNotNull(error.getException());
+      throwIfInstanceOf(e, NoSuchPackageException.class);
+      throw new IllegalStateException("Unexpected Exception type from PackageValue for '"
+          + pkgId + "'' with root causes: " + Iterables.toString(error.getRootCauses()), e);
+    }
+    return result.get(key).getPackage();
+  }
+
+  private BuildDriver makeFreshDriver() {
+    return new SequentialBuildDriver(
+        InMemoryMemoizingEvaluator.SUPPLIER.create(
+            makeFreshSkyFunctions(),
+            preinjectedDifferencer,
+            /*progressReceiver=*/ null,
+            new MemoizingEvaluator.EmittedEventState(),
+            /*keepEdges=*/ false));
+  }
+
+  protected abstract String getName();
+  protected abstract ImmutableList<EnvironmentExtension> getEnvironmentExtensions();
+  protected abstract PackageLookupFunction makePackageLookupFunction();
+  protected abstract ImmutableMap<SkyFunctionName, SkyFunction> getExtraExtraSkyFunctions();
+
+  protected final ImmutableMap<SkyFunctionName, SkyFunction> makeFreshSkyFunctions() {
+    AtomicReference<TimestampGranularityMonitor> tsgm =
+        new AtomicReference<>(new TimestampGranularityMonitor(BlazeClock.instance()));
+    Cache<PackageIdentifier, CacheEntryWithGlobDeps<Package.Builder>> packageFunctionCache =
+        CacheBuilder.newBuilder().build();
+    Cache<PackageIdentifier, CacheEntryWithGlobDeps<Preprocessor.AstAfterPreprocessing>> astCache =
+        CacheBuilder.newBuilder().build();
+    PackageFactory pkgFactory = new PackageFactory(
+        ruleClassProvider,
+        null,
+        AttributeContainer.ATTRIBUTE_CONTAINER_FACTORY,
+        getEnvironmentExtensions(),
+        getName(),
+        Package.Builder.DefaultHelper.INSTANCE);
+    ImmutableMap.Builder<SkyFunctionName, SkyFunction> builder = ImmutableMap.builder();
+    builder
+        .put(SkyFunctions.PRECOMPUTED, new PrecomputedFunction())
+        .put(SkyFunctions.FILE_STATE, new FileStateFunction(tsgm, externalFilesHelper))
+        .put(
+            SkyFunctions.DIRECTORY_LISTING_STATE,
+            new DirectoryListingStateFunction(externalFilesHelper))
+        .put(SkyFunctions.FILE_SYMLINK_CYCLE_UNIQUENESS, new FileSymlinkCycleUniquenessFunction())
+        .put(
+            SkyFunctions.FILE_SYMLINK_INFINITE_EXPANSION_UNIQUENESS,
+            new FileSymlinkInfiniteExpansionUniquenessFunction())
+        .put(SkyFunctions.FILE, new FileFunction(pkgLocatorRef))
+        .put(SkyFunctions.DIRECTORY_LISTING, new DirectoryListingFunction())
+        .put(SkyFunctions.PACKAGE_LOOKUP, makePackageLookupFunction())
+        .put(SkyFunctions.BLACKLISTED_PACKAGE_PREFIXES, new BlacklistedPackagePrefixesFunction())
+        .put(SkyFunctions.CONTAINING_PACKAGE_LOOKUP, new ContainingPackageLookupFunction())
+        .put(SkyFunctions.AST_FILE_LOOKUP, new ASTFileLookupFunction(ruleClassProvider))
+        .put(
+            SkyFunctions.SKYLARK_IMPORTS_LOOKUP,
+            new SkylarkImportLookupFunction(ruleClassProvider, pkgFactory))
+        .put(SkyFunctions.WORKSPACE_NAME, new WorkspaceNameFunction())
+        .put(SkyFunctions.WORKSPACE_AST, new WorkspaceASTFunction(ruleClassProvider))
+        .put(
+            SkyFunctions.WORKSPACE_FILE,
+            new WorkspaceFileFunction(ruleClassProvider, pkgFactory, directories))
+        .put(SkyFunctions.EXTERNAL_PACKAGE, new ExternalPackageFunction())
+        .put(SkyFunctions.GLOB, new GlobFunction(/*alwaysUseDirListing=*/ false))
+        .put(
+            SkyFunctions.PACKAGE,
+            new PackageFunction(
+                pkgFactory,
+                packageManager,
+                /*showLoadingProgress=*/ new AtomicBoolean(false),
+                packageFunctionCache,
+                astCache,
+                /*numPackagesLoaded=*/ new AtomicInteger(0),
+                null))
+        .putAll(extraSkyFunctions)
+        .putAll(getExtraExtraSkyFunctions());
+    return builder.build();
+  }
+}