// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.skyframe;
import static com.google.devtools.build.lib.concurrent.Uninterruptibles.callUninterruptibly;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.base.Predicates;
import com.google.common.base.Stopwatch;
import com.google.common.base.Supplier;
import com.google.common.base.Throwables;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Maps;
import com.google.common.collect.Range;
import com.google.common.collect.Sets;
import com.google.common.eventbus.EventBus;
import com.google.devtools.build.lib.actions.Action;
import com.google.devtools.build.lib.actions.ActionCacheChecker;
import com.google.devtools.build.lib.actions.ActionExecutionContextFactory;
import com.google.devtools.build.lib.actions.ActionExecutionStatusReporter;
import com.google.devtools.build.lib.actions.ActionGraph;
import com.google.devtools.build.lib.actions.ActionInputFileCache;
import com.google.devtools.build.lib.actions.ActionLogBufferPathGenerator;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.ArtifactFactory;
import com.google.devtools.build.lib.actions.ArtifactOwner;
import com.google.devtools.build.lib.actions.Executor;
import com.google.devtools.build.lib.actions.PackageRootResolutionException;
import com.google.devtools.build.lib.actions.ResourceManager;
import com.google.devtools.build.lib.actions.Root;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.analysis.BuildView.Options;
import com.google.devtools.build.lib.analysis.ConfiguredAspect;
import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
import com.google.devtools.build.lib.analysis.ConfiguredTarget;
import com.google.devtools.build.lib.analysis.DependencyResolver.Dependency;
import com.google.devtools.build.lib.analysis.RuleConfiguredTarget;
import com.google.devtools.build.lib.analysis.TopLevelArtifactContext;
import com.google.devtools.build.lib.analysis.WorkspaceStatusAction;
import com.google.devtools.build.lib.analysis.WorkspaceStatusAction.Factory;
import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory;
import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory.BuildInfoKey;
import com.google.devtools.build.lib.analysis.config.BinTools;
import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection;
import com.google.devtools.build.lib.analysis.config.BuildOptions;
import com.google.devtools.build.lib.analysis.config.ConfigurationFactory;
import com.google.devtools.build.lib.analysis.config.ConfigurationFragmentFactory;
import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
import com.google.devtools.build.lib.analysis.config.PatchTransition;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.cmdline.PackageIdentifier;
import com.google.devtools.build.lib.cmdline.TargetParsingException;
import com.google.devtools.build.lib.concurrent.ThreadSafety;
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
import com.google.devtools.build.lib.events.EventHandler;
import com.google.devtools.build.lib.events.Reporter;
import com.google.devtools.build.lib.packages.Aspect;
import com.google.devtools.build.lib.packages.Attribute;
import com.google.devtools.build.lib.packages.BuildFileContainsErrorsException;
import com.google.devtools.build.lib.packages.NoSuchPackageException;
import com.google.devtools.build.lib.packages.NoSuchThingException;
import com.google.devtools.build.lib.packages.Package;
import com.google.devtools.build.lib.packages.Package.LegacyBuilder;
import com.google.devtools.build.lib.packages.PackageFactory;
import com.google.devtools.build.lib.packages.Preprocessor;
import com.google.devtools.build.lib.packages.Preprocessor.AstAfterPreprocessing;
import com.google.devtools.build.lib.packages.RuleClassProvider;
import com.google.devtools.build.lib.packages.RuleVisibility;
import com.google.devtools.build.lib.packages.Target;
import com.google.devtools.build.lib.pkgcache.LegacyLoadingPhaseRunner;
import com.google.devtools.build.lib.pkgcache.LoadingCallback;
import com.google.devtools.build.lib.pkgcache.LoadingFailedException;
import com.google.devtools.build.lib.pkgcache.LoadingOptions;
import com.google.devtools.build.lib.pkgcache.LoadingPhaseCompleteEvent;
import com.google.devtools.build.lib.pkgcache.LoadingPhaseRunner;
import com.google.devtools.build.lib.pkgcache.LoadingResult;
import com.google.devtools.build.lib.pkgcache.PackageCacheOptions;
import com.google.devtools.build.lib.pkgcache.PackageManager;
import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
import com.google.devtools.build.lib.pkgcache.TargetParsingCompleteEvent;
import com.google.devtools.build.lib.pkgcache.TargetPatternEvaluator;
import com.google.devtools.build.lib.pkgcache.TestFilter;
import com.google.devtools.build.lib.pkgcache.TransitivePackageLoader;
import com.google.devtools.build.lib.profiler.AutoProfiler;
import com.google.devtools.build.lib.skyframe.AspectValue.AspectValueKey;
import com.google.devtools.build.lib.skyframe.DirtinessCheckerUtils.FileDirtinessChecker;
import com.google.devtools.build.lib.skyframe.SkyframeActionExecutor.ActionCompletedReceiver;
import com.google.devtools.build.lib.skyframe.SkyframeActionExecutor.ProgressSupplier;
import com.google.devtools.build.lib.util.AbruptExitException;
import com.google.devtools.build.lib.util.ExitCode;
import com.google.devtools.build.lib.util.ResourceUsage;
import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
import com.google.devtools.build.lib.vfs.BatchStat;
import com.google.devtools.build.lib.vfs.Dirent;
import com.google.devtools.build.lib.vfs.ModifiedFileSet;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.RootedPath;
import com.google.devtools.build.lib.vfs.UnixGlob;
import com.google.devtools.build.skyframe.BuildDriver;
import com.google.devtools.build.skyframe.CycleInfo;
import com.google.devtools.build.skyframe.CyclesReporter;
import com.google.devtools.build.skyframe.Differencer;
import com.google.devtools.build.skyframe.Differencer.DiffWithDelta.Delta;
import com.google.devtools.build.skyframe.ErrorInfo;
import com.google.devtools.build.skyframe.EvaluationProgressReceiver;
import com.google.devtools.build.skyframe.EvaluationResult;
import com.google.devtools.build.skyframe.ImmutableDiff;
import com.google.devtools.build.skyframe.Injectable;
import com.google.devtools.build.skyframe.MemoizingEvaluator;
import com.google.devtools.build.skyframe.MemoizingEvaluator.EvaluatorSupplier;
import com.google.devtools.build.skyframe.SkyFunction;
import com.google.devtools.build.skyframe.SkyFunctionName;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
import com.google.devtools.build.skyframe.WalkableGraph.WalkableGraphFactory;
import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.logging.Logger;
import javax.annotation.Nullable;
/**
* A helper object to support Skyframe-driven execution.
*
*
This object is mostly used to inject external state, such as the executor engine or
* some additional artifacts (workspace status and build info artifacts) into SkyFunctions
* for use during the build.
*/
public abstract class SkyframeExecutor implements WalkableGraphFactory {
private final EvaluatorSupplier evaluatorSupplier;
protected MemoizingEvaluator memoizingEvaluator;
private final MemoizingEvaluator.EmittedEventState emittedEventState =
new MemoizingEvaluator.EmittedEventState();
private final PackageFactory pkgFactory;
private final WorkspaceStatusAction.Factory workspaceStatusActionFactory;
private final BlazeDirectories directories;
@Nullable
private BatchStat batchStatter;
// TODO(bazel-team): Figure out how to handle value builders that block internally. Blocking
// operations may need to be handled in another (bigger?) thread pool. Also, we should detect
// the number of cores and use that as the thread-pool size for CPU-bound operations.
// I just bumped this to 200 to get reasonable execution phase performance; that may cause
// significant overhead for CPU-bound processes (i.e. analysis). [skyframe-analysis]
@VisibleForTesting
public static final int DEFAULT_THREAD_COUNT =
// Reduce thread count while running tests of Bazel. Test cases are typically small, and large
// thread pools vying for a relatively small number of CPU cores may induce non-optimal
// performance.
System.getenv("TEST_TMPDIR") == null ? 200 : 5;
// Cache of partially constructed Package instances, stored between reruns of the PackageFunction
// (because of missing dependencies, within the same evaluate() run) to avoid loading the same
// package twice (first time loading to find subincludes and declare value dependencies).
// TODO(bazel-team): remove this cache once we have skyframe-native package loading
// [skyframe-loading]
private final Cache packageFunctionCache =
newPkgFunctionCache();
private final Cache astCache = newAstCache();
private final AtomicInteger numPackagesLoaded = new AtomicInteger(0);
protected SkyframeBuildView skyframeBuildView;
private ActionLogBufferPathGenerator actionLogBufferPathGenerator;
protected BuildDriver buildDriver;
// AtomicReferences are used here as mutable boxes shared with value builders.
private final AtomicBoolean showLoadingProgress = new AtomicBoolean();
protected final AtomicReference syscalls =
new AtomicReference<>(UnixGlob.DEFAULT_SYSCALLS);
protected final AtomicReference pkgLocator =
new AtomicReference<>();
protected final AtomicReference> deletedPackages =
new AtomicReference<>(ImmutableSet.of());
private final AtomicReference eventBus = new AtomicReference<>();
private final ImmutableList buildInfoFactories;
// Under normal circumstances, the artifact factory persists for the life of a Blaze server, but
// since it is not yet created when we create the value builders, we have to use a supplier,
// initialized when the build view is created.
private final MutableSupplier artifactFactory = new MutableSupplier<>();
// Used to give to WriteBuildInfoAction via a supplier. Relying on BuildVariableValue.BUILD_ID
// would be preferable, but we have no way to have the Action depend on that value directly.
// Having the BuildInfoFunction own the supplier is currently not possible either, because then
// it would be invalidated on every build, since it would depend on the build id value.
private MutableSupplier buildId = new MutableSupplier<>();
protected boolean active = true;
private final SkyframePackageManager packageManager;
private final Preprocessor.Factory.Supplier preprocessorFactorySupplier;
private Preprocessor.Factory preprocessorFactory;
protected final TimestampGranularityMonitor tsgm;
private final ResourceManager resourceManager;
/** Used to lock evaluator on legacy calls to get existing values. */
private final Object valueLookupLock = new Object();
private final AtomicReference statusReporterRef =
new AtomicReference<>();
private final SkyframeActionExecutor skyframeActionExecutor;
private CompletionReceiver actionExecutionFunction;
protected SkyframeProgressReceiver progressReceiver;
private final AtomicReference cyclesReporter = new AtomicReference<>();
private final Set immutableDirectories;
private final BinTools binTools;
private boolean needToInjectEmbeddedArtifacts = true;
private boolean needToInjectPrecomputedValuesForAnalysis = true;
protected int modifiedFiles;
protected int outputDirtyFiles;
protected int modifiedFilesDuringPreviousBuild;
private final Predicate allowedMissingInputs;
private final boolean errorOnExternalFiles;
private final ImmutableMap extraSkyFunctions;
private final ImmutableList extraPrecomputedValues;
protected SkyframeIncrementalBuildMonitor incrementalBuildMonitor =
new SkyframeIncrementalBuildMonitor();
private MutableSupplier configurationFactory = new MutableSupplier<>();
private MutableSupplier> configurationFragments =
new MutableSupplier<>();
private static final Logger LOG = Logger.getLogger(SkyframeExecutor.class.getName());
protected SkyframeExecutor(
EvaluatorSupplier evaluatorSupplier,
PackageFactory pkgFactory,
TimestampGranularityMonitor tsgm,
BlazeDirectories directories,
BinTools binTools,
Factory workspaceStatusActionFactory,
ImmutableList buildInfoFactories,
Set immutableDirectories,
Predicate allowedMissingInputs,
Preprocessor.Factory.Supplier preprocessorFactorySupplier,
ImmutableMap extraSkyFunctions,
ImmutableList extraPrecomputedValues,
boolean errorOnExternalFiles) {
// Strictly speaking, these arguments are not required for initialization, but all current
// callsites have them at hand, so we might as well set them during construction.
this.evaluatorSupplier = evaluatorSupplier;
this.pkgFactory = pkgFactory;
this.pkgFactory.setSyscalls(syscalls);
this.tsgm = tsgm;
this.workspaceStatusActionFactory = workspaceStatusActionFactory;
this.packageManager = new SkyframePackageManager(
new SkyframePackageLoader(), new SkyframeTransitivePackageLoader(),
syscalls, cyclesReporter, pkgLocator, numPackagesLoaded, this);
this.resourceManager = ResourceManager.instance();
this.skyframeActionExecutor = new SkyframeActionExecutor(
resourceManager, eventBus, statusReporterRef);
this.directories = Preconditions.checkNotNull(directories);
this.buildInfoFactories = buildInfoFactories;
this.immutableDirectories = immutableDirectories;
this.allowedMissingInputs = allowedMissingInputs;
this.preprocessorFactorySupplier = preprocessorFactorySupplier;
this.extraSkyFunctions = extraSkyFunctions;
this.extraPrecomputedValues = extraPrecomputedValues;
this.errorOnExternalFiles = errorOnExternalFiles;
this.binTools = binTools;
this.skyframeBuildView = new SkyframeBuildView(
directories,
this,
binTools,
(ConfiguredRuleClassProvider) pkgFactory.getRuleClassProvider());
this.artifactFactory.set(skyframeBuildView.getArtifactFactory());
}
private ImmutableMap skyFunctions(
Root buildDataDirectory,
PackageFactory pkgFactory,
Predicate allowedMissingInputs) {
ExternalFilesHelper externalFilesHelper = new ExternalFilesHelper(pkgLocator,
immutableDirectories, errorOnExternalFiles);
RuleClassProvider ruleClassProvider = pkgFactory.getRuleClassProvider();
// We use an immutable map builder for the nice side effect that it throws if a duplicate key
// is inserted.
ImmutableMap.Builder map = ImmutableMap.builder();
map.put(SkyFunctions.PRECOMPUTED, new PrecomputedFunction());
map.put(SkyFunctions.FILE_STATE, new FileStateFunction(tsgm, externalFilesHelper));
map.put(SkyFunctions.DIRECTORY_LISTING_STATE,
new DirectoryListingStateFunction(externalFilesHelper));
map.put(SkyFunctions.FILE_SYMLINK_CYCLE_UNIQUENESS,
new FileSymlinkCycleUniquenessFunction());
map.put(SkyFunctions.FILE_SYMLINK_INFINITE_EXPANSION_UNIQUENESS,
new FileSymlinkInfiniteExpansionUniquenessFunction());
map.put(SkyFunctions.FILE, new FileFunction(pkgLocator, tsgm, externalFilesHelper));
map.put(SkyFunctions.DIRECTORY_LISTING, new DirectoryListingFunction());
map.put(SkyFunctions.PACKAGE_LOOKUP, new PackageLookupFunction(deletedPackages));
map.put(SkyFunctions.CONTAINING_PACKAGE_LOOKUP, new ContainingPackageLookupFunction());
map.put(SkyFunctions.AST_FILE_LOOKUP, new ASTFileLookupFunction(pkgLocator, ruleClassProvider));
map.put(
SkyFunctions.SKYLARK_IMPORTS_LOOKUP,
newSkylarkImportLookupFunction(ruleClassProvider, pkgFactory));
map.put(SkyFunctions.SKYLARK_IMPORT_CYCLE, new SkylarkImportUniqueCycleFunction());
map.put(SkyFunctions.GLOB, newGlobFunction());
map.put(SkyFunctions.TARGET_PATTERN, new TargetPatternFunction());
map.put(SkyFunctions.PREPARE_DEPS_OF_PATTERNS, new PrepareDepsOfPatternsFunction());
map.put(SkyFunctions.PREPARE_DEPS_OF_PATTERN, new PrepareDepsOfPatternFunction(pkgLocator));
map.put(SkyFunctions.PREPARE_DEPS_OF_TARGETS_UNDER_DIRECTORY,
new PrepareDepsOfTargetsUnderDirectoryFunction());
map.put(SkyFunctions.BLACKLISTED_PACKAGE_PREFIXES, new BlacklistedPackagePrefixesFunction());
map.put(SkyFunctions.TESTS_IN_SUITE, new TestsInSuiteFunction());
map.put(SkyFunctions.TEST_SUITE_EXPANSION, new TestSuiteExpansionFunction());
map.put(SkyFunctions.TARGET_PATTERN_PHASE, new TargetPatternPhaseFunction());
map.put(SkyFunctions.RECURSIVE_PKG, new RecursivePkgFunction());
map.put(
SkyFunctions.PACKAGE,
newPackageFunction(
pkgFactory,
packageManager,
showLoadingProgress,
packageFunctionCache,
astCache,
numPackagesLoaded,
ruleClassProvider));
map.put(SkyFunctions.PACKAGE_ERROR, new PackageErrorFunction());
map.put(SkyFunctions.TARGET_MARKER, new TargetMarkerFunction());
map.put(SkyFunctions.TRANSITIVE_TARGET, new TransitiveTargetFunction(ruleClassProvider));
map.put(SkyFunctions.TRANSITIVE_TRAVERSAL, new TransitiveTraversalFunction());
map.put(SkyFunctions.CONFIGURED_TARGET,
new ConfiguredTargetFunction(new BuildViewProvider(), ruleClassProvider));
map.put(SkyFunctions.ASPECT, new AspectFunction(new BuildViewProvider(), ruleClassProvider));
map.put(SkyFunctions.LOAD_SKYLARK_ASPECT, new ToplevelSkylarkAspectFunction());
map.put(SkyFunctions.POST_CONFIGURED_TARGET,
new PostConfiguredTargetFunction(new BuildViewProvider(), ruleClassProvider));
map.put(SkyFunctions.BUILD_CONFIGURATION,
new BuildConfigurationFunction(directories, ruleClassProvider));
map.put(SkyFunctions.CONFIGURATION_COLLECTION, new ConfigurationCollectionFunction(
configurationFactory, ruleClassProvider));
map.put(SkyFunctions.CONFIGURATION_FRAGMENT, new ConfigurationFragmentFunction(
configurationFragments, ruleClassProvider));
map.put(
SkyFunctions.WORKSPACE_FILE,
new WorkspaceFileFunction(ruleClassProvider, pkgFactory, directories));
map.put(SkyFunctions.TARGET_COMPLETION, CompletionFunction.targetCompletionFunction(eventBus));
map.put(SkyFunctions.ASPECT_COMPLETION, CompletionFunction.aspectCompletionFunction(eventBus));
map.put(SkyFunctions.TEST_COMPLETION, new TestCompletionFunction());
map.put(SkyFunctions.ARTIFACT, new ArtifactFunction(allowedMissingInputs));
map.put(SkyFunctions.BUILD_INFO_COLLECTION, new BuildInfoCollectionFunction(artifactFactory,
buildDataDirectory));
map.put(SkyFunctions.BUILD_INFO, new WorkspaceStatusFunction());
map.put(SkyFunctions.COVERAGE_REPORT, new CoverageReportFunction());
ActionExecutionFunction actionExecutionFunction =
new ActionExecutionFunction(skyframeActionExecutor, tsgm);
map.put(SkyFunctions.ACTION_EXECUTION, actionExecutionFunction);
this.actionExecutionFunction = actionExecutionFunction;
map.put(SkyFunctions.RECURSIVE_FILESYSTEM_TRAVERSAL,
new RecursiveFilesystemTraversalFunction());
map.put(SkyFunctions.FILESET_ENTRY, new FilesetEntryFunction());
map.putAll(extraSkyFunctions);
return map.build();
}
protected SkyFunction newGlobFunction() {
return new GlobFunction(/*alwaysUseDirListing=*/false);
}
protected PackageFunction newPackageFunction(
PackageFactory pkgFactory,
PackageManager packageManager,
AtomicBoolean showLoadingProgress,
Cache packageFunctionCache,
Cache astCache,
AtomicInteger numPackagesLoaded,
RuleClassProvider ruleClassProvider) {
return new PackageFunction(
pkgFactory,
packageManager,
showLoadingProgress,
packageFunctionCache,
astCache,
numPackagesLoaded,
null);
}
protected SkyFunction newSkylarkImportLookupFunction(
RuleClassProvider ruleClassProvider, PackageFactory pkgFactory) {
return new SkylarkImportLookupFunction(ruleClassProvider, this.pkgFactory);
}
protected PerBuildSyscallCache newPerBuildSyscallCache(int concurrencyLevel) {
return PerBuildSyscallCache.newBuilder().setConcurrencyLevel(concurrencyLevel).build();
}
@ThreadCompatible
public void setActive(boolean active) {
this.active = active;
}
protected void checkActive() {
Preconditions.checkState(active);
}
public void setFileCache(ActionInputFileCache fileCache) {
this.skyframeActionExecutor.setFileCache(fileCache);
}
public void dump(boolean summarize, PrintStream out) {
memoizingEvaluator.dump(summarize, out);
}
public abstract void dumpPackages(PrintStream out);
public void setBatchStatter(@Nullable BatchStat batchStatter) {
this.batchStatter = batchStatter;
}
/**
* Notify listeners about changed files, and release any associated memory afterwards.
*/
public void drainChangedFiles() {
incrementalBuildMonitor.alertListeners(getEventBus());
incrementalBuildMonitor = null;
}
@VisibleForTesting
public BuildDriver getDriverForTesting() {
return buildDriver;
}
/**
* This method exists only to allow a module to make a top-level Skyframe call during the
* transition to making it fully Skyframe-compatible. Do not add additional callers!
*/
public SkyValue evaluateSkyKeyForCodeMigration(
final EventHandler eventHandler, final SkyKey key, final Class clazz) throws E {
try {
return callUninterruptibly(new Callable() {
@Override
public SkyValue call() throws E, InterruptedException {
synchronized (valueLookupLock) {
// We evaluate in keepGoing mode because in the case that the graph does not store its
// edges, nokeepGoing builds are not allowed, whereas keepGoing builds are always
// permitted.
EvaluationResult result = buildDriver.evaluate(
ImmutableList.of(key), true, ResourceUsage.getAvailableProcessors(),
eventHandler);
if (!result.hasError()) {
return Preconditions.checkNotNull(result.get(key), "%s %s", result, key);
}
ErrorInfo errorInfo = Preconditions.checkNotNull(result.getError(key),
"%s %s", key, result);
Throwables.propagateIfPossible(errorInfo.getException(), clazz);
if (errorInfo.getException() != null) {
throw new IllegalStateException(errorInfo.getException());
}
throw new IllegalStateException(errorInfo.toString());
}
}
});
} catch (Exception e) {
Throwables.propagateIfPossible(e, clazz);
throw new IllegalStateException(e);
}
}
protected PathFragment getBlacklistedPackagePrefixesFile() {
return PathFragment.EMPTY_FRAGMENT;
}
class BuildViewProvider {
/**
* Returns the current {@link SkyframeBuildView} instance.
*/
SkyframeBuildView getSkyframeBuildView() {
return skyframeBuildView;
}
}
/**
* Must be called before the {@link SkyframeExecutor} can be used (should only be called in
* factory methods and as an implementation detail of {@link #resetEvaluator}).
*/
protected void init() {
progressReceiver = newSkyframeProgressReceiver();
Map skyFunctions = skyFunctions(
directories.getBuildDataDirectory(), pkgFactory, allowedMissingInputs);
memoizingEvaluator = evaluatorSupplier.create(
skyFunctions, evaluatorDiffer(), progressReceiver, emittedEventState,
hasIncrementalState());
buildDriver = newBuildDriver();
}
protected SkyframeProgressReceiver newSkyframeProgressReceiver() {
return new SkyframeProgressReceiver();
}
/**
* Reinitializes the Skyframe evaluator, dropping all previously computed values.
*
*
Be careful with this method as it also deletes all injected values. You need to make sure
* that any necessary precomputed values are reinjected before the next build. Constants can be
* put in {@link #reinjectConstantValuesLazily}.
*/
public void resetEvaluator() {
init();
emittedEventState.clear();
skyframeBuildView.clearLegacyData();
reinjectConstantValuesLazily();
}
protected abstract Differencer evaluatorDiffer();
protected abstract BuildDriver newBuildDriver();
/**
* Values whose values are known at startup and guaranteed constant are still wiped from the
* evaluator when we create a new one, so they must be re-injected each time we create a new
* evaluator.
*/
private void reinjectConstantValuesLazily() {
needToInjectEmbeddedArtifacts = true;
needToInjectPrecomputedValuesForAnalysis = true;
}
/**
* Deletes all ConfiguredTarget values from the Skyframe cache. This is done to save memory (e.g.
* on a configuration change); since the configuration is part of the key, these key/value pairs
* will be sitting around doing nothing until the configuration changes back to the previous
* value.
*
*
The next evaluation will delete all invalid values.
*/
public abstract void dropConfiguredTargets();
/**
* Removes ConfigurationFragmentValuess and ConfigurationCollectionValues from the cache.
*/
@VisibleForTesting
public void invalidateConfigurationCollection() {
invalidate(SkyFunctionName.functionIsIn(ImmutableSet.of(SkyFunctions.CONFIGURATION_FRAGMENT,
SkyFunctions.CONFIGURATION_COLLECTION)));
}
/**
* Decides if graph edges should be stored for this build. If not, re-creates the graph to not
* store graph edges. Necessary conditions to not store graph edges are:
* (1) batch (since incremental builds are not possible);
* (2) skyframe build (since otherwise the memory savings are too slight to bother);
* (3) keep-going (since otherwise bubbling errors up may require edges of done nodes);
* (4) discard_analysis_cache (since otherwise user isn't concerned about saving memory this way).
*/
public void decideKeepIncrementalState(boolean batch, Options viewOptions) {
// Assume incrementality.
}
public boolean hasIncrementalState() {
return true;
}
@VisibleForTesting
protected abstract Injectable injectable();
/**
* Saves memory by clearing analysis objects from Skyframe. If using legacy execution, actually
* deletes the relevant values. If using Skyframe execution, clears their data without deleting
* them (they will be deleted on the next build).
*/
public abstract void clearAnalysisCache(Collection topLevelTargets);
/**
* Injects the contents of the computed tools/defaults package.
*/
@VisibleForTesting
public void setupDefaultPackage(String defaultsPackageContents) {
PrecomputedValue.DEFAULTS_PACKAGE_CONTENTS.set(injectable(), defaultsPackageContents);
}
/**
* Injects the top-level artifact options.
*/
public void injectTopLevelContext(TopLevelArtifactContext options) {
PrecomputedValue.TOP_LEVEL_CONTEXT.set(injectable(), options);
}
public void injectWorkspaceStatusData() {
PrecomputedValue.WORKSPACE_STATUS_KEY.set(injectable(),
workspaceStatusActionFactory.createWorkspaceStatusAction(
artifactFactory.get(), WorkspaceStatusValue.ARTIFACT_OWNER, buildId));
}
public void injectCoverageReportData(ImmutableList actions) {
PrecomputedValue.COVERAGE_REPORT_KEY.set(injectable(), actions);
}
/**
* Sets the default visibility.
*/
private void setDefaultVisibility(RuleVisibility defaultVisibility) {
PrecomputedValue.DEFAULT_VISIBILITY.set(injectable(), defaultVisibility);
}
private void maybeInjectPrecomputedValuesForAnalysis() {
if (needToInjectPrecomputedValuesForAnalysis) {
injectBuildInfoFactories();
injectExtraPrecomputedValues();
needToInjectPrecomputedValuesForAnalysis = false;
}
}
private void injectExtraPrecomputedValues() {
for (PrecomputedValue.Injected injected : extraPrecomputedValues) {
injected.inject(injectable());
}
}
protected Cache newPkgFunctionCache() {
return CacheBuilder.newBuilder().build();
}
protected Cache newAstCache() {
return CacheBuilder.newBuilder().build();
}
/**
* Injects the build info factory map that will be used when constructing build info
* actions/artifacts. Unchanged across the life of the Blaze server, although it must be injected
* each time the evaluator is created.
*/
private void injectBuildInfoFactories() {
ImmutableMap.Builder factoryMapBuilder =
ImmutableMap.builder();
for (BuildInfoFactory factory : buildInfoFactories) {
factoryMapBuilder.put(factory.getKey(), factory);
}
PrecomputedValue.BUILD_INFO_FACTORIES.set(injectable(), factoryMapBuilder.build());
}
private void setShowLoadingProgress(boolean showLoadingProgressValue) {
showLoadingProgress.set(showLoadingProgressValue);
}
@VisibleForTesting
public void setCommandId(UUID commandId) {
PrecomputedValue.BUILD_ID.set(injectable(), commandId);
buildId.set(commandId);
}
/** Returns the build-info.txt and build-changelist.txt artifacts. */
public Collection getWorkspaceStatusArtifacts(EventHandler eventHandler)
throws InterruptedException {
// Should already be present, unless the user didn't request any targets for analysis.
EvaluationResult result = buildDriver.evaluate(
ImmutableList.of(WorkspaceStatusValue.SKY_KEY), /*keepGoing=*/true, /*numThreads=*/1,
eventHandler);
WorkspaceStatusValue value =
Preconditions.checkNotNull(result.get(WorkspaceStatusValue.SKY_KEY));
return ImmutableList.of(value.getStableArtifact(), value.getVolatileArtifact());
}
// TODO(bazel-team): Make this take a PackageIdentifier.
public Map getArtifactRoots(final EventHandler eventHandler,
Iterable execPaths) throws PackageRootResolutionException {
final List packageKeys = new ArrayList<>();
for (PathFragment execPath : execPaths) {
PathFragment parent = Preconditions.checkNotNull(
execPath.getParentDirectory(), "Must pass in files, not root directory");
Preconditions.checkArgument(!parent.isAbsolute(), execPath);
packageKeys.add(ContainingPackageLookupValue.key(
PackageIdentifier.createInDefaultRepo(parent)));
}
EvaluationResult result;
try {
result = callUninterruptibly(new Callable>() {
@Override
public EvaluationResult call() throws InterruptedException {
synchronized (valueLookupLock) {
return buildDriver.evaluate(
packageKeys, /*keepGoing=*/true, /*numThreads=*/1, eventHandler);
}
}
});
} catch (Exception e) {
throw new IllegalStateException(e); // Should never happen.
}
if (result.hasError()) {
throw new PackageRootResolutionException("Exception encountered determining package roots",
result.getError().getException());
}
Map roots = new HashMap<>();
for (PathFragment execPath : execPaths) {
ContainingPackageLookupValue value = result.get(ContainingPackageLookupValue.key(
PackageIdentifier.createInDefaultRepo(execPath.getParentDirectory())));
if (value.hasContainingPackage()) {
roots.put(execPath, Root.asSourceRoot(value.getContainingPackageRoot()));
} else {
roots.put(execPath, null);
}
}
return roots;
}
@VisibleForTesting
public WorkspaceStatusAction getLastWorkspaceStatusActionForTesting() {
PrecomputedValue value = (PrecomputedValue) buildDriver.getGraphForTesting()
.getExistingValueForTesting(PrecomputedValue.WORKSPACE_STATUS_KEY.getKeyForTesting());
return (WorkspaceStatusAction) value.get();
}
/**
* Informs user about number of modified files (source and output files).
*/
// Note, that number of modified files in some cases can be bigger than actual number of
// modified files for targets in current request. Skyframe may check for modification all files
// from previous requests.
protected void informAboutNumberOfModifiedFiles() {
LOG.info(String.format("Found %d modified files from last build", modifiedFiles));
}
public EventBus getEventBus() {
return eventBus.get();
}
public ActionExecutionContextFactory getActionExecutionContextFactory() {
return skyframeActionExecutor;
}
@VisibleForTesting
ImmutableList getPathEntries() {
return pkgLocator.get().getPathEntries();
}
protected abstract void invalidate(Predicate pred);
private static boolean compatibleFileTypes(Dirent.Type oldType, FileStateValue.Type newType) {
return (oldType.equals(Dirent.Type.FILE) && newType.equals(FileStateValue.Type.REGULAR_FILE))
|| (oldType.equals(Dirent.Type.UNKNOWN)
&& newType.equals(FileStateValue.Type.SPECIAL_FILE))
|| (oldType.equals(Dirent.Type.DIRECTORY) && newType.equals(FileStateValue.Type.DIRECTORY))
|| (oldType.equals(Dirent.Type.SYMLINK) && newType.equals(FileStateValue.Type.SYMLINK));
}
protected Differencer.Diff getDiff(Iterable modifiedSourceFiles,
final Path pathEntry) throws InterruptedException {
if (Iterables.isEmpty(modifiedSourceFiles)) {
return new ImmutableDiff(ImmutableList.of(), ImmutableMap.of());
}
// TODO(bazel-team): change ModifiedFileSet to work with RootedPaths instead of PathFragments.
Iterable dirtyFileStateSkyKeys = Iterables.transform(modifiedSourceFiles,
new Function() {
@Override
public SkyKey apply(PathFragment pathFragment) {
Preconditions.checkState(!pathFragment.isAbsolute(),
"found absolute PathFragment: %s", pathFragment);
return FileStateValue.key(RootedPath.toRootedPath(pathEntry, pathFragment));
}
});
// We only need to invalidate directory values when a file has been created or deleted or
// changes type, not when it has merely been modified. Unfortunately we do not have that
// information here, so we compute it ourselves.
// TODO(bazel-team): Fancy filesystems could provide it with a hypothetically modified
// DiffAwareness interface.
FilesystemValueChecker fsvc = new FilesystemValueChecker(tsgm, null);
Map valuesMap = memoizingEvaluator.getValues();
Differencer.DiffWithDelta diff =
fsvc.getNewAndOldValues(valuesMap, dirtyFileStateSkyKeys, new FileDirtinessChecker());
Set valuesToInvalidate = new HashSet<>();
Map valuesToInject = new HashMap<>();
for (Map.Entry entry : diff.changedKeysWithNewAndOldValues().entrySet()) {
SkyKey key = entry.getKey();
Preconditions.checkState(key.functionName().equals(SkyFunctions.FILE_STATE), key);
RootedPath rootedPath = (RootedPath) key.argument();
Delta delta = entry.getValue();
FileStateValue oldValue = (FileStateValue) delta.getOldValue();
FileStateValue newValue = (FileStateValue) delta.getNewValue();
if (newValue != null) {
valuesToInject.put(key, newValue);
} else {
valuesToInvalidate.add(key);
}
SkyKey dirListingStateKey = parentDirectoryListingStateKey(rootedPath);
// Invalidate the directory listing for the path's parent directory if the change was
// relevant (e.g. path turned from a symlink into a directory) OR if we don't have enough
// information to determine it was irrelevant.
boolean changedType = false;
if (newValue == null) {
changedType = true;
} else if (oldValue != null) {
changedType = !oldValue.getType().equals(newValue.getType());
} else {
DirectoryListingStateValue oldDirListingStateValue =
(DirectoryListingStateValue) valuesMap.get(dirListingStateKey);
if (oldDirListingStateValue != null) {
String baseName = rootedPath.getRelativePath().getBaseName();
Dirent oldDirent = oldDirListingStateValue.getDirents().maybeGetDirent(baseName);
changedType = (oldDirent == null)
|| !compatibleFileTypes(oldDirent.getType(), newValue.getType());
} else {
changedType = true;
}
}
if (changedType) {
valuesToInvalidate.add(dirListingStateKey);
}
}
for (SkyKey key : diff.changedKeysWithoutNewValues()) {
Preconditions.checkState(key.functionName().equals(SkyFunctions.FILE_STATE), key);
RootedPath rootedPath = (RootedPath) key.argument();
valuesToInvalidate.add(parentDirectoryListingStateKey(rootedPath));
}
return new ImmutableDiff(valuesToInvalidate, valuesToInject);
}
private static SkyKey parentDirectoryListingStateKey(RootedPath rootedPath) {
RootedPath parentDirRootedPath = RootedPath.toRootedPath(
rootedPath.getRoot(), rootedPath.getRelativePath().getParentDirectory());
return DirectoryListingStateValue.key(parentDirRootedPath);
}
/**
* Sets the packages that should be treated as deleted and ignored.
*/
@VisibleForTesting // productionVisibility = Visibility.PRIVATE
public abstract void setDeletedPackages(Iterable pkgs);
@VisibleForTesting
public final void setBlacklistedPackagePrefixesFile(PathFragment blacklistedPkgFile) {
PrecomputedValue.BLACKLISTED_PACKAGE_PREFIXES_FILE.set(injectable(), blacklistedPkgFile);
}
/**
* Prepares the evaluator for loading.
*
*
MUST be run before every incremental build.
*/
@VisibleForTesting // productionVisibility = Visibility.PRIVATE
public void preparePackageLoading(PathPackageLocator pkgLocator, RuleVisibility defaultVisibility,
boolean showLoadingProgress, int globbingThreads,
String defaultsPackageContents, UUID commandId) {
Preconditions.checkNotNull(pkgLocator);
setActive(true);
maybeInjectPrecomputedValuesForAnalysis();
setCommandId(commandId);
setBlacklistedPackagePrefixesFile(getBlacklistedPackagePrefixesFile());
setShowLoadingProgress(showLoadingProgress);
setDefaultVisibility(defaultVisibility);
setupDefaultPackage(defaultsPackageContents);
setPackageLocator(pkgLocator);
syscalls.set(newPerBuildSyscallCache(globbingThreads));
this.pkgFactory.setGlobbingThreads(globbingThreads);
checkPreprocessorFactory();
emittedEventState.clear();
// If the PackageFunction was interrupted, there may be stale entries here.
packageFunctionCache.invalidateAll();
astCache.invalidateAll();
numPackagesLoaded.set(0);
// Reset the stateful SkyframeCycleReporter, which contains cycles from last run.
cyclesReporter.set(createCyclesReporter());
}
@SuppressWarnings("unchecked")
private void setPackageLocator(PathPackageLocator pkgLocator) {
PathPackageLocator oldLocator = this.pkgLocator.getAndSet(pkgLocator);
PrecomputedValue.PATH_PACKAGE_LOCATOR.set(injectable(), pkgLocator);
if (!pkgLocator.equals(oldLocator)) {
// The package path is read not only by SkyFunctions but also by some other code paths.
// We need to take additional steps to keep the corresponding data structures in sync.
// (Some of the additional steps are carried out by ConfiguredTargetValueInvalidationListener,
// and some by BuildView#buildHasIncompatiblePackageRoots and #updateSkyframe.)
onNewPackageLocator(oldLocator, pkgLocator);
}
}
protected abstract void onNewPackageLocator(PathPackageLocator oldLocator,
PathPackageLocator pkgLocator);
private void checkPreprocessorFactory() {
if (preprocessorFactory == null) {
Preprocessor.Factory newPreprocessorFactory = preprocessorFactorySupplier.getFactory(
packageManager);
pkgFactory.setPreprocessorFactory(newPreprocessorFactory);
preprocessorFactory = newPreprocessorFactory;
} else if (!preprocessorFactory.isStillValid()) {
Preprocessor.Factory newPreprocessorFactory = preprocessorFactorySupplier.getFactory(
packageManager);
invalidate(SkyFunctionName.functionIs(SkyFunctions.PACKAGE));
pkgFactory.setPreprocessorFactory(newPreprocessorFactory);
preprocessorFactory = newPreprocessorFactory;
}
}
public SkyframeBuildView getSkyframeBuildView() {
return skyframeBuildView;
}
/**
* Sets the eventBus to use for posting events.
*/
public void setEventBus(EventBus eventBus) {
this.eventBus.set(eventBus);
}
/**
* Sets the path for action log buffers.
*/
public void setActionOutputRoot(Path actionOutputRoot) {
Preconditions.checkNotNull(actionOutputRoot);
this.actionLogBufferPathGenerator = new ActionLogBufferPathGenerator(actionOutputRoot);
this.skyframeActionExecutor.setActionLogBufferPathGenerator(actionLogBufferPathGenerator);
}
@VisibleForTesting
public void setConfigurationDataForTesting(BlazeDirectories directories,
ConfigurationFactory configurationFactory) {
PrecomputedValue.BLAZE_DIRECTORIES.set(injectable(), directories);
this.configurationFactory.set(configurationFactory);
this.configurationFragments.set(ImmutableList.copyOf(configurationFactory.getFactories()));
}
/**
* Asks the Skyframe evaluator to build the value for BuildConfigurationCollection and returns the
* result. Also invalidates {@link PrecomputedValue#BLAZE_DIRECTORIES} if it has changed.
*/
public BuildConfigurationCollection createConfigurations(
EventHandler eventHandler, ConfigurationFactory configurationFactory,
BuildOptions buildOptions, BlazeDirectories directories, Set multiCpu,
boolean keepGoing)
throws InvalidConfigurationException, InterruptedException {
this.configurationFactory.set(configurationFactory);
this.configurationFragments.set(ImmutableList.copyOf(configurationFactory.getFactories()));
// TODO(bazel-team): find a way to use only BuildConfigurationKey instead of
// BlazeDirectories.
PrecomputedValue.BLAZE_DIRECTORIES.set(injectable(), directories);
SkyKey skyKey = ConfigurationCollectionValue.key(
buildOptions, ImmutableSortedSet.copyOf(multiCpu));
EvaluationResult result = buildDriver.evaluate(
Arrays.asList(skyKey), keepGoing, DEFAULT_THREAD_COUNT, eventHandler);
if (result.hasError()) {
Throwable e = result.getError(skyKey).getException();
// Wrap loading failed exceptions
if (e instanceof NoSuchThingException) {
e = new InvalidConfigurationException(e);
}
Throwables.propagateIfInstanceOf(e, InvalidConfigurationException.class);
throw new IllegalStateException(
"Unknown error during ConfigurationCollectionValue evaluation", e);
}
Preconditions.checkState(result.values().size() == 1,
"Result of evaluate() must contain exactly one value %s", result);
ConfigurationCollectionValue configurationValue =
Iterables.getOnlyElement(result.values());
return configurationValue.getConfigurationCollection();
}
private Iterable getActionLookupValues() {
// This filter keeps subclasses of ActionLookupValue.
return Iterables.filter(memoizingEvaluator.getDoneValues().values(), ActionLookupValue.class);
}
@SuppressWarnings({"unchecked", "rawtypes"})
Map getActionLookupValueMap() {
return (Map) Maps.filterValues(memoizingEvaluator.getDoneValues(),
Predicates.instanceOf(ActionLookupValue.class));
}
/**
* Checks the actions in Skyframe for conflicts between their output artifacts. Delegates to
* {@link SkyframeActionExecutor#findAndStoreArtifactConflicts} to do the work, since any
* conflicts found will only be reported during execution.
*/
ImmutableMap findArtifactConflicts()
throws InterruptedException {
if (skyframeBuildView.isSomeConfiguredTargetEvaluated()
|| skyframeBuildView.isSomeConfiguredTargetInvalidated()) {
// This operation is somewhat expensive, so we only do it if the graph might have changed in
// some way -- either we analyzed a new target or we invalidated an old one.
try (AutoProfiler p = AutoProfiler.logged("discovering artifact conflicts", LOG)) {
skyframeActionExecutor.findAndStoreArtifactConflicts(getActionLookupValues());
skyframeBuildView.resetEvaluatedConfiguredTargetFlag();
// The invalidated configured targets flag will be reset later in the evaluate() call.
}
}
return skyframeActionExecutor.badActions();
}
/**
* Asks the Skyframe evaluator to build the given artifacts and targets, and to test the
* given test targets.
*/
public EvaluationResult> buildArtifacts(
Reporter reporter,
Executor executor,
Set artifactsToBuild,
Collection targetsToBuild,
Collection aspects,
Collection targetsToTest,
boolean exclusiveTesting,
boolean keepGoing,
boolean explain,
int numJobs,
ActionCacheChecker actionCacheChecker,
@Nullable EvaluationProgressReceiver executionProgressReceiver)
throws InterruptedException {
checkActive();
Preconditions.checkState(actionLogBufferPathGenerator != null);
skyframeActionExecutor.prepareForExecution(
reporter, executor, keepGoing, explain, actionCacheChecker);
resourceManager.resetResourceUsage();
try {
progressReceiver.executionProgressReceiver = executionProgressReceiver;
Iterable artifactKeys = ArtifactValue.mandatoryKeys(artifactsToBuild);
Iterable targetKeys = TargetCompletionValue.keys(targetsToBuild);
Iterable aspectKeys = AspectCompletionValue.keys(aspects);
Iterable testKeys = TestCompletionValue.keys(targetsToTest, exclusiveTesting);
return buildDriver.evaluate(
Iterables.concat(artifactKeys, targetKeys, aspectKeys, testKeys),
keepGoing,
numJobs,
reporter);
} finally {
progressReceiver.executionProgressReceiver = null;
// Also releases thread locks.
resourceManager.resetResourceUsage();
skyframeActionExecutor.executionOver();
actionExecutionFunction.complete();
}
}
@VisibleForTesting
public void prepareBuildingForTestingOnly(Reporter reporter, Executor executor, boolean keepGoing,
boolean explain, ActionCacheChecker checker) {
skyframeActionExecutor.prepareForExecution(reporter, executor, keepGoing, explain, checker);
}
EvaluationResult targetPatterns(Iterable patternSkyKeys,
int numThreads, boolean keepGoing, EventHandler eventHandler) throws InterruptedException {
checkActive();
return buildDriver.evaluate(patternSkyKeys, keepGoing, numThreads, eventHandler);
}
/**
* Returns the {@link ConfiguredTarget}s corresponding to the given keys.
*
*
For use for legacy support from {@code BuildView} only.
*
*
If a requested configured target is in error, the corresponding value is omitted from the
* returned list.
*/
@ThreadSafety.ThreadSafe
public ImmutableList getConfiguredTargets(
EventHandler eventHandler, BuildConfiguration originalConfig, Iterable keys,
boolean useOriginalConfig) {
return getConfiguredTargetMap(
eventHandler, originalConfig, keys, useOriginalConfig).values().asList();
}
@ThreadSafety.ThreadSafe
public ImmutableMap getConfiguredTargetMap(
EventHandler eventHandler, BuildConfiguration originalConfig, Iterable keys,
boolean useOriginalConfig) {
checkActive();
Map configs;
if (originalConfig != null) {
if (useOriginalConfig) {
// This flag is used because of some unfortunate complexity in the configuration machinery:
// Most callers of this method pass a