// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.skyframe;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.base.Predicates;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import com.google.common.base.Throwables;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
import com.google.common.collect.Range;
import com.google.common.collect.Sets;
import com.google.common.eventbus.EventBus;
import com.google.devtools.build.lib.actions.Action;
import com.google.devtools.build.lib.actions.ActionCacheChecker;
import com.google.devtools.build.lib.actions.ActionExecutionContextFactory;
import com.google.devtools.build.lib.actions.ActionExecutionStatusReporter;
import com.google.devtools.build.lib.actions.ActionInputFileCache;
import com.google.devtools.build.lib.actions.ActionLogBufferPathGenerator;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.ArtifactFactory;
import com.google.devtools.build.lib.actions.ArtifactOwner;
import com.google.devtools.build.lib.actions.Executor;
import com.google.devtools.build.lib.actions.PackageRootResolutionException;
import com.google.devtools.build.lib.actions.ResourceManager;
import com.google.devtools.build.lib.actions.Root;
import com.google.devtools.build.lib.analysis.Aspect;
import com.google.devtools.build.lib.analysis.AspectWithParameters;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.analysis.BuildView.Options;
import com.google.devtools.build.lib.analysis.ConfiguredTarget;
import com.google.devtools.build.lib.analysis.DependencyResolver.Dependency;
import com.google.devtools.build.lib.analysis.RuleConfiguredTarget;
import com.google.devtools.build.lib.analysis.TopLevelArtifactContext;
import com.google.devtools.build.lib.analysis.WorkspaceStatusAction;
import com.google.devtools.build.lib.analysis.WorkspaceStatusAction.Factory;
import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory;
import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory.BuildInfoKey;
import com.google.devtools.build.lib.analysis.config.BinTools;
import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection;
import com.google.devtools.build.lib.analysis.config.BuildOptions;
import com.google.devtools.build.lib.analysis.config.ConfigurationFactory;
import com.google.devtools.build.lib.analysis.config.ConfigurationFragmentFactory;
import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
import com.google.devtools.build.lib.analysis.config.PatchTransition;
import com.google.devtools.build.lib.cmdline.PackageIdentifier;
import com.google.devtools.build.lib.concurrent.ThreadSafety;
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
import com.google.devtools.build.lib.events.EventHandler;
import com.google.devtools.build.lib.events.Reporter;
import com.google.devtools.build.lib.packages.Attribute;
import com.google.devtools.build.lib.packages.BuildFileContainsErrorsException;
import com.google.devtools.build.lib.packages.NoSuchPackageException;
import com.google.devtools.build.lib.packages.NoSuchThingException;
import com.google.devtools.build.lib.packages.Package;
import com.google.devtools.build.lib.packages.PackageFactory;
import com.google.devtools.build.lib.packages.Preprocessor;
import com.google.devtools.build.lib.packages.RuleClassProvider;
import com.google.devtools.build.lib.packages.RuleVisibility;
import com.google.devtools.build.lib.packages.Target;
import com.google.devtools.build.lib.pkgcache.PackageCacheOptions;
import com.google.devtools.build.lib.pkgcache.PackageManager;
import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
import com.google.devtools.build.lib.pkgcache.TransitivePackageLoader;
import com.google.devtools.build.lib.profiler.AutoProfiler;
import com.google.devtools.build.lib.skyframe.AspectValue.AspectKey;
import com.google.devtools.build.lib.skyframe.DirtinessCheckerUtils.FileDirtinessChecker;
import com.google.devtools.build.lib.skyframe.SkyframeActionExecutor.ActionCompletedReceiver;
import com.google.devtools.build.lib.skyframe.SkyframeActionExecutor.ProgressSupplier;
import com.google.devtools.build.lib.syntax.Label;
import com.google.devtools.build.lib.util.AbruptExitException;
import com.google.devtools.build.lib.util.ExitCode;
import com.google.devtools.build.lib.util.ResourceUsage;
import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
import com.google.devtools.build.lib.vfs.BatchStat;
import com.google.devtools.build.lib.vfs.Dirent;
import com.google.devtools.build.lib.vfs.ModifiedFileSet;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.RootedPath;
import com.google.devtools.build.lib.vfs.UnixGlob;
import com.google.devtools.build.skyframe.BuildDriver;
import com.google.devtools.build.skyframe.CycleInfo;
import com.google.devtools.build.skyframe.CyclesReporter;
import com.google.devtools.build.skyframe.Differencer;
import com.google.devtools.build.skyframe.Differencer.DiffWithDelta.Delta;
import com.google.devtools.build.skyframe.ErrorInfo;
import com.google.devtools.build.skyframe.EvaluationProgressReceiver;
import com.google.devtools.build.skyframe.EvaluationResult;
import com.google.devtools.build.skyframe.ImmutableDiff;
import com.google.devtools.build.skyframe.Injectable;
import com.google.devtools.build.skyframe.MemoizingEvaluator;
import com.google.devtools.build.skyframe.MemoizingEvaluator.EvaluatorSupplier;
import com.google.devtools.build.skyframe.SkyFunction;
import com.google.devtools.build.skyframe.SkyFunctionName;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
import com.google.devtools.build.skyframe.WalkableGraph.WalkableGraphFactory;
import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.logging.Logger;
import javax.annotation.Nullable;
/**
* A helper object to support Skyframe-driven execution.
*
*
This object is mostly used to inject external state, such as the executor engine or
* some additional artifacts (workspace status and build info artifacts) into SkyFunctions
* for use during the build.
*/
public abstract class SkyframeExecutor implements WalkableGraphFactory {
private final EvaluatorSupplier evaluatorSupplier;
protected MemoizingEvaluator memoizingEvaluator;
private final MemoizingEvaluator.EmittedEventState emittedEventState =
new MemoizingEvaluator.EmittedEventState();
protected final Reporter reporter;
private final PackageFactory pkgFactory;
private final WorkspaceStatusAction.Factory workspaceStatusActionFactory;
private final BlazeDirectories directories;
@Nullable
private BatchStat batchStatter;
// TODO(bazel-team): Figure out how to handle value builders that block internally. Blocking
// operations may need to be handled in another (bigger?) thread pool. Also, we should detect
// the number of cores and use that as the thread-pool size for CPU-bound operations.
// I just bumped this to 200 to get reasonable execution phase performance; that may cause
// significant overhead for CPU-bound processes (i.e. analysis). [skyframe-analysis]
@VisibleForTesting
public static final int DEFAULT_THREAD_COUNT =
// Reduce thread count while running tests of Bazel. Test cases are typically small, and large
// thread pools vying for a relatively small number of CPU cores may induce non-optimal
// performance.
System.getenv("TEST_TMPDIR") == null ? 200 : 5;
// Cache of partially constructed Package instances, stored between reruns of the PackageFunction
// (because of missing dependencies, within the same evaluate() run) to avoid loading the same
// package twice (first time loading to find subincludes and declare value dependencies).
// TODO(bazel-team): remove this cache once we have skyframe-native package loading
// [skyframe-loading]
private final Cache packageFunctionCache =
newPkgFunctionCache();
private final Cache preprocessCache =
newPreprocessCache();
private final AtomicInteger numPackagesLoaded = new AtomicInteger(0);
protected SkyframeBuildView skyframeBuildView;
private EventHandler errorEventListener;
private ActionLogBufferPathGenerator actionLogBufferPathGenerator;
protected BuildDriver buildDriver;
// AtomicReferences are used here as mutable boxes shared with value builders.
private final AtomicBoolean showLoadingProgress = new AtomicBoolean();
protected final AtomicReference syscalls =
new AtomicReference<>(UnixGlob.DEFAULT_SYSCALLS);
protected final AtomicReference pkgLocator =
new AtomicReference<>();
protected final AtomicReference> deletedPackages =
new AtomicReference<>(ImmutableSet.of());
private final AtomicReference eventBus = new AtomicReference<>();
private final ImmutableList buildInfoFactories;
// Under normal circumstances, the artifact factory persists for the life of a Blaze server, but
// since it is not yet created when we create the value builders, we have to use a supplier,
// initialized when the build view is created.
private final MutableSupplier artifactFactory = new MutableSupplier<>();
// Used to give to WriteBuildInfoAction via a supplier. Relying on BuildVariableValue.BUILD_ID
// would be preferable, but we have no way to have the Action depend on that value directly.
// Having the BuildInfoFunction own the supplier is currently not possible either, because then
// it would be invalidated on every build, since it would depend on the build id value.
private MutableSupplier buildId = new MutableSupplier<>();
protected boolean active = true;
private final PackageManager packageManager;
private final Preprocessor.Factory.Supplier preprocessorFactorySupplier;
private Preprocessor.Factory preprocessorFactory;
protected final TimestampGranularityMonitor tsgm;
private final ResourceManager resourceManager;
/** Used to lock evaluator on legacy calls to get existing values. */
private final Object valueLookupLock = new Object();
private final AtomicReference statusReporterRef =
new AtomicReference<>();
private final SkyframeActionExecutor skyframeActionExecutor;
private CompletionReceiver actionExecutionFunction;
protected SkyframeProgressReceiver progressReceiver;
private final AtomicReference cyclesReporter = new AtomicReference<>();
private final Set immutableDirectories;
private BinTools binTools = null;
private boolean needToInjectEmbeddedArtifacts = true;
private boolean needToInjectPrecomputedValuesForAnalysis = true;
protected int modifiedFiles;
protected int outputDirtyFiles;
protected int modifiedFilesDuringPreviousBuild;
private final Predicate allowedMissingInputs;
private final boolean errorOnExternalFiles;
private final ImmutableMap extraSkyFunctions;
private final ImmutableList extraPrecomputedValues;
protected SkyframeIncrementalBuildMonitor incrementalBuildMonitor =
new SkyframeIncrementalBuildMonitor();
private MutableSupplier configurationFactory = new MutableSupplier<>();
private MutableSupplier> configurationFragments =
new MutableSupplier<>();
private static final Logger LOG = Logger.getLogger(SkyframeExecutor.class.getName());
protected SkyframeExecutor(
Reporter reporter,
EvaluatorSupplier evaluatorSupplier,
PackageFactory pkgFactory,
TimestampGranularityMonitor tsgm,
BlazeDirectories directories,
Factory workspaceStatusActionFactory,
ImmutableList buildInfoFactories,
Set immutableDirectories,
Predicate allowedMissingInputs,
Preprocessor.Factory.Supplier preprocessorFactorySupplier,
ImmutableMap extraSkyFunctions,
ImmutableList extraPrecomputedValues,
boolean errorOnExternalFiles) {
// Strictly speaking, these arguments are not required for initialization, but all current
// callsites have them at hand, so we might as well set them during construction.
this.reporter = Preconditions.checkNotNull(reporter);
this.evaluatorSupplier = evaluatorSupplier;
this.pkgFactory = pkgFactory;
this.pkgFactory.setSyscalls(syscalls);
this.tsgm = tsgm;
this.workspaceStatusActionFactory = workspaceStatusActionFactory;
this.packageManager = new SkyframePackageManager(
new SkyframePackageLoader(), new SkyframeTransitivePackageLoader(),
new SkyframeTargetPatternEvaluator(this), syscalls, cyclesReporter, pkgLocator,
numPackagesLoaded, this);
this.errorEventListener = this.reporter;
this.resourceManager = ResourceManager.instance();
this.skyframeActionExecutor = new SkyframeActionExecutor(reporter, resourceManager, eventBus,
statusReporterRef);
this.directories = Preconditions.checkNotNull(directories);
this.buildInfoFactories = buildInfoFactories;
this.immutableDirectories = immutableDirectories;
this.allowedMissingInputs = allowedMissingInputs;
this.preprocessorFactorySupplier = preprocessorFactorySupplier;
this.extraSkyFunctions = extraSkyFunctions;
this.extraPrecomputedValues = extraPrecomputedValues;
this.errorOnExternalFiles = errorOnExternalFiles;
}
private ImmutableMap skyFunctions(
Root buildDataDirectory,
PackageFactory pkgFactory,
Predicate allowedMissingInputs) {
ExternalFilesHelper externalFilesHelper = new ExternalFilesHelper(pkgLocator,
immutableDirectories, errorOnExternalFiles);
RuleClassProvider ruleClassProvider = pkgFactory.getRuleClassProvider();
// We use an immutable map builder for the nice side effect that it throws if a duplicate key
// is inserted.
ImmutableMap.Builder map = ImmutableMap.builder();
map.put(SkyFunctions.PRECOMPUTED, new PrecomputedFunction());
map.put(SkyFunctions.FILE_STATE, new FileStateFunction(tsgm, externalFilesHelper));
map.put(SkyFunctions.DIRECTORY_LISTING_STATE,
new DirectoryListingStateFunction(externalFilesHelper));
map.put(SkyFunctions.FILE_SYMLINK_CYCLE_UNIQUENESS,
new FileSymlinkCycleUniquenessFunction());
map.put(SkyFunctions.FILE_SYMLINK_INFINITE_EXPANSION_UNIQUENESS,
new FileSymlinkInfiniteExpansionUniquenessFunction());
map.put(SkyFunctions.FILE, new FileFunction(pkgLocator, tsgm, externalFilesHelper));
map.put(SkyFunctions.DIRECTORY_LISTING, new DirectoryListingFunction());
map.put(SkyFunctions.PACKAGE_LOOKUP, new PackageLookupFunction(deletedPackages));
map.put(SkyFunctions.CONTAINING_PACKAGE_LOOKUP, new ContainingPackageLookupFunction());
map.put(SkyFunctions.AST_FILE_LOOKUP, new ASTFileLookupFunction(
pkgLocator, packageManager, ruleClassProvider));
map.put(SkyFunctions.SKYLARK_IMPORTS_LOOKUP, new SkylarkImportLookupFunction(
ruleClassProvider, pkgFactory));
map.put(SkyFunctions.GLOB, newGlobFunction());
map.put(SkyFunctions.TARGET_PATTERN, new TargetPatternFunction(pkgLocator));
map.put(SkyFunctions.PREPARE_DEPS_OF_PATTERNS, new PrepareDepsOfPatternsFunction());
map.put(SkyFunctions.PREPARE_DEPS_OF_PATTERN, new PrepareDepsOfPatternFunction(pkgLocator));
map.put(SkyFunctions.PREPARE_DEPS_OF_TARGETS_UNDER_DIRECTORY,
new PrepareDepsOfTargetsUnderDirectoryFunction());
map.put(SkyFunctions.RECURSIVE_PKG, new RecursivePkgFunction());
map.put(SkyFunctions.PACKAGE, new PackageFunction(
reporter, pkgFactory, packageManager, showLoadingProgress, packageFunctionCache,
preprocessCache, numPackagesLoaded));
map.put(SkyFunctions.TARGET_MARKER, new TargetMarkerFunction());
map.put(SkyFunctions.TRANSITIVE_TARGET, new TransitiveTargetFunction(ruleClassProvider));
map.put(SkyFunctions.TRANSITIVE_TRAVERSAL, new TransitiveTraversalFunction());
map.put(SkyFunctions.CONFIGURED_TARGET,
new ConfiguredTargetFunction(new BuildViewProvider(), ruleClassProvider));
map.put(SkyFunctions.ASPECT, new AspectFunction(new BuildViewProvider(), ruleClassProvider));
map.put(SkyFunctions.POST_CONFIGURED_TARGET,
new PostConfiguredTargetFunction(new BuildViewProvider(), ruleClassProvider));
map.put(SkyFunctions.BUILD_CONFIGURATION,
new BuildConfigurationFunction(directories, ruleClassProvider));
map.put(SkyFunctions.CONFIGURATION_COLLECTION, new ConfigurationCollectionFunction(
configurationFactory));
map.put(SkyFunctions.CONFIGURATION_FRAGMENT, new ConfigurationFragmentFunction(
configurationFragments));
map.put(
SkyFunctions.WORKSPACE_FILE,
new WorkspaceFileFunction(ruleClassProvider, pkgFactory, directories));
map.put(SkyFunctions.TARGET_COMPLETION, CompletionFunction.targetCompletionFunction(eventBus));
map.put(SkyFunctions.ASPECT_COMPLETION, CompletionFunction.aspectCompletionFunction(eventBus));
map.put(SkyFunctions.TEST_COMPLETION, new TestCompletionFunction());
map.put(SkyFunctions.ARTIFACT, new ArtifactFunction(allowedMissingInputs));
map.put(SkyFunctions.BUILD_INFO_COLLECTION, new BuildInfoCollectionFunction(artifactFactory,
buildDataDirectory));
map.put(SkyFunctions.BUILD_INFO, new WorkspaceStatusFunction());
map.put(SkyFunctions.COVERAGE_REPORT, new CoverageReportFunction());
ActionExecutionFunction actionExecutionFunction =
new ActionExecutionFunction(skyframeActionExecutor, tsgm);
map.put(SkyFunctions.ACTION_EXECUTION, actionExecutionFunction);
this.actionExecutionFunction = actionExecutionFunction;
map.put(SkyFunctions.RECURSIVE_FILESYSTEM_TRAVERSAL,
new RecursiveFilesystemTraversalFunction());
map.put(SkyFunctions.FILESET_ENTRY, new FilesetEntryFunction());
map.putAll(extraSkyFunctions);
return map.build();
}
protected SkyFunction newGlobFunction() {
return new GlobFunction(/*alwaysUseDirListing=*/false);
}
protected PerBuildSyscallCache newPerBuildSyscallCache() {
return PerBuildSyscallCache.newUnboundedCache();
}
@ThreadCompatible
public void setActive(boolean active) {
this.active = active;
}
protected void checkActive() {
Preconditions.checkState(active);
}
public void setFileCache(ActionInputFileCache fileCache) {
this.skyframeActionExecutor.setFileCache(fileCache);
}
public void dump(boolean summarize, PrintStream out) {
memoizingEvaluator.dump(summarize, out);
}
public abstract void dumpPackages(PrintStream out);
public void setBatchStatter(@Nullable BatchStat batchStatter) {
this.batchStatter = batchStatter;
}
/**
* Notify listeners about changed files, and release any associated memory afterwards.
*/
public void drainChangedFiles() {
incrementalBuildMonitor.alertListeners(getEventBus());
incrementalBuildMonitor = null;
}
@VisibleForTesting
public BuildDriver getDriverForTesting() {
return buildDriver;
}
/**
* This method exists only to allow a module to make a top-level Skyframe call during the
* transition to making it fully Skyframe-compatible. Do not add additional callers!
*/
public SkyValue evaluateSkyKeyForCodeMigration(final SkyKey key,
final Class clazz) throws E {
try {
return callUninterruptibly(new Callable() {
@Override
public SkyValue call() throws E, InterruptedException {
synchronized (valueLookupLock) {
// We evaluate in keepGoing mode because in the case that the graph does not store its
// edges, nokeepGoing builds are not allowed, whereas keepGoing builds are always
// permitted.
EvaluationResult result = buildDriver.evaluate(
ImmutableList.of(key), true, ResourceUsage.getAvailableProcessors(),
errorEventListener);
if (!result.hasError()) {
return Preconditions.checkNotNull(result.get(key), "%s %s", result, key);
}
ErrorInfo errorInfo = Preconditions.checkNotNull(result.getError(key),
"%s %s", key, result);
Throwables.propagateIfPossible(errorInfo.getException(), clazz);
if (errorInfo.getException() != null) {
throw new IllegalStateException(errorInfo.getException());
}
throw new IllegalStateException(errorInfo.toString());
}
}
});
} catch (Exception e) {
Throwables.propagateIfPossible(e, clazz);
throw new IllegalStateException(e);
}
}
class BuildViewProvider {
/**
* Returns the current {@link SkyframeBuildView} instance.
*/
SkyframeBuildView getSkyframeBuildView() {
return skyframeBuildView;
}
}
/**
* Must be called before the {@link SkyframeExecutor} can be used (should only be called in
* factory methods and as an implementation detail of {@link #resetEvaluator}).
*/
protected void init() {
progressReceiver = newSkyframeProgressReceiver();
Map skyFunctions = skyFunctions(
directories.getBuildDataDirectory(), pkgFactory, allowedMissingInputs);
memoizingEvaluator = evaluatorSupplier.create(
skyFunctions, evaluatorDiffer(), progressReceiver, emittedEventState,
hasIncrementalState());
buildDriver = newBuildDriver();
}
protected SkyframeProgressReceiver newSkyframeProgressReceiver() {
return new SkyframeProgressReceiver();
}
/**
* Reinitializes the Skyframe evaluator, dropping all previously computed values.
*
*
Be careful with this method as it also deletes all injected values. You need to make sure
* that any necessary precomputed values are reinjected before the next build. Constants can be
* put in {@link #reinjectConstantValuesLazily}.
*/
public void resetEvaluator() {
init();
emittedEventState.clear();
if (skyframeBuildView != null) {
skyframeBuildView.clearLegacyData();
}
reinjectConstantValuesLazily();
}
protected abstract Differencer evaluatorDiffer();
protected abstract BuildDriver newBuildDriver();
/**
* Values whose values are known at startup and guaranteed constant are still wiped from the
* evaluator when we create a new one, so they must be re-injected each time we create a new
* evaluator.
*/
private void reinjectConstantValuesLazily() {
needToInjectEmbeddedArtifacts = true;
needToInjectPrecomputedValuesForAnalysis = true;
}
/**
* Deletes all ConfiguredTarget values from the Skyframe cache. This is done to save memory (e.g.
* on a configuration change); since the configuration is part of the key, these key/value pairs
* will be sitting around doing nothing until the configuration changes back to the previous
* value.
*
*
The next evaluation will delete all invalid values.
*/
public abstract void dropConfiguredTargets();
/**
* Removes ConfigurationFragmentValuess and ConfigurationCollectionValues from the cache.
*/
@VisibleForTesting
public void invalidateConfigurationCollection() {
invalidate(SkyFunctionName.functionIsIn(ImmutableSet.of(SkyFunctions.CONFIGURATION_FRAGMENT,
SkyFunctions.CONFIGURATION_COLLECTION)));
}
/**
* Decides if graph edges should be stored for this build. If not, re-creates the graph to not
* store graph edges. Necessary conditions to not store graph edges are:
* (1) batch (since incremental builds are not possible);
* (2) skyframe build (since otherwise the memory savings are too slight to bother);
* (3) keep-going (since otherwise bubbling errors up may require edges of done nodes);
* (4) discard_analysis_cache (since otherwise user isn't concerned about saving memory this way).
*/
public void decideKeepIncrementalState(boolean batch, Options viewOptions) {
// Assume incrementality.
}
public boolean hasIncrementalState() {
return true;
}
@VisibleForTesting
protected abstract Injectable injectable();
/**
* Saves memory by clearing analysis objects from Skyframe. If using legacy execution, actually
* deletes the relevant values. If using Skyframe execution, clears their data without deleting
* them (they will be deleted on the next build).
*/
public abstract void clearAnalysisCache(Collection topLevelTargets);
/**
* Injects the contents of the computed tools/defaults package.
*/
@VisibleForTesting
public void setupDefaultPackage(String defaultsPackageContents) {
PrecomputedValue.DEFAULTS_PACKAGE_CONTENTS.set(injectable(), defaultsPackageContents);
}
/**
* Injects the top-level artifact options.
*/
public void injectTopLevelContext(TopLevelArtifactContext options) {
PrecomputedValue.TOP_LEVEL_CONTEXT.set(injectable(), options);
}
public void injectWorkspaceStatusData() {
PrecomputedValue.WORKSPACE_STATUS_KEY.set(injectable(),
workspaceStatusActionFactory.createWorkspaceStatusAction(
artifactFactory.get(), WorkspaceStatusValue.ARTIFACT_OWNER, buildId));
}
public void injectCoverageReportData(ImmutableList actions) {
PrecomputedValue.COVERAGE_REPORT_KEY.set(injectable(), actions);
}
/**
* Sets the default visibility.
*/
private void setDefaultVisibility(RuleVisibility defaultVisibility) {
PrecomputedValue.DEFAULT_VISIBILITY.set(injectable(), defaultVisibility);
}
private void maybeInjectPrecomputedValuesForAnalysis() {
if (needToInjectPrecomputedValuesForAnalysis) {
injectBuildInfoFactories();
injectExtraPrecomputedValues();
needToInjectPrecomputedValuesForAnalysis = false;
}
}
private void injectExtraPrecomputedValues() {
for (PrecomputedValue.Injected injected : extraPrecomputedValues) {
injected.inject(injectable());
}
}
protected Cache newPkgFunctionCache() {
return CacheBuilder.newBuilder().build();
}
protected Cache newPreprocessCache() {
return CacheBuilder.newBuilder().build();
}
/**
* Injects the build info factory map that will be used when constructing build info
* actions/artifacts. Unchanged across the life of the Blaze server, although it must be injected
* each time the evaluator is created.
*/
private void injectBuildInfoFactories() {
ImmutableMap.Builder factoryMapBuilder =
ImmutableMap.builder();
for (BuildInfoFactory factory : buildInfoFactories) {
factoryMapBuilder.put(factory.getKey(), factory);
}
PrecomputedValue.BUILD_INFO_FACTORIES.set(injectable(), factoryMapBuilder.build());
}
private void setShowLoadingProgress(boolean showLoadingProgressValue) {
showLoadingProgress.set(showLoadingProgressValue);
}
@VisibleForTesting
public void setCommandId(UUID commandId) {
PrecomputedValue.BUILD_ID.set(injectable(), commandId);
buildId.set(commandId);
}
/** Returns the build-info.txt and build-changelist.txt artifacts. */
public Collection getWorkspaceStatusArtifacts() throws InterruptedException {
// Should already be present, unless the user didn't request any targets for analysis.
EvaluationResult result = buildDriver.evaluate(
ImmutableList.of(WorkspaceStatusValue.SKY_KEY), /*keepGoing=*/true, /*numThreads=*/1,
reporter);
WorkspaceStatusValue value =
Preconditions.checkNotNull(result.get(WorkspaceStatusValue.SKY_KEY));
return ImmutableList.of(value.getStableArtifact(), value.getVolatileArtifact());
}
// TODO(bazel-team): Make this take a PackageIdentifier.
public Map getArtifactRoots(Iterable execPaths)
throws PackageRootResolutionException {
final List packageKeys = new ArrayList<>();
for (PathFragment execPath : execPaths) {
Preconditions.checkArgument(!execPath.isAbsolute(), execPath);
packageKeys.add(ContainingPackageLookupValue.key(
PackageIdentifier.createInDefaultRepo(execPath)));
}
EvaluationResult result;
try {
result = callUninterruptibly(new Callable>() {
@Override
public EvaluationResult call() throws InterruptedException {
synchronized (valueLookupLock) {
return buildDriver.evaluate(
packageKeys, /*keepGoing=*/true, /*numThreads=*/1, reporter);
}
}
});
} catch (Exception e) {
throw new IllegalStateException(e); // Should never happen.
}
if (result.hasError()) {
throw new PackageRootResolutionException("Exception encountered determining package roots",
result.getError().getException());
}
Map roots = new HashMap<>();
for (PathFragment execPath : execPaths) {
ContainingPackageLookupValue value = result.get(ContainingPackageLookupValue.key(
PackageIdentifier.createInDefaultRepo(execPath)));
if (value.hasContainingPackage()) {
roots.put(execPath, Root.asSourceRoot(value.getContainingPackageRoot()));
} else {
roots.put(execPath, null);
}
}
return roots;
}
@VisibleForTesting
public WorkspaceStatusAction getLastWorkspaceStatusActionForTesting() {
PrecomputedValue value = (PrecomputedValue) buildDriver.getGraphForTesting()
.getExistingValueForTesting(PrecomputedValue.WORKSPACE_STATUS_KEY.getKeyForTesting());
return (WorkspaceStatusAction) value.get();
}
/**
* Informs user about number of modified files (source and output files).
*/
// Note, that number of modified files in some cases can be bigger than actual number of
// modified files for targets in current request. Skyframe may check for modification all files
// from previous requests.
protected void informAboutNumberOfModifiedFiles() {
LOG.info(String.format("Found %d modified files from last build", modifiedFiles));
}
public Reporter getReporter() {
return reporter;
}
public EventBus getEventBus() {
return eventBus.get();
}
public ActionExecutionContextFactory getActionExecutionContextFactory() {
return skyframeActionExecutor;
}
@VisibleForTesting
ImmutableList getPathEntries() {
return pkgLocator.get().getPathEntries();
}
protected abstract void invalidate(Predicate pred);
private static boolean compatibleFileTypes(Dirent.Type oldType, FileStateValue.Type newType) {
return (oldType.equals(Dirent.Type.FILE) && newType.equals(FileStateValue.Type.FILE))
|| (oldType.equals(Dirent.Type.DIRECTORY) && newType.equals(FileStateValue.Type.DIRECTORY))
|| (oldType.equals(Dirent.Type.SYMLINK) && newType.equals(FileStateValue.Type.SYMLINK));
}
protected Differencer.Diff getDiff(Iterable modifiedSourceFiles,
final Path pathEntry) throws InterruptedException {
// TODO(bazel-team): change ModifiedFileSet to work with RootedPaths instead of PathFragments.
Iterable dirtyFileStateSkyKeys = Iterables.transform(modifiedSourceFiles,
new Function() {
@Override
public SkyKey apply(PathFragment pathFragment) {
Preconditions.checkState(!pathFragment.isAbsolute(),
"found absolute PathFragment: %s", pathFragment);
return FileStateValue.key(RootedPath.toRootedPath(pathEntry, pathFragment));
}
});
// We only need to invalidate directory values when a file has been created or deleted or
// changes type, not when it has merely been modified. Unfortunately we do not have that
// information here, so we compute it ourselves.
// TODO(bazel-team): Fancy filesystems could provide it with a hypothetically modified
// DiffAwareness interface.
Supplier