// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.skyframe;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.base.Predicates;
import com.google.common.base.Throwables;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
import com.google.common.collect.Range;
import com.google.common.collect.Sets;
import com.google.common.eventbus.EventBus;
import com.google.devtools.build.lib.actions.Action;
import com.google.devtools.build.lib.actions.ActionCacheChecker;
import com.google.devtools.build.lib.actions.ActionExecutionContextFactory;
import com.google.devtools.build.lib.actions.ActionExecutionStatusReporter;
import com.google.devtools.build.lib.actions.ActionInputFileCache;
import com.google.devtools.build.lib.actions.ActionLogBufferPathGenerator;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.ArtifactFactory;
import com.google.devtools.build.lib.actions.ArtifactOwner;
import com.google.devtools.build.lib.actions.Executor;
import com.google.devtools.build.lib.actions.PackageRootResolutionException;
import com.google.devtools.build.lib.actions.ResourceManager;
import com.google.devtools.build.lib.actions.Root;
import com.google.devtools.build.lib.analysis.Aspect;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.analysis.BuildView.Options;
import com.google.devtools.build.lib.analysis.ConfiguredAspectFactory;
import com.google.devtools.build.lib.analysis.ConfiguredTarget;
import com.google.devtools.build.lib.analysis.DependencyResolver.Dependency;
import com.google.devtools.build.lib.analysis.RuleConfiguredTarget;
import com.google.devtools.build.lib.analysis.TopLevelArtifactContext;
import com.google.devtools.build.lib.analysis.WorkspaceStatusAction;
import com.google.devtools.build.lib.analysis.WorkspaceStatusAction.Factory;
import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory;
import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory.BuildInfoKey;
import com.google.devtools.build.lib.analysis.config.BinTools;
import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection;
import com.google.devtools.build.lib.analysis.config.BuildOptions;
import com.google.devtools.build.lib.analysis.config.ConfigurationFactory;
import com.google.devtools.build.lib.analysis.config.ConfigurationFragmentFactory;
import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
import com.google.devtools.build.lib.concurrent.ThreadSafety;
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
import com.google.devtools.build.lib.events.EventHandler;
import com.google.devtools.build.lib.events.Reporter;
import com.google.devtools.build.lib.packages.BuildFileContainsErrorsException;
import com.google.devtools.build.lib.packages.NoSuchPackageException;
import com.google.devtools.build.lib.packages.NoSuchThingException;
import com.google.devtools.build.lib.packages.Package;
import com.google.devtools.build.lib.packages.PackageFactory;
import com.google.devtools.build.lib.packages.PackageIdentifier;
import com.google.devtools.build.lib.packages.Preprocessor;
import com.google.devtools.build.lib.packages.RuleClassProvider;
import com.google.devtools.build.lib.packages.RuleVisibility;
import com.google.devtools.build.lib.packages.Target;
import com.google.devtools.build.lib.pkgcache.PackageCacheOptions;
import com.google.devtools.build.lib.pkgcache.PackageManager;
import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
import com.google.devtools.build.lib.pkgcache.TransitivePackageLoader;
import com.google.devtools.build.lib.profiler.Profiler;
import com.google.devtools.build.lib.skyframe.SkyframeActionExecutor.ActionCompletedReceiver;
import com.google.devtools.build.lib.skyframe.SkyframeActionExecutor.ProgressSupplier;
import com.google.devtools.build.lib.syntax.Label;
import com.google.devtools.build.lib.util.AbruptExitException;
import com.google.devtools.build.lib.util.ExitCode;
import com.google.devtools.build.lib.util.ResourceUsage;
import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
import com.google.devtools.build.lib.vfs.BatchStat;
import com.google.devtools.build.lib.vfs.ModifiedFileSet;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.RootedPath;
import com.google.devtools.build.lib.vfs.UnixGlob;
import com.google.devtools.build.skyframe.BuildDriver;
import com.google.devtools.build.skyframe.CycleInfo;
import com.google.devtools.build.skyframe.CyclesReporter;
import com.google.devtools.build.skyframe.Differencer;
import com.google.devtools.build.skyframe.ErrorInfo;
import com.google.devtools.build.skyframe.EvaluationProgressReceiver;
import com.google.devtools.build.skyframe.EvaluationResult;
import com.google.devtools.build.skyframe.Injectable;
import com.google.devtools.build.skyframe.MemoizingEvaluator;
import com.google.devtools.build.skyframe.MemoizingEvaluator.EvaluatorSupplier;
import com.google.devtools.build.skyframe.SkyFunction;
import com.google.devtools.build.skyframe.SkyFunctionName;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
import com.google.devtools.build.skyframe.WalkableGraph.WalkableGraphFactory;
import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.logging.Logger;
import javax.annotation.Nullable;
/**
* A helper object to support Skyframe-driven execution.
*
*
This object is mostly used to inject external state, such as the executor engine or
* some additional artifacts (workspace status and build info artifacts) into SkyFunctions
* for use during the build.
*/
public abstract class SkyframeExecutor implements WalkableGraphFactory {
private final EvaluatorSupplier evaluatorSupplier;
protected MemoizingEvaluator memoizingEvaluator;
private final MemoizingEvaluator.EmittedEventState emittedEventState =
new MemoizingEvaluator.EmittedEventState();
protected final Reporter reporter;
private final PackageFactory pkgFactory;
private final WorkspaceStatusAction.Factory workspaceStatusActionFactory;
private final BlazeDirectories directories;
@Nullable
private BatchStat batchStatter;
// TODO(bazel-team): Figure out how to handle value builders that block internally. Blocking
// operations may need to be handled in another (bigger?) thread pool. Also, we should detect
// the number of cores and use that as the thread-pool size for CPU-bound operations.
// I just bumped this to 200 to get reasonable execution phase performance; that may cause
// significant overhead for CPU-bound processes (i.e. analysis). [skyframe-analysis]
@VisibleForTesting
public static final int DEFAULT_THREAD_COUNT =
// Reduce thread count while running tests of Bazel. Test cases are typically small, and large
// thread pools vying for a relatively small number of CPU cores may induce non-optimal
// performance.
System.getenv("TEST_TMPDIR") == null ? 200 : 5;
// Cache of partially constructed Package instances, stored between reruns of the PackageFunction
// (because of missing dependencies, within the same evaluate() run) to avoid loading the same
// package twice (first time loading to find subincludes and declare value dependencies).
// TODO(bazel-team): remove this cache once we have skyframe-native package loading
// [skyframe-loading]
private final Cache packageFunctionCache =
newPkgFunctionCache();
private final AtomicInteger numPackagesLoaded = new AtomicInteger(0);
protected SkyframeBuildView skyframeBuildView;
private EventHandler errorEventListener;
private ActionLogBufferPathGenerator actionLogBufferPathGenerator;
protected BuildDriver buildDriver;
// AtomicReferences are used here as mutable boxes shared with value builders.
private final AtomicBoolean showLoadingProgress = new AtomicBoolean();
protected final AtomicReference syscalls =
new AtomicReference<>(UnixGlob.DEFAULT_SYSCALLS);
protected final AtomicReference pkgLocator =
new AtomicReference<>();
protected final AtomicReference> deletedPackages =
new AtomicReference<>(ImmutableSet.of());
private final AtomicReference eventBus = new AtomicReference<>();
private final ImmutableList buildInfoFactories;
// Under normal circumstances, the artifact factory persists for the life of a Blaze server, but
// since it is not yet created when we create the value builders, we have to use a supplier,
// initialized when the build view is created.
private final MutableSupplier artifactFactory = new MutableSupplier<>();
// Used to give to WriteBuildInfoAction via a supplier. Relying on BuildVariableValue.BUILD_ID
// would be preferable, but we have no way to have the Action depend on that value directly.
// Having the BuildInfoFunction own the supplier is currently not possible either, because then
// it would be invalidated on every build, since it would depend on the build id value.
private MutableSupplier buildId = new MutableSupplier<>();
protected boolean active = true;
private final PackageManager packageManager;
private final Preprocessor.Factory.Supplier preprocessorFactorySupplier;
private Preprocessor.Factory preprocessorFactory;
protected final TimestampGranularityMonitor tsgm;
private final ResourceManager resourceManager;
/** Used to lock evaluator on legacy calls to get existing values. */
private final Object valueLookupLock = new Object();
private final AtomicReference statusReporterRef =
new AtomicReference<>();
private final SkyframeActionExecutor skyframeActionExecutor;
private CompletionReceiver actionExecutionFunction;
protected SkyframeProgressReceiver progressReceiver;
private final AtomicReference cyclesReporter = new AtomicReference<>();
private final Set immutableDirectories;
private BinTools binTools = null;
private boolean needToInjectEmbeddedArtifacts = true;
private boolean needToInjectPrecomputedValuesForAnalysis = true;
protected int modifiedFiles;
protected int outputDirtyFiles;
protected int modifiedFilesDuringPreviousBuild;
private final Predicate allowedMissingInputs;
private final boolean errorOnExternalFiles;
private final ImmutableMap extraSkyFunctions;
private final ImmutableList extraPrecomputedValues;
protected SkyframeIncrementalBuildMonitor incrementalBuildMonitor =
new SkyframeIncrementalBuildMonitor();
private MutableSupplier configurationFactory = new MutableSupplier<>();
private MutableSupplier> configurationFragments =
new MutableSupplier<>();
private MutableSupplier> configurationPackages = new MutableSupplier<>();
private static final Logger LOG = Logger.getLogger(SkyframeExecutor.class.getName());
protected SkyframeExecutor(
Reporter reporter,
EvaluatorSupplier evaluatorSupplier,
PackageFactory pkgFactory,
TimestampGranularityMonitor tsgm,
BlazeDirectories directories,
Factory workspaceStatusActionFactory,
ImmutableList buildInfoFactories,
Set immutableDirectories,
Predicate allowedMissingInputs,
Preprocessor.Factory.Supplier preprocessorFactorySupplier,
ImmutableMap extraSkyFunctions,
ImmutableList extraPrecomputedValues,
boolean errorOnExternalFiles) {
// Strictly speaking, these arguments are not required for initialization, but all current
// callsites have them at hand, so we might as well set them during construction.
this.reporter = Preconditions.checkNotNull(reporter);
this.evaluatorSupplier = evaluatorSupplier;
this.pkgFactory = pkgFactory;
this.pkgFactory.setSyscalls(syscalls);
this.tsgm = tsgm;
this.workspaceStatusActionFactory = workspaceStatusActionFactory;
this.packageManager = new SkyframePackageManager(
new SkyframePackageLoader(), new SkyframeTransitivePackageLoader(),
new SkyframeTargetPatternEvaluator(this), syscalls, cyclesReporter, pkgLocator,
numPackagesLoaded, this);
this.errorEventListener = this.reporter;
this.resourceManager = ResourceManager.instance();
this.skyframeActionExecutor = new SkyframeActionExecutor(reporter, resourceManager, eventBus,
statusReporterRef);
this.directories = Preconditions.checkNotNull(directories);
this.buildInfoFactories = buildInfoFactories;
this.immutableDirectories = immutableDirectories;
this.allowedMissingInputs = allowedMissingInputs;
this.preprocessorFactorySupplier = preprocessorFactorySupplier;
this.extraSkyFunctions = extraSkyFunctions;
this.extraPrecomputedValues = extraPrecomputedValues;
this.errorOnExternalFiles = errorOnExternalFiles;
}
private ImmutableMap skyFunctions(
Root buildDataDirectory,
PackageFactory pkgFactory,
Predicate allowedMissingInputs) {
ExternalFilesHelper externalFilesHelper = new ExternalFilesHelper(pkgLocator,
immutableDirectories, errorOnExternalFiles);
RuleClassProvider ruleClassProvider = pkgFactory.getRuleClassProvider();
// We use an immutable map builder for the nice side effect that it throws if a duplicate key
// is inserted.
ImmutableMap.Builder map = ImmutableMap.builder();
map.put(SkyFunctions.PRECOMPUTED, new PrecomputedFunction());
map.put(SkyFunctions.FILE_STATE, new FileStateFunction(tsgm, externalFilesHelper));
map.put(SkyFunctions.DIRECTORY_LISTING_STATE,
new DirectoryListingStateFunction(externalFilesHelper));
map.put(SkyFunctions.FILE_SYMLINK_CYCLE_UNIQUENESS,
new FileSymlinkCycleUniquenessFunction());
map.put(SkyFunctions.FILE_SYMLINK_INFINITE_EXPANSION_UNIQUENESS,
new FileSymlinkInfiniteExpansionUniquenessFunction());
map.put(SkyFunctions.FILE, new FileFunction(pkgLocator, tsgm, externalFilesHelper));
map.put(SkyFunctions.DIRECTORY_LISTING, new DirectoryListingFunction());
map.put(SkyFunctions.PACKAGE_LOOKUP, new PackageLookupFunction(deletedPackages));
map.put(SkyFunctions.CONTAINING_PACKAGE_LOOKUP, new ContainingPackageLookupFunction());
map.put(SkyFunctions.AST_FILE_LOOKUP, new ASTFileLookupFunction(
pkgLocator, packageManager, ruleClassProvider));
map.put(SkyFunctions.SKYLARK_IMPORTS_LOOKUP, new SkylarkImportLookupFunction(
ruleClassProvider, pkgFactory));
map.put(SkyFunctions.GLOB, new GlobFunction());
map.put(SkyFunctions.TARGET_PATTERN, new TargetPatternFunction(pkgLocator));
map.put(SkyFunctions.PREPARE_DEPS_OF_PATTERNS, new PrepareDepsOfPatternsFunction());
map.put(SkyFunctions.PREPARE_DEPS_OF_PATTERN, new PrepareDepsOfPatternFunction(pkgLocator));
map.put(SkyFunctions.PREPARE_DEPS_OF_TARGETS_UNDER_DIRECTORY,
new PrepareDepsOfTargetsUnderDirectoryFunction());
map.put(SkyFunctions.RECURSIVE_PKG, new RecursivePkgFunction());
map.put(SkyFunctions.PACKAGE, new PackageFunction(
reporter, pkgFactory, packageManager, showLoadingProgress, packageFunctionCache,
numPackagesLoaded));
map.put(SkyFunctions.TARGET_MARKER, new TargetMarkerFunction());
map.put(SkyFunctions.TRANSITIVE_TARGET, new TransitiveTargetFunction(ruleClassProvider));
map.put(SkyFunctions.TRANSITIVE_TRAVERSAL, new TransitiveTraversalFunction());
map.put(SkyFunctions.CONFIGURED_TARGET,
new ConfiguredTargetFunction(new BuildViewProvider()));
map.put(SkyFunctions.ASPECT, new AspectFunction(new BuildViewProvider()));
map.put(SkyFunctions.POST_CONFIGURED_TARGET,
new PostConfiguredTargetFunction(new BuildViewProvider()));
map.put(SkyFunctions.BUILD_CONFIGURATION, new BuildConfigurationFunction(directories));
map.put(SkyFunctions.CONFIGURATION_COLLECTION, new ConfigurationCollectionFunction(
configurationFactory, configurationPackages));
map.put(SkyFunctions.CONFIGURATION_FRAGMENT, new ConfigurationFragmentFunction(
configurationFragments, configurationPackages));
map.put(
SkyFunctions.WORKSPACE_FILE,
new WorkspaceFileFunction(ruleClassProvider, pkgFactory, directories));
map.put(SkyFunctions.TARGET_COMPLETION, new TargetCompletionFunction(eventBus));
map.put(SkyFunctions.TEST_COMPLETION, new TestCompletionFunction());
map.put(SkyFunctions.ARTIFACT, new ArtifactFunction(allowedMissingInputs));
map.put(SkyFunctions.BUILD_INFO_COLLECTION, new BuildInfoCollectionFunction(artifactFactory,
buildDataDirectory));
map.put(SkyFunctions.BUILD_INFO, new WorkspaceStatusFunction());
map.put(SkyFunctions.COVERAGE_REPORT, new CoverageReportFunction());
ActionExecutionFunction actionExecutionFunction =
new ActionExecutionFunction(skyframeActionExecutor, tsgm);
map.put(SkyFunctions.ACTION_EXECUTION, actionExecutionFunction);
this.actionExecutionFunction = actionExecutionFunction;
map.put(SkyFunctions.RECURSIVE_FILESYSTEM_TRAVERSAL,
new RecursiveFilesystemTraversalFunction());
map.put(SkyFunctions.FILESET_ENTRY, new FilesetEntryFunction());
map.putAll(extraSkyFunctions);
return map.build();
}
@ThreadCompatible
public void setActive(boolean active) {
this.active = active;
}
protected void checkActive() {
Preconditions.checkState(active);
}
public void setFileCache(ActionInputFileCache fileCache) {
this.skyframeActionExecutor.setFileCache(fileCache);
}
public void dump(boolean summarize, PrintStream out) {
memoizingEvaluator.dump(summarize, out);
}
public abstract void dumpPackages(PrintStream out);
public void setBatchStatter(@Nullable BatchStat batchStatter) {
this.batchStatter = batchStatter;
}
/**
* Notify listeners about changed files, and release any associated memory afterwards.
*/
public void drainChangedFiles() {
incrementalBuildMonitor.alertListeners(getEventBus());
incrementalBuildMonitor = null;
}
@VisibleForTesting
public BuildDriver getDriverForTesting() {
return buildDriver;
}
/**
* This method exists only to allow a module to make a top-level Skyframe call during the
* transition to making it fully Skyframe-compatible. Do not add additional callers!
*/
public SkyValue evaluateSkyKeyForCodeMigration(final SkyKey key,
final Class clazz) throws E {
try {
return callUninterruptibly(new Callable() {
@Override
public SkyValue call() throws E, InterruptedException {
synchronized (valueLookupLock) {
// We evaluate in keepGoing mode because in the case that the graph does not store its
// edges, nokeepGoing builds are not allowed, whereas keepGoing builds are always
// permitted.
EvaluationResult result = buildDriver.evaluate(
ImmutableList.of(key), true, ResourceUsage.getAvailableProcessors(),
errorEventListener);
if (!result.hasError()) {
return Preconditions.checkNotNull(result.get(key), "%s %s", result, key);
}
ErrorInfo errorInfo = Preconditions.checkNotNull(result.getError(key),
"%s %s", key, result);
Throwables.propagateIfPossible(errorInfo.getException(), clazz);
if (errorInfo.getException() != null) {
throw new IllegalStateException(errorInfo.getException());
}
throw new IllegalStateException(errorInfo.toString());
}
}
});
} catch (Exception e) {
Throwables.propagateIfPossible(e, clazz);
throw new IllegalStateException(e);
}
}
class BuildViewProvider {
/**
* Returns the current {@link SkyframeBuildView} instance.
*/
SkyframeBuildView getSkyframeBuildView() {
return skyframeBuildView;
}
}
/**
* Must be called before the {@link SkyframeExecutor} can be used (should only be called in
* factory methods and as an implementation detail of {@link #resetEvaluator}).
*/
protected void init() {
progressReceiver = newSkyframeProgressReceiver();
Map skyFunctions = skyFunctions(
directories.getBuildDataDirectory(), pkgFactory, allowedMissingInputs);
memoizingEvaluator = evaluatorSupplier.create(
skyFunctions, evaluatorDiffer(), progressReceiver, emittedEventState,
hasIncrementalState());
buildDriver = newBuildDriver();
}
protected SkyframeProgressReceiver newSkyframeProgressReceiver() {
return new SkyframeProgressReceiver();
}
/**
* Reinitializes the Skyframe evaluator, dropping all previously computed values.
*
*
Be careful with this method as it also deletes all injected values. You need to make sure
* that any necessary precomputed values are reinjected before the next build. Constants can be
* put in {@link #reinjectConstantValuesLazily}.
*/
public void resetEvaluator() {
init();
emittedEventState.clear();
if (skyframeBuildView != null) {
skyframeBuildView.clearLegacyData();
}
reinjectConstantValuesLazily();
}
protected abstract Differencer evaluatorDiffer();
protected abstract BuildDriver newBuildDriver();
/**
* Values whose values are known at startup and guaranteed constant are still wiped from the
* evaluator when we create a new one, so they must be re-injected each time we create a new
* evaluator.
*/
private void reinjectConstantValuesLazily() {
needToInjectEmbeddedArtifacts = true;
needToInjectPrecomputedValuesForAnalysis = true;
}
/**
* Deletes all ConfiguredTarget values from the Skyframe cache. This is done to save memory (e.g.
* on a configuration change); since the configuration is part of the key, these key/value pairs
* will be sitting around doing nothing until the configuration changes back to the previous
* value.
*
*
The next evaluation will delete all invalid values.
*/
public abstract void dropConfiguredTargets();
/**
* Removes ConfigurationFragmentValuess and ConfigurationCollectionValues from the cache.
*/
@VisibleForTesting
public void invalidateConfigurationCollection() {
invalidate(SkyFunctionName.functionIsIn(ImmutableSet.of(SkyFunctions.CONFIGURATION_FRAGMENT,
SkyFunctions.CONFIGURATION_COLLECTION)));
}
/**
* Decides if graph edges should be stored for this build. If not, re-creates the graph to not
* store graph edges. Necessary conditions to not store graph edges are:
* (1) batch (since incremental builds are not possible);
* (2) skyframe build (since otherwise the memory savings are too slight to bother);
* (3) keep-going (since otherwise bubbling errors up may require edges of done nodes);
* (4) discard_analysis_cache (since otherwise user isn't concerned about saving memory this way).
*/
public void decideKeepIncrementalState(boolean batch, Options viewOptions) {
// Assume incrementality.
}
public boolean hasIncrementalState() {
return true;
}
@VisibleForTesting
protected abstract Injectable injectable();
/**
* Saves memory by clearing analysis objects from Skyframe. If using legacy execution, actually
* deletes the relevant values. If using Skyframe execution, clears their data without deleting
* them (they will be deleted on the next build).
*/
public abstract void clearAnalysisCache(Collection topLevelTargets);
/**
* Injects the contents of the computed tools/defaults package.
*/
@VisibleForTesting
public void setupDefaultPackage(String defaultsPackageContents) {
PrecomputedValue.DEFAULTS_PACKAGE_CONTENTS.set(injectable(), defaultsPackageContents);
}
/**
* Injects the top-level artifact options.
*/
public void injectTopLevelContext(TopLevelArtifactContext options) {
PrecomputedValue.TOP_LEVEL_CONTEXT.set(injectable(), options);
}
public void injectWorkspaceStatusData() {
PrecomputedValue.WORKSPACE_STATUS_KEY.set(injectable(),
workspaceStatusActionFactory.createWorkspaceStatusAction(
artifactFactory.get(), WorkspaceStatusValue.ARTIFACT_OWNER, buildId));
}
public void injectCoverageReportData(ImmutableList actions) {
PrecomputedValue.COVERAGE_REPORT_KEY.set(injectable(), actions);
}
/**
* Sets the default visibility.
*/
private void setDefaultVisibility(RuleVisibility defaultVisibility) {
PrecomputedValue.DEFAULT_VISIBILITY.set(injectable(), defaultVisibility);
}
private void maybeInjectPrecomputedValuesForAnalysis() {
if (needToInjectPrecomputedValuesForAnalysis) {
injectBuildInfoFactories();
injectExtraPrecomputedValues();
needToInjectPrecomputedValuesForAnalysis = false;
}
}
private void injectExtraPrecomputedValues() {
for (PrecomputedValue.Injected injected : extraPrecomputedValues) {
injected.inject(injectable());
}
}
protected Cache newPkgFunctionCache() {
return CacheBuilder.newBuilder().build();
}
/**
* Injects the build info factory map that will be used when constructing build info
* actions/artifacts. Unchanged across the life of the Blaze server, although it must be injected
* each time the evaluator is created.
*/
private void injectBuildInfoFactories() {
ImmutableMap.Builder factoryMapBuilder =
ImmutableMap.builder();
for (BuildInfoFactory factory : buildInfoFactories) {
factoryMapBuilder.put(factory.getKey(), factory);
}
PrecomputedValue.BUILD_INFO_FACTORIES.set(injectable(), factoryMapBuilder.build());
}
private void setShowLoadingProgress(boolean showLoadingProgressValue) {
showLoadingProgress.set(showLoadingProgressValue);
}
@VisibleForTesting
public void setCommandId(UUID commandId) {
PrecomputedValue.BUILD_ID.set(injectable(), commandId);
buildId.set(commandId);
}
/** Returns the build-info.txt and build-changelist.txt artifacts. */
public Collection getWorkspaceStatusArtifacts() throws InterruptedException {
// Should already be present, unless the user didn't request any targets for analysis.
EvaluationResult result = buildDriver.evaluate(
ImmutableList.of(WorkspaceStatusValue.SKY_KEY), /*keepGoing=*/true, /*numThreads=*/1,
reporter);
WorkspaceStatusValue value =
Preconditions.checkNotNull(result.get(WorkspaceStatusValue.SKY_KEY));
return ImmutableList.of(value.getStableArtifact(), value.getVolatileArtifact());
}
// TODO(bazel-team): Make this take a PackageIdentifier.
public Map getArtifactRoots(Iterable execPaths)
throws PackageRootResolutionException {
final List packageKeys = new ArrayList<>();
for (PathFragment execPath : execPaths) {
Preconditions.checkArgument(!execPath.isAbsolute(), execPath);
packageKeys.add(ContainingPackageLookupValue.key(
PackageIdentifier.createInDefaultRepo(execPath)));
}
EvaluationResult result;
try {
result = callUninterruptibly(new Callable>() {
@Override
public EvaluationResult call() throws InterruptedException {
synchronized (valueLookupLock) {
return buildDriver.evaluate(
packageKeys, /*keepGoing=*/true, /*numThreads=*/1, reporter);
}
}
});
} catch (Exception e) {
throw new IllegalStateException(e); // Should never happen.
}
if (result.hasError()) {
throw new PackageRootResolutionException("Exception encountered determining package roots",
result.getError().getException());
}
Map roots = new HashMap<>();
for (PathFragment execPath : execPaths) {
ContainingPackageLookupValue value = result.get(ContainingPackageLookupValue.key(
PackageIdentifier.createInDefaultRepo(execPath)));
if (value.hasContainingPackage()) {
roots.put(execPath, Root.asSourceRoot(value.getContainingPackageRoot()));
} else {
roots.put(execPath, null);
}
}
return roots;
}
@VisibleForTesting
public WorkspaceStatusAction getLastWorkspaceStatusActionForTesting() {
PrecomputedValue value = (PrecomputedValue) buildDriver.getGraphForTesting()
.getExistingValueForTesting(PrecomputedValue.WORKSPACE_STATUS_KEY.getKeyForTesting());
return (WorkspaceStatusAction) value.get();
}
/**
* Informs user about number of modified files (source and output files).
*/
// Note, that number of modified files in some cases can be bigger than actual number of
// modified files for targets in current request. Skyframe may check for modification all files
// from previous requests.
protected void informAboutNumberOfModifiedFiles() {
LOG.info(String.format("Found %d modified files from last build", modifiedFiles));
}
public Reporter getReporter() {
return reporter;
}
public EventBus getEventBus() {
return eventBus.get();
}
public ActionExecutionContextFactory getActionExecutionContextFactory() {
return skyframeActionExecutor;
}
/**
* The map from package names to the package root where each package was found; this is used to
* set up the symlink tree.
*/
public ImmutableMap getPackageRoots() {
// Make a map of the package names to their root paths.
ImmutableMap.Builder packageRoots = ImmutableMap.builder();
for (Package pkg : configurationPackages.get()) {
packageRoots.put(pkg.getPackageIdentifier(), pkg.getSourceRoot());
}
return packageRoots.build();
}
@VisibleForTesting
ImmutableList getPathEntries() {
return pkgLocator.get().getPathEntries();
}
protected abstract void invalidate(Predicate pred);
protected static Iterable getSkyKeysPotentiallyAffected(
Iterable modifiedSourceFiles, final Path pathEntry) {
// TODO(bazel-team): change ModifiedFileSet to work with RootedPaths instead of PathFragments.
Iterable fileStateSkyKeys = Iterables.transform(modifiedSourceFiles,
new Function() {
@Override
public SkyKey apply(PathFragment pathFragment) {
Preconditions.checkState(!pathFragment.isAbsolute(),
"found absolute PathFragment: %s", pathFragment);
return FileStateValue.key(RootedPath.toRootedPath(pathEntry, pathFragment));
}
});
// TODO(bazel-team): Strictly speaking, we only need to invalidate directory values when a file
// has been created or deleted, not when it has been modified. Unfortunately we
// do not have that information here, although fancy filesystems could provide it with a
// hypothetically modified DiffAwareness interface.
// TODO(bazel-team): Even if we don't have that information, we could avoid invalidating
// directories when the state of a file does not change by statting them and comparing
// the new filetype (nonexistent/file/symlink/directory) with the old one.
Iterable dirListingStateSkyKeys = Iterables.transform(
modifiedSourceFiles,
new Function() {
@Override
public SkyKey apply(PathFragment pathFragment) {
Preconditions.checkState(!pathFragment.isAbsolute(),
"found absolute PathFragment: %s", pathFragment);
return DirectoryListingStateValue.key(RootedPath.toRootedPath(pathEntry,
pathFragment.getParentDirectory()));
}
});
return Iterables.concat(fileStateSkyKeys, dirListingStateSkyKeys);
}
protected static SkyKey createFileStateKey(RootedPath rootedPath) {
return FileStateValue.key(rootedPath);
}
protected static SkyKey createDirectoryListingStateKey(RootedPath rootedPath) {
return DirectoryListingStateValue.key(rootedPath);
}
/**
* Creates a FileValue pointing of type directory. No matter that the rootedPath points to a
* symlink.
*
*
Use it with caution as it would prevent invalidation when the destination file in the
* symlink changes.
*/
protected static FileValue createFileDirValue(RootedPath rootedPath) {
return FileValue.value(rootedPath, FileStateValue.DIRECTORY_FILE_STATE_NODE,
rootedPath, FileStateValue.DIRECTORY_FILE_STATE_NODE);
}
/**
* Sets the packages that should be treated as deleted and ignored.
*/
@VisibleForTesting // productionVisibility = Visibility.PRIVATE
public abstract void setDeletedPackages(Iterable pkgs);
/**
* Prepares the evaluator for loading.
*
*
MUST be run before every incremental build.
*/
@VisibleForTesting // productionVisibility = Visibility.PRIVATE
public void preparePackageLoading(PathPackageLocator pkgLocator, RuleVisibility defaultVisibility,
boolean showLoadingProgress, int globbingThreads,
String defaultsPackageContents, UUID commandId) {
Preconditions.checkNotNull(pkgLocator);
setActive(true);
maybeInjectPrecomputedValuesForAnalysis();
setCommandId(commandId);
setShowLoadingProgress(showLoadingProgress);
setDefaultVisibility(defaultVisibility);
setupDefaultPackage(defaultsPackageContents);
setPackageLocator(pkgLocator);
syscalls.set(new PerBuildSyscallCache());
this.pkgFactory.setGlobbingThreads(globbingThreads);
checkPreprocessorFactory();
emittedEventState.clear();
// If the PackageFunction was interrupted, there may be stale entries here.
packageFunctionCache.invalidateAll();
numPackagesLoaded.set(0);
// Reset the stateful SkyframeCycleReporter, which contains cycles from last run.
cyclesReporter.set(createCyclesReporter());
}
@SuppressWarnings("unchecked")
private void setPackageLocator(PathPackageLocator pkgLocator) {
PathPackageLocator oldLocator = this.pkgLocator.getAndSet(pkgLocator);
PrecomputedValue.PATH_PACKAGE_LOCATOR.set(injectable(), pkgLocator);
if (!pkgLocator.equals(oldLocator)) {
// The package path is read not only by SkyFunctions but also by some other code paths.
// We need to take additional steps to keep the corresponding data structures in sync.
// (Some of the additional steps are carried out by ConfiguredTargetValueInvalidationListener,
// and some by BuildView#buildHasIncompatiblePackageRoots and #updateSkyframe.)
onNewPackageLocator(oldLocator, pkgLocator);
}
}
protected abstract void onNewPackageLocator(PathPackageLocator oldLocator,
PathPackageLocator pkgLocator);
private void checkPreprocessorFactory() {
if (preprocessorFactory == null) {
Preprocessor.Factory newPreprocessorFactory = preprocessorFactorySupplier.getFactory(
packageManager);
pkgFactory.setPreprocessorFactory(newPreprocessorFactory);
preprocessorFactory = newPreprocessorFactory;
} else if (!preprocessorFactory.isStillValid()) {
Preprocessor.Factory newPreprocessorFactory = preprocessorFactorySupplier.getFactory(
packageManager);
invalidate(SkyFunctionName.functionIs(SkyFunctions.PACKAGE));
pkgFactory.setPreprocessorFactory(newPreprocessorFactory);
preprocessorFactory = newPreprocessorFactory;
}
}
/**
* Specifies the current {@link SkyframeBuildView} instance. This should only be set once over the
* lifetime of the Blaze server, except in tests.
*/
public void setSkyframeBuildView(SkyframeBuildView skyframeBuildView) {
this.skyframeBuildView = skyframeBuildView;
this.artifactFactory.set(skyframeBuildView.getArtifactFactory());
}
/**
* Sets the eventBus to use for posting events.
*/
public void setEventBus(EventBus eventBus) {
this.eventBus.set(eventBus);
}
/**
* Sets the path for action log buffers.
*/
public void setActionOutputRoot(Path actionOutputRoot) {
Preconditions.checkNotNull(actionOutputRoot);
this.actionLogBufferPathGenerator = new ActionLogBufferPathGenerator(actionOutputRoot);
this.skyframeActionExecutor.setActionLogBufferPathGenerator(actionLogBufferPathGenerator);
}
@VisibleForTesting
public void setConfigurationDataForTesting(BlazeDirectories directories,
ConfigurationFactory configurationFactory) {
PrecomputedValue.BLAZE_DIRECTORIES.set(injectable(), directories);
this.configurationFactory.set(configurationFactory);
this.configurationFragments.set(ImmutableList.copyOf(configurationFactory.getFactories()));
this.configurationPackages.set(Sets.newConcurrentHashSet());
}
/**
* Asks the Skyframe evaluator to build the value for BuildConfigurationCollection and returns the
* result. Also invalidates {@link PrecomputedValue#BLAZE_DIRECTORIES} if it has changed.
*/
public BuildConfigurationCollection createConfigurations(
ConfigurationFactory configurationFactory, BuildOptions buildOptions,
BlazeDirectories directories, Set multiCpu, boolean keepGoing)
throws InvalidConfigurationException, InterruptedException {
this.configurationPackages.set(Sets.newConcurrentHashSet());
this.configurationFactory.set(configurationFactory);
this.configurationFragments.set(ImmutableList.copyOf(configurationFactory.getFactories()));
// TODO(bazel-team): find a way to use only BuildConfigurationKey instead of
// BlazeDirectories.
PrecomputedValue.BLAZE_DIRECTORIES.set(injectable(), directories);
SkyKey skyKey = ConfigurationCollectionValue.key(
buildOptions, ImmutableSortedSet.copyOf(multiCpu));
EvaluationResult result = buildDriver.evaluate(
Arrays.asList(skyKey), keepGoing, DEFAULT_THREAD_COUNT, errorEventListener);
if (result.hasError()) {
Throwable e = result.getError(skyKey).getException();
// Wrap loading failed exceptions
if (e instanceof NoSuchThingException) {
e = new InvalidConfigurationException(e);
}
Throwables.propagateIfInstanceOf(e, InvalidConfigurationException.class);
throw new IllegalStateException(
"Unknown error during ConfigurationCollectionValue evaluation", e);
}
Preconditions.checkState(result.values().size() == 1,
"Result of evaluate() must contain exactly one value %s", result);
ConfigurationCollectionValue configurationValue =
Iterables.getOnlyElement(result.values());
this.configurationPackages.set(
Sets.newConcurrentHashSet(configurationValue.getConfigurationPackages()));
return configurationValue.getConfigurationCollection();
}
private Iterable getActionLookupValues() {
// This filter keeps subclasses of ActionLookupValue.
return Iterables.filter(memoizingEvaluator.getDoneValues().values(), ActionLookupValue.class);
}
@SuppressWarnings({"unchecked", "rawtypes"})
Map getActionLookupValueMap() {
return (Map) Maps.filterValues(memoizingEvaluator.getDoneValues(),
Predicates.instanceOf(ActionLookupValue.class));
}
/**
* Checks the actions in Skyframe for conflicts between their output artifacts. Delegates to
* {@link SkyframeActionExecutor#findAndStoreArtifactConflicts} to do the work, since any
* conflicts found will only be reported during execution.
*/
ImmutableMap findArtifactConflicts()
throws InterruptedException {
if (skyframeBuildView.isSomeConfiguredTargetEvaluated()
|| skyframeBuildView.isSomeConfiguredTargetInvalidated()) {
// This operation is somewhat expensive, so we only do it if the graph might have changed in
// some way -- either we analyzed a new target or we invalidated an old one.
long startTime = Profiler.nanoTimeMaybe();
skyframeActionExecutor.findAndStoreArtifactConflicts(getActionLookupValues());
skyframeBuildView.resetEvaluatedConfiguredTargetFlag();
// The invalidated configured targets flag will be reset later in the evaluate() call.
long duration = Profiler.nanoTimeMaybe() - startTime;
if (duration > 0) {
LOG.info("Spent " + (duration / 1000 / 1000) + " ms discovering artifact conflicts");
}
}
return skyframeActionExecutor.badActions();
}
/**
* Asks the Skyframe evaluator to build the given artifacts and targets, and to test the
* given test targets.
*/
public EvaluationResult> buildArtifacts(
Executor executor,
Set artifactsToBuild,
Collection targetsToBuild,
Collection targetsToTest,
boolean exclusiveTesting,
boolean keepGoing,
boolean explain,
int numJobs,
ActionCacheChecker actionCacheChecker,
@Nullable EvaluationProgressReceiver executionProgressReceiver) throws InterruptedException {
checkActive();
Preconditions.checkState(actionLogBufferPathGenerator != null);
skyframeActionExecutor.prepareForExecution(executor, keepGoing, explain, actionCacheChecker);
resourceManager.resetResourceUsage();
try {
progressReceiver.executionProgressReceiver = executionProgressReceiver;
Iterable artifactKeys = ArtifactValue.mandatoryKeys(artifactsToBuild);
Iterable targetKeys = TargetCompletionValue.keys(targetsToBuild);
Iterable testKeys = TestCompletionValue.keys(targetsToTest, exclusiveTesting);
return buildDriver.evaluate(Iterables.concat(artifactKeys, targetKeys, testKeys), keepGoing,
numJobs, errorEventListener);
} finally {
progressReceiver.executionProgressReceiver = null;
// Also releases thread locks.
resourceManager.resetResourceUsage();
skyframeActionExecutor.executionOver();
actionExecutionFunction.complete();
}
}
@VisibleForTesting
public void prepareBuildingForTestingOnly(Executor executor, boolean keepGoing, boolean explain,
ActionCacheChecker checker) {
skyframeActionExecutor.prepareForExecution(executor, keepGoing, explain, checker);
}
EvaluationResult targetPatterns(Iterable patternSkyKeys,
int numThreads, boolean keepGoing, EventHandler eventHandler) throws InterruptedException {
checkActive();
return buildDriver.evaluate(patternSkyKeys, keepGoing, numThreads, eventHandler);
}
/**
* Returns the {@link ConfiguredTarget}s corresponding to the given keys.
*
*
For use for legacy support from {@code BuildView} only.
*
*
If a requested configured target is in error, the corresponding value is omitted from the
* returned list.
*/
@ThreadSafety.ThreadSafe
public ImmutableList getConfiguredTargets(Iterable keys) {
checkActive();
if (skyframeBuildView == null) {
// If build view has not yet been initialized, no configured targets can have been created.
// This is most likely to happen after a failed loading phase.
return ImmutableList.of();
}
final List skyKeys = new ArrayList<>();
for (Dependency key : keys) {
skyKeys.add(ConfiguredTargetValue.key(key.getLabel(), key.getConfiguration()));
for (Class extends ConfiguredAspectFactory> aspect : key.getAspects()) {
skyKeys.add(AspectValue.key(key.getLabel(), key.getConfiguration(), aspect));
}
}
EvaluationResult result;
try {
result = callUninterruptibly(new Callable>() {
@Override
public EvaluationResult call() throws Exception {
synchronized (valueLookupLock) {
try {
skyframeBuildView.enableAnalysis(true);
return buildDriver.evaluate(skyKeys, false, DEFAULT_THREAD_COUNT,
errorEventListener);
} finally {
skyframeBuildView.enableAnalysis(false);
}
}
}
});
} catch (Exception e) {
throw new IllegalStateException(e); // Should never happen.
}
ImmutableList.Builder cts = ImmutableList.builder();
DependentNodeLoop:
for (Dependency key : keys) {
SkyKey configuredTargetKey = ConfiguredTargetValue.key(
key.getLabel(), key.getConfiguration());
if (result.get(configuredTargetKey) == null) {
continue;
}
ConfiguredTarget configuredTarget =
((ConfiguredTargetValue) result.get(configuredTargetKey)).getConfiguredTarget();
List aspects = new ArrayList<>();
for (Class extends ConfiguredAspectFactory> aspect : key.getAspects()) {
SkyKey aspectKey = AspectValue.key(key.getLabel(), key.getConfiguration(), aspect);
if (result.get(aspectKey) == null) {
continue DependentNodeLoop;
}
aspects.add(((AspectValue) result.get(aspectKey)).get());
}
cts.add(RuleConfiguredTarget.mergeAspects(configuredTarget, aspects));
}
return cts.build();
}
/**
* Returns a particular configured target.
*
*
Used only for testing.
*/
@VisibleForTesting
@Nullable
public ConfiguredTarget getConfiguredTargetForTesting(
Label label, BuildConfiguration configuration) {
if (memoizingEvaluator.getExistingValueForTesting(
PrecomputedValue.WORKSPACE_STATUS_KEY.getKeyForTesting()) == null) {
injectWorkspaceStatusData();
}
return Iterables.getFirst(getConfiguredTargets(ImmutableList.of(
new Dependency(label, configuration))), null);
}
/**
* Invalidates Skyframe values corresponding to the given set of modified files under the given
* path entry.
*
*
May throw an {@link InterruptedException}, which means that no values have been invalidated.
*/
@VisibleForTesting
public abstract void invalidateFilesUnderPathForTesting(ModifiedFileSet modifiedFileSet,
Path pathEntry) throws InterruptedException;
/**
* Invalidates SkyFrame values that may have failed for transient reasons.
*/
public abstract void invalidateTransientErrors();
@VisibleForTesting
public TimestampGranularityMonitor getTimestampGranularityMonitorForTesting() {
return tsgm;
}
/**
* Configures a given set of configured targets.
*/
public EvaluationResult configureTargets(
List values, boolean keepGoing) throws InterruptedException {
checkActive();
// Make sure to not run too many analysis threads. This can cause memory thrashing.
return buildDriver.evaluate(ConfiguredTargetValue.keys(values), keepGoing,
ResourceUsage.getAvailableProcessors(), errorEventListener);
}
/**
* Post-process the targets. Values in the EvaluationResult are known to be transitively
* error-free from action conflicts.
*/
public EvaluationResult postConfigureTargets(
List values, boolean keepGoing,
ImmutableMap badActions)
throws InterruptedException {
checkActive();
PrecomputedValue.BAD_ACTIONS.set(injectable(), badActions);
// Make sure to not run too many analysis threads. This can cause memory thrashing.
EvaluationResult result =
buildDriver.evaluate(PostConfiguredTargetValue.keys(values), keepGoing,
ResourceUsage.getAvailableProcessors(), errorEventListener);
// Remove all post-configured target values immediately for memory efficiency. We are OK with
// this mini-phase being non-incremental as the failure mode of action conflict is rare.
memoizingEvaluator.delete(SkyFunctionName.functionIs(SkyFunctions.POST_CONFIGURED_TARGET));
return result;
}
/**
* Returns a Skyframe-based {@link SkyframeTransitivePackageLoader} implementation.
*/
@VisibleForTesting
public TransitivePackageLoader pkgLoader() {
checkActive();
return new SkyframeLabelVisitor(new SkyframeTransitivePackageLoader(), cyclesReporter);
}
class SkyframeTransitivePackageLoader {
/**
* Loads the specified {@link TransitiveTargetValue}s.
*/
EvaluationResult loadTransitiveTargets(
Iterable targetsToVisit, Iterable