// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.skyframe;
import static com.google.devtools.build.lib.concurrent.Uninterruptibles.callUninterruptibly;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.base.Predicate;
import com.google.common.base.Predicates;
import com.google.common.base.Stopwatch;
import com.google.common.base.Supplier;
import com.google.common.base.Throwables;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
import com.google.common.collect.Range;
import com.google.common.eventbus.EventBus;
import com.google.devtools.build.lib.actions.ActionAnalysisMetadata;
import com.google.devtools.build.lib.actions.ActionCacheChecker;
import com.google.devtools.build.lib.actions.ActionExecutionContextFactory;
import com.google.devtools.build.lib.actions.ActionExecutionStatusReporter;
import com.google.devtools.build.lib.actions.ActionGraph;
import com.google.devtools.build.lib.actions.ActionInputFileCache;
import com.google.devtools.build.lib.actions.ActionLogBufferPathGenerator;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.ArtifactFactory;
import com.google.devtools.build.lib.actions.ArtifactOwner;
import com.google.devtools.build.lib.actions.EnvironmentalExecException;
import com.google.devtools.build.lib.actions.Executor;
import com.google.devtools.build.lib.actions.PackageRootResolutionException;
import com.google.devtools.build.lib.actions.ResourceManager;
import com.google.devtools.build.lib.actions.Root;
import com.google.devtools.build.lib.analysis.AspectDescriptor;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.analysis.BuildView.Options;
import com.google.devtools.build.lib.analysis.ConfiguredAspect;
import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
import com.google.devtools.build.lib.analysis.ConfiguredTarget;
import com.google.devtools.build.lib.analysis.Dependency;
import com.google.devtools.build.lib.analysis.MergedConfiguredTarget;
import com.google.devtools.build.lib.analysis.MergedConfiguredTarget.DuplicateException;
import com.google.devtools.build.lib.analysis.TopLevelArtifactContext;
import com.google.devtools.build.lib.analysis.WorkspaceStatusAction;
import com.google.devtools.build.lib.analysis.WorkspaceStatusAction.Factory;
import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory;
import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory.BuildInfoKey;
import com.google.devtools.build.lib.analysis.config.BinTools;
import com.google.devtools.build.lib.analysis.config.BuildConfiguration;
import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection;
import com.google.devtools.build.lib.analysis.config.BuildOptions;
import com.google.devtools.build.lib.analysis.config.ConfigurationFactory;
import com.google.devtools.build.lib.analysis.config.ConfigurationFragmentFactory;
import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.cmdline.PackageIdentifier;
import com.google.devtools.build.lib.cmdline.TargetParsingException;
import com.google.devtools.build.lib.concurrent.ThreadSafety;
import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
import com.google.devtools.build.lib.events.EventHandler;
import com.google.devtools.build.lib.events.Reporter;
import com.google.devtools.build.lib.exec.OutputService;
import com.google.devtools.build.lib.packages.Attribute;
import com.google.devtools.build.lib.packages.BuildFileContainsErrorsException;
import com.google.devtools.build.lib.packages.NoSuchPackageException;
import com.google.devtools.build.lib.packages.NoSuchThingException;
import com.google.devtools.build.lib.packages.Package;
import com.google.devtools.build.lib.packages.Package.Builder;
import com.google.devtools.build.lib.packages.PackageFactory;
import com.google.devtools.build.lib.packages.Preprocessor;
import com.google.devtools.build.lib.packages.Preprocessor.AstAfterPreprocessing;
import com.google.devtools.build.lib.packages.RuleClassProvider;
import com.google.devtools.build.lib.packages.RuleVisibility;
import com.google.devtools.build.lib.packages.Target;
import com.google.devtools.build.lib.pkgcache.LoadingCallback;
import com.google.devtools.build.lib.pkgcache.LoadingFailedException;
import com.google.devtools.build.lib.pkgcache.LoadingOptions;
import com.google.devtools.build.lib.pkgcache.LoadingPhaseCompleteEvent;
import com.google.devtools.build.lib.pkgcache.LoadingPhaseRunner;
import com.google.devtools.build.lib.pkgcache.LoadingResult;
import com.google.devtools.build.lib.pkgcache.PackageCacheOptions;
import com.google.devtools.build.lib.pkgcache.PackageManager;
import com.google.devtools.build.lib.pkgcache.PathPackageLocator;
import com.google.devtools.build.lib.pkgcache.TargetParsingCompleteEvent;
import com.google.devtools.build.lib.pkgcache.TestFilter;
import com.google.devtools.build.lib.pkgcache.TransitivePackageLoader;
import com.google.devtools.build.lib.profiler.AutoProfiler;
import com.google.devtools.build.lib.skyframe.AspectValue.AspectValueKey;
import com.google.devtools.build.lib.skyframe.DirtinessCheckerUtils.FileDirtinessChecker;
import com.google.devtools.build.lib.skyframe.ExternalFilesHelper.ExternalFileAction;
import com.google.devtools.build.lib.skyframe.PackageFunction.CacheEntryWithGlobDeps;
import com.google.devtools.build.lib.skyframe.PackageLookupFunction.CrossRepositoryLabelViolationStrategy;
import com.google.devtools.build.lib.skyframe.PackageLookupValue.BuildFileName;
import com.google.devtools.build.lib.skyframe.SkyframeActionExecutor.ActionCompletedReceiver;
import com.google.devtools.build.lib.skyframe.SkyframeActionExecutor.ProgressSupplier;
import com.google.devtools.build.lib.util.AbruptExitException;
import com.google.devtools.build.lib.util.ExitCode;
import com.google.devtools.build.lib.util.Preconditions;
import com.google.devtools.build.lib.util.ResourceUsage;
import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
import com.google.devtools.build.lib.vfs.BatchStat;
import com.google.devtools.build.lib.vfs.Dirent;
import com.google.devtools.build.lib.vfs.ModifiedFileSet;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.RootedPath;
import com.google.devtools.build.lib.vfs.UnixGlob;
import com.google.devtools.build.skyframe.BuildDriver;
import com.google.devtools.build.skyframe.CycleInfo;
import com.google.devtools.build.skyframe.CyclesReporter;
import com.google.devtools.build.skyframe.Differencer;
import com.google.devtools.build.skyframe.Differencer.DiffWithDelta.Delta;
import com.google.devtools.build.skyframe.ErrorInfo;
import com.google.devtools.build.skyframe.EvaluationProgressReceiver;
import com.google.devtools.build.skyframe.EvaluationResult;
import com.google.devtools.build.skyframe.ImmutableDiff;
import com.google.devtools.build.skyframe.Injectable;
import com.google.devtools.build.skyframe.MemoizingEvaluator;
import com.google.devtools.build.skyframe.MemoizingEvaluator.EvaluatorSupplier;
import com.google.devtools.build.skyframe.SkyFunction;
import com.google.devtools.build.skyframe.SkyFunctionName;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
import com.google.devtools.build.skyframe.WalkableGraph.WalkableGraphFactory;
import com.google.devtools.common.options.OptionsClassProvider;
import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.logging.Logger;
import javax.annotation.Nullable;
/**
* A helper object to support Skyframe-driven execution.
*
*
This object is mostly used to inject external state, such as the executor engine or
* some additional artifacts (workspace status and build info artifacts) into SkyFunctions
* for use during the build.
*/
public abstract class SkyframeExecutor implements WalkableGraphFactory {
private final EvaluatorSupplier evaluatorSupplier;
protected MemoizingEvaluator memoizingEvaluator;
private final MemoizingEvaluator.EmittedEventState emittedEventState =
new MemoizingEvaluator.EmittedEventState();
private final PackageFactory pkgFactory;
private final WorkspaceStatusAction.Factory workspaceStatusActionFactory;
private final BlazeDirectories directories;
protected final ExternalFilesHelper externalFilesHelper;
@Nullable
private OutputService outputService;
// TODO(bazel-team): Figure out how to handle value builders that block internally. Blocking
// operations may need to be handled in another (bigger?) thread pool. Also, we should detect
// the number of cores and use that as the thread-pool size for CPU-bound operations.
// I just bumped this to 200 to get reasonable execution phase performance; that may cause
// significant overhead for CPU-bound processes (i.e. analysis). [skyframe-analysis]
@VisibleForTesting
public static final int DEFAULT_THREAD_COUNT =
// Reduce thread count while running tests of Bazel. Test cases are typically small, and large
// thread pools vying for a relatively small number of CPU cores may induce non-optimal
// performance.
System.getenv("TEST_TMPDIR") == null ? 200 : 5;
// Cache of partially constructed Package instances, stored between reruns of the PackageFunction
// (because of missing dependencies, within the same evaluate() run) to avoid loading the same
// package twice (first time loading to find subincludes and declare value dependencies).
// TODO(bazel-team): remove this cache once we have skyframe-native package loading
// [skyframe-loading]
private final Cache>
packageFunctionCache = newPkgFunctionCache();
private final Cache> astCache =
newAstCache();
private final AtomicInteger numPackagesLoaded = new AtomicInteger(0);
private final PackageProgressReceiver packageProgress = new PackageProgressReceiver();
protected SkyframeBuildView skyframeBuildView;
private ActionLogBufferPathGenerator actionLogBufferPathGenerator;
protected BuildDriver buildDriver;
// AtomicReferences are used here as mutable boxes shared with value builders.
private final AtomicBoolean showLoadingProgress = new AtomicBoolean();
protected final AtomicReference syscalls =
new AtomicReference<>(UnixGlob.DEFAULT_SYSCALLS);
protected final AtomicReference pkgLocator =
new AtomicReference<>();
protected final AtomicReference> deletedPackages =
new AtomicReference<>(ImmutableSet.of());
private final AtomicReference eventBus = new AtomicReference<>();
protected final AtomicReference tsgm =
new AtomicReference<>();
private final ImmutableList buildInfoFactories;
// Under normal circumstances, the artifact factory persists for the life of a Blaze server, but
// since it is not yet created when we create the value builders, we have to use a supplier,
// initialized when the build view is created.
private final MutableSupplier artifactFactory = new MutableSupplier<>();
// Used to give to WriteBuildInfoAction via a supplier. Relying on BuildVariableValue.BUILD_ID
// would be preferable, but we have no way to have the Action depend on that value directly.
// Having the BuildInfoFunction own the supplier is currently not possible either, because then
// it would be invalidated on every build, since it would depend on the build id value.
private MutableSupplier buildId = new MutableSupplier<>();
protected boolean active = true;
private final SkyframePackageManager packageManager;
private final Preprocessor.Factory.Supplier preprocessorFactorySupplier;
private Preprocessor.Factory preprocessorFactory;
private final ResourceManager resourceManager;
/** Used to lock evaluator on legacy calls to get existing values. */
private final Object valueLookupLock = new Object();
private final AtomicReference statusReporterRef =
new AtomicReference<>();
private final SkyframeActionExecutor skyframeActionExecutor;
private CompletionReceiver actionExecutionFunction;
protected SkyframeProgressReceiver progressReceiver;
private final AtomicReference cyclesReporter = new AtomicReference<>();
private final BinTools binTools;
private boolean needToInjectEmbeddedArtifacts = true;
private boolean needToInjectPrecomputedValuesForAnalysis = true;
protected int modifiedFiles;
protected int outputDirtyFiles;
protected int modifiedFilesDuringPreviousBuild;
private final Predicate allowedMissingInputs;
private final ExternalFileAction externalFileAction;
private final ImmutableMap extraSkyFunctions;
private final ImmutableList extraPrecomputedValues;
private final String productName;
protected SkyframeIncrementalBuildMonitor incrementalBuildMonitor =
new SkyframeIncrementalBuildMonitor();
private MutableSupplier configurationFactory = new MutableSupplier<>();
private MutableSupplier> configurationFragments =
new MutableSupplier<>();
private final PathFragment blacklistedPackagePrefixesFile;
private final RuleClassProvider ruleClassProvider;
private final CrossRepositoryLabelViolationStrategy crossRepositoryLabelViolationStrategy;
private final List buildFilesByPriority;
private PerBuildSyscallCache perBuildSyscallCache;
private int lastConcurrencyLevel = -1;
private static final Logger LOG = Logger.getLogger(SkyframeExecutor.class.getName());
protected SkyframeExecutor(
EvaluatorSupplier evaluatorSupplier,
PackageFactory pkgFactory,
BlazeDirectories directories,
BinTools binTools,
Factory workspaceStatusActionFactory,
ImmutableList buildInfoFactories,
Predicate allowedMissingInputs,
Preprocessor.Factory.Supplier preprocessorFactorySupplier,
ImmutableMap extraSkyFunctions,
ImmutableList extraPrecomputedValues,
ExternalFileAction externalFileAction,
PathFragment blacklistedPackagePrefixesFile,
String productName,
CrossRepositoryLabelViolationStrategy crossRepositoryLabelViolationStrategy,
List buildFilesByPriority) {
// Strictly speaking, these arguments are not required for initialization, but all current
// callsites have them at hand, so we might as well set them during construction.
this.evaluatorSupplier = evaluatorSupplier;
this.pkgFactory = pkgFactory;
this.pkgFactory.setSyscalls(syscalls);
this.workspaceStatusActionFactory = workspaceStatusActionFactory;
this.packageManager = new SkyframePackageManager(
new SkyframePackageLoader(), new SkyframeTransitivePackageLoader(),
syscalls, cyclesReporter, pkgLocator, numPackagesLoaded, this);
this.resourceManager = ResourceManager.instance();
this.skyframeActionExecutor = new SkyframeActionExecutor(
resourceManager, eventBus, statusReporterRef);
this.directories = Preconditions.checkNotNull(directories);
this.buildInfoFactories = buildInfoFactories;
this.allowedMissingInputs = allowedMissingInputs;
this.preprocessorFactorySupplier = preprocessorFactorySupplier;
this.extraSkyFunctions = extraSkyFunctions;
this.extraPrecomputedValues = extraPrecomputedValues;
this.externalFileAction = externalFileAction;
this.blacklistedPackagePrefixesFile = blacklistedPackagePrefixesFile;
this.binTools = binTools;
this.ruleClassProvider = pkgFactory.getRuleClassProvider();
this.skyframeBuildView = new SkyframeBuildView(
directories,
this,
binTools,
(ConfiguredRuleClassProvider) ruleClassProvider);
this.artifactFactory.set(skyframeBuildView.getArtifactFactory());
this.externalFilesHelper = new ExternalFilesHelper(
pkgLocator, this.externalFileAction, directories);
this.productName = productName;
this.crossRepositoryLabelViolationStrategy = crossRepositoryLabelViolationStrategy;
this.buildFilesByPriority = buildFilesByPriority;
}
private ImmutableMap skyFunctions(
PackageFactory pkgFactory,
Predicate allowedMissingInputs) {
ConfiguredRuleClassProvider ruleClassProvider =
(ConfiguredRuleClassProvider) pkgFactory.getRuleClassProvider();
// TODO(janakr): use this semaphore to bound memory usage for SkyFunctions besides
// ConfiguredTargetFunction that may have a large temporary memory blow-up.
Semaphore cpuBoundSemaphore = new Semaphore(ResourceUsage.getAvailableProcessors());
// We use an immutable map builder for the nice side effect that it throws if a duplicate key
// is inserted.
ImmutableMap.Builder map = ImmutableMap.builder();
map.put(SkyFunctions.PRECOMPUTED, new PrecomputedFunction());
map.put(SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE, new ClientEnvironmentFunction());
map.put(SkyFunctions.FILE_STATE, new FileStateFunction(tsgm, externalFilesHelper));
map.put(SkyFunctions.DIRECTORY_LISTING_STATE,
new DirectoryListingStateFunction(externalFilesHelper));
map.put(SkyFunctions.FILE_SYMLINK_CYCLE_UNIQUENESS,
new FileSymlinkCycleUniquenessFunction());
map.put(SkyFunctions.FILE_SYMLINK_INFINITE_EXPANSION_UNIQUENESS,
new FileSymlinkInfiniteExpansionUniquenessFunction());
map.put(SkyFunctions.FILE, new FileFunction(pkgLocator));
map.put(SkyFunctions.DIRECTORY_LISTING, new DirectoryListingFunction());
map.put(
SkyFunctions.PACKAGE_LOOKUP,
new PackageLookupFunction(
deletedPackages, crossRepositoryLabelViolationStrategy, buildFilesByPriority));
map.put(SkyFunctions.CONTAINING_PACKAGE_LOOKUP, new ContainingPackageLookupFunction());
map.put(SkyFunctions.AST_FILE_LOOKUP, new ASTFileLookupFunction(ruleClassProvider));
map.put(
SkyFunctions.SKYLARK_IMPORTS_LOOKUP,
newSkylarkImportLookupFunction(ruleClassProvider, pkgFactory));
map.put(SkyFunctions.GLOB, newGlobFunction());
map.put(SkyFunctions.TARGET_PATTERN, new TargetPatternFunction());
map.put(SkyFunctions.PREPARE_DEPS_OF_PATTERNS, new PrepareDepsOfPatternsFunction());
map.put(SkyFunctions.PREPARE_DEPS_OF_PATTERN, new PrepareDepsOfPatternFunction(pkgLocator));
map.put(
SkyFunctions.PREPARE_DEPS_OF_TARGETS_UNDER_DIRECTORY,
new PrepareDepsOfTargetsUnderDirectoryFunction(directories));
map.put(SkyFunctions.COLLECT_TARGETS_IN_PACKAGE, new CollectTargetsInPackageFunction());
map.put(
SkyFunctions.COLLECT_PACKAGES_UNDER_DIRECTORY,
new CollectPackagesUnderDirectoryFunction(directories));
map.put(SkyFunctions.BLACKLISTED_PACKAGE_PREFIXES, new BlacklistedPackagePrefixesFunction());
map.put(SkyFunctions.TESTS_IN_SUITE, new TestsInSuiteFunction());
map.put(SkyFunctions.TEST_SUITE_EXPANSION, new TestSuiteExpansionFunction());
map.put(SkyFunctions.TARGET_PATTERN_PHASE, new TargetPatternPhaseFunction());
map.put(SkyFunctions.RECURSIVE_PKG, new RecursivePkgFunction(directories));
map.put(
SkyFunctions.PACKAGE,
newPackageFunction(
pkgFactory,
packageManager,
showLoadingProgress,
packageFunctionCache,
astCache,
numPackagesLoaded,
ruleClassProvider,
packageProgress));
map.put(SkyFunctions.PACKAGE_ERROR, new PackageErrorFunction());
map.put(SkyFunctions.TARGET_MARKER, new TargetMarkerFunction());
map.put(SkyFunctions.TRANSITIVE_TARGET, new TransitiveTargetFunction(ruleClassProvider));
map.put(SkyFunctions.TRANSITIVE_TRAVERSAL, new TransitiveTraversalFunction());
map.put(
SkyFunctions.CONFIGURED_TARGET,
new ConfiguredTargetFunction(
new BuildViewProvider(), ruleClassProvider, cpuBoundSemaphore));
map.put(SkyFunctions.ASPECT, new AspectFunction(new BuildViewProvider(), ruleClassProvider));
map.put(SkyFunctions.LOAD_SKYLARK_ASPECT, new ToplevelSkylarkAspectFunction());
map.put(SkyFunctions.POST_CONFIGURED_TARGET,
new PostConfiguredTargetFunction(new BuildViewProvider(), ruleClassProvider));
map.put(SkyFunctions.BUILD_CONFIGURATION,
new BuildConfigurationFunction(directories, ruleClassProvider));
map.put(SkyFunctions.CONFIGURATION_COLLECTION, new ConfigurationCollectionFunction(
configurationFactory, ruleClassProvider));
map.put(SkyFunctions.CONFIGURATION_FRAGMENT, new ConfigurationFragmentFunction(
configurationFragments, ruleClassProvider));
map.put(SkyFunctions.WORKSPACE_AST, new WorkspaceASTFunction(ruleClassProvider));
map.put(
SkyFunctions.WORKSPACE_FILE,
new WorkspaceFileFunction(ruleClassProvider, pkgFactory, directories));
map.put(SkyFunctions.EXTERNAL_PACKAGE, new ExternalPackageFunction());
map.put(SkyFunctions.TARGET_COMPLETION, CompletionFunction.targetCompletionFunction(eventBus));
map.put(SkyFunctions.ASPECT_COMPLETION, CompletionFunction.aspectCompletionFunction(eventBus));
map.put(SkyFunctions.TEST_COMPLETION, new TestCompletionFunction());
map.put(SkyFunctions.ARTIFACT, new ArtifactFunction(allowedMissingInputs));
map.put(SkyFunctions.BUILD_INFO_COLLECTION, new BuildInfoCollectionFunction(artifactFactory));
map.put(SkyFunctions.BUILD_INFO, new WorkspaceStatusFunction());
map.put(SkyFunctions.COVERAGE_REPORT, new CoverageReportFunction());
ActionExecutionFunction actionExecutionFunction =
new ActionExecutionFunction(skyframeActionExecutor, tsgm);
map.put(SkyFunctions.ACTION_EXECUTION, actionExecutionFunction);
this.actionExecutionFunction = actionExecutionFunction;
map.put(SkyFunctions.RECURSIVE_FILESYSTEM_TRAVERSAL,
new RecursiveFilesystemTraversalFunction());
map.put(SkyFunctions.FILESET_ENTRY, new FilesetEntryFunction());
map.put(SkyFunctions.ACTION_TEMPLATE_EXPANSION, new ActionTemplateExpansionFunction());
map.put(SkyFunctions.LOCAL_REPOSITORY_LOOKUP, new LocalRepositoryLookupFunction());
map.putAll(extraSkyFunctions);
return map.build();
}
protected SkyFunction newGlobFunction() {
return new GlobFunction(/*alwaysUseDirListing=*/false);
}
protected PackageFunction newPackageFunction(
PackageFactory pkgFactory,
PackageManager packageManager,
AtomicBoolean showLoadingProgress,
Cache> packageFunctionCache,
Cache> astCache,
AtomicInteger numPackagesLoaded,
RuleClassProvider ruleClassProvider,
PackageProgressReceiver packageProgress) {
return new PackageFunction(
pkgFactory,
packageManager,
showLoadingProgress,
packageFunctionCache,
astCache,
numPackagesLoaded,
null,
packageProgress);
}
protected SkyFunction newSkylarkImportLookupFunction(
RuleClassProvider ruleClassProvider, PackageFactory pkgFactory) {
return new SkylarkImportLookupFunction(ruleClassProvider, this.pkgFactory);
}
protected PerBuildSyscallCache newPerBuildSyscallCache(int concurrencyLevel) {
return PerBuildSyscallCache.newBuilder().setConcurrencyLevel(concurrencyLevel).build();
}
/**
* Gets a (possibly cached) syscalls cache, re-initialized each build.
*
*
We cache the syscalls cache if possible because construction of the cache is surprisingly
* expensive, and is on the critical path of null builds.
*/
protected final PerBuildSyscallCache getPerBuildSyscallCache(int concurrencyLevel) {
if (perBuildSyscallCache != null && lastConcurrencyLevel == concurrencyLevel) {
perBuildSyscallCache.clear();
return perBuildSyscallCache;
}
lastConcurrencyLevel = concurrencyLevel;
perBuildSyscallCache = newPerBuildSyscallCache(concurrencyLevel);
return perBuildSyscallCache;
}
@ThreadCompatible
public void setActive(boolean active) {
this.active = active;
}
protected void checkActive() {
Preconditions.checkState(active);
}
public void setFileCache(ActionInputFileCache fileCache) {
this.skyframeActionExecutor.setFileCache(fileCache);
}
public void dump(boolean summarize, PrintStream out) {
memoizingEvaluator.dump(summarize, out);
}
public abstract void dumpPackages(PrintStream out);
public void setOutputService(OutputService outputService) {
this.outputService = outputService;
}
/**
* Notify listeners about changed files, and release any associated memory afterwards.
*/
public void drainChangedFiles() {
incrementalBuildMonitor.alertListeners(getEventBus());
incrementalBuildMonitor = null;
}
@VisibleForTesting
public BuildDriver getDriverForTesting() {
return buildDriver;
}
/**
* This method exists only to allow a module to make a top-level Skyframe call during the
* transition to making it fully Skyframe-compatible. Do not add additional callers!
*/
public SkyValue evaluateSkyKeyForExecutionSetup(final EventHandler eventHandler, final SkyKey key)
throws EnvironmentalExecException, InterruptedException {
synchronized (valueLookupLock) {
// We evaluate in keepGoing mode because in the case that the graph does not store its
// edges, nokeepGoing builds are not allowed, whereas keepGoing builds are always
// permitted.
EvaluationResult result =
buildDriver.evaluate(
ImmutableList.of(key), true, ResourceUsage.getAvailableProcessors(), eventHandler);
if (!result.hasError()) {
return Preconditions.checkNotNull(result.get(key), "%s %s", result, key);
}
ErrorInfo errorInfo = Preconditions.checkNotNull(result.getError(key), "%s %s", key, result);
Throwables.propagateIfPossible(errorInfo.getException(), EnvironmentalExecException.class);
if (errorInfo.getException() != null) {
throw new IllegalStateException(errorInfo.getException());
}
throw new IllegalStateException(errorInfo.toString());
}
}
@VisibleForTesting
public PathFragment getBlacklistedPackagePrefixesFile() {
return blacklistedPackagePrefixesFile;
}
class BuildViewProvider {
/**
* Returns the current {@link SkyframeBuildView} instance.
*/
SkyframeBuildView getSkyframeBuildView() {
return skyframeBuildView;
}
}
/**
* Must be called before the {@link SkyframeExecutor} can be used (should only be called in
* factory methods and as an implementation detail of {@link #resetEvaluator}).
*/
protected void init() {
progressReceiver = newSkyframeProgressReceiver();
ImmutableMap skyFunctions =
skyFunctions(pkgFactory, allowedMissingInputs);
memoizingEvaluator = evaluatorSupplier.create(
skyFunctions, evaluatorDiffer(), progressReceiver, emittedEventState,
hasIncrementalState());
buildDriver = getBuildDriver();
}
protected SkyframeProgressReceiver newSkyframeProgressReceiver() {
return new SkyframeProgressReceiver();
}
/**
* Reinitializes the Skyframe evaluator, dropping all previously computed values.
*
*
Be careful with this method as it also deletes all injected values. You need to make sure
* that any necessary precomputed values are reinjected before the next build. Constants can be
* put in {@link #reinjectConstantValuesLazily}.
*/
public void resetEvaluator() {
init();
emittedEventState.clear();
skyframeBuildView.clearLegacyData();
reinjectConstantValuesLazily();
}
protected abstract Differencer evaluatorDiffer();
protected abstract BuildDriver getBuildDriver();
/**
* Values whose values are known at startup and guaranteed constant are still wiped from the
* evaluator when we create a new one, so they must be re-injected each time we create a new
* evaluator.
*/
private void reinjectConstantValuesLazily() {
needToInjectEmbeddedArtifacts = true;
needToInjectPrecomputedValuesForAnalysis = true;
}
/**
* Deletes all ConfiguredTarget values from the Skyframe cache. This is done to save memory (e.g.
* on a configuration change); since the configuration is part of the key, these key/value pairs
* will be sitting around doing nothing until the configuration changes back to the previous
* value.
*
*
The next evaluation will delete all invalid values.
*/
public abstract void dropConfiguredTargets();
/**
* Removes ConfigurationFragmentValuess and ConfigurationCollectionValues from the cache.
*/
@VisibleForTesting
public void invalidateConfigurationCollection() {
invalidate(SkyFunctionName.functionIsIn(ImmutableSet.of(SkyFunctions.CONFIGURATION_FRAGMENT,
SkyFunctions.CONFIGURATION_COLLECTION)));
}
/**
* Decides if graph edges should be stored for this build. If not, re-creates the graph to not
* store graph edges. Necessary conditions to not store graph edges are:
* (1) batch (since incremental builds are not possible);
* (2) skyframe build (since otherwise the memory savings are too slight to bother);
* (3) keep-going (since otherwise bubbling errors up may require edges of done nodes);
* (4) discard_analysis_cache (since otherwise user isn't concerned about saving memory this way).
*/
public void decideKeepIncrementalState(boolean batch, Options viewOptions) {
// Assume incrementality.
}
public boolean hasIncrementalState() {
return true;
}
@VisibleForTesting
protected abstract Injectable injectable();
/**
* Saves memory by clearing analysis objects from Skyframe. If using legacy execution, actually
* deletes the relevant values. If using Skyframe execution, clears their data without deleting
* them (they will be deleted on the next build).
*/
public abstract void clearAnalysisCache(Collection topLevelTargets);
/**
* Injects the contents of the computed tools/defaults package.
*/
@VisibleForTesting
public void setupDefaultPackage(String defaultsPackageContents) {
PrecomputedValue.DEFAULTS_PACKAGE_CONTENTS.set(injectable(), defaultsPackageContents);
}
public void injectWorkspaceStatusData(String workspaceName) {
PrecomputedValue.WORKSPACE_STATUS_KEY.set(injectable(),
workspaceStatusActionFactory.createWorkspaceStatusAction(
artifactFactory.get(), WorkspaceStatusValue.ARTIFACT_OWNER, buildId, workspaceName));
}
public void injectCoverageReportData(ImmutableList actions) {
PrecomputedValue.COVERAGE_REPORT_KEY.set(injectable(), actions);
}
/**
* Sets the default visibility.
*/
private void setDefaultVisibility(RuleVisibility defaultVisibility) {
PrecomputedValue.DEFAULT_VISIBILITY.set(injectable(), defaultVisibility);
}
protected void maybeInjectPrecomputedValuesForAnalysis() {
if (needToInjectPrecomputedValuesForAnalysis) {
PrecomputedValue.BLAZE_DIRECTORIES.set(injectable(), directories);
PrecomputedValue.PRODUCT_NAME.set(injectable(), productName);
injectBuildInfoFactories();
injectExtraPrecomputedValues();
needToInjectPrecomputedValuesForAnalysis = false;
}
}
private void injectExtraPrecomputedValues() {
for (PrecomputedValue.Injected injected : extraPrecomputedValues) {
injected.inject(injectable());
}
}
protected Cache>
newPkgFunctionCache() {
return CacheBuilder.newBuilder().build();
}
protected Cache> newAstCache() {
return CacheBuilder.newBuilder().build();
}
/**
* Injects the build info factory map that will be used when constructing build info
* actions/artifacts. Unchanged across the life of the Blaze server, although it must be injected
* each time the evaluator is created.
*/
private void injectBuildInfoFactories() {
ImmutableMap.Builder factoryMapBuilder =
ImmutableMap.builder();
for (BuildInfoFactory factory : buildInfoFactories) {
factoryMapBuilder.put(factory.getKey(), factory);
}
PrecomputedValue.BUILD_INFO_FACTORIES.set(injectable(), factoryMapBuilder.build());
}
private void setShowLoadingProgress(boolean showLoadingProgressValue) {
showLoadingProgress.set(showLoadingProgressValue);
}
@VisibleForTesting
public void setCommandId(UUID commandId) {
PrecomputedValue.BUILD_ID.set(injectable(), commandId);
buildId.set(commandId);
}
protected void setPrecomputedClientEnv(Map clientEnv) {
PrecomputedValue.CLIENT_ENV.set(injectable(), clientEnv);
}
/** Returns the build-info.txt and build-changelist.txt artifacts. */
public Collection getWorkspaceStatusArtifacts(EventHandler eventHandler)
throws InterruptedException {
// Should already be present, unless the user didn't request any targets for analysis.
EvaluationResult result = buildDriver.evaluate(
ImmutableList.of(WorkspaceStatusValue.SKY_KEY), /*keepGoing=*/true, /*numThreads=*/1,
eventHandler);
WorkspaceStatusValue value =
Preconditions.checkNotNull(result.get(WorkspaceStatusValue.SKY_KEY));
return ImmutableList.of(value.getStableArtifact(), value.getVolatileArtifact());
}
// TODO(bazel-team): Make this take a PackageIdentifier.
public Map getArtifactRootsForFiles(
final EventHandler eventHandler, Iterable execPaths)
throws PackageRootResolutionException, InterruptedException {
return getArtifactRoots(eventHandler, execPaths, true);
}
public Map getArtifactRoots(
final EventHandler eventHandler, Iterable execPaths)
throws PackageRootResolutionException, InterruptedException {
return getArtifactRoots(eventHandler, execPaths, false);
}
private Map getArtifactRoots(
final EventHandler eventHandler, Iterable execPaths, boolean forFiles)
throws PackageRootResolutionException, InterruptedException {
final List packageKeys = new ArrayList<>();
if (forFiles) {
for (PathFragment execPath : execPaths) {
PathFragment parent = Preconditions.checkNotNull(
execPath.getParentDirectory(), "Must pass in files, not root directory");
Preconditions.checkArgument(!parent.isAbsolute(), execPath);
packageKeys.add(ContainingPackageLookupValue.key(
PackageIdentifier.createInMainRepo(parent)));
}
} else {
for (PathFragment execPath : execPaths) {
Preconditions.checkArgument(!execPath.isAbsolute(), execPath);
packageKeys.add(ContainingPackageLookupValue.key(
PackageIdentifier.createInMainRepo(execPath)));
}
}
EvaluationResult result;
synchronized (valueLookupLock) {
result =
buildDriver.evaluate(packageKeys, /*keepGoing=*/ true, /*numThreads=*/ 1, eventHandler);
}
if (result.hasError()) {
throw new PackageRootResolutionException("Exception encountered determining package roots",
result.getError().getException());
}
Map roots = new HashMap<>();
for (PathFragment execPath : execPaths) {
ContainingPackageLookupValue value = result.get(ContainingPackageLookupValue.key(
PackageIdentifier.createInMainRepo(forFiles ? execPath.getParentDirectory() : execPath)));
if (value.hasContainingPackage()) {
roots.put(execPath, Root.asSourceRoot(value.getContainingPackageRoot(),
value.getContainingPackageName().getRepository().isMain()));
} else {
roots.put(execPath, null);
}
}
return roots;
}
@VisibleForTesting
public WorkspaceStatusAction getLastWorkspaceStatusActionForTesting() {
PrecomputedValue value = (PrecomputedValue) buildDriver.getGraphForTesting()
.getExistingValueForTesting(PrecomputedValue.WORKSPACE_STATUS_KEY.getKeyForTesting());
return (WorkspaceStatusAction) value.get();
}
/**
* Informs user about number of modified files (source and output files).
*/
// Note, that number of modified files in some cases can be bigger than actual number of
// modified files for targets in current request. Skyframe may check for modification all files
// from previous requests.
protected void informAboutNumberOfModifiedFiles() {
LOG.info(String.format("Found %d modified files from last build", modifiedFiles));
}
public EventBus getEventBus() {
return eventBus.get();
}
public ActionExecutionContextFactory getActionExecutionContextFactory() {
return skyframeActionExecutor;
}
@VisibleForTesting
ImmutableList getPathEntries() {
return pkgLocator.get().getPathEntries();
}
protected abstract void invalidate(Predicate pred);
private static boolean compatibleFileTypes(Dirent.Type oldType, FileStateValue.Type newType) {
return (oldType.equals(Dirent.Type.FILE) && newType.equals(FileStateValue.Type.REGULAR_FILE))
|| (oldType.equals(Dirent.Type.UNKNOWN)
&& newType.equals(FileStateValue.Type.SPECIAL_FILE))
|| (oldType.equals(Dirent.Type.DIRECTORY) && newType.equals(FileStateValue.Type.DIRECTORY))
|| (oldType.equals(Dirent.Type.SYMLINK) && newType.equals(FileStateValue.Type.SYMLINK));
}
protected Differencer.Diff getDiff(TimestampGranularityMonitor tsgm,
Iterable modifiedSourceFiles, final Path pathEntry)
throws InterruptedException {
if (Iterables.isEmpty(modifiedSourceFiles)) {
return new ImmutableDiff(ImmutableList.of(), ImmutableMap.of());
}
// TODO(bazel-team): change ModifiedFileSet to work with RootedPaths instead of PathFragments.
Iterable dirtyFileStateSkyKeys = Iterables.transform(modifiedSourceFiles,
new Function() {
@Override
public SkyKey apply(PathFragment pathFragment) {
Preconditions.checkState(!pathFragment.isAbsolute(),
"found absolute PathFragment: %s", pathFragment);
return FileStateValue.key(RootedPath.toRootedPath(pathEntry, pathFragment));
}
});
// We only need to invalidate directory values when a file has been created or deleted or
// changes type, not when it has merely been modified. Unfortunately we do not have that
// information here, so we compute it ourselves.
// TODO(bazel-team): Fancy filesystems could provide it with a hypothetically modified
// DiffAwareness interface.
FilesystemValueChecker fsvc = new FilesystemValueChecker(tsgm, null);
Map valuesMap = memoizingEvaluator.getValues();
Differencer.DiffWithDelta diff =
fsvc.getNewAndOldValues(valuesMap, dirtyFileStateSkyKeys, new FileDirtinessChecker());
Set valuesToInvalidate = new HashSet<>();
Map valuesToInject = new HashMap<>();
for (Map.Entry entry : diff.changedKeysWithNewAndOldValues().entrySet()) {
SkyKey key = entry.getKey();
Preconditions.checkState(key.functionName().equals(SkyFunctions.FILE_STATE), key);
RootedPath rootedPath = (RootedPath) key.argument();
Delta delta = entry.getValue();
FileStateValue oldValue = (FileStateValue) delta.getOldValue();
FileStateValue newValue = (FileStateValue) delta.getNewValue();
if (newValue != null) {
valuesToInject.put(key, newValue);
} else {
valuesToInvalidate.add(key);
}
SkyKey dirListingStateKey = parentDirectoryListingStateKey(rootedPath);
// Invalidate the directory listing for the path's parent directory if the change was
// relevant (e.g. path turned from a symlink into a directory) OR if we don't have enough
// information to determine it was irrelevant.
boolean changedType = false;
if (newValue == null) {
changedType = true;
} else if (oldValue != null) {
changedType = !oldValue.getType().equals(newValue.getType());
} else {
DirectoryListingStateValue oldDirListingStateValue =
(DirectoryListingStateValue) valuesMap.get(dirListingStateKey);
if (oldDirListingStateValue != null) {
String baseName = rootedPath.getRelativePath().getBaseName();
Dirent oldDirent = oldDirListingStateValue.getDirents().maybeGetDirent(baseName);
changedType = (oldDirent == null)
|| !compatibleFileTypes(oldDirent.getType(), newValue.getType());
} else {
changedType = true;
}
}
if (changedType) {
valuesToInvalidate.add(dirListingStateKey);
}
}
for (SkyKey key : diff.changedKeysWithoutNewValues()) {
Preconditions.checkState(key.functionName().equals(SkyFunctions.FILE_STATE), key);
RootedPath rootedPath = (RootedPath) key.argument();
valuesToInvalidate.add(parentDirectoryListingStateKey(rootedPath));
}
return new ImmutableDiff(valuesToInvalidate, valuesToInject);
}
private static SkyKey parentDirectoryListingStateKey(RootedPath rootedPath) {
RootedPath parentDirRootedPath = RootedPath.toRootedPath(
rootedPath.getRoot(), rootedPath.getRelativePath().getParentDirectory());
return DirectoryListingStateValue.key(parentDirRootedPath);
}
/**
* Sets the packages that should be treated as deleted and ignored.
*/
@VisibleForTesting // productionVisibility = Visibility.PRIVATE
public abstract void setDeletedPackages(Iterable pkgs);
@VisibleForTesting
public final void setBlacklistedPackagePrefixesFile(PathFragment blacklistedPkgFile) {
PrecomputedValue.BLACKLISTED_PACKAGE_PREFIXES_FILE.set(injectable(), blacklistedPkgFile);
}
/**
* Prepares the evaluator for loading.
*
*
MUST be run before every incremental build.
*/
@VisibleForTesting // productionVisibility = Visibility.PRIVATE
public void preparePackageLoading(
PathPackageLocator pkgLocator,
PackageCacheOptions packageCacheOptions,
String defaultsPackageContents,
UUID commandId,
Map clientEnv,
TimestampGranularityMonitor tsgm) {
Preconditions.checkNotNull(pkgLocator);
Preconditions.checkNotNull(tsgm);
setActive(true);
this.tsgm.set(tsgm);
maybeInjectPrecomputedValuesForAnalysis();
setCommandId(commandId);
setPrecomputedClientEnv(clientEnv);
setBlacklistedPackagePrefixesFile(getBlacklistedPackagePrefixesFile());
setShowLoadingProgress(packageCacheOptions.showLoadingProgress);
setDefaultVisibility(packageCacheOptions.defaultVisibility);
setupDefaultPackage(defaultsPackageContents);
setPackageLocator(pkgLocator);
syscalls.set(getPerBuildSyscallCache(packageCacheOptions.globbingThreads));
this.pkgFactory.setGlobbingThreads(packageCacheOptions.globbingThreads);
this.pkgFactory.setMaxDirectoriesToEagerlyVisitInGlobbing(
packageCacheOptions.maxDirectoriesToEagerlyVisitInGlobbing);
checkPreprocessorFactory();
emittedEventState.clear();
// If the PackageFunction was interrupted, there may be stale entries here.
packageFunctionCache.invalidateAll();
astCache.invalidateAll();
numPackagesLoaded.set(0);
packageProgress.reset();
// Reset the stateful SkyframeCycleReporter, which contains cycles from last run.
cyclesReporter.set(createCyclesReporter());
}
@SuppressWarnings("unchecked")
private void setPackageLocator(PathPackageLocator pkgLocator) {
PathPackageLocator oldLocator = this.pkgLocator.getAndSet(pkgLocator);
PrecomputedValue.PATH_PACKAGE_LOCATOR.set(injectable(), pkgLocator);
if (!pkgLocator.equals(oldLocator)) {
// The package path is read not only by SkyFunctions but also by some other code paths.
// We need to take additional steps to keep the corresponding data structures in sync.
// (Some of the additional steps are carried out by ConfiguredTargetValueInvalidationListener,
// and some by BuildView#buildHasIncompatiblePackageRoots and #updateSkyframe.)
onNewPackageLocator(oldLocator, pkgLocator);
}
}
protected abstract void onNewPackageLocator(PathPackageLocator oldLocator,
PathPackageLocator pkgLocator);
private void checkPreprocessorFactory() {
if (preprocessorFactory == null) {
Preprocessor.Factory newPreprocessorFactory = preprocessorFactorySupplier.getFactory(
packageManager, directories.getOutputBase());
pkgFactory.setPreprocessorFactory(newPreprocessorFactory);
preprocessorFactory = newPreprocessorFactory;
} else if (!preprocessorFactory.isStillValid()) {
Preprocessor.Factory newPreprocessorFactory = preprocessorFactorySupplier.getFactory(
packageManager, directories.getOutputBase());
invalidate(SkyFunctionName.functionIs(SkyFunctions.PACKAGE));
pkgFactory.setPreprocessorFactory(newPreprocessorFactory);
preprocessorFactory = newPreprocessorFactory;
}
}
public SkyframeBuildView getSkyframeBuildView() {
return skyframeBuildView;
}
/**
* Sets the eventBus to use for posting events.
*/
public void setEventBus(EventBus eventBus) {
this.eventBus.set(eventBus);
}
public void setClientEnv(Map clientEnv) {
this.skyframeActionExecutor.setClientEnv(clientEnv);
}
/**
* Sets the path for action log buffers.
*/
public void setActionOutputRoot(Path actionOutputRoot) {
Preconditions.checkNotNull(actionOutputRoot);
this.actionLogBufferPathGenerator = new ActionLogBufferPathGenerator(actionOutputRoot);
this.skyframeActionExecutor.setActionLogBufferPathGenerator(actionLogBufferPathGenerator);
}
/**
* Asks the Skyframe evaluator to build the value for BuildConfigurationCollection and returns the
* result. Also invalidates {@link PrecomputedValue#BLAZE_DIRECTORIES} if it has changed.
*/
public BuildConfigurationCollection createConfigurations(
EventHandler eventHandler, ConfigurationFactory configurationFactory,
BuildOptions buildOptions, Set multiCpu,
boolean keepGoing)
throws InvalidConfigurationException, InterruptedException {
this.configurationFactory.set(configurationFactory);
this.configurationFragments.set(ImmutableList.copyOf(configurationFactory.getFactories()));
SkyKey skyKey = ConfigurationCollectionValue.key(
buildOptions, ImmutableSortedSet.copyOf(multiCpu));
EvaluationResult result =
buildDriver.evaluate(Arrays.asList(skyKey), keepGoing, DEFAULT_THREAD_COUNT, eventHandler);
if (result.hasError()) {
Throwable e = result.getError(skyKey).getException();
// Wrap loading failed exceptions
if (e instanceof NoSuchThingException) {
e = new InvalidConfigurationException(e);
}
Throwables.propagateIfInstanceOf(e, InvalidConfigurationException.class);
throw new IllegalStateException(
"Unknown error during ConfigurationCollectionValue evaluation", e);
}
ConfigurationCollectionValue configurationValue = result.get(skyKey);
return configurationValue.getConfigurationCollection();
}
private Iterable getActionLookupValues() {
// This filter keeps subclasses of ActionLookupValue.
return Iterables.filter(memoizingEvaluator.getDoneValues().values(), ActionLookupValue.class);
}
@SuppressWarnings({"unchecked", "rawtypes"})
Map getActionLookupValueMap() {
return (Map) Maps.filterValues(memoizingEvaluator.getDoneValues(),
Predicates.instanceOf(ActionLookupValue.class));
}
/**
* Checks the actions in Skyframe for conflicts between their output artifacts. Delegates to
* {@link SkyframeActionExecutor#findAndStoreArtifactConflicts} to do the work, since any
* conflicts found will only be reported during execution.
*/
ImmutableMap
findArtifactConflicts() throws InterruptedException {
if (skyframeBuildView.isSomeConfiguredTargetEvaluated()
|| skyframeBuildView.isSomeConfiguredTargetInvalidated()) {
// This operation is somewhat expensive, so we only do it if the graph might have changed in
// some way -- either we analyzed a new target or we invalidated an old one.
try (AutoProfiler p = AutoProfiler.logged("discovering artifact conflicts", LOG)) {
skyframeActionExecutor.findAndStoreArtifactConflicts(getActionLookupValues());
skyframeBuildView.resetEvaluatedConfiguredTargetFlag();
// The invalidated configured targets flag will be reset later in the evaluate() call.
}
}
return skyframeActionExecutor.badActions();
}
/**
* Asks the Skyframe evaluator to build the given artifacts and targets, and to test the
* given test targets.
*/
public EvaluationResult> buildArtifacts(
Reporter reporter,
Executor executor,
Set artifactsToBuild,
Collection targetsToBuild,
Collection aspects,
Collection targetsToTest,
boolean exclusiveTesting,
boolean keepGoing,
boolean explain,
boolean finalizeActionsToOutputService,
int numJobs,
ActionCacheChecker actionCacheChecker,
@Nullable EvaluationProgressReceiver executionProgressReceiver,
TopLevelArtifactContext topLevelArtifactContext)
throws InterruptedException {
checkActive();
Preconditions.checkState(actionLogBufferPathGenerator != null);
skyframeActionExecutor.prepareForExecution(
reporter, executor, keepGoing, explain, actionCacheChecker,
finalizeActionsToOutputService ? outputService : null);
resourceManager.resetResourceUsage();
try {
progressReceiver.executionProgressReceiver = executionProgressReceiver;
Iterable artifactKeys = ArtifactSkyKey.mandatoryKeys(artifactsToBuild);
Iterable targetKeys =
TargetCompletionValue.keys(targetsToBuild, topLevelArtifactContext);
Iterable aspectKeys = AspectCompletionValue.keys(aspects, topLevelArtifactContext);
Iterable testKeys =
TestCompletionValue.keys(targetsToTest, topLevelArtifactContext, exclusiveTesting);
return buildDriver.evaluate(
Iterables.concat(artifactKeys, targetKeys, aspectKeys, testKeys),
keepGoing,
numJobs,
reporter);
} finally {
progressReceiver.executionProgressReceiver = null;
// Also releases thread locks.
resourceManager.resetResourceUsage();
skyframeActionExecutor.executionOver();
actionExecutionFunction.complete();
}
}
@VisibleForTesting
public void prepareBuildingForTestingOnly(Reporter reporter, Executor executor, boolean keepGoing,
boolean explain, ActionCacheChecker checker) {
skyframeActionExecutor.prepareForExecution(reporter, executor, keepGoing, explain, checker,
outputService);
}
EvaluationResult targetPatterns(Iterable patternSkyKeys,
int numThreads, boolean keepGoing, EventHandler eventHandler) throws InterruptedException {
checkActive();
return buildDriver.evaluate(patternSkyKeys, keepGoing, numThreads, eventHandler);
}
/**
* Returns the {@link ConfiguredTarget}s corresponding to the given keys.
*
*
For use for legacy support and tests calling through {@code BuildView} only.
*
*
If a requested configured target is in error, the corresponding value is omitted from the
* returned list.
*/
@ThreadSafety.ThreadSafe
// TODO(bazel-team): rename this and below methods to something that discourages general use
public ImmutableList getConfiguredTargets(
EventHandler eventHandler, BuildConfiguration originalConfig, Iterable keys,
boolean useOriginalConfig) {
return getConfiguredTargetMap(
eventHandler, originalConfig, keys, useOriginalConfig).values().asList();
}
/**
* Returns a map from {@link Dependency} inputs to the {@link ConfiguredTarget}s corresponding
* to those dependencies.
*
*
For use for legacy support and tests calling through {@code BuildView} only.
*
*
If a requested configured target is in error, the corresponding value is omitted from the
* returned list.
*/
@ThreadSafety.ThreadSafe
public ImmutableMultimap getConfiguredTargetMap(
EventHandler eventHandler, BuildConfiguration originalConfig, Iterable keys,
boolean useOriginalConfig) {
checkActive();
Multimap configs;
if (originalConfig != null) {
if (useOriginalConfig) {
// This flag is used because of some unfortunate complexity in the configuration machinery:
// Most callers of this method pass a