// Copyright 2014 The Bazel Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package com.google.devtools.build.lib.skyframe; import static com.google.devtools.build.lib.concurrent.Uninterruptibles.callUninterruptibly; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Function; import com.google.common.base.Predicate; import com.google.common.base.Predicates; import com.google.common.base.Stopwatch; import com.google.common.base.Supplier; import com.google.common.base.Throwables; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSortedSet; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.collect.Multimap; import com.google.common.collect.Range; import com.google.common.eventbus.EventBus; import com.google.devtools.build.lib.actions.ActionAnalysisMetadata; import com.google.devtools.build.lib.actions.ActionCacheChecker; import com.google.devtools.build.lib.actions.ActionExecutionContextFactory; import com.google.devtools.build.lib.actions.ActionExecutionStatusReporter; import com.google.devtools.build.lib.actions.ActionGraph; import com.google.devtools.build.lib.actions.ActionInputFileCache; import com.google.devtools.build.lib.actions.ActionLogBufferPathGenerator; import com.google.devtools.build.lib.actions.ActionLookupValue; import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.actions.ArtifactFactory; import com.google.devtools.build.lib.actions.ArtifactOwner; import com.google.devtools.build.lib.actions.EnvironmentalExecException; import com.google.devtools.build.lib.actions.Executor; import com.google.devtools.build.lib.actions.ResourceManager; import com.google.devtools.build.lib.actions.Root; import com.google.devtools.build.lib.analysis.AspectCollection; import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.analysis.BuildView.Options; import com.google.devtools.build.lib.analysis.ConfiguredAspect; import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; import com.google.devtools.build.lib.analysis.ConfiguredTarget; import com.google.devtools.build.lib.analysis.Dependency; import com.google.devtools.build.lib.analysis.MergedConfiguredTarget; import com.google.devtools.build.lib.analysis.MergedConfiguredTarget.DuplicateException; import com.google.devtools.build.lib.analysis.TopLevelArtifactContext; import com.google.devtools.build.lib.analysis.WorkspaceStatusAction; import com.google.devtools.build.lib.analysis.WorkspaceStatusAction.Factory; import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory; import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory.BuildInfoKey; import com.google.devtools.build.lib.analysis.config.BinTools; import com.google.devtools.build.lib.analysis.config.BuildConfiguration; import com.google.devtools.build.lib.analysis.config.BuildConfigurationCollection; import com.google.devtools.build.lib.analysis.config.BuildOptions; import com.google.devtools.build.lib.analysis.config.ConfigurationFactory; import com.google.devtools.build.lib.analysis.config.ConfigurationFragmentFactory; import com.google.devtools.build.lib.analysis.config.HostTransition; import com.google.devtools.build.lib.analysis.config.InvalidConfigurationException; import com.google.devtools.build.lib.analysis.config.PatchTransition; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.cmdline.LabelSyntaxException; import com.google.devtools.build.lib.cmdline.PackageIdentifier; import com.google.devtools.build.lib.cmdline.TargetParsingException; import com.google.devtools.build.lib.concurrent.ThreadSafety; import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible; import com.google.devtools.build.lib.events.ErrorSensingEventHandler; import com.google.devtools.build.lib.events.ExtendedEventHandler; import com.google.devtools.build.lib.events.Reporter; import com.google.devtools.build.lib.exec.OutputService; import com.google.devtools.build.lib.packages.AspectDescriptor; import com.google.devtools.build.lib.packages.AstAfterPreprocessing; import com.google.devtools.build.lib.packages.Attribute; import com.google.devtools.build.lib.packages.Attribute.ConfigurationTransition; import com.google.devtools.build.lib.packages.BuildFileContainsErrorsException; import com.google.devtools.build.lib.packages.NoSuchPackageException; import com.google.devtools.build.lib.packages.NoSuchThingException; import com.google.devtools.build.lib.packages.Package; import com.google.devtools.build.lib.packages.Package.Builder; import com.google.devtools.build.lib.packages.PackageFactory; import com.google.devtools.build.lib.packages.RuleClassProvider; import com.google.devtools.build.lib.packages.RuleVisibility; import com.google.devtools.build.lib.pkgcache.LoadingCallback; import com.google.devtools.build.lib.pkgcache.LoadingFailedException; import com.google.devtools.build.lib.pkgcache.LoadingOptions; import com.google.devtools.build.lib.pkgcache.LoadingPhaseCompleteEvent; import com.google.devtools.build.lib.pkgcache.LoadingPhaseRunner; import com.google.devtools.build.lib.pkgcache.LoadingResult; import com.google.devtools.build.lib.pkgcache.PackageCacheOptions; import com.google.devtools.build.lib.pkgcache.PackageManager; import com.google.devtools.build.lib.pkgcache.PackageManager.PackageManagerStatistics; import com.google.devtools.build.lib.pkgcache.PathPackageLocator; import com.google.devtools.build.lib.pkgcache.TargetParsingCompleteEvent; import com.google.devtools.build.lib.pkgcache.TestFilter; import com.google.devtools.build.lib.pkgcache.TransitivePackageLoader; import com.google.devtools.build.lib.profiler.AutoProfiler; import com.google.devtools.build.lib.skyframe.AspectValue.AspectValueKey; import com.google.devtools.build.lib.skyframe.DirtinessCheckerUtils.FileDirtinessChecker; import com.google.devtools.build.lib.skyframe.ExternalFilesHelper.ExternalFileAction; import com.google.devtools.build.lib.skyframe.PackageFunction.ActionOnIOExceptionReadingBuildFile; import com.google.devtools.build.lib.skyframe.PackageFunction.CacheEntryWithGlobDeps; import com.google.devtools.build.lib.skyframe.PackageLookupFunction.CrossRepositoryLabelViolationStrategy; import com.google.devtools.build.lib.skyframe.PackageLookupValue.BuildFileName; import com.google.devtools.build.lib.skyframe.SkyframeActionExecutor.ActionCompletedReceiver; import com.google.devtools.build.lib.skyframe.SkyframeActionExecutor.ProgressSupplier; import com.google.devtools.build.lib.syntax.SkylarkSemanticsOptions; import com.google.devtools.build.lib.util.AbruptExitException; import com.google.devtools.build.lib.util.ExitCode; import com.google.devtools.build.lib.util.Preconditions; import com.google.devtools.build.lib.util.ResourceUsage; import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor; import com.google.devtools.build.lib.vfs.BatchStat; import com.google.devtools.build.lib.vfs.Dirent; import com.google.devtools.build.lib.vfs.ModifiedFileSet; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.lib.vfs.RootedPath; import com.google.devtools.build.lib.vfs.UnixGlob; import com.google.devtools.build.skyframe.BuildDriver; import com.google.devtools.build.skyframe.CycleInfo; import com.google.devtools.build.skyframe.CyclesReporter; import com.google.devtools.build.skyframe.Differencer; import com.google.devtools.build.skyframe.Differencer.DiffWithDelta.Delta; import com.google.devtools.build.skyframe.ErrorInfo; import com.google.devtools.build.skyframe.EvaluationProgressReceiver; import com.google.devtools.build.skyframe.EvaluationResult; import com.google.devtools.build.skyframe.ImmutableDiff; import com.google.devtools.build.skyframe.Injectable; import com.google.devtools.build.skyframe.MemoizingEvaluator; import com.google.devtools.build.skyframe.MemoizingEvaluator.EvaluatorSupplier; import com.google.devtools.build.skyframe.SkyFunction; import com.google.devtools.build.skyframe.SkyFunctionName; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; import com.google.devtools.build.skyframe.WalkableGraph.WalkableGraphFactory; import com.google.devtools.common.options.OptionsClassProvider; import java.io.IOException; import java.io.PrintStream; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.UUID; import java.util.concurrent.Callable; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Logger; import javax.annotation.Nullable; /** * A helper object to support Skyframe-driven execution. * *

This object is mostly used to inject external state, such as the executor engine or * some additional artifacts (workspace status and build info artifacts) into SkyFunctions * for use during the build. */ public abstract class SkyframeExecutor implements WalkableGraphFactory { private final EvaluatorSupplier evaluatorSupplier; protected MemoizingEvaluator memoizingEvaluator; private final MemoizingEvaluator.EmittedEventState emittedEventState = new MemoizingEvaluator.EmittedEventState(); private final PackageFactory pkgFactory; private final WorkspaceStatusAction.Factory workspaceStatusActionFactory; private final BlazeDirectories directories; protected final ExternalFilesHelper externalFilesHelper; @Nullable private OutputService outputService; // TODO(bazel-team): Figure out how to handle value builders that block internally. Blocking // operations may need to be handled in another (bigger?) thread pool. Also, we should detect // the number of cores and use that as the thread-pool size for CPU-bound operations. // I just bumped this to 200 to get reasonable execution phase performance; that may cause // significant overhead for CPU-bound processes (i.e. analysis). [skyframe-analysis] @VisibleForTesting public static final int DEFAULT_THREAD_COUNT = // Reduce thread count while running tests of Bazel. Test cases are typically small, and large // thread pools vying for a relatively small number of CPU cores may induce non-optimal // performance. System.getenv("TEST_TMPDIR") == null ? 200 : 5; // Cache of partially constructed Package instances, stored between reruns of the PackageFunction // (because of missing dependencies, within the same evaluate() run) to avoid loading the same // package twice (first time loading to find subincludes and declare value dependencies). // TODO(bazel-team): remove this cache once we have skyframe-native package loading // [skyframe-loading] private final Cache> packageFunctionCache = newPkgFunctionCache(); private final Cache> astCache = newAstCache(); private final AtomicInteger numPackagesLoaded = new AtomicInteger(0); private final PackageProgressReceiver packageProgress = new PackageProgressReceiver(); protected SkyframeBuildView skyframeBuildView; private ActionLogBufferPathGenerator actionLogBufferPathGenerator; protected BuildDriver buildDriver; // AtomicReferences are used here as mutable boxes shared with value builders. private final AtomicBoolean showLoadingProgress = new AtomicBoolean(); protected final AtomicReference syscalls = new AtomicReference<>(UnixGlob.DEFAULT_SYSCALLS); protected final AtomicReference pkgLocator = new AtomicReference<>(); protected final AtomicReference> deletedPackages = new AtomicReference<>(ImmutableSet.of()); private final AtomicReference eventBus = new AtomicReference<>(); protected final AtomicReference tsgm = new AtomicReference<>(); protected final AtomicReference> clientEnv = new AtomicReference<>(); private final ImmutableList buildInfoFactories; // Under normal circumstances, the artifact factory persists for the life of a Blaze server, but // since it is not yet created when we create the value builders, we have to use a supplier, // initialized when the build view is created. private final MutableSupplier artifactFactory = new MutableSupplier<>(); // Used to give to WriteBuildInfoAction via a supplier. Relying on BuildVariableValue.BUILD_ID // would be preferable, but we have no way to have the Action depend on that value directly. // Having the BuildInfoFunction own the supplier is currently not possible either, because then // it would be invalidated on every build, since it would depend on the build id value. private MutableSupplier buildId = new MutableSupplier<>(); protected boolean active = true; private final SkyframePackageManager packageManager; private final ResourceManager resourceManager; /** Used to lock evaluator on legacy calls to get existing values. */ private final Object valueLookupLock = new Object(); private final AtomicReference statusReporterRef = new AtomicReference<>(); private final SkyframeActionExecutor skyframeActionExecutor; private CompletionReceiver actionExecutionFunction; protected SkyframeProgressReceiver progressReceiver; private final AtomicReference cyclesReporter = new AtomicReference<>(); private final BinTools binTools; private boolean needToInjectEmbeddedArtifacts = true; private boolean needToInjectPrecomputedValuesForAnalysis = true; protected int modifiedFiles; protected int outputDirtyFiles; protected int modifiedFilesDuringPreviousBuild; private final Predicate allowedMissingInputs; private final ExternalFileAction externalFileAction; private final ImmutableMap extraSkyFunctions; private final ImmutableList extraPrecomputedValues; private final String productName; protected SkyframeIncrementalBuildMonitor incrementalBuildMonitor = new SkyframeIncrementalBuildMonitor(); protected final MutableSupplier removeActionsAfterEvaluation = new MutableSupplier<>(); private MutableSupplier configurationFactory = new MutableSupplier<>(); private MutableSupplier> configurationFragments = new MutableSupplier<>(); private final PathFragment blacklistedPackagePrefixesFile; private final RuleClassProvider ruleClassProvider; private final CrossRepositoryLabelViolationStrategy crossRepositoryLabelViolationStrategy; private final List buildFilesByPriority; private final ActionOnIOExceptionReadingBuildFile actionOnIOExceptionReadingBuildFile; private PerBuildSyscallCache perBuildSyscallCache; private int lastConcurrencyLevel = -1; private static final Logger LOG = Logger.getLogger(SkyframeExecutor.class.getName()); protected SkyframeExecutor( EvaluatorSupplier evaluatorSupplier, PackageFactory pkgFactory, BlazeDirectories directories, BinTools binTools, Factory workspaceStatusActionFactory, ImmutableList buildInfoFactories, Predicate allowedMissingInputs, ImmutableMap extraSkyFunctions, ImmutableList extraPrecomputedValues, ExternalFileAction externalFileAction, PathFragment blacklistedPackagePrefixesFile, String productName, CrossRepositoryLabelViolationStrategy crossRepositoryLabelViolationStrategy, List buildFilesByPriority, ActionOnIOExceptionReadingBuildFile actionOnIOExceptionReadingBuildFile) { // Strictly speaking, these arguments are not required for initialization, but all current // callsites have them at hand, so we might as well set them during construction. this.evaluatorSupplier = evaluatorSupplier; this.pkgFactory = pkgFactory; this.pkgFactory.setSyscalls(syscalls); this.workspaceStatusActionFactory = workspaceStatusActionFactory; this.packageManager = new SkyframePackageManager( new SkyframePackageLoader(), new SkyframeTransitivePackageLoader(), syscalls, cyclesReporter, pkgLocator, numPackagesLoaded, this); this.resourceManager = ResourceManager.instance(); this.skyframeActionExecutor = new SkyframeActionExecutor(eventBus, statusReporterRef); this.directories = Preconditions.checkNotNull(directories); this.buildInfoFactories = buildInfoFactories; this.allowedMissingInputs = allowedMissingInputs; this.extraSkyFunctions = extraSkyFunctions; this.extraPrecomputedValues = extraPrecomputedValues; this.externalFileAction = externalFileAction; this.blacklistedPackagePrefixesFile = blacklistedPackagePrefixesFile; this.binTools = binTools; this.ruleClassProvider = pkgFactory.getRuleClassProvider(); this.skyframeBuildView = new SkyframeBuildView( directories, this, (ConfiguredRuleClassProvider) ruleClassProvider); this.artifactFactory.set(skyframeBuildView.getArtifactFactory()); this.externalFilesHelper = new ExternalFilesHelper( pkgLocator, this.externalFileAction, directories); this.productName = productName; this.crossRepositoryLabelViolationStrategy = crossRepositoryLabelViolationStrategy; this.buildFilesByPriority = buildFilesByPriority; this.actionOnIOExceptionReadingBuildFile = actionOnIOExceptionReadingBuildFile; this.removeActionsAfterEvaluation.set(false); } private ImmutableMap skyFunctions( PackageFactory pkgFactory, Predicate allowedMissingInputs) { ConfiguredRuleClassProvider ruleClassProvider = (ConfiguredRuleClassProvider) pkgFactory.getRuleClassProvider(); // TODO(janakr): use this semaphore to bound memory usage for SkyFunctions besides // ConfiguredTargetFunction that may have a large temporary memory blow-up. Semaphore cpuBoundSemaphore = new Semaphore(ResourceUsage.getAvailableProcessors()); // We use an immutable map builder for the nice side effect that it throws if a duplicate key // is inserted. ImmutableMap.Builder map = ImmutableMap.builder(); map.put(SkyFunctions.PRECOMPUTED, new PrecomputedFunction()); map.put(SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE, new ClientEnvironmentFunction(clientEnv)); map.put(SkyFunctions.ACTION_ENVIRONMENT_VARIABLE, new ActionEnvironmentFunction()); map.put(SkyFunctions.FILE_STATE, new FileStateFunction(tsgm, externalFilesHelper)); map.put(SkyFunctions.DIRECTORY_LISTING_STATE, new DirectoryListingStateFunction(externalFilesHelper)); map.put(SkyFunctions.FILE_SYMLINK_CYCLE_UNIQUENESS, new FileSymlinkCycleUniquenessFunction()); map.put(SkyFunctions.FILE_SYMLINK_INFINITE_EXPANSION_UNIQUENESS, new FileSymlinkInfiniteExpansionUniquenessFunction()); map.put(SkyFunctions.FILE, new FileFunction(pkgLocator)); map.put(SkyFunctions.DIRECTORY_LISTING, new DirectoryListingFunction()); map.put( SkyFunctions.PACKAGE_LOOKUP, new PackageLookupFunction( deletedPackages, crossRepositoryLabelViolationStrategy, buildFilesByPriority)); map.put(SkyFunctions.CONTAINING_PACKAGE_LOOKUP, new ContainingPackageLookupFunction()); map.put(SkyFunctions.AST_FILE_LOOKUP, new ASTFileLookupFunction(ruleClassProvider)); map.put( SkyFunctions.SKYLARK_IMPORTS_LOOKUP, newSkylarkImportLookupFunction(ruleClassProvider, pkgFactory)); map.put(SkyFunctions.GLOB, newGlobFunction()); map.put(SkyFunctions.TARGET_PATTERN, new TargetPatternFunction()); map.put(SkyFunctions.PREPARE_DEPS_OF_PATTERNS, new PrepareDepsOfPatternsFunction()); map.put(SkyFunctions.PREPARE_DEPS_OF_PATTERN, new PrepareDepsOfPatternFunction(pkgLocator)); map.put( SkyFunctions.PREPARE_DEPS_OF_TARGETS_UNDER_DIRECTORY, new PrepareDepsOfTargetsUnderDirectoryFunction(directories)); map.put(SkyFunctions.COLLECT_TARGETS_IN_PACKAGE, new CollectTargetsInPackageFunction()); map.put( SkyFunctions.COLLECT_PACKAGES_UNDER_DIRECTORY, new CollectPackagesUnderDirectoryFunction(directories)); map.put(SkyFunctions.BLACKLISTED_PACKAGE_PREFIXES, new BlacklistedPackagePrefixesFunction()); map.put(SkyFunctions.TESTS_IN_SUITE, new TestsInSuiteFunction()); map.put(SkyFunctions.TEST_SUITE_EXPANSION, new TestSuiteExpansionFunction()); map.put(SkyFunctions.TARGET_PATTERN_PHASE, new TargetPatternPhaseFunction()); map.put(SkyFunctions.RECURSIVE_PKG, new RecursivePkgFunction(directories)); map.put( SkyFunctions.PACKAGE, newPackageFunction( pkgFactory, packageManager, showLoadingProgress, packageFunctionCache, astCache, numPackagesLoaded, ruleClassProvider, packageProgress)); map.put(SkyFunctions.PACKAGE_ERROR, new PackageErrorFunction()); map.put(SkyFunctions.TARGET_MARKER, new TargetMarkerFunction()); map.put(SkyFunctions.TRANSITIVE_TARGET, new TransitiveTargetFunction(ruleClassProvider)); map.put(Label.TRANSITIVE_TRAVERSAL, new TransitiveTraversalFunction()); map.put( SkyFunctions.CONFIGURED_TARGET, new ConfiguredTargetFunction( new BuildViewProvider(), ruleClassProvider, cpuBoundSemaphore, removeActionsAfterEvaluation)); map.put( SkyFunctions.ASPECT, new AspectFunction( new BuildViewProvider(), ruleClassProvider, removeActionsAfterEvaluation)); map.put(SkyFunctions.LOAD_SKYLARK_ASPECT, new ToplevelSkylarkAspectFunction()); map.put(SkyFunctions.POST_CONFIGURED_TARGET, new PostConfiguredTargetFunction(new BuildViewProvider(), ruleClassProvider)); map.put(SkyFunctions.BUILD_CONFIGURATION, new BuildConfigurationFunction(directories, ruleClassProvider)); map.put(SkyFunctions.CONFIGURATION_COLLECTION, new ConfigurationCollectionFunction( configurationFactory, ruleClassProvider)); map.put(SkyFunctions.CONFIGURATION_FRAGMENT, new ConfigurationFragmentFunction( configurationFragments, ruleClassProvider)); map.put(SkyFunctions.WORKSPACE_NAME, new WorkspaceNameFunction()); map.put(SkyFunctions.WORKSPACE_AST, new WorkspaceASTFunction(ruleClassProvider)); map.put( SkyFunctions.WORKSPACE_FILE, new WorkspaceFileFunction(ruleClassProvider, pkgFactory, directories)); map.put(SkyFunctions.EXTERNAL_PACKAGE, new ExternalPackageFunction()); map.put(SkyFunctions.TARGET_COMPLETION, CompletionFunction.targetCompletionFunction(eventBus)); map.put(SkyFunctions.ASPECT_COMPLETION, CompletionFunction.aspectCompletionFunction(eventBus)); map.put(SkyFunctions.TEST_COMPLETION, new TestCompletionFunction()); map.put(SkyFunctions.ARTIFACT, new ArtifactFunction(allowedMissingInputs)); map.put( SkyFunctions.BUILD_INFO_COLLECTION, new BuildInfoCollectionFunction(artifactFactory, removeActionsAfterEvaluation)); map.put(SkyFunctions.BUILD_INFO, new WorkspaceStatusFunction(removeActionsAfterEvaluation)); map.put(SkyFunctions.COVERAGE_REPORT, new CoverageReportFunction(removeActionsAfterEvaluation)); ActionExecutionFunction actionExecutionFunction = new ActionExecutionFunction(skyframeActionExecutor, tsgm); map.put(SkyFunctions.ACTION_EXECUTION, actionExecutionFunction); this.actionExecutionFunction = actionExecutionFunction; map.put(SkyFunctions.RECURSIVE_FILESYSTEM_TRAVERSAL, new RecursiveFilesystemTraversalFunction()); map.put(SkyFunctions.FILESET_ENTRY, new FilesetEntryFunction()); map.put( SkyFunctions.ACTION_TEMPLATE_EXPANSION, new ActionTemplateExpansionFunction(removeActionsAfterEvaluation)); map.put(SkyFunctions.LOCAL_REPOSITORY_LOOKUP, new LocalRepositoryLookupFunction()); map.put(SkyFunctions.REGISTERED_TOOLCHAINS, new RegisteredToolchainsFunction()); map.put(SkyFunctions.TOOLCHAIN_RESOLUTION, new ToolchainResolutionFunction()); map.putAll(extraSkyFunctions); return map.build(); } protected SkyFunction newGlobFunction() { return new GlobFunction(/*alwaysUseDirListing=*/false); } protected PackageFunction newPackageFunction( PackageFactory pkgFactory, PackageManager packageManager, AtomicBoolean showLoadingProgress, Cache> packageFunctionCache, Cache> astCache, AtomicInteger numPackagesLoaded, RuleClassProvider ruleClassProvider, PackageProgressReceiver packageProgress) { return new PackageFunction( pkgFactory, packageManager, showLoadingProgress, packageFunctionCache, astCache, numPackagesLoaded, null, packageProgress, actionOnIOExceptionReadingBuildFile); } protected SkyFunction newSkylarkImportLookupFunction( RuleClassProvider ruleClassProvider, PackageFactory pkgFactory) { return new SkylarkImportLookupFunction(ruleClassProvider, this.pkgFactory); } protected PerBuildSyscallCache newPerBuildSyscallCache(int concurrencyLevel) { return PerBuildSyscallCache.newBuilder().setConcurrencyLevel(concurrencyLevel).build(); } /** * Gets a (possibly cached) syscalls cache, re-initialized each build. * *

We cache the syscalls cache if possible because construction of the cache is surprisingly * expensive, and is on the critical path of null builds. */ protected final PerBuildSyscallCache getPerBuildSyscallCache(int concurrencyLevel) { if (perBuildSyscallCache != null && lastConcurrencyLevel == concurrencyLevel) { perBuildSyscallCache.clear(); return perBuildSyscallCache; } lastConcurrencyLevel = concurrencyLevel; perBuildSyscallCache = newPerBuildSyscallCache(concurrencyLevel); return perBuildSyscallCache; } @ThreadCompatible public void setActive(boolean active) { this.active = active; } protected void checkActive() { Preconditions.checkState(active); } public void setFileCache(ActionInputFileCache fileCache) { this.skyframeActionExecutor.setFileCache(fileCache); } public void dump(boolean summarize, PrintStream out) { memoizingEvaluator.dump(summarize, out); } public abstract void dumpPackages(PrintStream out); public void setOutputService(OutputService outputService) { this.outputService = outputService; } /** * Notify listeners about changed files, and release any associated memory afterwards. */ public void drainChangedFiles() { incrementalBuildMonitor.alertListeners(getEventBus()); incrementalBuildMonitor = null; } @VisibleForTesting public BuildDriver getDriverForTesting() { return buildDriver; } /** * This method exists only to allow a module to make a top-level Skyframe call during the * transition to making it fully Skyframe-compatible. Do not add additional callers! */ public SkyValue evaluateSkyKeyForExecutionSetup( final ExtendedEventHandler eventHandler, final SkyKey key) throws EnvironmentalExecException, InterruptedException { synchronized (valueLookupLock) { // We evaluate in keepGoing mode because in the case that the graph does not store its // edges, nokeepGoing builds are not allowed, whereas keepGoing builds are always // permitted. EvaluationResult result = buildDriver.evaluate( ImmutableList.of(key), true, ResourceUsage.getAvailableProcessors(), eventHandler); if (!result.hasError()) { return Preconditions.checkNotNull(result.get(key), "%s %s", result, key); } ErrorInfo errorInfo = Preconditions.checkNotNull(result.getError(key), "%s %s", key, result); Throwables.propagateIfPossible(errorInfo.getException(), EnvironmentalExecException.class); if (errorInfo.getException() != null) { throw new IllegalStateException(errorInfo.getException()); } throw new IllegalStateException(errorInfo.toString()); } } @VisibleForTesting public PathFragment getBlacklistedPackagePrefixesFile() { return blacklistedPackagePrefixesFile; } class BuildViewProvider { /** * Returns the current {@link SkyframeBuildView} instance. */ SkyframeBuildView getSkyframeBuildView() { return skyframeBuildView; } } /** * Must be called before the {@link SkyframeExecutor} can be used (should only be called in * factory methods and as an implementation detail of {@link #resetEvaluator}). */ protected void init() { progressReceiver = newSkyframeProgressReceiver(); ImmutableMap skyFunctions = skyFunctions(pkgFactory, allowedMissingInputs); memoizingEvaluator = evaluatorSupplier.create( skyFunctions, evaluatorDiffer(), progressReceiver, emittedEventState, hasIncrementalState()); buildDriver = getBuildDriver(); } protected SkyframeProgressReceiver newSkyframeProgressReceiver() { return new SkyframeProgressReceiver(); } /** * Reinitializes the Skyframe evaluator, dropping all previously computed values. * *

Be careful with this method as it also deletes all injected values. You need to make sure * that any necessary precomputed values are reinjected before the next build. Constants can be * put in {@link #reinjectConstantValuesLazily}. */ public void resetEvaluator() { init(); emittedEventState.clear(); skyframeBuildView.clearLegacyData(); reinjectConstantValuesLazily(); } protected abstract Differencer evaluatorDiffer(); protected abstract BuildDriver getBuildDriver(); /** * Values whose values are known at startup and guaranteed constant are still wiped from the * evaluator when we create a new one, so they must be re-injected each time we create a new * evaluator. */ private void reinjectConstantValuesLazily() { needToInjectEmbeddedArtifacts = true; needToInjectPrecomputedValuesForAnalysis = true; } /** * Deletes all ConfiguredTarget values from the Skyframe cache. This is done to save memory (e.g. * on a configuration change); since the configuration is part of the key, these key/value pairs * will be sitting around doing nothing until the configuration changes back to the previous * value. * *

The next evaluation will delete all invalid values. */ public abstract void dropConfiguredTargets(); /** * Removes ConfigurationFragmentValuess and ConfigurationCollectionValues from the cache. */ @VisibleForTesting public void invalidateConfigurationCollection() { invalidate(SkyFunctionName.functionIsIn(ImmutableSet.of(SkyFunctions.CONFIGURATION_FRAGMENT, SkyFunctions.CONFIGURATION_COLLECTION))); } /** * Decides if graph edges should be stored for this build. If not, re-creates the graph to not * store graph edges. Necessary conditions to not store graph edges are: * *

    *
  1. batch (since incremental builds are not possible); *
  2. keep-going (since otherwise bubbling errors up may require edges of done nodes); *
  3. discard_analysis_cache (since otherwise user isn't concerned about saving memory this * way). *
*/ public void decideKeepIncrementalState(boolean batch, Options viewOptions) { // Assume incrementality. } public boolean hasIncrementalState() { return true; } @VisibleForTesting protected abstract Injectable injectable(); /** * Saves memory by clearing analysis objects from Skyframe. If using legacy execution, actually * deletes the relevant values. If using Skyframe execution, clears their data without deleting * them (they will be deleted on the next build). */ public abstract void clearAnalysisCache( Collection topLevelTargets, Collection topLevelAspects); /** * Injects the contents of the computed tools/defaults package. */ @VisibleForTesting public void setupDefaultPackage(String defaultsPackageContents) { PrecomputedValue.DEFAULTS_PACKAGE_CONTENTS.set(injectable(), defaultsPackageContents); } public void injectWorkspaceStatusData(String workspaceName) { PrecomputedValue.WORKSPACE_STATUS_KEY.set(injectable(), workspaceStatusActionFactory.createWorkspaceStatusAction( artifactFactory.get(), WorkspaceStatusValue.ARTIFACT_OWNER, buildId, workspaceName)); } public void injectCoverageReportData(ImmutableList actions) { PrecomputedValue.COVERAGE_REPORT_KEY.set(injectable(), actions); } private void setDefaultVisibility(RuleVisibility defaultVisibility) { PrecomputedValue.DEFAULT_VISIBILITY.set(injectable(), defaultVisibility); } private void setSkylarkSemantics(SkylarkSemanticsOptions skylarkSemanticsOptions) { PrecomputedValue.SKYLARK_SEMANTICS.set(injectable(), skylarkSemanticsOptions); } protected void maybeInjectPrecomputedValuesForAnalysis() { if (needToInjectPrecomputedValuesForAnalysis) { PrecomputedValue.BLAZE_DIRECTORIES.set(injectable(), directories); PrecomputedValue.PRODUCT_NAME.set(injectable(), productName); injectBuildInfoFactories(); injectExtraPrecomputedValues(extraPrecomputedValues); needToInjectPrecomputedValuesForAnalysis = false; } } public void injectExtraPrecomputedValues( List extraPrecomputedValues) { for (PrecomputedValue.Injected injected : extraPrecomputedValues) { injected.inject(injectable()); } } protected Cache> newPkgFunctionCache() { return CacheBuilder.newBuilder().build(); } protected Cache> newAstCache() { return CacheBuilder.newBuilder().build(); } /** * Injects the build info factory map that will be used when constructing build info * actions/artifacts. Unchanged across the life of the Blaze server, although it must be injected * each time the evaluator is created. */ private void injectBuildInfoFactories() { ImmutableMap.Builder factoryMapBuilder = ImmutableMap.builder(); for (BuildInfoFactory factory : buildInfoFactories) { factoryMapBuilder.put(factory.getKey(), factory); } PrecomputedValue.BUILD_INFO_FACTORIES.set(injectable(), factoryMapBuilder.build()); } private void setShowLoadingProgress(boolean showLoadingProgressValue) { showLoadingProgress.set(showLoadingProgressValue); } @VisibleForTesting public void setCommandId(UUID commandId) { PrecomputedValue.BUILD_ID.set(injectable(), commandId); buildId.set(commandId); } /** Returns the build-info.txt and build-changelist.txt artifacts. */ public Collection getWorkspaceStatusArtifacts(ExtendedEventHandler eventHandler) throws InterruptedException { // Should already be present, unless the user didn't request any targets for analysis. EvaluationResult result = buildDriver.evaluate( ImmutableList.of(WorkspaceStatusValue.SKY_KEY), /*keepGoing=*/true, /*numThreads=*/1, eventHandler); WorkspaceStatusValue value = Preconditions.checkNotNull(result.get(WorkspaceStatusValue.SKY_KEY)); return ImmutableList.of(value.getStableArtifact(), value.getVolatileArtifact()); } public Map getArtifactRootsForFiles( final ExtendedEventHandler eventHandler, Iterable execPaths) throws InterruptedException { return getArtifactRoots(eventHandler, execPaths, true); } public Map getArtifactRoots( final ExtendedEventHandler eventHandler, Iterable execPaths) throws InterruptedException { return getArtifactRoots(eventHandler, execPaths, false); } private Map getArtifactRoots( final ExtendedEventHandler eventHandler, Iterable execPaths, boolean forFiles) throws InterruptedException { final Map packageKeys = new HashMap<>(); for (PathFragment execPath : execPaths) { try { PackageIdentifier pkgIdentifier = PackageIdentifier.discoverFromExecPath(execPath, forFiles); packageKeys.put(execPath, ContainingPackageLookupValue.key(pkgIdentifier)); } catch (LabelSyntaxException e) { continue; } } EvaluationResult result; synchronized (valueLookupLock) { result = buildDriver.evaluate( packageKeys.values(), /*keepGoing=*/ true, /*numThreads=*/ 1, eventHandler); } if (result.hasError()) { return new HashMap<>(); } Map roots = new HashMap<>(); for (PathFragment execPath : execPaths) { ContainingPackageLookupValue value = result.get(packageKeys.get(execPath)); if (value.hasContainingPackage()) { roots.put(execPath, Root.computeSourceRoot( value.getContainingPackageRoot(), value.getContainingPackageName().getRepository())); } else { roots.put(execPath, null); } } return roots; } @VisibleForTesting public WorkspaceStatusAction getLastWorkspaceStatusActionForTesting() { PrecomputedValue value = (PrecomputedValue) buildDriver.getGraphForTesting() .getExistingValueForTesting(PrecomputedValue.WORKSPACE_STATUS_KEY.getKeyForTesting()); return (WorkspaceStatusAction) value.get(); } /** * Informs user about number of modified files (source and output files). */ // Note, that number of modified files in some cases can be bigger than actual number of // modified files for targets in current request. Skyframe may check for modification all files // from previous requests. protected void informAboutNumberOfModifiedFiles() { LOG.info(String.format("Found %d modified files from last build", modifiedFiles)); } public EventBus getEventBus() { return eventBus.get(); } public ActionExecutionContextFactory getActionExecutionContextFactory() { return skyframeActionExecutor; } @VisibleForTesting ImmutableList getPathEntries() { return pkgLocator.get().getPathEntries(); } protected abstract void invalidate(Predicate pred); private static boolean compatibleFileTypes(Dirent.Type oldType, FileStateValue.Type newType) { return (oldType.equals(Dirent.Type.FILE) && newType.equals(FileStateValue.Type.REGULAR_FILE)) || (oldType.equals(Dirent.Type.UNKNOWN) && newType.equals(FileStateValue.Type.SPECIAL_FILE)) || (oldType.equals(Dirent.Type.DIRECTORY) && newType.equals(FileStateValue.Type.DIRECTORY)) || (oldType.equals(Dirent.Type.SYMLINK) && newType.equals(FileStateValue.Type.SYMLINK)); } protected Differencer.Diff getDiff(TimestampGranularityMonitor tsgm, Iterable modifiedSourceFiles, final Path pathEntry) throws InterruptedException { if (Iterables.isEmpty(modifiedSourceFiles)) { return new ImmutableDiff(ImmutableList.of(), ImmutableMap.of()); } // TODO(bazel-team): change ModifiedFileSet to work with RootedPaths instead of PathFragments. Iterable dirtyFileStateSkyKeys = Iterables.transform(modifiedSourceFiles, new Function() { @Override public SkyKey apply(PathFragment pathFragment) { Preconditions.checkState(!pathFragment.isAbsolute(), "found absolute PathFragment: %s", pathFragment); return FileStateValue.key(RootedPath.toRootedPath(pathEntry, pathFragment)); } }); // We only need to invalidate directory values when a file has been created or deleted or // changes type, not when it has merely been modified. Unfortunately we do not have that // information here, so we compute it ourselves. // TODO(bazel-team): Fancy filesystems could provide it with a hypothetically modified // DiffAwareness interface. LOG.info("About to recompute filesystem nodes corresponding to files that are known to have " + "changed"); FilesystemValueChecker fsvc = new FilesystemValueChecker(tsgm, null); Map valuesMap = memoizingEvaluator.getValues(); Differencer.DiffWithDelta diff = fsvc.getNewAndOldValues(valuesMap, dirtyFileStateSkyKeys, new FileDirtinessChecker()); Set valuesToInvalidate = new HashSet<>(); Map valuesToInject = new HashMap<>(); for (Map.Entry entry : diff.changedKeysWithNewAndOldValues().entrySet()) { SkyKey key = entry.getKey(); Preconditions.checkState(key.functionName().equals(SkyFunctions.FILE_STATE), key); RootedPath rootedPath = (RootedPath) key.argument(); Delta delta = entry.getValue(); FileStateValue oldValue = (FileStateValue) delta.getOldValue(); FileStateValue newValue = (FileStateValue) delta.getNewValue(); if (newValue != null) { valuesToInject.put(key, newValue); } else { valuesToInvalidate.add(key); } SkyKey dirListingStateKey = parentDirectoryListingStateKey(rootedPath); // Invalidate the directory listing for the path's parent directory if the change was // relevant (e.g. path turned from a symlink into a directory) OR if we don't have enough // information to determine it was irrelevant. boolean changedType = false; if (newValue == null) { changedType = true; } else if (oldValue != null) { changedType = !oldValue.getType().equals(newValue.getType()); } else { DirectoryListingStateValue oldDirListingStateValue = (DirectoryListingStateValue) valuesMap.get(dirListingStateKey); if (oldDirListingStateValue != null) { String baseName = rootedPath.getRelativePath().getBaseName(); Dirent oldDirent = oldDirListingStateValue.getDirents().maybeGetDirent(baseName); changedType = (oldDirent == null) || !compatibleFileTypes(oldDirent.getType(), newValue.getType()); } else { changedType = true; } } if (changedType) { valuesToInvalidate.add(dirListingStateKey); } } for (SkyKey key : diff.changedKeysWithoutNewValues()) { Preconditions.checkState(key.functionName().equals(SkyFunctions.FILE_STATE), key); RootedPath rootedPath = (RootedPath) key.argument(); valuesToInvalidate.add(parentDirectoryListingStateKey(rootedPath)); } return new ImmutableDiff(valuesToInvalidate, valuesToInject); } private static SkyKey parentDirectoryListingStateKey(RootedPath rootedPath) { RootedPath parentDirRootedPath = RootedPath.toRootedPath( rootedPath.getRoot(), rootedPath.getRelativePath().getParentDirectory()); return DirectoryListingStateValue.key(parentDirRootedPath); } /** * Sets the packages that should be treated as deleted and ignored. */ @VisibleForTesting // productionVisibility = Visibility.PRIVATE public abstract void setDeletedPackages(Iterable pkgs); @VisibleForTesting public final void setBlacklistedPackagePrefixesFile(PathFragment blacklistedPkgFile) { PrecomputedValue.BLACKLISTED_PACKAGE_PREFIXES_FILE.set(injectable(), blacklistedPkgFile); } /** * Prepares the evaluator for loading. * *

MUST be run before every incremental build. */ @VisibleForTesting // productionVisibility = Visibility.PRIVATE public void preparePackageLoading( PathPackageLocator pkgLocator, PackageCacheOptions packageCacheOptions, SkylarkSemanticsOptions skylarkSemanticsOptions, String defaultsPackageContents, UUID commandId, Map clientEnv, Map actionEnv, TimestampGranularityMonitor tsgm) { Preconditions.checkNotNull(pkgLocator); Preconditions.checkNotNull(tsgm); setActive(true); this.tsgm.set(tsgm); maybeInjectPrecomputedValuesForAnalysis(); setCommandId(commandId); PrecomputedValue.ACTION_ENV.set(injectable(), actionEnv); this.clientEnv.set(clientEnv); setBlacklistedPackagePrefixesFile(getBlacklistedPackagePrefixesFile()); setShowLoadingProgress(packageCacheOptions.showLoadingProgress); setDefaultVisibility(packageCacheOptions.defaultVisibility); setSkylarkSemantics(skylarkSemanticsOptions); setupDefaultPackage(defaultsPackageContents); setPackageLocator(pkgLocator); syscalls.set(getPerBuildSyscallCache(packageCacheOptions.globbingThreads)); this.pkgFactory.setGlobbingThreads(packageCacheOptions.globbingThreads); this.pkgFactory.setMaxDirectoriesToEagerlyVisitInGlobbing( packageCacheOptions.maxDirectoriesToEagerlyVisitInGlobbing); emittedEventState.clear(); // If the PackageFunction was interrupted, there may be stale entries here. packageFunctionCache.invalidateAll(); astCache.invalidateAll(); numPackagesLoaded.set(0); packageProgress.reset(); // Reset the stateful SkyframeCycleReporter, which contains cycles from last run. cyclesReporter.set(createCyclesReporter()); } @SuppressWarnings("unchecked") private void setPackageLocator(PathPackageLocator pkgLocator) { PathPackageLocator oldLocator = this.pkgLocator.getAndSet(pkgLocator); PrecomputedValue.PATH_PACKAGE_LOCATOR.set(injectable(), pkgLocator); if (!pkgLocator.equals(oldLocator)) { // The package path is read not only by SkyFunctions but also by some other code paths. // We need to take additional steps to keep the corresponding data structures in sync. // (Some of the additional steps are carried out by ConfiguredTargetValueInvalidationListener, // and some by BuildView#buildHasIncompatiblePackageRoots and #updateSkyframe.) onNewPackageLocator(oldLocator, pkgLocator); } } protected abstract void onNewPackageLocator(PathPackageLocator oldLocator, PathPackageLocator pkgLocator); public SkyframeBuildView getSkyframeBuildView() { return skyframeBuildView; } /** * Sets the eventBus to use for posting events. */ public void setEventBus(EventBus eventBus) { this.eventBus.set(eventBus); } public void setClientEnv(Map clientEnv) { this.skyframeActionExecutor.setClientEnv(clientEnv); } /** * Sets the path for action log buffers. */ public void setActionOutputRoot(Path actionOutputRoot) { Preconditions.checkNotNull(actionOutputRoot); this.actionLogBufferPathGenerator = new ActionLogBufferPathGenerator(actionOutputRoot); this.skyframeActionExecutor.setActionLogBufferPathGenerator(actionLogBufferPathGenerator); } /** * Sets the configuration factory and known fragment set. */ public void setConfigurationFactory(ConfigurationFactory configurationFactory) { this.configurationFactory.set(configurationFactory); this.configurationFragments.set(ImmutableList.copyOf(configurationFactory.getFactories())); } /** * Asks the Skyframe evaluator to build the value for BuildConfigurationCollection and returns the * result. Also invalidates {@link PrecomputedValue#BLAZE_DIRECTORIES} if it has changed. */ public BuildConfigurationCollection createConfigurations( ExtendedEventHandler eventHandler, ConfigurationFactory configurationFactory, BuildOptions buildOptions, Set multiCpu, boolean keepGoing) throws InvalidConfigurationException, InterruptedException { setConfigurationFactory(configurationFactory); if (buildOptions.get(BuildConfiguration.Options.class).useDynamicConfigurations == BuildConfiguration.Options.DynamicConfigsMode.OFF) { return createStaticConfigurations(eventHandler, buildOptions, multiCpu, keepGoing); } else { return createDynamicConfigurations(eventHandler, buildOptions, multiCpu); } } /** * {@link #createConfigurations} implementation that creates the configurations statically. */ private BuildConfigurationCollection createStaticConfigurations( ExtendedEventHandler eventHandler, BuildOptions buildOptions, Set multiCpu, boolean keepGoing) throws InvalidConfigurationException, InterruptedException { SkyKey skyKey = ConfigurationCollectionValue.key( buildOptions, ImmutableSortedSet.copyOf(multiCpu)); EvaluationResult result = buildDriver.evaluate(Arrays.asList(skyKey), keepGoing, DEFAULT_THREAD_COUNT, eventHandler); if (result.hasError()) { ErrorInfo error = result.getError(skyKey); Throwable e = error.getException(); // Wrap loading failed exceptions if (e instanceof NoSuchThingException) { e = new InvalidConfigurationException(e); } else if (e == null && !Iterables.isEmpty(error.getCycleInfo())) { getCyclesReporter().reportCycles(error.getCycleInfo(), skyKey, eventHandler); e = new InvalidConfigurationException( "cannot load build configuration because of this cycle"); } Throwables.propagateIfInstanceOf(e, InvalidConfigurationException.class); throw new IllegalStateException( "Unknown error during ConfigurationCollectionValue evaluation", e); } ConfigurationCollectionValue configurationValue = result.get(skyKey); return configurationValue.getConfigurationCollection(); } /** * {@link #createConfigurations} implementation that creates the configurations dynamically. */ private BuildConfigurationCollection createDynamicConfigurations( ExtendedEventHandler eventHandler, BuildOptions buildOptions, Set multiCpu) throws InvalidConfigurationException, InterruptedException { List topLevelTargetConfigs = getConfigurations(eventHandler, getTopLevelBuildOptions(buildOptions, multiCpu)); // The host configuration inherits the data, not target options. This is so host tools don't // apply LIPO. BuildConfiguration firstTargetConfig = topLevelTargetConfigs.get(0); Attribute.Transition dataTransition = ((ConfiguredRuleClassProvider) ruleClassProvider) .getDynamicTransitionMapper() .map(Attribute.ConfigurationTransition.DATA); BuildOptions dataOptions = dataTransition != Attribute.ConfigurationTransition.NONE ? ((PatchTransition) dataTransition).apply(firstTargetConfig.getOptions()) : firstTargetConfig.getOptions(); BuildOptions hostOptions = dataOptions.get(BuildConfiguration.Options.class).useDistinctHostConfiguration ? HostTransition.INSTANCE.apply(dataOptions) : dataOptions; BuildConfiguration hostConfig = getConfiguration(eventHandler, hostOptions); // TODO(gregce): cache invalid option errors in BuildConfigurationFunction, then use a dedicated // accessor (i.e. not the event handler) to trigger the exception below. ErrorSensingEventHandler nosyEventHandler = new ErrorSensingEventHandler(eventHandler); topLevelTargetConfigs.forEach(config -> config.reportInvalidOptions(nosyEventHandler)); if (nosyEventHandler.hasErrors()) { throw new InvalidConfigurationException("Build options are invalid"); } return new BuildConfigurationCollection(topLevelTargetConfigs, hostConfig); } /** * Returns the {@link BuildOptions} to apply to the top-level build configurations. This can be * plural because of {@code multiCpu}. */ private static List getTopLevelBuildOptions(BuildOptions buildOptions, Set multiCpu) { if (multiCpu.isEmpty()) { return ImmutableList.of(buildOptions); } ImmutableList.Builder multiCpuOptions = ImmutableList.builder(); for (String cpu : multiCpu) { BuildOptions clonedOptions = buildOptions.clone(); clonedOptions.get(BuildConfiguration.Options.class).cpu = cpu; clonedOptions.get(BuildConfiguration.Options.class).experimentalMultiCpuDistinguisher = cpu; multiCpuOptions.add(clonedOptions); } return multiCpuOptions.build(); } private Iterable getActionLookupValues() { // This filter keeps subclasses of ActionLookupValue. return Iterables.filter(memoizingEvaluator.getDoneValues().values(), ActionLookupValue.class); } @SuppressWarnings({"unchecked", "rawtypes"}) Map getActionLookupValueMap() { return (Map) Maps.filterValues(memoizingEvaluator.getDoneValues(), Predicates.instanceOf(ActionLookupValue.class)); } /** * Checks the actions in Skyframe for conflicts between their output artifacts. Delegates to * {@link SkyframeActionExecutor#findAndStoreArtifactConflicts} to do the work, since any * conflicts found will only be reported during execution. */ ImmutableMap findArtifactConflicts() throws InterruptedException { if (skyframeBuildView.isSomeConfiguredTargetEvaluated() || skyframeBuildView.isSomeConfiguredTargetInvalidated()) { // This operation is somewhat expensive, so we only do it if the graph might have changed in // some way -- either we analyzed a new target or we invalidated an old one. try (AutoProfiler p = AutoProfiler.logged("discovering artifact conflicts", LOG)) { skyframeActionExecutor.findAndStoreArtifactConflicts(getActionLookupValues()); skyframeBuildView.resetEvaluatedConfiguredTargetFlag(); // The invalidated configured targets flag will be reset later in the evaluate() call. } } return skyframeActionExecutor.badActions(); } /** * Asks the Skyframe evaluator to build the given artifacts and targets, and to test the * given test targets. */ public EvaluationResult buildArtifacts( Reporter reporter, Executor executor, Set artifactsToBuild, Collection targetsToBuild, Collection aspects, Collection targetsToTest, boolean exclusiveTesting, boolean keepGoing, boolean explain, boolean finalizeActionsToOutputService, int numJobs, ActionCacheChecker actionCacheChecker, @Nullable EvaluationProgressReceiver executionProgressReceiver, TopLevelArtifactContext topLevelArtifactContext) throws InterruptedException { checkActive(); Preconditions.checkState(actionLogBufferPathGenerator != null); skyframeActionExecutor.prepareForExecution( reporter, executor, keepGoing, explain, actionCacheChecker, finalizeActionsToOutputService ? outputService : null); resourceManager.resetResourceUsage(); try { progressReceiver.executionProgressReceiver = executionProgressReceiver; Iterable artifactKeys = ArtifactSkyKey.mandatoryKeys(artifactsToBuild); Iterable targetKeys = TargetCompletionValue.keys(targetsToBuild, topLevelArtifactContext); Iterable aspectKeys = AspectCompletionValue.keys(aspects, topLevelArtifactContext); Iterable testKeys = TestCompletionValue.keys(targetsToTest, topLevelArtifactContext, exclusiveTesting); return buildDriver.evaluate( Iterables.concat(artifactKeys, targetKeys, aspectKeys, testKeys), keepGoing, numJobs, reporter); } finally { progressReceiver.executionProgressReceiver = null; // Also releases thread locks. resourceManager.resetResourceUsage(); skyframeActionExecutor.executionOver(); actionExecutionFunction.complete(); } } @VisibleForTesting public void prepareBuildingForTestingOnly(Reporter reporter, Executor executor, boolean keepGoing, boolean explain, ActionCacheChecker checker) { skyframeActionExecutor.prepareForExecution(reporter, executor, keepGoing, explain, checker, outputService); } EvaluationResult targetPatterns( Iterable patternSkyKeys, int numThreads, boolean keepGoing, ExtendedEventHandler eventHandler) throws InterruptedException { checkActive(); return buildDriver.evaluate(patternSkyKeys, keepGoing, numThreads, eventHandler); } /** * Returns the {@link ConfiguredTarget}s corresponding to the given keys. * *

For use for legacy support and tests calling through {@code BuildView} only. * *

If a requested configured target is in error, the corresponding value is omitted from the * returned list. */ @ThreadSafety.ThreadSafe // TODO(bazel-team): rename this and below methods to something that discourages general use public ImmutableList getConfiguredTargets( ExtendedEventHandler eventHandler, BuildConfiguration originalConfig, Iterable keys, boolean useOriginalConfig) { return getConfiguredTargetMap( eventHandler, originalConfig, keys, useOriginalConfig).values().asList(); } /** * Returns a map from {@link Dependency} inputs to the {@link ConfiguredTarget}s corresponding to * those dependencies. * *

For use for legacy support and tests calling through {@code BuildView} only. * *

If a requested configured target is in error, the corresponding value is omitted from the * returned list. */ @ThreadSafety.ThreadSafe public ImmutableMultimap getConfiguredTargetMap( ExtendedEventHandler eventHandler, BuildConfiguration originalConfig, Iterable keys, boolean useOriginalConfig) { checkActive(); Multimap configs; if (originalConfig != null) { if (useOriginalConfig) { // This flag is used because of some unfortunate complexity in the configuration machinery: // Most callers of this method pass a pair to directly create a // ConfiguredTarget from, but happen to use the Dependency data structure to pass that // info (even though the data has nothing to do with dependencies). If this configuration // includes a split transition, a dynamic configuration created from it will *not* // include that transition (because dynamic configurations don't embed transitions to // other configurations. In that case, we need to preserve the original configuration. // TODO(bazel-team); make this unnecessary once split transition logic is properly ported // out of configurations. configs = ArrayListMultimap.create(); configs.put(Iterables.getOnlyElement(keys), originalConfig); } else { configs = getConfigurations(eventHandler, originalConfig.getOptions(), keys); } } else { configs = ArrayListMultimap.create(); for (Dependency key : keys) { configs.put(key, null); } } final List skyKeys = new ArrayList<>(); for (Dependency key : keys) { if (!configs.containsKey(key)) { // If we couldn't compute a configuration for this target, the target was in error (e.g. // it couldn't be loaded). Exclude it from the results. continue; } for (BuildConfiguration depConfig : configs.get(key)) { skyKeys.add(ConfiguredTargetValue.key(key.getLabel(), depConfig)); for (AspectDescriptor aspectDescriptor : key.getAspects().getAllAspects()) { skyKeys.add(ActionLookupValue.key(AspectValue.createAspectKey(key.getLabel(), depConfig, aspectDescriptor, depConfig))); } } } EvaluationResult result = evaluateSkyKeys(eventHandler, skyKeys); for (Map.Entry entry : result.errorMap().entrySet()) { reportCycles(eventHandler, entry.getValue().getCycleInfo(), entry.getKey()); } ImmutableMultimap.Builder cts = ImmutableMultimap.builder(); DependentNodeLoop: for (Dependency key : keys) { if (!configs.containsKey(key)) { // If we couldn't compute a configuration for this target, the target was in error (e.g. // it couldn't be loaded). Exclude it from the results. continue; } for (BuildConfiguration depConfig : configs.get(key)) { SkyKey configuredTargetKey = ConfiguredTargetValue.key( key.getLabel(), depConfig); if (result.get(configuredTargetKey) == null) { continue; } ConfiguredTarget configuredTarget = ((ConfiguredTargetValue) result.get(configuredTargetKey)).getConfiguredTarget(); List configuredAspects = new ArrayList<>(); for (AspectDescriptor aspectDescriptor : key.getAspects().getAllAspects()) { SkyKey aspectKey = ActionLookupValue.key(AspectValue.createAspectKey(key.getLabel(), depConfig, aspectDescriptor, depConfig)); if (result.get(aspectKey) == null) { continue DependentNodeLoop; } configuredAspects.add(((AspectValue) result.get(aspectKey)).getConfiguredAspect()); } try { cts.put(key, MergedConfiguredTarget.of(configuredTarget, configuredAspects)); } catch (DuplicateException e) { throw new IllegalStateException( String.format("Error creating %s", configuredTarget.getTarget().getLabel()), e); } } } return cts.build(); } /** * Returns the configuration corresponding to the given set of build options. * * @throws InvalidConfigurationException if the build options produces an invalid configuration */ public BuildConfiguration getConfiguration(ExtendedEventHandler eventHandler, BuildOptions options) throws InvalidConfigurationException { return Iterables.getOnlyElement( getConfigurations(eventHandler, ImmutableList.of(options))); } /** * Returns the configurations corresponding to the given sets of build options. Output order is * the same as input order. * * @throws InvalidConfigurationException if any build options produces an invalid configuration */ public List getConfigurations(ExtendedEventHandler eventHandler, List optionsList) throws InvalidConfigurationException { Preconditions.checkArgument(!Iterables.isEmpty(optionsList)); // Prepare the Skyframe inputs. // TODO(gregce): support trimmed configs. Set> allFragments = configurationFragments.get() .stream() .map(factory -> factory.creates()) .collect(ImmutableSet.toImmutableSet()); final ImmutableList configSkyKeys = optionsList .stream() .map(elem -> BuildConfigurationValue.key(allFragments, elem)) .collect(ImmutableList.toImmutableList()); // Skyframe-evaluate the configurations and throw errors if any. EvaluationResult evalResult = evaluateSkyKeys(eventHandler, configSkyKeys, /*keepGoing=*/true); if (evalResult.hasError()) { Map.Entry firstError = Iterables.get(evalResult.errorMap().entrySet(), 0); ErrorInfo error = firstError.getValue(); Throwable e = error.getException(); // Wrap loading failed exceptions if (e instanceof NoSuchThingException) { e = new InvalidConfigurationException(e); } else if (e == null && !Iterables.isEmpty(error.getCycleInfo())) { getCyclesReporter().reportCycles(error.getCycleInfo(), firstError.getKey(), eventHandler); e = new InvalidConfigurationException( "cannot load build configuration because of this cycle"); } if (e != null) { Throwables.throwIfInstanceOf(e, InvalidConfigurationException.class); } throw new IllegalStateException( "Unknown error during ConfigurationCollectionValue evaluation", e); } // Prepare and return the results. return configSkyKeys .stream() .map(key -> ((BuildConfigurationValue) evalResult.get(key)).getConfiguration()) .collect(ImmutableList.toImmutableList()); } /** * Retrieves the configurations needed for the given deps. If {@link * BuildConfiguration.Options#trimConfigurations()} is true, trims their fragments to only those * needed by their transitive closures. Else unconditionally includes all fragments. * *

Skips targets with loading phase errors. */ public Multimap getConfigurations( ExtendedEventHandler eventHandler, BuildOptions fromOptions, Iterable keys) { Multimap builder = ArrayListMultimap.create(); Set depsToEvaluate = new HashSet<>(); // Check: if !Configuration.useDynamicConfigs then just return the original configs. Set> allFragments = null; if (useUntrimmedDynamicConfigs(fromOptions)) { allFragments = ((ConfiguredRuleClassProvider) ruleClassProvider).getAllFragments(); } // Get the fragments needed for dynamic configuration nodes. final List transitiveFragmentSkyKeys = new ArrayList<>(); Map>> fragmentsMap = new HashMap<>(); Set

May throw an {@link InterruptedException}, which means that no values have been invalidated. */ @VisibleForTesting public abstract void invalidateFilesUnderPathForTesting( ExtendedEventHandler eventHandler, ModifiedFileSet modifiedFileSet, Path pathEntry) throws InterruptedException; /** * Invalidates SkyFrame values that may have failed for transient reasons. */ public abstract void invalidateTransientErrors(); /** Configures a given set of configured targets. */ EvaluationResult configureTargets( ExtendedEventHandler eventHandler, List values, List aspectKeys, boolean keepGoing, int numThreads) throws InterruptedException { checkActive(); List keys = new ArrayList<>(ConfiguredTargetValue.keys(values)); for (AspectValueKey aspectKey : aspectKeys) { keys.add(aspectKey.getSkyKey()); } EvaluationResult result = buildDriver.evaluate(keys, keepGoing, numThreads, eventHandler); // Get rid of any memory retained by the cache -- all loading is done. perBuildSyscallCache.clear(); return result; } /** * Post-process the targets. Values in the EvaluationResult are known to be transitively * error-free from action conflicts. */ public EvaluationResult postConfigureTargets( ExtendedEventHandler eventHandler, List values, boolean keepGoing, ImmutableMap badActions) throws InterruptedException { checkActive(); PrecomputedValue.BAD_ACTIONS.set(injectable(), badActions); // Make sure to not run too many analysis threads. This can cause memory thrashing. EvaluationResult result = buildDriver.evaluate(PostConfiguredTargetValue.keys(values), keepGoing, ResourceUsage.getAvailableProcessors(), eventHandler); // Remove all post-configured target values immediately for memory efficiency. We are OK with // this mini-phase being non-incremental as the failure mode of action conflict is rare. memoizingEvaluator.delete(SkyFunctionName.functionIs(SkyFunctions.POST_CONFIGURED_TARGET)); return result; } /** * Returns a Skyframe-based {@link SkyframeTransitivePackageLoader} implementation. */ @VisibleForTesting public TransitivePackageLoader pkgLoader() { checkActive(); return new SkyframeLabelVisitor(new SkyframeTransitivePackageLoader(), cyclesReporter); } class SkyframeTransitivePackageLoader { /** Loads the specified {@link TransitiveTargetValue}s. */ EvaluationResult loadTransitiveTargets( ExtendedEventHandler eventHandler, Iterable

The graph update is unconditionally done in keep-going mode, so that the query is guaranteed * a complete graph to work on. */ @Override public EvaluationResult prepareAndGet( SkyKey universeKey, int numThreads, ExtendedEventHandler eventHandler) throws InterruptedException { EvaluationResult evaluationResult = buildDriver.evaluate(ImmutableList.of(universeKey), true, numThreads, eventHandler); Preconditions.checkNotNull(evaluationResult.getWalkableGraph(), universeKey); return evaluationResult; } @Override public boolean isUpToDate(SkyKey universeKey) { return buildDriver.alreadyEvaluated(ImmutableList.of(universeKey)); } /** * Get metadata related to the prepareAndGet() lookup. Resulting data is specific to the * underlying evaluation implementation. */ public String prepareAndGetMetadata(Collection patterns, String offset, OptionsClassProvider options) throws AbruptExitException, InterruptedException { return buildDriver.meta(ImmutableList.of(getUniverseKey(patterns, offset)), options); } @Override public SkyKey getUniverseKey(Collection patterns, String offset) { return PrepareDepsOfPatternsValue.key(ImmutableList.copyOf(patterns), offset); } /** Returns the generating action of a given artifact ({@code null} if it's a source artifact). */ private ActionAnalysisMetadata getGeneratingAction( ExtendedEventHandler eventHandler, Artifact artifact) throws InterruptedException { if (artifact.isSourceArtifact()) { return null; } ArtifactOwner artifactOwner = artifact.getArtifactOwner(); Preconditions.checkState(artifactOwner instanceof ActionLookupValue.ActionLookupKey, "%s %s", artifact, artifactOwner); SkyKey actionLookupKey = ActionLookupValue.key((ActionLookupValue.ActionLookupKey) artifactOwner); synchronized (valueLookupLock) { // Note that this will crash (attempting to run a configured target value builder after // analysis) after a failed --nokeep_going analysis in which the configured target that // failed was a (transitive) dependency of the configured target that should generate // this action. We don't expect callers to query generating actions in such cases. EvaluationResult result = buildDriver.evaluate( ImmutableList.of(actionLookupKey), false, ResourceUsage.getAvailableProcessors(), eventHandler); return result.hasError() ? null : result.get(actionLookupKey).getGeneratingActionDangerousReadJavadoc(artifact); } } /** * Returns an action graph. * *

For legacy compatibility only. */ public ActionGraph getActionGraph(final ExtendedEventHandler eventHandler) { return new ActionGraph() { @Override public ActionAnalysisMetadata getGeneratingAction(final Artifact artifact) { try { return callUninterruptibly(new Callable() { @Override public ActionAnalysisMetadata call() throws InterruptedException { return SkyframeExecutor.this.getGeneratingAction(eventHandler, artifact); } }); } catch (Exception e) { throw new IllegalStateException("Error getting generating action: " + artifact.prettyPrint(), e); } } }; } public PackageManager getPackageManager() { return packageManager; } class SkyframePackageLoader { /** * Looks up a particular package (mostly used after the loading phase, so packages should * already be present, but occasionally used pre-loading phase). Use should be discouraged, * since this cannot be used inside a Skyframe evaluation, and concurrent calls are * synchronized. * *

Note that this method needs to be synchronized since InMemoryMemoizingEvaluator.evaluate() * method does not support concurrent calls. */ Package getPackage(ExtendedEventHandler eventHandler, PackageIdentifier pkgName) throws InterruptedException, NoSuchPackageException { synchronized (valueLookupLock) { SkyKey key = PackageValue.key(pkgName); // Any call to this method post-loading phase should either be error-free or be in a // keep_going build, since otherwise the build would have failed during loading. Thus // we set keepGoing=true unconditionally. EvaluationResult result = buildDriver.evaluate(ImmutableList.of(key), /*keepGoing=*/true, DEFAULT_THREAD_COUNT, eventHandler); ErrorInfo error = result.getError(key); if (error != null) { if (!Iterables.isEmpty(error.getCycleInfo())) { reportCycles(eventHandler, result.getError().getCycleInfo(), key); // This can only happen if a package is freshly loaded outside of the target parsing // or loading phase throw new BuildFileContainsErrorsException( pkgName, "Cycle encountered while loading package " + pkgName); } Throwable e = error.getException(); // PackageFunction should be catching, swallowing, and rethrowing all transitive // errors as NoSuchPackageExceptions or constructing packages with errors, since we're in // keep_going mode. Throwables.propagateIfInstanceOf(e, NoSuchPackageException.class); throw new IllegalStateException("Unexpected Exception type from PackageValue for '" + pkgName + "'' with root causes: " + Iterables.toString(error.getRootCauses()), e); } return result.get(key).getPackage(); } } /** * Returns whether the given package should be consider deleted and thus should be ignored. */ public boolean isPackageDeleted(PackageIdentifier packageName) { Preconditions.checkState(!packageName.getRepository().isDefault(), "package must be absolute: %s", packageName); return deletedPackages.get().contains(packageName); } } @VisibleForTesting public MemoizingEvaluator getEvaluatorForTesting() { return memoizingEvaluator; } public void sync( ExtendedEventHandler eventHandler, PackageCacheOptions packageCacheOptions, SkylarkSemanticsOptions skylarkSemanticsOptions, Path outputBase, Path workingDirectory, String defaultsPackageContents, UUID commandId, Map clientEnv, TimestampGranularityMonitor tsgm, OptionsClassProvider options) throws InterruptedException, AbruptExitException { // ImmutableMap does not support null values, so use a LinkedHashMap instead. LinkedHashMap actionEnvironment = new LinkedHashMap<>(); BuildConfiguration.Options opt = options.getOptions(BuildConfiguration.Options.class); if (opt != null) { for (Entry v : opt.actionEnvironment) { actionEnvironment.put(v.getKey(), v.getValue()); } } preparePackageLoading( createPackageLocator( eventHandler, packageCacheOptions, outputBase, directories.getWorkspace(), workingDirectory), packageCacheOptions, skylarkSemanticsOptions, defaultsPackageContents, commandId, clientEnv, Collections.unmodifiableMap(actionEnvironment), tsgm); setDeletedPackages(packageCacheOptions.getDeletedPackages()); incrementalBuildMonitor = new SkyframeIncrementalBuildMonitor(); invalidateTransientErrors(); } protected PathPackageLocator createPackageLocator( ExtendedEventHandler eventHandler, PackageCacheOptions packageCacheOptions, Path outputBase, Path workspace, Path workingDirectory) throws AbruptExitException { return PathPackageLocator.create( outputBase, packageCacheOptions.packagePath, eventHandler, workspace, workingDirectory); } private CyclesReporter createCyclesReporter() { return new CyclesReporter( new TransitiveTargetCycleReporter(packageManager), new ActionArtifactCycleReporter(packageManager), // TODO(ulfjack): The SkylarkModuleCycleReporter swallows previously reported cycles // unconditionally! Is that intentional? new ConfiguredTargetCycleReporter(packageManager), new SkylarkModuleCycleReporter()); } CyclesReporter getCyclesReporter() { return cyclesReporter.get(); } /** Convenience method with same semantics as {@link CyclesReporter#reportCycles}. */ public void reportCycles( ExtendedEventHandler eventHandler, Iterable cycles, SkyKey topLevelKey) { getCyclesReporter().reportCycles(cycles, topLevelKey, eventHandler); } public void setActionExecutionProgressReportingObjects(@Nullable ProgressSupplier supplier, @Nullable ActionCompletedReceiver completionReceiver, @Nullable ActionExecutionStatusReporter statusReporter) { skyframeActionExecutor.setActionExecutionProgressReportingObjects(supplier, completionReceiver); this.statusReporterRef.set(statusReporter); } public void prepareExecution(ModifiedFileSet modifiedOutputFiles, @Nullable Range lastExecutionTimeRange) throws AbruptExitException, InterruptedException { maybeInjectEmbeddedArtifacts(); // Detect external modifications in the output tree. FilesystemValueChecker fsvc = new FilesystemValueChecker(Preconditions.checkNotNull(tsgm.get()), lastExecutionTimeRange); BatchStat batchStatter = outputService == null ? null : outputService.getBatchStatter(); invalidateDirtyActions(fsvc.getDirtyActionValues(memoizingEvaluator.getValues(), batchStatter, modifiedOutputFiles)); modifiedFiles += fsvc.getNumberOfModifiedOutputFiles(); outputDirtyFiles += fsvc.getNumberOfModifiedOutputFiles(); modifiedFilesDuringPreviousBuild += fsvc.getNumberOfModifiedOutputFilesDuringPreviousBuild(); informAboutNumberOfModifiedFiles(); } protected abstract void invalidateDirtyActions(Iterable dirtyActionValues); @VisibleForTesting void maybeInjectEmbeddedArtifacts() throws AbruptExitException { // The blaze client already ensures that the contents of the embedded binaries never change, // so we just need to make sure that the appropriate artifacts are present in the skyframe // graph. if (!needToInjectEmbeddedArtifacts) { return; } Preconditions.checkNotNull(artifactFactory.get()); Preconditions.checkNotNull(binTools); Map values = Maps.newHashMap(); // Blaze separately handles the symlinks that target these binaries. See BinTools#setupTool. for (Artifact artifact : binTools.getAllEmbeddedArtifacts(artifactFactory.get())) { FileArtifactValue fileArtifactValue; try { fileArtifactValue = FileArtifactValue.create(artifact); } catch (IOException e) { // See ExtractData in blaze.cc. String message = "Error: corrupt installation: file " + artifact.getPath() + " missing. " + "Please remove '" + directories.getInstallBase() + "' and try again."; throw new AbruptExitException(message, ExitCode.LOCAL_ENVIRONMENTAL_ERROR, e); } values.put(ArtifactSkyKey.key(artifact, /*isMandatory=*/ true), fileArtifactValue); } injectable().inject(values); needToInjectEmbeddedArtifacts = false; } /** * Mark dirty values for deletion if they've been dirty for longer than N versions. * *

Specifying a value N means, if the current version is V and a value was dirtied (and * has remained so) in version U, and U + N <= V, then the value will be marked for deletion * and purged in version V+1. */ public abstract void deleteOldNodes(long versionWindowForDirtyGc); public LoadingPhaseRunner getLoadingPhaseRunner(Set ruleClassNames, boolean useNewImpl) { if (!useNewImpl) { return new LegacyLoadingPhaseRunner(packageManager, ruleClassNames); } else { return new SkyframeLoadingPhaseRunner(ruleClassNames); } } /** * Skyframe-based implementation of {@link LoadingPhaseRunner} based on {@link * TargetPatternPhaseFunction}. */ final class SkyframeLoadingPhaseRunner extends LoadingPhaseRunner { private final Set ruleClassNames; public SkyframeLoadingPhaseRunner(Set ruleClassNames) { this.ruleClassNames = ruleClassNames; } @Override public LoadingResult execute( ExtendedEventHandler eventHandler, List targetPatterns, PathFragment relativeWorkingDirectory, LoadingOptions options, boolean keepGoing, boolean determineTests, @Nullable LoadingCallback callback) throws TargetParsingException, LoadingFailedException, InterruptedException { Stopwatch timer = Stopwatch.createStarted(); SkyKey key = TargetPatternPhaseValue.key(ImmutableList.copyOf(targetPatterns), relativeWorkingDirectory.getPathString(), options.compileOneDependency, options.buildTestsOnly, determineTests, ImmutableList.copyOf(options.buildTagFilterList), options.buildManualTests, TestFilter.forOptions(options, eventHandler, ruleClassNames)); EvaluationResult evalResult; eventHandler.post(new LoadingPhaseStartedEvent(packageProgress)); evalResult = buildDriver.evaluate(ImmutableList.of(key), keepGoing, /*numThreads=*/ 10, eventHandler); if (evalResult.hasError()) { ErrorInfo errorInfo = evalResult.getError(key); if (!Iterables.isEmpty(errorInfo.getCycleInfo())) { String errorMessage = "cycles detected during target parsing"; getCyclesReporter().reportCycles(errorInfo.getCycleInfo(), key, eventHandler); throw new TargetParsingException(errorMessage); } if (errorInfo.getException() != null) { Exception e = errorInfo.getException(); Throwables.propagateIfInstanceOf(e, TargetParsingException.class); if (!keepGoing) { // This is the same code as in SkyframeTargetPatternEvaluator; we allow any exception // and turn it into a TargetParsingException here. throw new TargetParsingException(e.getMessage()); } throw new IllegalStateException("Unexpected Exception type from TargetPatternPhaseValue " + "for '" + targetPatterns + "'' with root causes: " + Iterables.toString(errorInfo.getRootCauses()), e); } } long time = timer.stop().elapsed(TimeUnit.MILLISECONDS); TargetPatternPhaseValue patternParsingValue = evalResult.get(key); eventHandler.post( new TargetParsingCompleteEvent( patternParsingValue.getOriginalTargets(), patternParsingValue.getFilteredTargets(), patternParsingValue.getTestFilteredTargets(), time, targetPatterns, patternParsingValue.getTargets())); if (callback != null) { callback.notifyTargets(patternParsingValue.getTargets()); } eventHandler.post(new LoadingPhaseCompleteEvent( patternParsingValue.getTargets(), patternParsingValue.getTestSuiteTargets(), PackageManagerStatistics.ZERO, /*timeInMs=*/0)); return patternParsingValue.toLoadingResult(); } } /** * A progress received to track analysis invalidation and update progress messages. */ protected class SkyframeProgressReceiver extends EvaluationProgressReceiver.NullEvaluationProgressReceiver { /** * This flag is needed in order to avoid invalidating legacy data when we clear the * analysis cache because of --discard_analysis_cache flag. For that case we want to keep * the legacy data but get rid of the Skyframe data. */ protected boolean ignoreInvalidations = false; /** This receiver is only needed for execution, so it is null otherwise. */ @Nullable EvaluationProgressReceiver executionProgressReceiver = null; /** This receiver is only needed for loading, so it is null otherwise. */ @Override public void invalidated(SkyKey skyKey, InvalidationState state) { if (ignoreInvalidations) { return; } skyframeBuildView.getProgressReceiver().invalidated(skyKey, state); } @Override public void enqueueing(SkyKey skyKey) { if (ignoreInvalidations) { return; } skyframeBuildView.getProgressReceiver().enqueueing(skyKey); if (executionProgressReceiver != null) { executionProgressReceiver.enqueueing(skyKey); } } @Override public void evaluated(SkyKey skyKey, Supplier valueSupplier, EvaluationState state) { if (ignoreInvalidations) { return; } skyframeBuildView.getProgressReceiver().evaluated(skyKey, valueSupplier, state); if (executionProgressReceiver != null) { executionProgressReceiver.evaluated(skyKey, valueSupplier, state); } } } public int getOutputDirtyFilesAndClear() { int result = outputDirtyFiles; outputDirtyFiles = 0; return result; } public int getModifiedFilesDuringPreviousBuildAndClear() { int result = modifiedFilesDuringPreviousBuild; modifiedFilesDuringPreviousBuild = 0; return result; } }