diff options
Diffstat (limited to 'src/main/java/com/google')
6 files changed, 148 insertions, 59 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/BuildView.java b/src/main/java/com/google/devtools/build/lib/analysis/BuildView.java index bf942365d1..65c4a05667 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/BuildView.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/BuildView.java @@ -154,25 +154,25 @@ public class BuildView { abbrev = 'k', defaultValue = "false", category = "strategy", - help = "Continue as much as possible after an error. While the " - + "target that failed, and those that depend on it, cannot be " - + "analyzed (or built), the other prerequisites of these " - + "targets can be analyzed (or built) all the same.") + help = "Continue as much as possible after an error. While the" + + " target that failed, and those that depend on it, cannot be" + + " analyzed (or built), the other prerequisites of these" + + " targets can be analyzed (or built) all the same.") public boolean keepGoing; @Option(name = "analysis_warnings_as_errors", deprecationWarning = "analysis_warnings_as_errors is now a no-op and will be removed in" - + " an upcoming Blaze release", + + " an upcoming Blaze release", defaultValue = "false", category = "strategy", help = "Treat visible analysis warnings as errors.") public boolean analysisWarningsAsErrors; @Option(name = "discard_analysis_cache", - defaultValue = "false", - category = "strategy", - help = "Discard the analysis cache immediately after the analysis phase completes. " - + "Reduces memory usage by ~10%, but makes further incremental builds slower.") + defaultValue = "false", + category = "strategy", + help = "Discard the analysis cache immediately after the analysis phase completes." + + " Reduces memory usage by ~10%, but makes further incremental builds slower.") public boolean discardAnalysisCache; @Option(name = "experimental_extra_action_filter", @@ -188,6 +188,13 @@ public class BuildView { help = "Only schedules extra_actions for top level targets.") public boolean extraActionTopLevelOnly; + @Option(name = "experimental_interleave_loading_and_analysis", + defaultValue = "false", + category = "experimental", + help = "Interleave loading and analysis phases, so that one target may be analyzed at" + + " the same time as an unrelated target is loaded.") + public boolean interleaveLoadingAndAnalysis; + @Option(name = "version_window_for_dirty_node_gc", defaultValue = "0", category = "undocumented", @@ -455,7 +462,8 @@ public class BuildView { ImmutableList.<Artifact>of(), ImmutableList.<ConfiguredTarget>of(), ImmutableList.<ConfiguredTarget>of(), - null); + null, + ImmutableMap.<PackageIdentifier, Path>of()); private final ImmutableList<ConfiguredTarget> targetsToBuild; @Nullable private final ImmutableList<ConfiguredTarget> targetsToTest; @@ -466,6 +474,7 @@ public class BuildView { private final ImmutableSet<ConfiguredTarget> exclusiveTests; @Nullable private final TopLevelArtifactContext topLevelContext; private final ImmutableList<AspectValue> aspects; + private final ImmutableMap<PackageIdentifier, Path> packageRoots; private AnalysisResult( Collection<ConfiguredTarget> targetsToBuild, @@ -476,7 +485,8 @@ public class BuildView { Collection<Artifact> artifactsToBuild, Collection<ConfiguredTarget> parallelTests, Collection<ConfiguredTarget> exclusiveTests, - TopLevelArtifactContext topLevelContext) { + TopLevelArtifactContext topLevelContext, + ImmutableMap<PackageIdentifier, Path> packageRoots) { this.targetsToBuild = ImmutableList.copyOf(targetsToBuild); this.aspects = ImmutableList.copyOf(aspects); this.targetsToTest = targetsToTest == null ? null : ImmutableList.copyOf(targetsToTest); @@ -486,6 +496,7 @@ public class BuildView { this.parallelTests = ImmutableSet.copyOf(parallelTests); this.exclusiveTests = ImmutableSet.copyOf(exclusiveTests); this.topLevelContext = topLevelContext; + this.packageRoots = packageRoots; } /** @@ -496,6 +507,14 @@ public class BuildView { } /** + * The map from package names to the package root where each package was found; this is used to + * set up the symlink tree. + */ + public ImmutableMap<PackageIdentifier, Path> getPackageRoots() { + return packageRoots; + } + + /** * Returns aspects of configured targets to build. * * <p>If this list is empty, build the targets returned by {@code getTargetsToBuild()}. @@ -581,7 +600,8 @@ public class BuildView { Options viewOptions, TopLevelArtifactContext topLevelOptions, EventHandler eventHandler, - EventBus eventBus) + EventBus eventBus, + boolean loadingEnabled) throws ViewCreationFailedException, InterruptedException { LOG.info("Starting analysis"); pollInterruptedStatus(); @@ -609,10 +629,9 @@ public class BuildView { clear(); } skyframeAnalysisWasDiscarded = false; - ImmutableMap<PackageIdentifier, Path> packageRoots = loadingResult.getPackageRoots(); this.configurations = configurations; skyframeBuildView.setTopLevelHostConfiguration(this.configurations.getHostConfiguration()); - setArtifactRoots(packageRoots); + // Determine the configurations. List<TargetAndConfiguration> nodes = nodesForTargets(targets); @@ -641,6 +660,12 @@ public class BuildView { } } + // Configuration of some BuildConfiguration.Fragments may require information about + // artifactRoots, so we need to set them before calling prepareToBuild. In that case loading + // phase has to be enabled. + if (loadingEnabled) { + setArtifactRoots(loadingResult.getPackageRoots()); + } prepareToBuild(new SkyframePackageRootResolver(skyframeExecutor)); skyframeExecutor.injectWorkspaceStatusData(); SkyframeAnalysisResult skyframeAnalysisResult; @@ -648,6 +673,7 @@ public class BuildView { skyframeAnalysisResult = skyframeBuildView.configureTargets( targetSpecs, aspectKeys, eventBus, viewOptions.keepGoing); + setArtifactRoots(skyframeAnalysisResult.getPackageRoots()); } finally { skyframeBuildView.clearInvalidatedConfiguredTargets(); } @@ -670,6 +696,7 @@ public class BuildView { skyframeAnalysisResult.getConfiguredTargets(), skyframeAnalysisResult.getAspects(), skyframeAnalysisResult.getWalkableGraph(), + skyframeAnalysisResult.getPackageRoots(), analysisSuccessful); LOG.info("Finished analysis"); return result; @@ -682,6 +709,7 @@ public class BuildView { Collection<ConfiguredTarget> configuredTargets, Collection<AspectValue> aspects, final WalkableGraph graph, + ImmutableMap<PackageIdentifier, Path> packageRoots, boolean analysisSuccessful) throws InterruptedException { Collection<Target> testsToRun = loadingResult.getTestsToRun(); @@ -752,7 +780,8 @@ public class BuildView { artifactsToBuild, parallelTests, exclusiveTests, - topLevelOptions); + topLevelOptions, + packageRoots); } private static NestedSet<Artifact> getBaselineCoverageArtifacts( diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java index af5e23899c..d646b5148a 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java @@ -399,7 +399,7 @@ public final class BuildTool { LoadingResult result = runtime.getLoadingPhaseRunner().execute(getReporter(), env.getEventBus(), request.getTargets(), request.getLoadingOptions(), runtime.createBuildOptions(request).getAllLabels(), keepGoing, - request.shouldRunTests(), callback); + isLoadingEnabled(request), request.shouldRunTests(), callback); runtime.throwPendingException(); return result; } @@ -448,7 +448,8 @@ public final class BuildTool { request.getViewOptions(), request.getTopLevelArtifactContext(), env.getReporter(), - env.getEventBus()); + env.getEventBus(), + isLoadingEnabled(request)); // TODO(bazel-team): Merge these into one event. env.getEventBus().post(new AnalysisPhaseCompleteEvent(analysisResult.getTargetsToBuild(), @@ -583,7 +584,7 @@ public final class BuildTool { if (!keepGoing) { throw new ViewCreationFailedException("Build aborted due to licensing error"); } - } + } } } } @@ -591,4 +592,11 @@ public final class BuildTool { private Reporter getReporter() { return env.getReporter(); } + + private static boolean isLoadingEnabled(BuildRequest request) { + boolean enableLoadingFlag = !request.getViewOptions().interleaveLoadingAndAnalysis; + // TODO(bazel-team): should return false when fdo optimization is enabled, because in that case, + // we would require packages to be set before analysis phase. See FdoSupport#prepareToBuild. + return enableLoadingFlag; + } } diff --git a/src/main/java/com/google/devtools/build/lib/pkgcache/LoadingPhaseRunner.java b/src/main/java/com/google/devtools/build/lib/pkgcache/LoadingPhaseRunner.java index 97f336f09d..8006899a3b 100644 --- a/src/main/java/com/google/devtools/build/lib/pkgcache/LoadingPhaseRunner.java +++ b/src/main/java/com/google/devtools/build/lib/pkgcache/LoadingPhaseRunner.java @@ -290,7 +290,7 @@ public class LoadingPhaseRunner { public LoadingResult execute(EventHandler eventHandler, EventBus eventBus, List<String> targetPatterns, Options options, ListMultimap<String, Label> labelsToLoadUnconditionally, boolean keepGoing, - boolean determineTests, @Nullable Callback callback) + boolean enableLoading, boolean determineTests, @Nullable Callback callback) throws TargetParsingException, LoadingFailedException, InterruptedException { LOG.info("Starting pattern evaluation"); Stopwatch timer = Stopwatch.createStarted(); @@ -371,14 +371,20 @@ public class LoadingPhaseRunner { callback.notifyTargets(targets.getTargets()); } maybeReportDeprecation(eventHandler, targets.getTargets()); - - BaseLoadingResult result = doLoadingPhase(eventHandler, eventBus, targets.getTargets(), - labelsToLoadUnconditionally, keepGoing, options.loadingPhaseThreads); - LoadingResult loadingResult = new LoadingResult(targets.hasError(), !result.isSuccesful(), - result.getTargets(), testsToRun, - collectPackageRoots(pkgLoader.getErrorFreeVisitedPackages())); - freeMemoryAfterLoading(callback, pkgLoader.getVisitedPackageNames()); - return loadingResult; + BaseLoadingResult result; + if (enableLoading) { + result = doLoadingPhase(eventHandler, eventBus, targets.getTargets(), + labelsToLoadUnconditionally, keepGoing, options.loadingPhaseThreads); + LoadingResult loadingResult = new LoadingResult(targets.hasError(), !result.isSuccesful(), + result.getTargets(), testsToRun, + collectPackageRoots(pkgLoader.getErrorFreeVisitedPackages())); + freeMemoryAfterLoading(callback, pkgLoader.getVisitedPackageNames()); + return loadingResult; + } else { + result = doSimpleLoadingPhase(eventHandler, eventBus, targets.getTargets(), keepGoing); + return new LoadingResult(targets.hasError(), !result.isSuccesful(), result.getTargets(), + testsToRun, collectPackageRoots(new ArrayList<Package>())); + } } private void freeMemoryAfterLoading(Callback callback, Set<PackageIdentifier> visitedPackages) { @@ -389,6 +395,27 @@ public class LoadingPhaseRunner { packageManager.partiallyClear(); } + /** + * Simplified version of {@code doLoadingPhase} method. This method does not load targets. + * It only does test_suite expansion and emits necessary events and logging messages for legacy + * support. + */ + private BaseLoadingResult doSimpleLoadingPhase(EventHandler eventHandler, EventBus eventBus, + ImmutableSet<Target> targetsToLoad, boolean keepGoing) + throws LoadingFailedException { + Stopwatch timer = preLoadingLogging(eventHandler); + + BaseLoadingResult expandedResult; + try { + expandedResult = expandTestSuites(eventHandler, targetsToLoad, keepGoing); + } catch (TargetParsingException e) { + throw new LoadingFailedException("Loading failed; build aborted", e); + } + + postLoadingLogging(eventBus, targetsToLoad, expandedResult.getTargets(), timer); + return new BaseLoadingResult(expandedResult.getTargets(), expandedResult.isSuccesful()); + } + /** * Visit the transitive closure of the targets, populating the package cache * and ensuring that all labels can be resolved and all rules were free from @@ -404,25 +431,38 @@ public class LoadingPhaseRunner { ImmutableSet<Target> targetsToLoad, ListMultimap<String, Label> labelsToLoadUnconditionally, boolean keepGoing, int loadingPhaseThreads) throws InterruptedException, LoadingFailedException { - eventHandler.handle(Event.progress("Loading...")); - Stopwatch timer = Stopwatch.createStarted(); - LOG.info("Starting loading phase"); + Stopwatch timer = preLoadingLogging(eventHandler); BaseLoadingResult baseResult = performLoadingOfTargets(eventHandler, eventBus, targetsToLoad, labelsToLoadUnconditionally, keepGoing, loadingPhaseThreads); - BaseLoadingResult expandedResult = expandTestSuites(eventHandler, baseResult.getTargets(), - keepGoing); - - Set<Target> testSuiteTargets = Sets.difference(baseResult.getTargets(), - expandedResult.getTargets()); - eventBus.post(new LoadingPhaseCompleteEvent(expandedResult.getTargets(), testSuiteTargets, - packageManager.getStatistics(), timer.stop().elapsed(TimeUnit.MILLISECONDS))); - LOG.info("Loading phase finished"); + BaseLoadingResult expandedResult; + try { + expandedResult = expandTestSuites(eventHandler, baseResult.getTargets(), + keepGoing); + } catch (TargetParsingException e) { + // This shouldn't happen, because we've already loaded the targets successfully. + throw (AssertionError) (new AssertionError("Unexpected target failure").initCause(e)); + } + postLoadingLogging(eventBus, baseResult.getTargets(), expandedResult.getTargets(), timer); return new BaseLoadingResult(expandedResult.getTargets(), baseResult.isSuccesful() && expandedResult.isSuccesful()); } + private Stopwatch preLoadingLogging(EventHandler eventHandler) { + eventHandler.handle(Event.progress("Loading...")); + LOG.info("Starting loading phase"); + return Stopwatch.createStarted(); + } + + private void postLoadingLogging(EventBus eventBus, ImmutableSet<Target> originalTargetsToLoad, + ImmutableSet<Target> expandedTargetsToLoad, Stopwatch timer) { + Set<Target> testSuiteTargets = Sets.difference(originalTargetsToLoad, expandedTargetsToLoad); + eventBus.post(new LoadingPhaseCompleteEvent(expandedTargetsToLoad, testSuiteTargets, + packageManager.getStatistics(), timer.stop().elapsed(TimeUnit.MILLISECONDS))); + LOG.info("Loading phase finished"); + } + private BaseLoadingResult performLoadingOfTargets(EventHandler eventHandler, EventBus eventBus, ImmutableSet<Target> targetsToLoad, ListMultimap<String, Label> labelsToLoadUnconditionally, boolean keepGoing, int loadingPhaseThreads) throws InterruptedException, @@ -468,19 +508,15 @@ public class LoadingPhaseRunner { } private BaseLoadingResult expandTestSuites(EventHandler eventHandler, - ImmutableSet<Target> targets, boolean keepGoing) throws LoadingFailedException { - try { - // We use strict test_suite expansion here to match the analysis-time checks. - ResolvedTargets<Target> expandedResult = TestTargetUtils.expandTestSuites( - packageManager, eventHandler, targets, /*strict=*/true, /*keepGoing=*/true); - if (expandedResult.hasError() && !keepGoing) { - throw new LoadingFailedException("Could not expand test suite target"); - } - return new BaseLoadingResult(expandedResult.getTargets(), !expandedResult.hasError()); - } catch (TargetParsingException e) { - // This shouldn't happen, because we've already loaded the targets successfully. - throw (AssertionError) (new AssertionError("Unexpected target failure").initCause(e)); + ImmutableSet<Target> targets, boolean keepGoing) + throws LoadingFailedException, TargetParsingException { + // We use strict test_suite expansion here to match the analysis-time checks. + ResolvedTargets<Target> expandedResult = TestTargetUtils.expandTestSuites( + packageManager, eventHandler, targets, /*strict=*/true, /*keepGoing=*/true); + if (expandedResult.hasError() && !keepGoing) { + throw new LoadingFailedException("Could not expand test suite target"); } + return new BaseLoadingResult(expandedResult.getTargets(), !expandedResult.hasError()); } private static class BaseLoadingResult { @@ -543,7 +579,7 @@ public class LoadingPhaseRunner { /** * Returns a map of collected package names to root paths. */ - private static ImmutableMap<PackageIdentifier, Path> collectPackageRoots( + public static ImmutableMap<PackageIdentifier, Path> collectPackageRoots( Collection<Package> packages) { // Make a map of the package names to their root paths. ImmutableMap.Builder<PackageIdentifier, Path> packageRoots = ImmutableMap.builder(); diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeAnalysisResult.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeAnalysisResult.java index 323dc3d88a..acd6d747ce 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeAnalysisResult.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeAnalysisResult.java @@ -14,7 +14,10 @@ package com.google.devtools.build.lib.skyframe; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.cmdline.PackageIdentifier; +import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.skyframe.WalkableGraph; import java.util.Collection; @@ -26,14 +29,17 @@ public class SkyframeAnalysisResult { private final ImmutableList<ConfiguredTarget> configuredTargets; private final WalkableGraph walkableGraph; private final ImmutableList<AspectValue> aspects; + private final ImmutableMap<PackageIdentifier, Path> packageRoots; public SkyframeAnalysisResult( ImmutableList<ConfiguredTarget> configuredTargets, WalkableGraph walkableGraph, - ImmutableList<AspectValue> aspects) { + ImmutableList<AspectValue> aspects, + ImmutableMap<PackageIdentifier, Path> packageRoots) { this.configuredTargets = configuredTargets; this.walkableGraph = walkableGraph; this.aspects = aspects; + this.packageRoots = packageRoots; } public Collection<ConfiguredTarget> getConfiguredTargets() { @@ -47,4 +53,8 @@ public class SkyframeAnalysisResult { public Collection<AspectValue> getAspects() { return aspects; } + + public ImmutableMap<PackageIdentifier, Path> getPackageRoots() { + return packageRoots; + } } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeBuildView.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeBuildView.java index e507d6697d..40bbeb4f9e 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeBuildView.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeBuildView.java @@ -46,12 +46,15 @@ import com.google.devtools.build.lib.analysis.buildinfo.BuildInfoFactory.BuildIn import com.google.devtools.build.lib.analysis.config.BinTools; import com.google.devtools.build.lib.analysis.config.BuildConfiguration; import com.google.devtools.build.lib.analysis.config.ConfigMatchingProvider; +import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.EventHandler; import com.google.devtools.build.lib.packages.AspectParameters; import com.google.devtools.build.lib.packages.Attribute; +import com.google.devtools.build.lib.packages.Package; import com.google.devtools.build.lib.packages.RuleClassProvider; import com.google.devtools.build.lib.packages.Target; +import com.google.devtools.build.lib.pkgcache.LoadingPhaseRunner; import com.google.devtools.build.lib.skyframe.ActionLookupValue.ActionLookupKey; import com.google.devtools.build.lib.skyframe.AspectValue.AspectKey; import com.google.devtools.build.lib.skyframe.BuildInfoCollectionValue.BuildInfoKeyAndConfig; @@ -205,6 +208,7 @@ public final class SkyframeBuildView { ImmutableMap<Action, ConflictException> badActions = skyframeExecutor.findArtifactConflicts(); Collection<AspectValue> goodAspects = Lists.newArrayListWithCapacity(values.size()); + NestedSetBuilder<Package> packages = NestedSetBuilder.stableOrder(); for (AspectKey aspectKey : aspectKeys) { AspectValue value = (AspectValue) result.get(AspectValue.key(aspectKey)); if (value == null) { @@ -212,6 +216,7 @@ public final class SkyframeBuildView { continue; } goodAspects.add(value); + packages.addTransitive(value.getTransitivePackages()); } // Filter out all CTs that have a bad action and convert to a list of configured targets. This @@ -225,15 +230,16 @@ public final class SkyframeBuildView { continue; } goodCts.add(ctValue.getConfiguredTarget()); + packages.addTransitive(ctValue.getTransitivePackages()); } - if (!result.hasError() && badActions.isEmpty()) { setDeserializedArtifactOwners(); return new SkyframeAnalysisResult( ImmutableList.copyOf(goodCts), result.getWalkableGraph(), - ImmutableList.copyOf(goodAspects)); + ImmutableList.copyOf(goodAspects), + LoadingPhaseRunner.collectPackageRoots(packages.build().toCollection())); } // --nokeep_going so we fail with an exception for the first error. @@ -341,7 +347,8 @@ public final class SkyframeBuildView { return new SkyframeAnalysisResult( ImmutableList.copyOf(goodCts), result.getWalkableGraph(), - ImmutableList.copyOf(goodAspects)); + ImmutableList.copyOf(goodAspects), + LoadingPhaseRunner.collectPackageRoots(packages.build().toCollection())); } @Nullable diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java index 050497fc64..9a4f082abd 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java @@ -1287,17 +1287,16 @@ public abstract class SkyframeExecutor implements WalkableGraphFactory { /** Configures a given set of configured targets. */ public EvaluationResult<ActionLookupValue> configureTargets( List<ConfiguredTargetKey> values, List<AspectKey> aspectKeys, boolean keepGoing) - throws InterruptedException { + throws InterruptedException { checkActive(); List<SkyKey> keys = new ArrayList<>(ConfiguredTargetValue.keys(values)); for (AspectKey aspectKey : aspectKeys) { keys.add(AspectValue.key(aspectKey)); } - // Make sure to not run too many analysis threads. This can cause memory thrashing. - return buildDriver.evaluate( - keys, keepGoing, ResourceUsage.getAvailableProcessors(), errorEventListener); + return buildDriver.evaluate(keys, keepGoing, ResourceUsage.getAvailableProcessors(), + errorEventListener); } /** |