From 93e3eeadc24197c62a12fc843db319cb58d98d84 Mon Sep 17 00:00:00 2001 From: janakr Date: Thu, 30 Mar 2017 22:09:37 +0000 Subject: If --batch, --keep_going, --discard_analysis_cache, and the new --noexperimental_enable_critical_path_profiling flags are all specified, then Bazel will delete Actions from ActionLookupValues as they are executed in order to save memory. Because an already-run action may output an artifact that is only requested later in the build, we need to maintain a way for the artifact to look up the action. But in most cases we don't need to keep the action itself, just its output metadata. Some actions unfortunately are needed post-execution, and so we special-case them. Also includes dependency change with description: Move action out of key. This keeps action references from polluting the graph -- actions are just stored in one SkyValue, instead of being present in SkyKeys. This does mean additional memory used: we have a separate ActionLookupData object per Action executed. That may reach ~24M for million-action builds. PiperOrigin-RevId: 151756383 --- .../google/devtools/build/lib/actions/Actions.java | 63 +++++++++++++++------- 1 file changed, 43 insertions(+), 20 deletions(-) (limited to 'src/main/java/com/google/devtools/build/lib/actions/Actions.java') diff --git a/src/main/java/com/google/devtools/build/lib/actions/Actions.java b/src/main/java/com/google/devtools/build/lib/actions/Actions.java index b8bd981156..8a49af4bad 100644 --- a/src/main/java/com/google/devtools/build/lib/actions/Actions.java +++ b/src/main/java/com/google/devtools/build/lib/actions/Actions.java @@ -14,6 +14,7 @@ package com.google.devtools.build.lib.actions; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.escape.Escaper; @@ -23,14 +24,13 @@ import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe; import com.google.devtools.build.lib.util.Preconditions; import com.google.devtools.build.lib.vfs.PathFragment; - import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.SortedMap; import java.util.TreeMap; - import javax.annotation.Nullable; /** @@ -85,12 +85,12 @@ public final class Actions { * artifact. Shared actions are tolerated. See {@link #canBeShared} for details. * * @param actions a list of actions to check for action conflicts - * @return a map between generated artifacts and their associated generating actions. If there is - * more than one action generating the same output artifact, only one action is chosen. + * @return a structure giving the mapping between artifacts and generating actions, with a level + * of indirection. * @throws ActionConflictException iff there are two actions generate the same output */ - public static ImmutableMap findAndThrowActionConflict( - Iterable actions) throws ActionConflictException { + public static GeneratingActions findAndThrowActionConflict(List actions) + throws ActionConflictException { return Actions.maybeFilterSharedActionsAndThrowIfConflict( actions, /*allowSharedAction=*/ false); } @@ -100,34 +100,34 @@ public final class Actions { * artifact. Shared actions are tolerated. See {@link #canBeShared} for details. * * @param actions a list of actions to check for action conflicts - * @return a map between generated artifacts and their associated generating actions. If there is - * more than one action generating the same output artifact, only one action is chosen. + * @return a structure giving the mapping between artifacts and generating actions, with a level + * of indirection. * @throws ActionConflictException iff there are two unshareable actions generating the same * output */ - public static ImmutableMap - filterSharedActionsAndThrowActionConflict( - Iterable actions) throws ActionConflictException { + public static GeneratingActions filterSharedActionsAndThrowActionConflict( + List actions) throws ActionConflictException { return Actions.maybeFilterSharedActionsAndThrowIfConflict( actions, /*allowSharedAction=*/ true); } - private static ImmutableMap - maybeFilterSharedActionsAndThrowIfConflict( - Iterable actions, boolean allowSharedAction) + private static GeneratingActions maybeFilterSharedActionsAndThrowIfConflict( + List actions, boolean allowSharedAction) throws ActionConflictException { - Map generatingActions = new HashMap<>(); + Map generatingActions = new HashMap<>(); + int actionIndex = 0; for (ActionAnalysisMetadata action : actions) { for (Artifact artifact : action.getOutputs()) { - ActionAnalysisMetadata previousAction = generatingActions.put(artifact, action); - if (previousAction != null && previousAction != action) { - if (!allowSharedAction || !Actions.canBeShared(previousAction, action)) { - throw new ActionConflictException(artifact, previousAction, action); + Integer previousIndex = generatingActions.put(artifact, actionIndex); + if (previousIndex != null && previousIndex != actionIndex) { + if (!allowSharedAction || !Actions.canBeShared(actions.get(previousIndex), action)) { + throw new ActionConflictException(artifact, actions.get(previousIndex), action); } } } + actionIndex++; } - return ImmutableMap.copyOf(generatingActions); + return new GeneratingActions(actions, ImmutableMap.copyOf(generatingActions)); } /** @@ -252,4 +252,27 @@ public final class Actions { return generatingActions.get(artifact); } } + + /** Container class for actions and the artifacts they generate. */ + @VisibleForTesting + public static class GeneratingActions { + private final List actions; + private final ImmutableMap generatingActionIndex; + + @VisibleForTesting + public GeneratingActions( + List actions, + ImmutableMap generatingActionIndex) { + this.actions = actions; + this.generatingActionIndex = generatingActionIndex; + } + + public ImmutableMap getGeneratingActionIndex() { + return generatingActionIndex; + } + + public List getActions() { + return actions; + } + } } -- cgit v1.2.3