diff options
Diffstat (limited to 'src/main/java/com/google/devtools/build')
12 files changed, 162 insertions, 100 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/BuildView.java b/src/main/java/com/google/devtools/build/lib/analysis/BuildView.java index 61dd9f199c..02a6dfd684 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/BuildView.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/BuildView.java @@ -322,7 +322,7 @@ public class BuildView { } @VisibleForTesting - WorkspaceStatusAction getLastWorkspaceBuildInfoActionForTesting() { + WorkspaceStatusAction getLastWorkspaceBuildInfoActionForTesting() throws InterruptedException { return skyframeExecutor.getLastWorkspaceStatusAction(); } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java index 1f4f8039a2..406de7850d 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java @@ -66,6 +66,7 @@ import com.google.devtools.build.skyframe.LegacySkyKey; import com.google.devtools.build.skyframe.MemoizingEvaluator.EvaluatorSupplier; import com.google.devtools.build.skyframe.NodeEntry; import com.google.devtools.build.skyframe.RecordingDifferencer; +import com.google.devtools.build.skyframe.SequencedRecordingDifferencer; import com.google.devtools.build.skyframe.SequentialBuildDriver; import com.google.devtools.build.skyframe.SkyFunction; import com.google.devtools.build.skyframe.SkyFunctionName; @@ -184,7 +185,7 @@ public final class SequencedSkyframeExecutor extends SkyframeExecutor { protected void init() { // Note that we need to set recordingDiffer first since SkyframeExecutor#init calls // SkyframeExecutor#evaluatorDiffer. - recordingDiffer = new RecordingDifferencer(); + recordingDiffer = new SequencedRecordingDifferencer(); super.init(); } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java index 273ab775e1..0891d0602d 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java @@ -677,7 +677,8 @@ public abstract class SkyframeExecutor implements WalkableGraphFactory { PrecomputedValue.DEFAULTS_PACKAGE_CONTENTS.set(injectable(), defaultsPackageContents); } - public void maybeInvalidateWorkspaceStatusValue(String workspaceName) { + public void maybeInvalidateWorkspaceStatusValue(String workspaceName) + throws InterruptedException { WorkspaceStatusAction newWorkspaceStatusAction = makeWorkspaceStatusAction(workspaceName); WorkspaceStatusAction oldWorkspaceStatusAction = getLastWorkspaceStatusAction(); if (oldWorkspaceStatusAction != null @@ -696,7 +697,7 @@ public abstract class SkyframeExecutor implements WalkableGraphFactory { @VisibleForTesting @Nullable - public WorkspaceStatusAction getLastWorkspaceStatusAction() { + public WorkspaceStatusAction getLastWorkspaceStatusAction() throws InterruptedException { WorkspaceStatusValue workspaceStatusValue = (WorkspaceStatusValue) memoizingEvaluator.getExistingValue(WorkspaceStatusValue.SKY_KEY); return workspaceStatusValue == null diff --git a/src/main/java/com/google/devtools/build/skyframe/DelegatingNodeEntry.java b/src/main/java/com/google/devtools/build/skyframe/DelegatingNodeEntry.java index e0b3579d09..e884954845 100644 --- a/src/main/java/com/google/devtools/build/skyframe/DelegatingNodeEntry.java +++ b/src/main/java/com/google/devtools/build/skyframe/DelegatingNodeEntry.java @@ -28,7 +28,7 @@ public abstract class DelegatingNodeEntry implements NodeEntry { } @Override - public boolean keepEdges() { + public KeepEdgesPolicy keepEdges() { return getDelegate().keepEdges(); } diff --git a/src/main/java/com/google/devtools/build/skyframe/DirtyBuildingState.java b/src/main/java/com/google/devtools/build/skyframe/DirtyBuildingState.java index 2b8dc9aef6..30884fcbe5 100644 --- a/src/main/java/com/google/devtools/build/skyframe/DirtyBuildingState.java +++ b/src/main/java/com/google/devtools/build/skyframe/DirtyBuildingState.java @@ -43,8 +43,10 @@ public abstract class DirtyBuildingState { * build to check whether this node has changed in {@link NodeEntry#setValue}. If they are null, * it means that this node is being built for the first time. See {@link * InMemoryNodeEntry#directDeps} for more on dependency group storage. + * + * <p>Public only for the use of alternative graph implementations. */ - protected abstract GroupedList<SkyKey> getLastBuildDirectDeps() throws InterruptedException; + public abstract GroupedList<SkyKey> getLastBuildDirectDeps() throws InterruptedException; /** * The number of groups of the dependencies requested last time when the node was built. @@ -53,8 +55,12 @@ public abstract class DirtyBuildingState { */ protected abstract int getNumOfGroupsInLastBuildDirectDeps(); - /** The value of the node the last time it was built. */ - protected abstract SkyValue getLastBuildValue() throws InterruptedException; + /** + * The value of the node the last time it was built. + * + * <p>Public only for the use of alternative graph implementations. + */ + public abstract SkyValue getLastBuildValue() throws InterruptedException; /** * Group of children to be checked next in the process of determining if this entry needs to be @@ -216,12 +222,12 @@ public abstract class DirtyBuildingState { } @Override - protected SkyValue getLastBuildValue() { + public SkyValue getLastBuildValue() { return lastBuildValue; } @Override - protected GroupedList<SkyKey> getLastBuildDirectDeps() throws InterruptedException { + public GroupedList<SkyKey> getLastBuildDirectDeps() throws InterruptedException { return lastBuildDirectDeps; } diff --git a/src/main/java/com/google/devtools/build/skyframe/EdgelessInMemoryNodeEntry.java b/src/main/java/com/google/devtools/build/skyframe/EdgelessInMemoryNodeEntry.java index daa24ac64d..b0c2b2e714 100644 --- a/src/main/java/com/google/devtools/build/skyframe/EdgelessInMemoryNodeEntry.java +++ b/src/main/java/com/google/devtools/build/skyframe/EdgelessInMemoryNodeEntry.java @@ -26,7 +26,13 @@ package com.google.devtools.build.skyframe; */ class EdgelessInMemoryNodeEntry extends InMemoryNodeEntry { @Override - public boolean keepEdges() { - return false; + public KeepEdgesPolicy keepEdges() { + return KeepEdgesPolicy.NONE; + } + + @Override + protected void postProcessAfterDone() { + this.directDeps = null; + this.reverseDeps = null; } } diff --git a/src/main/java/com/google/devtools/build/skyframe/InMemoryNodeEntry.java b/src/main/java/com/google/devtools/build/skyframe/InMemoryNodeEntry.java index 007cfefa12..67e1867b7f 100644 --- a/src/main/java/com/google/devtools/build/skyframe/InMemoryNodeEntry.java +++ b/src/main/java/com/google/devtools/build/skyframe/InMemoryNodeEntry.java @@ -73,7 +73,7 @@ import javax.annotation.Nullable; public class InMemoryNodeEntry implements NodeEntry { /** Actual data stored in this entry when it is done. */ - private SkyValue value = null; + protected SkyValue value = null; /** * The last version of the graph at which this node's value was changed. In {@link #setValue} it @@ -95,13 +95,13 @@ public class InMemoryNodeEntry implements NodeEntry { /** * This object represents the direct deps of the node, in groups if the {@code SkyFunction} - * requested them that way. It contains either the in-progress direct deps, stored as a - * {@code GroupedList<SkyKey>} before the node is finished building, or the full direct deps, - * compressed in a memory-efficient way (via {@link GroupedList#compress}, after the node is done. + * requested them that way. It contains either the in-progress direct deps, stored as a {@code + * GroupedList<SkyKey>} before the node is finished building, or the full direct deps, compressed + * in a memory-efficient way (via {@link GroupedList#compress}, after the node is done. * * <p>It is initialized lazily in getTemporaryDirectDeps() to save a little bit more memory. */ - private Object directDeps = null; + protected Object directDeps = null; /** * This list stores the reverse dependencies of this node that have been declared so far. @@ -175,8 +175,12 @@ public class InMemoryNodeEntry implements NodeEntry { } @Override - public boolean keepEdges() { - return true; + public KeepEdgesPolicy keepEdges() { + return KeepEdgesPolicy.ALL; + } + + private boolean keepReverseDeps() { + return keepEdges() == KeepEdgesPolicy.ALL; } @Override @@ -224,7 +228,7 @@ public class InMemoryNodeEntry implements NodeEntry { * added in {@link #addTemporaryDirectDeps}. */ public synchronized GroupedList<SkyKey> getGroupedDirectDeps() { - assertKeepEdges(); + assertKeepDeps(); Preconditions.checkState(isDone(), "no deps until done. NodeEntry: %s", this); return GroupedList.create(directDeps); } @@ -236,7 +240,7 @@ public class InMemoryNodeEntry implements NodeEntry { return ValueWithMetadata.getMaybeErrorInfo(value); } - private DirtyBuildingState getDirtyBuildingState() { + protected DirtyBuildingState getDirtyBuildingState() { return Preconditions.checkNotNull(dirtyBuildingState, "Didn't have state: %s", this); } @@ -249,20 +253,18 @@ public class InMemoryNodeEntry implements NodeEntry { signaledDeps = NOT_EVALUATING_SENTINEL; } - protected synchronized Set<SkyKey> setStateFinishedAndReturnReverseDepsToSignal() { + protected final synchronized Set<SkyKey> setStateFinishedAndReturnReverseDepsToSignal() { Set<SkyKey> reverseDepsToSignal = ReverseDepsUtility.consolidateDataAndReturnNewElements(this, getOpToStoreBare()); this.directDeps = getTemporaryDirectDeps().compress(); markDone(); - - if (!keepEdges()) { - this.directDeps = null; - this.reverseDeps = null; - } + postProcessAfterDone(); return reverseDepsToSignal; } + protected void postProcessAfterDone() {} + @Override public synchronized Set<SkyKey> getInProgressReverseDeps() { Preconditions.checkState(!isDone(), this); @@ -298,7 +300,7 @@ public class InMemoryNodeEntry implements NodeEntry { public synchronized DependencyState addReverseDepAndCheckIfDone(SkyKey reverseDep) { if (reverseDep != null) { if (isDone()) { - if (keepEdges()) { + if (keepReverseDeps()) { ReverseDepsUtility.addReverseDeps(this, ImmutableList.of(reverseDep)); } } else { @@ -356,7 +358,13 @@ public class InMemoryNodeEntry implements NodeEntry { @Override public synchronized DependencyState checkIfDoneForDirtyReverseDep(SkyKey reverseDep) { Preconditions.checkNotNull(reverseDep, this); - Preconditions.checkState(keepEdges(), "%s %s", reverseDep, this); + // Note that implementations of InMemoryNodeEntry that have + // #keepEdges == KeepEdgesPolicy.JUST_DEPS may override this entire method. + Preconditions.checkState( + keepEdges() == KeepEdgesPolicy.ALL, + "Incremental means keeping edges %s %s", + reverseDep, + this); if (isDone()) { ReverseDepsUtility.checkReverseDep(this, reverseDep); } else { @@ -367,7 +375,7 @@ public class InMemoryNodeEntry implements NodeEntry { @Override public synchronized void removeReverseDep(SkyKey reverseDep) { - if (!keepEdges()) { + if (!keepReverseDeps()) { return; } if (isDone()) { @@ -386,14 +394,14 @@ public class InMemoryNodeEntry implements NodeEntry { @Override public synchronized Set<SkyKey> getReverseDepsForDoneEntry() { - assertKeepEdges(); + assertKeepRdeps(); Preconditions.checkState(isDone(), "Called on not done %s", this); return ReverseDepsUtility.getReverseDeps(this); } @Override public synchronized Set<SkyKey> getAllReverseDepsForNodeBeingDeleted() { - assertKeepEdges(); + assertKeepRdeps(); if (!isDone()) { // This consolidation loses information about pending reverse deps to signal, but that is // unimportant since this node is being deleted. @@ -429,13 +437,19 @@ public class InMemoryNodeEntry implements NodeEntry { } /** Checks that a caller is not trying to access not-stored graph edges. */ - private void assertKeepEdges() { - Preconditions.checkState(keepEdges(), "Graph edges not stored. %s", this); + private void assertKeepDeps() { + Preconditions.checkState(keepEdges() != KeepEdgesPolicy.NONE, "Not keeping deps: %s", this); + } + + /** Checks that a caller is not trying to access not-stored graph edges. */ + private void assertKeepRdeps() { + Preconditions.checkState(keepEdges() == KeepEdgesPolicy.ALL, "Not keeping rdeps: %s", this); } @Override public synchronized MarkedDirtyResult markDirty(boolean isChanged) { - assertKeepEdges(); + // Can't process a dirty node without its deps. + assertKeepDeps(); if (isDone()) { dirtyBuildingState = DirtyBuildingState.create(isChanged, GroupedList.<SkyKey>create(directDeps), value); @@ -601,21 +615,24 @@ public class InMemoryNodeEntry implements NodeEntry { .toString(); } + protected synchronized InMemoryNodeEntry cloneNodeEntry(InMemoryNodeEntry newEntry) { + // As this is temporary, for now let's limit to done nodes. + Preconditions.checkState(isDone(), "Only done nodes can be copied: %s", this); + newEntry.value = value; + newEntry.lastChangedVersion = this.lastChangedVersion; + newEntry.lastEvaluatedVersion = this.lastEvaluatedVersion; + ReverseDepsUtility.addReverseDeps(newEntry, ReverseDepsUtility.getReverseDeps(this)); + newEntry.directDeps = directDeps; + newEntry.dirtyBuildingState = null; + return newEntry; + } + /** * Do not use except in custom evaluator implementations! Added only temporarily. * * <p>Clones a InMemoryMutableNodeEntry iff it is a done node. Otherwise it fails. */ public synchronized InMemoryNodeEntry cloneNodeEntry() { - // As this is temporary, for now let's limit to done nodes. - Preconditions.checkState(isDone(), "Only done nodes can be copied: %s", this); - InMemoryNodeEntry nodeEntry = new InMemoryNodeEntry(); - nodeEntry.value = value; - nodeEntry.lastChangedVersion = this.lastChangedVersion; - nodeEntry.lastEvaluatedVersion = this.lastEvaluatedVersion; - ReverseDepsUtility.addReverseDeps(nodeEntry, ReverseDepsUtility.getReverseDeps(this)); - nodeEntry.directDeps = directDeps; - nodeEntry.dirtyBuildingState = null; - return nodeEntry; + return cloneNodeEntry(new InMemoryNodeEntry()); } } diff --git a/src/main/java/com/google/devtools/build/skyframe/MemoizingEvaluator.java b/src/main/java/com/google/devtools/build/skyframe/MemoizingEvaluator.java index 889c9df7b8..562c2eaebc 100644 --- a/src/main/java/com/google/devtools/build/skyframe/MemoizingEvaluator.java +++ b/src/main/java/com/google/devtools/build/skyframe/MemoizingEvaluator.java @@ -113,7 +113,7 @@ public interface MemoizingEvaluator { */ @VisibleForTesting @Nullable - SkyValue getExistingValue(SkyKey key); + SkyValue getExistingValue(SkyKey key) throws InterruptedException; /** * Returns an error if and only if an earlier call to {@link #evaluate} created it; null @@ -127,7 +127,7 @@ public interface MemoizingEvaluator { ErrorInfo getExistingErrorForTesting(SkyKey key) throws InterruptedException; @Nullable - NodeEntry getExistingEntryForTesting(SkyKey key); + NodeEntry getExistingEntryForTesting(SkyKey key) throws InterruptedException; /** * Tests that want finer control over the graph being used may provide a {@code transformer} here. diff --git a/src/main/java/com/google/devtools/build/skyframe/NodeEntry.java b/src/main/java/com/google/devtools/build/skyframe/NodeEntry.java index 57334d6384..7857915fa2 100644 --- a/src/main/java/com/google/devtools/build/skyframe/NodeEntry.java +++ b/src/main/java/com/google/devtools/build/skyframe/NodeEntry.java @@ -75,7 +75,7 @@ public interface NodeEntry extends ThinNodeEntry { REBUILDING } - boolean keepEdges(); + KeepEdgesPolicy keepEdges(); /** * Returns the value stored in this entry. This method may only be called after the evaluation of @@ -378,4 +378,17 @@ public interface NodeEntry extends ThinNodeEntry { */ @ThreadSafe boolean isReady(); + + /** Which edges a done NodeEntry stores (dependencies and/or reverse dependencies. */ + enum KeepEdgesPolicy { + /** Both deps and rdeps are stored. Incremental builds and sanity checks are possible. */ + ALL, + /** + * Only deps are stored. Incremental builds may be possible with a "top-down" evaluation + * framework. Sanity checking of reverse deps is not possible. + */ + JUST_DEPS, + /** Neither deps nor rdeps are stored. Incremental builds and sanity checking are disabled. */ + NONE + } } diff --git a/src/main/java/com/google/devtools/build/skyframe/RecordingDifferencer.java b/src/main/java/com/google/devtools/build/skyframe/RecordingDifferencer.java index a4d768d043..d92f93b4eb 100644 --- a/src/main/java/com/google/devtools/build/skyframe/RecordingDifferencer.java +++ b/src/main/java/com/google/devtools/build/skyframe/RecordingDifferencer.java @@ -1,4 +1,4 @@ -// Copyright 2014 The Bazel Authors. All rights reserved. +// Copyright 2017 The Bazel Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,47 +11,18 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + package com.google.devtools.build.skyframe; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; -import com.google.devtools.build.lib.concurrent.ThreadSafety; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * A simple Differencer which just records the invalidated values it's been given. - */ -@ThreadSafety.ThreadCompatible -public class RecordingDifferencer implements Differencer, Injectable { - - private List<SkyKey> valuesToInvalidate; - private Map<SkyKey, SkyValue> valuesToInject; - - public RecordingDifferencer() { - clear(); - } - - private void clear() { - valuesToInvalidate = new ArrayList<>(); - valuesToInject = new HashMap<>(); - } +/** A simple {@link Differencer} that is manually informed of invalid/injected nodes. */ +public interface RecordingDifferencer extends Differencer, Injectable { @Override - public Diff getDiff(WalkableGraph fromGraph, Version fromVersion, Version toVersion) { - Diff diff = new ImmutableDiff(valuesToInvalidate, valuesToInject); - clear(); - return diff; - } + Diff getDiff(WalkableGraph fromGraph, Version fromVersion, Version toVersion); - /** - * Store the given values for invalidation. - */ - public void invalidate(Iterable<SkyKey> values) { - Iterables.addAll(valuesToInvalidate, values); - } + /** Stores the given values for invalidation. */ + void invalidate(Iterable<SkyKey> values); /** * Invalidates the cached values of any values in error transiently. @@ -59,22 +30,9 @@ public class RecordingDifferencer implements Differencer, Injectable { * <p>If a future call to {@link MemoizingEvaluator#evaluate} requests a value that transitively * depends on any value that was in an error state (or is one of these), they will be re-computed. */ - public void invalidateTransientErrors() { + default void invalidateTransientErrors() { // All transient error values have a dependency on the single global ERROR_TRANSIENCE value, // so we only have to invalidate that one value to catch everything. invalidate(ImmutableList.of(ErrorTransienceValue.KEY)); } - - /** - * Store the given values for injection. - */ - @Override - public void inject(Map<SkyKey, ? extends SkyValue> values) { - valuesToInject.putAll(values); - } - - @Override - public void inject(SkyKey key, SkyValue value) { - valuesToInject.put(key, value); - } } diff --git a/src/main/java/com/google/devtools/build/skyframe/SequencedRecordingDifferencer.java b/src/main/java/com/google/devtools/build/skyframe/SequencedRecordingDifferencer.java new file mode 100644 index 0000000000..80aad4ec58 --- /dev/null +++ b/src/main/java/com/google/devtools/build/skyframe/SequencedRecordingDifferencer.java @@ -0,0 +1,60 @@ +// Copyright 2014 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.skyframe; + +import com.google.common.collect.Iterables; +import com.google.devtools.build.lib.concurrent.ThreadSafety; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** A simple Differencer which just records the invalidated values it's been given. */ +@ThreadSafety.ThreadCompatible +public class SequencedRecordingDifferencer implements RecordingDifferencer { + + private List<SkyKey> valuesToInvalidate; + private Map<SkyKey, SkyValue> valuesToInject; + + public SequencedRecordingDifferencer() { + clear(); + } + + private void clear() { + valuesToInvalidate = new ArrayList<>(); + valuesToInject = new HashMap<>(); + } + + @Override + public Diff getDiff(WalkableGraph fromGraph, Version fromVersion, Version toVersion) { + Diff diff = new ImmutableDiff(valuesToInvalidate, valuesToInject); + clear(); + return diff; + } + + @Override + public void invalidate(Iterable<SkyKey> values) { + Iterables.addAll(valuesToInvalidate, values); + } + + @Override + public void inject(Map<SkyKey, ? extends SkyValue> values) { + valuesToInject.putAll(values); + } + + @Override + public void inject(SkyKey key, SkyValue value) { + valuesToInject.put(key, value); + } +} diff --git a/src/main/java/com/google/devtools/build/skyframe/SequentialBuildDriver.java b/src/main/java/com/google/devtools/build/skyframe/SequentialBuildDriver.java index 82b4ecbb75..c4430db939 100644 --- a/src/main/java/com/google/devtools/build/skyframe/SequentialBuildDriver.java +++ b/src/main/java/com/google/devtools/build/skyframe/SequentialBuildDriver.java @@ -59,7 +59,7 @@ public class SequentialBuildDriver implements BuildDriver { @Nullable @Override - public SkyValue getExistingValueForTesting(SkyKey key) { + public SkyValue getExistingValueForTesting(SkyKey key) throws InterruptedException { return memoizingEvaluator.getExistingValue(key); } @@ -71,7 +71,7 @@ public class SequentialBuildDriver implements BuildDriver { @Nullable @Override - public NodeEntry getEntryForTesting(SkyKey key) { + public NodeEntry getEntryForTesting(SkyKey key) throws InterruptedException { return memoizingEvaluator.getExistingEntryForTesting(key); } } |