diff options
author | 2015-11-10 03:24:01 +0000 | |
---|---|---|
committer | 2015-11-10 10:28:30 +0000 | |
commit | 8cd2978e429bdc178e06c0cb42f7ba73cabaeeb5 (patch) | |
tree | da3d1149e3a485fc83686efe96c98cb9e4f456c4 /src/main/java | |
parent | e040fe994c2da2b211858f7985fb736962a4499b (diff) |
Allow FilesystemValueChecker to operate on a WalkableGraph and add TODOs for replacing MemoizingEvaluator#getValues et al with WalkableGraph usage. I initially attempted to do this but punted once I realized how much work it would be.
Also make DirectoryListingStateValue and FileStateValue public for use in outside callers of FilesystemValueChecker.
--
MOS_MIGRATED_REVID=107447425
Diffstat (limited to 'src/main/java')
9 files changed, 111 insertions, 85 deletions
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingStateValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingStateValue.java index 15abaf44e7..7a422e5388 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingStateValue.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/DirectoryListingStateValue.java @@ -39,7 +39,7 @@ import javax.annotation.Nullable; * * <p>This class is an implementation detail of {@link DirectoryListingValue}. */ -final class DirectoryListingStateValue implements SkyValue { +public final class DirectoryListingStateValue implements SkyValue { private final CompactSortedDirents compactSortedDirents; diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FileStateValue.java b/src/main/java/com/google/devtools/build/lib/skyframe/FileStateValue.java index b774c5848b..28d4808b15 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/FileStateValue.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/FileStateValue.java @@ -58,7 +58,8 @@ public abstract class FileStateValue implements SkyValue { public static final NonexistentFileStateValue NONEXISTENT_FILE_STATE_NODE = new NonexistentFileStateValue(); - enum Type { + /** Type of a path. */ + public enum Type { REGULAR_FILE, SPECIAL_FILE, DIRECTORY, @@ -69,7 +70,7 @@ public abstract class FileStateValue implements SkyValue { protected FileStateValue() { } - static FileStateValue create(RootedPath rootedPath, + public static FileStateValue create(RootedPath rootedPath, @Nullable TimestampGranularityMonitor tsgm) throws InconsistentFilesystemException, IOException { Path path = rootedPath.asPath(); @@ -105,7 +106,7 @@ public abstract class FileStateValue implements SkyValue { return new SkyKey(SkyFunctions.FILE_STATE, rootedPath); } - abstract Type getType(); + public abstract Type getType(); PathFragment getSymlinkTarget() { throw new IllegalStateException(); @@ -193,7 +194,7 @@ public abstract class FileStateValue implements SkyValue { } @Override - Type getType() { + public Type getType() { return Type.REGULAR_FILE; } @@ -261,7 +262,7 @@ public abstract class FileStateValue implements SkyValue { } @Override - Type getType() { + public Type getType() { return Type.SPECIAL_FILE; } @@ -307,7 +308,7 @@ public abstract class FileStateValue implements SkyValue { } @Override - Type getType() { + public Type getType() { return Type.DIRECTORY; } @@ -338,7 +339,7 @@ public abstract class FileStateValue implements SkyValue { } @Override - Type getType() { + public Type getType() { return Type.SYMLINK; } @@ -374,7 +375,7 @@ public abstract class FileStateValue implements SkyValue { } @Override - Type getType() { + public Type getType() { return Type.NONEXISTENT; } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FilesystemValueChecker.java b/src/main/java/com/google/devtools/build/lib/skyframe/FilesystemValueChecker.java index 4c7efc36a0..0e8a5cf1bc 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/FilesystemValueChecker.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/FilesystemValueChecker.java @@ -15,8 +15,6 @@ package com.google.devtools.build.lib.skyframe; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; -import com.google.common.base.Supplier; -import com.google.common.base.Suppliers; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.collect.Range; @@ -35,15 +33,16 @@ import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor; import com.google.devtools.build.lib.vfs.BatchStat; import com.google.devtools.build.lib.vfs.FileStatusWithDigest; import com.google.devtools.build.skyframe.Differencer; -import com.google.devtools.build.skyframe.MemoizingEvaluator; import com.google.devtools.build.skyframe.SkyFunctionName; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; +import com.google.devtools.build.skyframe.WalkableGraph; import java.io.IOException; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -61,7 +60,7 @@ import javax.annotation.Nullable; * A helper class to find dirty values by accessing the filesystem directly (contrast with * {@link DiffAwareness}). */ -class FilesystemValueChecker { +public class FilesystemValueChecker { private static final int DIRTINESS_CHECK_THREADS = 200; private static final Logger LOG = Logger.getLogger(FilesystemValueChecker.class.getName()); @@ -72,70 +71,102 @@ class FilesystemValueChecker { private final TimestampGranularityMonitor tsgm; @Nullable private final Range<Long> lastExecutionTimeRange; - private final Supplier<Map<SkyKey, SkyValue>> valuesSupplier; private AtomicInteger modifiedOutputFilesCounter = new AtomicInteger(0); private AtomicInteger modifiedOutputFilesIntraBuildCounter = new AtomicInteger(0); - FilesystemValueChecker(Supplier<Map<SkyKey, SkyValue>> valuesSupplier, - @Nullable TimestampGranularityMonitor tsgm, @Nullable Range<Long> lastExecutionTimeRange) { - this.valuesSupplier = valuesSupplier; + public FilesystemValueChecker(@Nullable TimestampGranularityMonitor tsgm, + @Nullable Range<Long> lastExecutionTimeRange) { this.tsgm = tsgm; this.lastExecutionTimeRange = lastExecutionTimeRange; } - FilesystemValueChecker(final MemoizingEvaluator evaluator, - @Nullable TimestampGranularityMonitor tsgm, @Nullable Range<Long> lastExecutionTimeRange) { - this.tsgm = tsgm; - this.lastExecutionTimeRange = lastExecutionTimeRange; - - // Construct the full map view of the entire graph at most once ("memoized"), lazily. If - // getDirtyFilesystemValues(Iterable<SkyKey>) is called on an empty Iterable, we avoid having - // to create the Map of value keys to values. This is useful in the case where the graph - // getValues() method could be slow. - this.valuesSupplier = Suppliers.memoize(new Supplier<Map<SkyKey, SkyValue>>() { - @Override - public Map<SkyKey, SkyValue> get() { - return evaluator.getValues(); - } - }); + /** + * Returns a {@link Differencer.DiffWithDelta} containing keys from the give map that are dirty + * according to the passed-in {@code dirtinessChecker}. + */ + // TODO(bazel-team): Refactor these methods so that FilesystemValueChecker only operates on a + // WalkableGraph. + Differencer.DiffWithDelta getDirtyKeys(Map<SkyKey, SkyValue> valuesMap, + SkyValueDirtinessChecker dirtinessChecker) throws InterruptedException { + return getDirtyValues(new MapBackedValueFetcher(valuesMap), valuesMap.keySet(), + dirtinessChecker, /*checkMissingValues=*/false); } /** - * Returns a {@link Differencer.DiffWithDelta} containing keys from the backing graph (of the - * {@link MemoizingEvaluator} given at construction time) that are dirty according to the + * Returns a {@link Differencer.DiffWithDelta} containing keys that are dirty according to the * passed-in {@code dirtinessChecker}. */ - Differencer.DiffWithDelta getDirtyKeys(SkyValueDirtinessChecker dirtinessChecker) - throws InterruptedException { - return getDirtyValues(valuesSupplier.get().keySet(), dirtinessChecker, - /*checkMissingValues=*/false); + public Differencer.DiffWithDelta getNewAndOldValues(Map<SkyKey, SkyValue> valuesMap, + Iterable<SkyKey> keys, SkyValueDirtinessChecker dirtinessChecker) + throws InterruptedException { + return getDirtyValues(new MapBackedValueFetcher(valuesMap), keys, + dirtinessChecker, /*checkMissingValues=*/true); } /** * Returns a {@link Differencer.DiffWithDelta} containing keys that are dirty according to the * passed-in {@code dirtinessChecker}. */ - Differencer.DiffWithDelta getNewAndOldValues(Iterable<SkyKey> keys, - SkyValueDirtinessChecker dirtinessChecker) throws InterruptedException { - return getDirtyValues(keys, dirtinessChecker, /*checkMissingValues=*/true); + public Differencer.DiffWithDelta getNewAndOldValues(WalkableGraph walkableGraph, + Iterable<SkyKey> keys, SkyValueDirtinessChecker dirtinessChecker) + throws InterruptedException { + return getDirtyValues(new WalkableGraphBackedValueFetcher(walkableGraph), keys, + dirtinessChecker, /*checkMissingValues=*/true); + } + + private static interface ValueFetcher { + @Nullable + SkyValue get(SkyKey key); + } + + private static class WalkableGraphBackedValueFetcher implements ValueFetcher { + private final WalkableGraph walkableGraph; + + private WalkableGraphBackedValueFetcher(WalkableGraph walkableGraph) { + this.walkableGraph = walkableGraph; + } + + @Override + @Nullable + public SkyValue get(SkyKey key) { + return walkableGraph.exists(key) ? walkableGraph.getValue(key) : null; + } + } + + private static class MapBackedValueFetcher implements ValueFetcher { + private final Map<SkyKey, SkyValue> valuesMap; + + private MapBackedValueFetcher(Map<SkyKey, SkyValue> valuesMap) { + this.valuesMap = valuesMap; + } + + @Override + @Nullable + public SkyValue get(SkyKey key) { + return valuesMap.get(key); + } } /** * Return a collection of action values which have output files that are not in-sync with * the on-disk file value (were modified externally). */ - Collection<SkyKey> getDirtyActionValues(@Nullable final BatchStat batchStatter) - throws InterruptedException { + Collection<SkyKey> getDirtyActionValues(Map<SkyKey, SkyValue> valuesMap, + @Nullable final BatchStat batchStatter) throws InterruptedException { // CPU-bound (usually) stat() calls, plus a fudge factor. LOG.info("Accumulating dirty actions"); final int numOutputJobs = Runtime.getRuntime().availableProcessors() * 4; - final Set<SkyKey> actionSkyKeys = - Sets.filter(valuesSupplier.get().keySet(), ACTION_FILTER); + final Set<SkyKey> actionSkyKeys = new HashSet<>(); + for (SkyKey key : valuesMap.keySet()) { + if (ACTION_FILTER.apply(key)) { + actionSkyKeys.add(key); + } + } final Sharder<Pair<SkyKey, ActionExecutionValue>> outputShards = new Sharder<>(numOutputJobs, actionSkyKeys.size()); for (SkyKey key : actionSkyKeys) { - outputShards.add(Pair.of(key, (ActionExecutionValue) valuesSupplier.get().get(key))); + outputShards.add(Pair.of(key, (ActionExecutionValue) valuesMap.get(key))); } LOG.info("Sharded action values for batching"); @@ -285,8 +316,8 @@ class FilesystemValueChecker { return isDirty; } - private BatchDirtyResult getDirtyValues( - Iterable<SkyKey> values, final SkyValueDirtinessChecker checker, + private BatchDirtyResult getDirtyValues(ValueFetcher fetcher, + Iterable<SkyKey> keys, final SkyValueDirtinessChecker checker, final boolean checkMissingValues) throws InterruptedException { ExecutorService executor = Executors.newFixedThreadPool( @@ -310,23 +341,24 @@ class FilesystemValueChecker { } }; try (AutoProfiler prof = AutoProfiler.create(elapsedTimeReceiver)) { - for (final SkyKey key : values) { + for (final SkyKey key : keys) { numKeysScanned.incrementAndGet(); if (!checker.applies(key)) { continue; } - final SkyValue value = valuesSupplier.get().get(key); + final SkyValue value = fetcher.get(key); + if (!checkMissingValues && value == null) { + continue; + } executor.execute( wrapper.wrap( new Runnable() { @Override public void run() { - if (value != null || checkMissingValues) { - numKeysChecked.incrementAndGet(); - DirtyResult result = checker.check(key, value, tsgm); - if (result.isDirty()) { - batchResult.add(key, value, result.getNewValue()); - } + numKeysChecked.incrementAndGet(); + DirtyResult result = checker.check(key, value, tsgm); + if (result.isDirty()) { + batchResult.add(key, value, result.getNewValue()); } } })); @@ -342,7 +374,7 @@ class FilesystemValueChecker { } /** - * Result of a batch call to {@link SkyValueDirtinessChecker#maybeCheck}. Partitions the dirty + * Result of a batch call to {@link SkyValueDirtinessChecker#check}. Partitions the dirty * values based on whether we have a new value available for them or not. */ private static class BatchDirtyResult implements Differencer.DiffWithDelta { diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java index 7a29573a78..c4b11d22bf 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java @@ -341,7 +341,7 @@ public final class SequencedSkyframeExecutor extends SkyframeExecutor { buildDriver.evaluate(ImmutableList.<SkyKey>of(), false, DEFAULT_THREAD_COUNT, eventHandler); - FilesystemValueChecker fsvc = new FilesystemValueChecker(memoizingEvaluator, tsgm, null); + FilesystemValueChecker fsvc = new FilesystemValueChecker(tsgm, null); // We need to manually check for changes to known files. This entails finding all dirty file // system values under package roots for which we don't have diff information. If at least // one path entry doesn't have diff information, then we're going to have to iterate over @@ -353,6 +353,7 @@ public final class SequencedSkyframeExecutor extends SkyframeExecutor { } Differencer.Diff diff = fsvc.getDirtyKeys( + memoizingEvaluator.getValues(), new UnionDirtinessChecker( Iterables.concat( customDirtinessCheckers, @@ -444,7 +445,7 @@ public final class SequencedSkyframeExecutor extends SkyframeExecutor { } Differencer.Diff diff; if (modifiedFileSet.treatEverythingAsModified()) { - diff = new FilesystemValueChecker(memoizingEvaluator, tsgm, null).getDirtyKeys( + diff = new FilesystemValueChecker(tsgm, null).getDirtyKeys(memoizingEvaluator.getValues(), new BasicFilesystemDirtinessChecker()); } else { diff = getDiff(modifiedFileSet.modifiedSourceFiles(), pathEntry); diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java index 28ac669e66..76ad0e7c22 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java @@ -22,7 +22,6 @@ import com.google.common.base.Predicate; import com.google.common.base.Predicates; import com.google.common.base.Stopwatch; import com.google.common.base.Supplier; -import com.google.common.base.Suppliers; import com.google.common.base.Throwables; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; @@ -779,6 +778,9 @@ public abstract class SkyframeExecutor implements WalkableGraphFactory { protected Differencer.Diff getDiff(Iterable<PathFragment> modifiedSourceFiles, final Path pathEntry) throws InterruptedException { + if (Iterables.isEmpty(modifiedSourceFiles)) { + return new ImmutableDiff(ImmutableList.<SkyKey>of(), ImmutableMap.<SkyKey, SkyValue>of()); + } // TODO(bazel-team): change ModifiedFileSet to work with RootedPaths instead of PathFragments. Iterable<SkyKey> dirtyFileStateSkyKeys = Iterables.transform(modifiedSourceFiles, new Function<PathFragment, SkyKey>() { @@ -794,16 +796,10 @@ public abstract class SkyframeExecutor implements WalkableGraphFactory { // information here, so we compute it ourselves. // TODO(bazel-team): Fancy filesystems could provide it with a hypothetically modified // DiffAwareness interface. - Supplier<Map<SkyKey, SkyValue>> valuesSupplier = Suppliers.memoize( - new Supplier<Map<SkyKey, SkyValue>>() { - @Override - public Map<SkyKey, SkyValue> get() { - return memoizingEvaluator.getValues(); - } - }); - FilesystemValueChecker fsvc = new FilesystemValueChecker(valuesSupplier, tsgm, null); + FilesystemValueChecker fsvc = new FilesystemValueChecker(tsgm, null); + Map<SkyKey, SkyValue> valuesMap = memoizingEvaluator.getValues(); Differencer.DiffWithDelta diff = - fsvc.getNewAndOldValues(dirtyFileStateSkyKeys, new FileDirtinessChecker()); + fsvc.getNewAndOldValues(valuesMap, dirtyFileStateSkyKeys, new FileDirtinessChecker()); Set<SkyKey> valuesToInvalidate = new HashSet<>(); Map<SkyKey, SkyValue> valuesToInject = new HashMap<>(); @@ -830,7 +826,7 @@ public abstract class SkyframeExecutor implements WalkableGraphFactory { changedType = !oldValue.getType().equals(newValue.getType()); } else { DirectoryListingStateValue oldDirListingStateValue = - (DirectoryListingStateValue) valuesSupplier.get().get(dirListingStateKey); + (DirectoryListingStateValue) valuesMap.get(dirListingStateKey); if (oldDirListingStateValue != null) { String baseName = rootedPath.getRelativePath().getBaseName(); Dirent oldDirent = oldDirListingStateValue.getDirents().maybeGetDirent(baseName); @@ -858,14 +854,6 @@ public abstract class SkyframeExecutor implements WalkableGraphFactory { return DirectoryListingStateValue.key(parentDirRootedPath); } - protected static SkyKey createFileStateKey(RootedPath rootedPath) { - return FileStateValue.key(rootedPath); - } - - protected static SkyKey createDirectoryListingStateKey(RootedPath rootedPath) { - return DirectoryListingStateValue.key(rootedPath); - } - /** * Sets the packages that should be treated as deleted and ignored. */ @@ -1653,9 +1641,9 @@ public abstract class SkyframeExecutor implements WalkableGraphFactory { if (checkOutputFiles) { // Detect external modifications in the output tree. - FilesystemValueChecker fsvc = new FilesystemValueChecker(memoizingEvaluator, tsgm, - lastExecutionTimeRange); - invalidateDirtyActions(fsvc.getDirtyActionValues(batchStatter)); + FilesystemValueChecker fsvc = new FilesystemValueChecker(tsgm, lastExecutionTimeRange); + invalidateDirtyActions(fsvc.getDirtyActionValues(memoizingEvaluator.getValues(), + batchStatter)); modifiedFiles += fsvc.getNumberOfModifiedOutputFiles(); outputDirtyFiles += fsvc.getNumberOfModifiedOutputFiles(); modifiedFilesDuringPreviousBuild += fsvc.getNumberOfModifiedOutputFilesDuringPreviousBuild(); diff --git a/src/main/java/com/google/devtools/build/skyframe/Differencer.java b/src/main/java/com/google/devtools/build/skyframe/Differencer.java index f30558cdd1..ddcb34fe0d 100644 --- a/src/main/java/com/google/devtools/build/skyframe/Differencer.java +++ b/src/main/java/com/google/devtools/build/skyframe/Differencer.java @@ -92,5 +92,6 @@ public interface Differencer { /** * Returns the value keys that have changed between the two Versions. */ - Diff getDiff(Version fromVersion, Version toVersion) throws InterruptedException; + Diff getDiff(WalkableGraph fromGraph, Version fromVersion, Version toVersion) + throws InterruptedException; } diff --git a/src/main/java/com/google/devtools/build/skyframe/InMemoryMemoizingEvaluator.java b/src/main/java/com/google/devtools/build/skyframe/InMemoryMemoizingEvaluator.java index e35cc54c3d..a16ff87355 100644 --- a/src/main/java/com/google/devtools/build/skyframe/InMemoryMemoizingEvaluator.java +++ b/src/main/java/com/google/devtools/build/skyframe/InMemoryMemoizingEvaluator.java @@ -153,7 +153,8 @@ public final class InMemoryMemoizingEvaluator implements MemoizingEvaluator { // It clears the internal data structures after getDiff is called and will not return // diffs for historical versions. This makes the following code sensitive to interrupts. // Ideally we would simply not update lastGraphVersion if an interrupt occurs. - Diff diff = differencer.getDiff(lastGraphVersion, version); + Diff diff = differencer.getDiff(new DelegatingWalkableGraph(graph), lastGraphVersion, + version); valuesToInject.putAll(diff.changedKeysWithNewValues()); invalidate(diff.changedKeysWithoutNewValues()); pruneInjectedValues(valuesToInject); diff --git a/src/main/java/com/google/devtools/build/skyframe/MemoizingEvaluator.java b/src/main/java/com/google/devtools/build/skyframe/MemoizingEvaluator.java index e54dc8d223..f00d88f48d 100644 --- a/src/main/java/com/google/devtools/build/skyframe/MemoizingEvaluator.java +++ b/src/main/java/com/google/devtools/build/skyframe/MemoizingEvaluator.java @@ -86,9 +86,11 @@ public interface MemoizingEvaluator { * * <p>The returned map may be a live view of the graph. */ + // TODO(bazel-team): Replace all usages of getValues, getDoneValues, getExistingValueForTesting, + // and getExistingErrorForTesting with usages of WalkableGraph. Changing the getValues usages + // require some care because getValues gives access to the previous value for changed/dirty nodes. Map<SkyKey, SkyValue> getValues(); - /** * Returns the done (without error) values in the graph. * diff --git a/src/main/java/com/google/devtools/build/skyframe/RecordingDifferencer.java b/src/main/java/com/google/devtools/build/skyframe/RecordingDifferencer.java index 1582e52b78..4d25e0e65a 100644 --- a/src/main/java/com/google/devtools/build/skyframe/RecordingDifferencer.java +++ b/src/main/java/com/google/devtools/build/skyframe/RecordingDifferencer.java @@ -41,7 +41,7 @@ public class RecordingDifferencer implements Differencer, Injectable { } @Override - public Diff getDiff(Version fromVersion, Version toVersion) { + public Diff getDiff(WalkableGraph fromGraph, Version fromVersion, Version toVersion) { Diff diff = new ImmutableDiff(valuesToInvalidate, valuesToInject); clear(); return diff; |