From 7cf2365a90ca3080eb89cbb8b746e7cc49d400eb Mon Sep 17 00:00:00 2001 From: Miguel Alcon Pinto Date: Tue, 10 Mar 2015 21:27:48 +0000 Subject: Record statistics about dirty output files detected in the output tree. -- MOS_MIGRATED_REVID=88257621 --- .../lib/buildtool/ExecutionFinishedEvent.java | 16 ++++++++++- .../build/lib/buildtool/ExecutionTool.java | 9 ++++--- .../build/lib/buildtool/SkyframeBuilder.java | 6 +++-- .../devtools/build/lib/runtime/BlazeRuntime.java | 14 ++++++++++ .../devtools/build/lib/skyframe/Builder.java | 18 ++++++++----- .../build/lib/skyframe/FilesystemValueChecker.java | 31 +++++++++++++++++++--- .../lib/skyframe/SequencedSkyframeExecutor.java | 4 +-- .../build/lib/skyframe/SkyframeExecutor.java | 18 +++++++++++-- 8 files changed, 96 insertions(+), 20 deletions(-) (limited to 'src/main/java') diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionFinishedEvent.java b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionFinishedEvent.java index 74143cc6df..9662e0817e 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionFinishedEvent.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionFinishedEvent.java @@ -23,6 +23,9 @@ import java.util.Map; * the metadata cache and about last file save times. */ public class ExecutionFinishedEvent { + + private final int outputDirtyFiles; + private final int outputModifiedFilesDuringPreviousBuild; /** The mtime of the most recently saved source file when the build starts. */ private long lastFileSaveTimeInMillis; @@ -34,7 +37,10 @@ public class ExecutionFinishedEvent { private Map changedFileSaveTimes = new HashMap<>(); public ExecutionFinishedEvent(Map changedFileSaveTimes, - long lastFileSaveTimeInMillis) { + long lastFileSaveTimeInMillis, int outputDirtyFiles, + int outputModifiedFilesDuringPreviousBuild) { + this.outputDirtyFiles = outputDirtyFiles; + this.outputModifiedFilesDuringPreviousBuild = outputModifiedFilesDuringPreviousBuild; this.changedFileSaveTimes = ImmutableMap.copyOf(changedFileSaveTimes); this.lastFileSaveTimeInMillis = lastFileSaveTimeInMillis; } @@ -43,6 +49,14 @@ public class ExecutionFinishedEvent { return lastFileSaveTimeInMillis; } + public int getOutputDirtyFiles() { + return outputDirtyFiles; + } + + public int getOutputModifiedFilesDuringPreviousBuild() { + return outputModifiedFilesDuringPreviousBuild; + } + public Map getChangedFileSaveTimes() { return changedFileSaveTimes; } diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java index bd48395081..d66da235c2 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java @@ -404,12 +404,14 @@ public class ExecutionTool { analysisResult.getExclusiveTests(), analysisResult.getTargetsToBuild(), executor, builtTargets, - request.getBuildOptions().explanationPath != null); + request.getBuildOptions().explanationPath != null, + runtime.getLastExecutionTimeRange()); } catch (InterruptedException e) { interrupted = true; throw e; } finally { + runtime.recordLastExecutionTime(); if (request.isRunningInEmacs()) { request.getOutErr().printErrLn("blaze: Leaving directory `" + getExecRoot() + "/'"); } @@ -417,8 +419,9 @@ public class ExecutionTool { getReporter().handle(Event.progress("Building complete.")); } - // Transfer over source file "last save time" stats so the remote logger can find them. - runtime.getEventBus().post(new ExecutionFinishedEvent(ImmutableMap. of(), 0)); + runtime.getEventBus().post(new ExecutionFinishedEvent(ImmutableMap. of(), 0L, + skyframeExecutor.getOutputDirtyFiles(), + skyframeExecutor.getModifiedFilesDuringPreviousBuild())); // Disable system load polling (noop if it was not enabled). ResourceManager.instance().setAutoSensing(false); diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/SkyframeBuilder.java b/src/main/java/com/google/devtools/build/lib/buildtool/SkyframeBuilder.java index 779515a28b..c96c178d7f 100644 --- a/src/main/java/com/google/devtools/build/lib/buildtool/SkyframeBuilder.java +++ b/src/main/java/com/google/devtools/build/lib/buildtool/SkyframeBuilder.java @@ -17,6 +17,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; +import com.google.common.collect.Range; import com.google.common.collect.Sets; import com.google.common.eventbus.EventBus; import com.google.devtools.build.lib.actions.Action; @@ -91,9 +92,10 @@ public class SkyframeBuilder implements Builder { Collection targetsToBuild, Executor executor, Set builtTargets, - boolean explain) + boolean explain, + Range lastExecutionTimeRange) throws BuildFailedException, AbruptExitException, TestExecException, InterruptedException { - skyframeExecutor.prepareExecution(checkOutputFiles); + skyframeExecutor.prepareExecution(checkOutputFiles, lastExecutionTimeRange); skyframeExecutor.setFileCache(fileCache); // Note that executionProgressReceiver accesses builtTargets concurrently (after wrapping in a // synchronized collection), so unsynchronized access to this variable is unsafe while it runs. diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java index d911820c37..b4308dbfc1 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java @@ -26,6 +26,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSortedSet; import com.google.common.collect.Lists; +import com.google.common.collect.Range; import com.google.common.collect.Sets; import com.google.common.eventbus.EventBus; import com.google.common.eventbus.SubscriberExceptionContext; @@ -181,6 +182,8 @@ public final class BlazeRuntime { private Path workingDirectory; private long commandStartTime; + private Range lastExecutionStartFinish = null; + private final SkyframeExecutor skyframeExecutor; private final Reporter reporter; @@ -418,6 +421,17 @@ public final class BlazeRuntime { } } + public void recordLastExecutionTime() { + lastExecutionStartFinish = Range.closed(commandStartTime, clock.currentTimeMillis()); + } + + /** + * Range that represents the last execution time of a build in millis since epoch. + */ + @Nullable + public Range getLastExecutionTimeRange() { + return lastExecutionStartFinish; + } public void recordCommandStartTime(long commandStartTime) { this.commandStartTime = commandStartTime; } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/Builder.java b/src/main/java/com/google/devtools/build/lib/skyframe/Builder.java index 7fdb55c897..f23b09f28b 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/Builder.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/Builder.java @@ -14,6 +14,7 @@ package com.google.devtools.build.lib.skyframe; +import com.google.common.collect.Range; import com.google.devtools.build.lib.actions.Artifact; import com.google.devtools.build.lib.actions.BuildFailedException; import com.google.devtools.build.lib.actions.Executor; @@ -25,6 +26,8 @@ import com.google.devtools.build.lib.util.AbruptExitException; import java.util.Collection; import java.util.Set; +import javax.annotation.Nullable; + /** * A Builder consumes top-level artifacts, targets, and tests,, and executes them in some * topological order, possibly concurrently, using some dependency-checking policy. @@ -57,6 +60,8 @@ public interface Builder { * @param builtTargets (out) set of successfully built subset of targetsToBuild. This set is * populated immediately upon confirmation that artifact is built so it will be * valid even if a future action throws ActionExecutionException + * @param lastExecutionTimeRange If not null, the start/finish time of the last build that + * run the execution phase. * @throws BuildFailedException if there were problems establishing the action execution * environment, if the the metadata of any file during the build could not be obtained, * if any input files are missing, or if an action fails during execution @@ -65,11 +70,12 @@ public interface Builder { */ @ThreadCompatible void buildArtifacts(Set artifacts, - Set parallelTests, - Set exclusiveTests, - Collection targetsToBuild, - Executor executor, - Set builtTargets, - boolean explain) + Set parallelTests, + Set exclusiveTests, + Collection targetsToBuild, + Executor executor, + Set builtTargets, + boolean explain, + @Nullable Range lastExecutionTimeRange) throws BuildFailedException, AbruptExitException, InterruptedException, TestExecException; } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FilesystemValueChecker.java b/src/main/java/com/google/devtools/build/lib/skyframe/FilesystemValueChecker.java index be4f4e8492..d2e962a6af 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/FilesystemValueChecker.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/FilesystemValueChecker.java @@ -21,6 +21,7 @@ import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; +import com.google.common.collect.Range; import com.google.common.collect.Sets; import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.devtools.build.lib.actions.Artifact; @@ -71,11 +72,15 @@ class FilesystemValueChecker { SkyFunctionName.functionIs(SkyFunctions.ACTION_EXECUTION); private final TimestampGranularityMonitor tsgm; + private final Range lastExecutionTimeRange; private final Supplier> valuesSupplier; private AtomicInteger modifiedOutputFilesCounter = new AtomicInteger(0); + private AtomicInteger modifiedOutputFilesIntraBuildCounter = new AtomicInteger(0); - FilesystemValueChecker(final MemoizingEvaluator evaluator, TimestampGranularityMonitor tsgm) { + FilesystemValueChecker(final MemoizingEvaluator evaluator, TimestampGranularityMonitor tsgm, + Range lastExecutionTimeRange) { this.tsgm = tsgm; + this.lastExecutionTimeRange = lastExecutionTimeRange; // Construct the full map view of the entire graph at most once ("memoized"), lazily. If // getDirtyFilesystemValues(Iterable) is called on an empty Iterable, we avoid having @@ -149,6 +154,7 @@ class FilesystemValueChecker { new ThrowableRecordingRunnableWrapper("FileSystemValueChecker#getDirtyActionValues"); modifiedOutputFilesCounter.set(0); + modifiedOutputFilesIntraBuildCounter.set(0); for (List> shard : outputShards) { Runnable job = (batchStatter == null) ? outputStatJob(dirtyKeys, shard) @@ -210,6 +216,7 @@ class FilesystemValueChecker { try { FileValue newData = FileAndMetadataCache.fileValueFromArtifact(artifact, stat, tsgm); if (!newData.equals(lastKnownData)) { + updateIntraBuildModifiedCounter(stat != null ? stat.getLastChangeTime() : -1); modifiedOutputFilesCounter.getAndIncrement(); dirtyKeys.add(key); } @@ -223,6 +230,12 @@ class FilesystemValueChecker { }; } + private void updateIntraBuildModifiedCounter(long time) throws IOException { + if (lastExecutionTimeRange != null && lastExecutionTimeRange.contains(time)) { + modifiedOutputFilesIntraBuildCounter.incrementAndGet(); + } + } + private Runnable outputStatJob(final Collection dirtyKeys, final List> shard) { return new Runnable() { @@ -239,12 +252,19 @@ class FilesystemValueChecker { } /** - * Returns number of modified output files inside of dirty actions. + * Returns the number of modified output files inside of dirty actions. */ int getNumberOfModifiedOutputFiles() { return modifiedOutputFilesCounter.get(); } + /** + * Returns the number of modified output files that occur during the previous build. + */ + public int getNumberOfModifiedOutputFilesDuringPreviousBuild() { + return modifiedOutputFilesIntraBuildCounter.get(); + } + private boolean actionValueIsDirtyWithDirectSystemCalls(ActionExecutionValue actionValue) { boolean isDirty = false; for (Map.Entry entry : @@ -252,8 +272,11 @@ class FilesystemValueChecker { Artifact artifact = entry.getKey(); FileValue lastKnownData = entry.getValue(); try { - if (!FileAndMetadataCache.fileValueFromArtifact(artifact, null, tsgm).equals( - lastKnownData)) { + FileValue fileValue = FileAndMetadataCache.fileValueFromArtifact(artifact, null, tsgm); + if (!fileValue.equals(lastKnownData)) { + updateIntraBuildModifiedCounter(fileValue.exists() + ? fileValue.realRootedPath().asPath().getLastModifiedTime() + : -1); modifiedOutputFilesCounter.getAndIncrement(); isDirty = true; } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java index e547166466..dc6dfac5ef 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java @@ -323,7 +323,7 @@ public final class SequencedSkyframeExecutor extends SkyframeExecutor { // next evaluate() call), because checking those is a waste of time. buildDriver.evaluate(ImmutableList.of(), false, DEFAULT_THREAD_COUNT, reporter); - FilesystemValueChecker fsnc = new FilesystemValueChecker(memoizingEvaluator, tsgm); + FilesystemValueChecker fsnc = new FilesystemValueChecker(memoizingEvaluator, tsgm, null); // We need to manually check for changes to known files. This entails finding all dirty file // system values under package roots for which we don't have diff information. If at least // one path entry doesn't have diff information, then we're going to have to iterate over @@ -422,7 +422,7 @@ public final class SequencedSkyframeExecutor extends SkyframeExecutor { Iterable keys; if (modifiedFileSet.treatEverythingAsModified()) { Differencer.Diff diff = - new FilesystemValueChecker(memoizingEvaluator, tsgm).getDirtyFilesystemSkyKeys(); + new FilesystemValueChecker(memoizingEvaluator, tsgm, null).getDirtyFilesystemSkyKeys(); keys = diff.changedKeysWithoutNewValues(); recordingDiffer.inject(diff.changedKeysWithNewValues()); } else { diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java index 0c5b503a89..5bf35799b5 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java @@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; +import com.google.common.collect.Range; import com.google.common.collect.Sets; import com.google.common.eventbus.EventBus; import com.google.devtools.build.lib.actions.Action; @@ -214,6 +215,8 @@ public abstract class SkyframeExecutor implements WalkableGraphFactory { private boolean needToInjectEmbeddedArtifacts = true; private boolean needToInjectPrecomputedValuesForAnalysis = true; protected int modifiedFiles; + protected int outputDirtyFiles; + protected int modifiedFilesDuringPreviousBuild; private final Predicate allowedMissingInputs; private final ImmutableMap extraSkyFunctions; @@ -1416,15 +1419,19 @@ public abstract class SkyframeExecutor implements WalkableGraphFactory { this.binTools = binTools; } - public void prepareExecution(boolean checkOutputFiles) throws AbruptExitException, + public void prepareExecution(boolean checkOutputFiles, + Range lastExecutionTimeRange) throws AbruptExitException, InterruptedException { maybeInjectEmbeddedArtifacts(); if (checkOutputFiles) { // Detect external modifications in the output tree. - FilesystemValueChecker fsnc = new FilesystemValueChecker(memoizingEvaluator, tsgm); + FilesystemValueChecker fsnc = new FilesystemValueChecker(memoizingEvaluator, tsgm, + lastExecutionTimeRange); invalidateDirtyActions(fsnc.getDirtyActionValues(batchStatter)); modifiedFiles += fsnc.getNumberOfModifiedOutputFiles(); + outputDirtyFiles += fsnc.getNumberOfModifiedOutputFiles(); + modifiedFilesDuringPreviousBuild += fsnc.getNumberOfModifiedOutputFilesDuringPreviousBuild(); } informAboutNumberOfModifiedFiles(); } @@ -1519,4 +1526,11 @@ public abstract class SkyframeExecutor implements WalkableGraphFactory { } } + public int getOutputDirtyFiles() { + return outputDirtyFiles; + } + + public int getModifiedFilesDuringPreviousBuild() { + return modifiedFilesDuringPreviousBuild; + } } -- cgit v1.2.3