Record statistics about dirty output files detected in the output tree.

--
MOS_MIGRATED_REVID=88257621

8 files changed, 96 insertions, 20 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionFinishedEvent.java b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionFinishedEvent.java
index 74143cc6df..9662e0817e 100644
--- a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionFinishedEvent.java
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionFinishedEvent.java
@@ -23,6 +23,9 @@ import java.util.Map;
  * the metadata cache and about last file save times.
  */
 public class ExecutionFinishedEvent {
+
+  private final int outputDirtyFiles;
+  private final int outputModifiedFilesDuringPreviousBuild;
   /** The mtime of the most recently saved source file when the build starts. */
   private long lastFileSaveTimeInMillis;
 
@@ -34,7 +37,10 @@ public class ExecutionFinishedEvent {
   private Map<String, Long> changedFileSaveTimes = new HashMap<>();
 
   public ExecutionFinishedEvent(Map<String, Long> changedFileSaveTimes,
-      long lastFileSaveTimeInMillis) {
+      long lastFileSaveTimeInMillis, int outputDirtyFiles,
+      int outputModifiedFilesDuringPreviousBuild) {
+    this.outputDirtyFiles = outputDirtyFiles;
+    this.outputModifiedFilesDuringPreviousBuild = outputModifiedFilesDuringPreviousBuild;
     this.changedFileSaveTimes = ImmutableMap.copyOf(changedFileSaveTimes);
     this.lastFileSaveTimeInMillis = lastFileSaveTimeInMillis;
   }
@@ -43,6 +49,14 @@ public class ExecutionFinishedEvent {
     return lastFileSaveTimeInMillis;
   }
 
+  public int getOutputDirtyFiles() {
+    return outputDirtyFiles;
+  }
+
+  public int getOutputModifiedFilesDuringPreviousBuild() {
+    return outputModifiedFilesDuringPreviousBuild;
+  }
+
   public Map<String, Long> getChangedFileSaveTimes() {
     return changedFileSaveTimes;
   }
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java
index bd48395081..d66da235c2 100644
--- a/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/ExecutionTool.java
@@ -404,12 +404,14 @@ public class ExecutionTool {
           analysisResult.getExclusiveTests(),
           analysisResult.getTargetsToBuild(),
           executor, builtTargets,
-          request.getBuildOptions().explanationPath != null);
+          request.getBuildOptions().explanationPath != null,
+          runtime.getLastExecutionTimeRange());
 
     } catch (InterruptedException e) {
       interrupted = true;
       throw e;
     } finally {
+      runtime.recordLastExecutionTime();
       if (request.isRunningInEmacs()) {
         request.getOutErr().printErrLn("blaze: Leaving directory `" + getExecRoot() + "/'");
       }
@@ -417,8 +419,9 @@ public class ExecutionTool {
         getReporter().handle(Event.progress("Building complete."));
       }
 
-      // Transfer over source file "last save time" stats so the remote logger can find them.
-      runtime.getEventBus().post(new ExecutionFinishedEvent(ImmutableMap.<String, Long> of(), 0));
+      runtime.getEventBus().post(new ExecutionFinishedEvent(ImmutableMap.<String, Long> of(), 0L,
+          skyframeExecutor.getOutputDirtyFiles(),
+          skyframeExecutor.getModifiedFilesDuringPreviousBuild()));
 
       // Disable system load polling (noop if it was not enabled).
       ResourceManager.instance().setAutoSensing(false);
diff --git a/src/main/java/com/google/devtools/build/lib/buildtool/SkyframeBuilder.java b/src/main/java/com/google/devtools/build/lib/buildtool/SkyframeBuilder.java
index 779515a28b..c96c178d7f 100644
--- a/src/main/java/com/google/devtools/build/lib/buildtool/SkyframeBuilder.java
+++ b/src/main/java/com/google/devtools/build/lib/buildtool/SkyframeBuilder.java
@@ -17,6 +17,7 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
+import com.google.common.collect.Range;
 import com.google.common.collect.Sets;
 import com.google.common.eventbus.EventBus;
 import com.google.devtools.build.lib.actions.Action;
@@ -91,9 +92,10 @@ public class SkyframeBuilder implements Builder {
       Collection<ConfiguredTarget> targetsToBuild,
       Executor executor,
       Set<ConfiguredTarget> builtTargets,
-      boolean explain)
+      boolean explain,
+      Range<Long> lastExecutionTimeRange)
       throws BuildFailedException, AbruptExitException, TestExecException, InterruptedException {
-    skyframeExecutor.prepareExecution(checkOutputFiles);
+    skyframeExecutor.prepareExecution(checkOutputFiles, lastExecutionTimeRange);
     skyframeExecutor.setFileCache(fileCache);
     // Note that executionProgressReceiver accesses builtTargets concurrently (after wrapping in a
     // synchronized collection), so unsynchronized access to this variable is unsafe while it runs.
diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java
index d911820c37..b4308dbfc1 100644
--- a/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java
+++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeRuntime.java
@@ -26,6 +26,7 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSortedSet;
 import com.google.common.collect.Lists;
+import com.google.common.collect.Range;
 import com.google.common.collect.Sets;
 import com.google.common.eventbus.EventBus;
 import com.google.common.eventbus.SubscriberExceptionContext;
@@ -181,6 +182,8 @@ public final class BlazeRuntime {
   private Path workingDirectory;
   private long commandStartTime;
 
+  private Range<Long> lastExecutionStartFinish = null;
+
   private final SkyframeExecutor skyframeExecutor;
 
   private final Reporter reporter;
@@ -418,6 +421,17 @@ public final class BlazeRuntime {
     }
   }
 
+  public void recordLastExecutionTime() {
+    lastExecutionStartFinish = Range.closed(commandStartTime, clock.currentTimeMillis());
+  }
+
+  /**
+   * Range that represents the last execution time of a build in millis since epoch.
+   */
+  @Nullable
+  public Range<Long> getLastExecutionTimeRange() {
+    return lastExecutionStartFinish;
+  }
   public void recordCommandStartTime(long commandStartTime) {
     this.commandStartTime = commandStartTime;
   }
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/Builder.java b/src/main/java/com/google/devtools/build/lib/skyframe/Builder.java
index 7fdb55c897..f23b09f28b 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/Builder.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/Builder.java
@@ -14,6 +14,7 @@
 
 package com.google.devtools.build.lib.skyframe;
 
+import com.google.common.collect.Range;
 import com.google.devtools.build.lib.actions.Artifact;
 import com.google.devtools.build.lib.actions.BuildFailedException;
 import com.google.devtools.build.lib.actions.Executor;
@@ -25,6 +26,8 @@ import com.google.devtools.build.lib.util.AbruptExitException;
 import java.util.Collection;
 import java.util.Set;
 
+import javax.annotation.Nullable;
+
 /**
  * A Builder consumes top-level artifacts, targets, and tests,, and executes them in some
  * topological order, possibly concurrently, using some dependency-checking policy.
@@ -57,6 +60,8 @@ public interface Builder {
    * @param builtTargets (out) set of successfully built subset of targetsToBuild. This set is
    *        populated immediately upon confirmation that artifact is built so it will be
    *        valid even if a future action throws ActionExecutionException
+   * @param lastExecutionTimeRange If not null, the start/finish time of the last build that
+   *        run the execution phase.
    * @throws BuildFailedException if there were problems establishing the action execution
    *         environment, if the the metadata of any file  during the build could not be obtained,
    *         if any input files are missing, or if an action fails during execution
@@ -65,11 +70,12 @@ public interface Builder {
    */
   @ThreadCompatible
   void buildArtifacts(Set<Artifact> artifacts,
-                      Set<ConfiguredTarget> parallelTests,
-                      Set<ConfiguredTarget> exclusiveTests,
-                      Collection<ConfiguredTarget> targetsToBuild,
-                      Executor executor,
-                      Set<ConfiguredTarget> builtTargets,
-                      boolean explain)
+      Set<ConfiguredTarget> parallelTests,
+      Set<ConfiguredTarget> exclusiveTests,
+      Collection<ConfiguredTarget> targetsToBuild,
+      Executor executor,
+      Set<ConfiguredTarget> builtTargets,
+      boolean explain,
+      @Nullable Range<Long> lastExecutionTimeRange)
       throws BuildFailedException, AbruptExitException, InterruptedException, TestExecException;
 }
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/FilesystemValueChecker.java b/src/main/java/com/google/devtools/build/lib/skyframe/FilesystemValueChecker.java
index be4f4e8492..d2e962a6af 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/FilesystemValueChecker.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/FilesystemValueChecker.java
@@ -21,6 +21,7 @@ import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
+import com.google.common.collect.Range;
 import com.google.common.collect.Sets;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import com.google.devtools.build.lib.actions.Artifact;
@@ -71,11 +72,15 @@ class FilesystemValueChecker {
       SkyFunctionName.functionIs(SkyFunctions.ACTION_EXECUTION);
 
   private final TimestampGranularityMonitor tsgm;
+  private final Range<Long> lastExecutionTimeRange;
   private final Supplier<Map<SkyKey, SkyValue>> valuesSupplier;
   private AtomicInteger modifiedOutputFilesCounter = new AtomicInteger(0);
+  private AtomicInteger modifiedOutputFilesIntraBuildCounter = new AtomicInteger(0);
 
-  FilesystemValueChecker(final MemoizingEvaluator evaluator, TimestampGranularityMonitor tsgm) {
+  FilesystemValueChecker(final MemoizingEvaluator evaluator, TimestampGranularityMonitor tsgm,
+      Range<Long> lastExecutionTimeRange) {
     this.tsgm = tsgm;
+    this.lastExecutionTimeRange = lastExecutionTimeRange;
 
     // Construct the full map view of the entire graph at most once ("memoized"), lazily. If
     // getDirtyFilesystemValues(Iterable<SkyKey>) is called on an empty Iterable, we avoid having
@@ -149,6 +154,7 @@ class FilesystemValueChecker {
         new ThrowableRecordingRunnableWrapper("FileSystemValueChecker#getDirtyActionValues");
 
     modifiedOutputFilesCounter.set(0);
+    modifiedOutputFilesIntraBuildCounter.set(0);
     for (List<Pair<SkyKey, ActionExecutionValue>> shard : outputShards) {
       Runnable job = (batchStatter == null)
           ? outputStatJob(dirtyKeys, shard)
@@ -210,6 +216,7 @@ class FilesystemValueChecker {
           try {
             FileValue newData = FileAndMetadataCache.fileValueFromArtifact(artifact, stat, tsgm);
             if (!newData.equals(lastKnownData)) {
+              updateIntraBuildModifiedCounter(stat != null ? stat.getLastChangeTime() : -1);
               modifiedOutputFilesCounter.getAndIncrement();
               dirtyKeys.add(key);
             }
@@ -223,6 +230,12 @@ class FilesystemValueChecker {
     };
   }
 
+  private void updateIntraBuildModifiedCounter(long time) throws IOException {
+    if (lastExecutionTimeRange != null && lastExecutionTimeRange.contains(time)) {
+      modifiedOutputFilesIntraBuildCounter.incrementAndGet();
+    }
+  }
+
   private Runnable outputStatJob(final Collection<SkyKey> dirtyKeys,
                                  final List<Pair<SkyKey, ActionExecutionValue>> shard) {
     return new Runnable() {
@@ -239,12 +252,19 @@ class FilesystemValueChecker {
   }
 
   /**
-   * Returns number of modified output files inside of dirty actions.
+   * Returns the number of modified output files inside of dirty actions.
    */
   int getNumberOfModifiedOutputFiles() {
     return modifiedOutputFilesCounter.get();
   }
 
+  /**
+   * Returns the number of modified output files that occur during the previous build.
+   */
+  public int getNumberOfModifiedOutputFilesDuringPreviousBuild() {
+    return modifiedOutputFilesIntraBuildCounter.get();
+  }
+
   private boolean actionValueIsDirtyWithDirectSystemCalls(ActionExecutionValue actionValue) {
     boolean isDirty = false;
     for (Map.Entry<Artifact, FileValue> entry :
@@ -252,8 +272,11 @@ class FilesystemValueChecker {
       Artifact artifact = entry.getKey();
       FileValue lastKnownData = entry.getValue();
       try {
-        if (!FileAndMetadataCache.fileValueFromArtifact(artifact, null, tsgm).equals(
-            lastKnownData)) {
+        FileValue fileValue = FileAndMetadataCache.fileValueFromArtifact(artifact, null, tsgm);
+        if (!fileValue.equals(lastKnownData)) {
+          updateIntraBuildModifiedCounter(fileValue.exists()
+              ? fileValue.realRootedPath().asPath().getLastModifiedTime()
+              : -1);
           modifiedOutputFilesCounter.getAndIncrement();
           isDirty = true;
         }
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java
index e547166466..dc6dfac5ef 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SequencedSkyframeExecutor.java
@@ -323,7 +323,7 @@ public final class SequencedSkyframeExecutor extends SkyframeExecutor {
     // next evaluate() call), because checking those is a waste of time.
     buildDriver.evaluate(ImmutableList.<SkyKey>of(), false,
         DEFAULT_THREAD_COUNT, reporter);
-    FilesystemValueChecker fsnc = new FilesystemValueChecker(memoizingEvaluator, tsgm);
+    FilesystemValueChecker fsnc = new FilesystemValueChecker(memoizingEvaluator, tsgm, null);
     // We need to manually check for changes to known files. This entails finding all dirty file
     // system values under package roots for which we don't have diff information. If at least
     // one path entry doesn't have diff information, then we're going to have to iterate over
@@ -422,7 +422,7 @@ public final class SequencedSkyframeExecutor extends SkyframeExecutor {
     Iterable<SkyKey> keys;
     if (modifiedFileSet.treatEverythingAsModified()) {
       Differencer.Diff diff =
-          new FilesystemValueChecker(memoizingEvaluator, tsgm).getDirtyFilesystemSkyKeys();
+          new FilesystemValueChecker(memoizingEvaluator, tsgm, null).getDirtyFilesystemSkyKeys();
       keys = diff.changedKeysWithoutNewValues();
       recordingDiffer.inject(diff.changedKeysWithNewValues());
     } else {
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
index 0c5b503a89..5bf35799b5 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java
@@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Maps;
+import com.google.common.collect.Range;
 import com.google.common.collect.Sets;
 import com.google.common.eventbus.EventBus;
 import com.google.devtools.build.lib.actions.Action;
@@ -214,6 +215,8 @@ public abstract class SkyframeExecutor implements WalkableGraphFactory {
   private boolean needToInjectEmbeddedArtifacts = true;
   private boolean needToInjectPrecomputedValuesForAnalysis = true;
   protected int modifiedFiles;
+  protected int outputDirtyFiles;
+  protected int modifiedFilesDuringPreviousBuild;
   private final Predicate<PathFragment> allowedMissingInputs;
 
   private final ImmutableMap<SkyFunctionName, SkyFunction> extraSkyFunctions;
@@ -1416,15 +1419,19 @@ public abstract class SkyframeExecutor implements WalkableGraphFactory {
     this.binTools = binTools;
   }
 
-  public void prepareExecution(boolean checkOutputFiles) throws AbruptExitException,
+  public void prepareExecution(boolean checkOutputFiles,
+      Range<Long> lastExecutionTimeRange) throws AbruptExitException,
       InterruptedException {
     maybeInjectEmbeddedArtifacts();
 
     if (checkOutputFiles) {
       // Detect external modifications in the output tree.
-      FilesystemValueChecker fsnc = new FilesystemValueChecker(memoizingEvaluator, tsgm);
+      FilesystemValueChecker fsnc = new FilesystemValueChecker(memoizingEvaluator, tsgm,
+          lastExecutionTimeRange);
       invalidateDirtyActions(fsnc.getDirtyActionValues(batchStatter));
       modifiedFiles += fsnc.getNumberOfModifiedOutputFiles();
+      outputDirtyFiles += fsnc.getNumberOfModifiedOutputFiles();
+      modifiedFilesDuringPreviousBuild += fsnc.getNumberOfModifiedOutputFilesDuringPreviousBuild();
     }
     informAboutNumberOfModifiedFiles();
   }
@@ -1519,4 +1526,11 @@ public abstract class SkyframeExecutor implements WalkableGraphFactory {
     }
   }
 
+  public int getOutputDirtyFiles() {
+    return outputDirtyFiles;
+  }
+
+  public int getModifiedFilesDuringPreviousBuild() {
+    return modifiedFilesDuringPreviousBuild;
+  }
 }