Improve space efficiency of Blaze action cache: For actions that don't perform input discovery, there is no need to store the full set of edges in the action cache. This data is only used to formulate the set of input files for an action prior to checking the validity of a cache entry. For non-input-discovering actions, the set of input files is known statically and the action cache data is not used.

--
MOS_MIGRATED_REVID=112704382

4 files changed, 45 insertions, 24 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java b/src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java
index 3c4e5ada78..72d5756c42 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/ActionCacheChecker.java
@@ -202,7 +202,7 @@ public class ActionCacheChecker {
     } else if (validateArtifacts(entry, action, actionInputs, metadataHandler, true)) {
       reportChanged(handler, action);
       return true; // files have changed
-    } else if (!entry.getActionKey().equals(action.getKey())){
+    } else if (!entry.getActionKey().equals(action.getKey())) {
       reportCommand(handler, action);
       return true; // must execute -- action key is different
     }
@@ -219,7 +219,8 @@ public class ActionCacheChecker {
       // This cache entry has already been updated by a shared action. We don't need to do it again.
       return;
     }
-    ActionCache.Entry entry = actionCache.createEntry(action.getKey());
+    ActionCache.Entry entry =
+        actionCache.createEntry(action.getKey(), action.discoversInputs());
     for (Artifact output : action.getOutputs()) {
       // Remove old records from the cache if they used different key.
       String execPath = output.getExecPathString();
@@ -297,7 +298,7 @@ public class ActionCacheChecker {
       // Compute the aggregated middleman digest.
       // Since we never validate action key for middlemen, we should not store
       // it in the cache entry and just use empty string instead.
-      entry = actionCache.createEntry("");
+      entry = actionCache.createEntry("", false);
       for (Artifact input : action.getInputs()) {
         entry.addFile(input.getExecPath(), metadataHandler.getMetadataMaybe(input));
       }
diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java b/src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java
index 964e6b09b6..d992fb6b13 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java
@@ -14,6 +14,7 @@
 
 package com.google.devtools.build.lib.actions.cache;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadCompatible;
 import com.google.devtools.build.lib.util.Preconditions;
@@ -28,6 +29,8 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import javax.annotation.Nullable;
+
 /**
  * An interface defining a cache of already-executed Actions.
  *
@@ -61,7 +64,7 @@ public interface ActionCache {
    * Returns a new Entry instance. This method allows ActionCache subclasses to
    * define their own Entry implementation.
    */
-  ActionCache.Entry createEntry(String key);
+  ActionCache.Entry createEntry(String key, boolean discoversInputs);
 
   /**
    * An entry in the ActionCache that contains all action input and output
@@ -71,20 +74,22 @@ public interface ActionCache {
    * and getFileDigest() method is called, it becomes logically immutable (all methods
    * will continue to return same result regardless of internal data transformations).
    */
-  public final class Entry {
+  final class Entry {
     private final String actionKey;
+    @Nullable
+    // Null iff the corresponding action does not do input discovery.
     private final List<String> files;
     // If null, digest is non-null and the entry is immutable.
     private Map<String, Metadata> mdMap;
     private Digest digest;
 
-    public Entry(String key) {
+    public Entry(String key, boolean discoversInputs) {
       actionKey = key;
-      files = new ArrayList<>();
+      files = discoversInputs ? new ArrayList<String>() : null;
       mdMap = new HashMap<>();
     }
 
-    public Entry(String key, List<String> files, Digest digest) {
+    public Entry(String key, @Nullable List<String> files, Digest digest) {
       actionKey = key;
       this.files = files;
       this.digest = digest;
@@ -101,7 +106,9 @@ public interface ActionCache {
       Preconditions.checkState(digest == null);
 
       String execPath = relativePath.getPathString();
-      files.add(execPath);
+      if (discoversInputs()) {
+        files.add(execPath);
+      }
       mdMap.put(execPath, md);
     }
 
@@ -134,10 +141,17 @@ public interface ActionCache {
     }
 
     /**
-     * @return stored path strings.
+     * @return stored path strings, or null if the corresponding action does not discover inputs.
      */
     public Collection<String> getPaths() {
-      return files;
+      return discoversInputs() ? files : ImmutableList.<String>of();
+    }
+
+    /**
+     * @return whether the corresponding action discovers input files dynamically.
+     */
+    public boolean discoversInputs() {
+      return files != null;
     }
 
     @Override
@@ -150,11 +164,14 @@ public interface ActionCache {
       } else {
         builder.append(digest).append("\n");
       }
-      List<String> fileInfo = Lists.newArrayListWithCapacity(files.size());
-      fileInfo.addAll(files);
-      Collections.sort(fileInfo);
-      for (String info : fileInfo) {
-        builder.append("      ").append(info).append("\n");
+
+      if (discoversInputs()) {
+        List<String> fileInfo = Lists.newArrayListWithCapacity(files.size());
+        fileInfo.addAll(files);
+        Collections.sort(fileInfo);
+        for (String info : fileInfo) {
+          builder.append("      ").append(info).append("\n");
+        }
       }
       return builder.toString();
     }
diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCache.java b/src/main/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCache.java
index 328a31d33f..faf5214b6b 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCache.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCache.java
@@ -56,7 +56,9 @@ public class CompactPersistentActionCache implements ActionCache {
   // cache records.
   private static final int VALIDATION_KEY = -10;
 
-  private static final int VERSION = 10;
+  private static final int NO_INPUT_DISCOVERY_COUNT = -1;
+
+  private static final int VERSION = 11;
 
   private final class ActionMap extends PersistentMap<Integer, byte[]> {
     private final Clock clock;
@@ -136,7 +138,7 @@ public class CompactPersistentActionCache implements ActionCache {
 
   private final PersistentMap<Integer, byte[]> map;
   private final PersistentStringIndexer indexer;
-  static final ActionCache.Entry CORRUPTED = new ActionCache.Entry(null);
+  static final ActionCache.Entry CORRUPTED = new ActionCache.Entry(null, false);
 
   public CompactPersistentActionCache(Path cacheRoot, Clock clock) throws IOException {
     Path cacheFile = cacheFile(cacheRoot);
@@ -222,8 +224,8 @@ public class CompactPersistentActionCache implements ActionCache {
   }
 
   @Override
-  public ActionCache.Entry createEntry(String key) {
-    return new ActionCache.Entry(key);
+  public ActionCache.Entry createEntry(String key, boolean discoversInputs) {
+    return new ActionCache.Entry(key, discoversInputs);
   }
 
   @Override
@@ -349,7 +351,7 @@ public class CompactPersistentActionCache implements ActionCache {
 
       entry.getFileDigest().write(sink);
 
-      VarInt.putVarInt(files.size(), sink);
+      VarInt.putVarInt(entry.discoversInputs() ? files.size() : NO_INPUT_DISCOVERY_COUNT, sink);
       for (String file : files) {
         VarInt.putVarInt(indexer.getOrCreateIndex(file), sink);
       }
@@ -388,7 +390,8 @@ public class CompactPersistentActionCache implements ActionCache {
       if (source.remaining() > 0) {
         throw new IOException("serialized entry data has not been fully decoded");
       }
-      return new Entry(actionKey, builder.build(), digest);
+      return new Entry(actionKey,
+          count == NO_INPUT_DISCOVERY_COUNT ? null : builder.build(), digest);
     } catch (BufferUnderflowException e) {
       throw new IOException("encoded entry data is incomplete", e);
     }
diff --git a/src/main/java/com/google/devtools/build/lib/actions/cache/NullActionCache.java b/src/main/java/com/google/devtools/build/lib/actions/cache/NullActionCache.java
index 6af10418b5..f3f12de553 100644
--- a/src/main/java/com/google/devtools/build/lib/actions/cache/NullActionCache.java
+++ b/src/main/java/com/google/devtools/build/lib/actions/cache/NullActionCache.java
@@ -36,8 +36,8 @@ public final class NullActionCache implements ActionCache {
   }
 
   @Override
-  public Entry createEntry(String key) {
-    return new ActionCache.Entry(key);
+  public Entry createEntry(String key, boolean discoversInputs) {
+    return new ActionCache.Entry(key, discoversInputs);
   }
 
   @Override