Optional reuse of classpath from previous compile saved in memory, as it

appears in the jdeps file. These are the minimum inputs needed by the
JavaCompileAction to obtain the correct result when classpath order and source
files remain unchanged. Uses existing --java_classpath flag to toggle between
'javabuilder' (current released behavior) and new 'experimental_blaze' (minimum classpath
and Spawn inputs). Handles failure of minimal action by falling back to existing
compilation behavior when needed. The new Spawn does not use a parameter file,
but puts all arguments directly on the command line.

--
MOS_MIGRATED_REVID=131109163

3 files changed, 150 insertions, 12 deletions

diff --git a/src/main/java/com/google/devtools/build/lib/BUILD b/src/main/java/com/google/devtools/build/lib/BUILD
index 5a501f8613..17104788af 100644
--- a/src/main/java/com/google/devtools/build/lib/BUILD
+++ b/src/main/java/com/google/devtools/build/lib/BUILD
@@ -843,6 +843,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/cmdline",
         "//src/main/java/com/google/devtools/build/lib/rules/cpp",
         "//src/main/java/com/google/devtools/common/options",
+        "//src/main/protobuf:deps_java_proto",
         "//src/main/protobuf:extra_actions_base_java_proto",
         "//third_party:guava",
         "//third_party:joda_time",
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompileAction.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompileAction.java
index c168642c88..eac7e6e80b 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompileAction.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaCompileAction.java
@@ -31,6 +31,7 @@ import com.google.devtools.build.lib.actions.AbstractAction;
 import com.google.devtools.build.lib.actions.Action;
 import com.google.devtools.build.lib.actions.ActionExecutionContext;
 import com.google.devtools.build.lib.actions.ActionExecutionException;
+import com.google.devtools.build.lib.actions.ActionInput;
 import com.google.devtools.build.lib.actions.ActionOwner;
 import com.google.devtools.build.lib.actions.Artifact;
 import com.google.devtools.build.lib.actions.ArtifactOwner;
@@ -65,15 +66,23 @@ import com.google.devtools.build.lib.util.Fingerprint;
 import com.google.devtools.build.lib.util.Preconditions;
 import com.google.devtools.build.lib.util.ShellEscaper;
 import com.google.devtools.build.lib.util.StringCanonicalizer;
+import com.google.devtools.build.lib.vfs.FileSystemUtils;
+import com.google.devtools.build.lib.vfs.Path;
 import com.google.devtools.build.lib.vfs.PathFragment;
+import com.google.devtools.build.lib.view.proto.Deps;
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.logging.Logger;
 
 /**
  * Action that represents a Java compilation.
@@ -85,6 +94,11 @@ public final class JavaCompileAction extends AbstractAction {
   private static final ResourceSet LOCAL_RESOURCES =
       ResourceSet.createWithRamCpuIo(750 /*MB*/, 0.5 /*CPU*/, 0.0 /*IO*/);
 
+  /** Default number of unused jars below which we ignore minimum classpath optimization. */
+  private static final int MINIMUM_REDUCTION_TO_SAVE_USED_INPUTS = 0;
+
+  private static final Logger logger = Logger.getLogger(JavaCompileAction.class.getName());
+
   private final CommandLine javaCompileCommandLine;
   private final CommandLine commandLine;
 
@@ -156,6 +170,20 @@ public final class JavaCompileAction extends AbstractAction {
    */
   private final ImmutableList<Artifact> compileTimeDependencyArtifacts;
 
+  /** Saved copy of the baseInputs needed to reconstruct the full inputs. */
+  private final Iterable<Artifact> baseInputs;
+
+  /** If non-null, use this command line base for calling JavaBuilder with minimum classpath. */
+  private final CommandLine minimumCommandLineBase;
+
+  private final String pathDelimiter;
+
+  /** Inputs that were actually used for the previous compilation, if successful. */
+  private Iterable<Artifact> usedInputs;
+
+  /** Actual, complete command line for minimum compile optimization. */
+  private CommandLine minCommandLine;
+
   /**
    * Constructs an action to compile a set of Java source files to class files.
    *
@@ -168,6 +196,7 @@ public final class JavaCompileAction extends AbstractAction {
    *     written to the parameter file, but other parts (for example, ide_build_info) need access to
    *     the data
    * @param commandLine the actual invocation command line
+   * @param minimumCommandLineBase minimum classpath invocation command line, without inputs
    * @param resourceCount the count of all resource inputs
    */
   private JavaCompileAction(
@@ -193,6 +222,8 @@ public final class JavaCompileAction extends AbstractAction {
       Map<String, String> executionInfo,
       BuildConfiguration.StrictDepsMode strictJavaDeps,
       Collection<Artifact> compileTimeDependencyArtifacts,
+      CommandLine minimumCommandLineBase,
+      String pathDelimiter,
       int resourceCount) {
     super(
         owner,
@@ -223,6 +254,9 @@ public final class JavaCompileAction extends AbstractAction {
     this.executionInfo = ImmutableMap.copyOf(executionInfo);
     this.strictJavaDeps = strictJavaDeps;
     this.compileTimeDependencyArtifacts = ImmutableList.copyOf(compileTimeDependencyArtifacts);
+    this.baseInputs = ImmutableList.copyOf(baseInputs);
+    this.minimumCommandLineBase = minimumCommandLineBase;
+    this.pathDelimiter = pathDelimiter;
     this.resourceCount = resourceCount;
   }
 
@@ -344,11 +378,11 @@ public final class JavaCompileAction extends AbstractAction {
     return javaCompileCommandLine.arguments();
   }
 
-  /**
-   * Returns the command and arguments for a java compile action.
-   */
+  /** Returns the command and arguments for a java compile action. */
   public List<String> getCommand() {
-    return ImmutableList.copyOf(commandLine.arguments());
+    // If available, use the saved minCommandLine, otherwise use the command line with full inputs.
+    return ImmutableList.copyOf(
+        (minCommandLine != null ? minCommandLine : commandLine).arguments());
   }
 
   @VisibleForTesting
@@ -358,7 +392,13 @@ public final class JavaCompileAction extends AbstractAction {
         ImmutableMap.of("LC_CTYPE", "en_US.UTF-8"),
         executionInfo,
         this,
-        LOCAL_RESOURCES);
+        LOCAL_RESOURCES) {
+      @Override
+      public Iterable<? extends ActionInput> getInputFiles() {
+        // Reduce inputs for minclasspath compile. Requires use of minCommandLine.
+        return usedInputs != null ? usedInputs : super.getInputFiles();
+      }
+    };
   }
 
   @Override
@@ -366,11 +406,70 @@ public final class JavaCompileAction extends AbstractAction {
   public void execute(ActionExecutionContext actionExecutionContext)
       throws ActionExecutionException, InterruptedException {
     Executor executor = actionExecutionContext.getExecutor();
-    try {
-      getContext(executor).exec(createSpawn(), actionExecutionContext);
-    } catch (ExecException e) {
-      throw e.toActionExecutionException("Java compilation in rule '" + getOwner().getLabel() + "'",
-          executor.getVerboseFailures(), this);
+    boolean executeFullCompile = true;
+    if (usedInputs != null) {
+      try {
+        getContext(executor).exec(createSpawn(), actionExecutionContext);
+        executeFullCompile = false;
+      } catch (ExecException e) {
+        logger.info("Minimum classpath failed for " + getOwner().getLabel().toShorthandString());
+        minCommandLine = null;
+        usedInputs = null;
+      }
+    }
+    if (executeFullCompile) {
+      try {
+        getContext(executor).exec(createSpawn(), actionExecutionContext);
+      } catch (ExecException e) {
+        throw e.toActionExecutionException(
+            "Java compilation in rule '" + getOwner().getLabel() + "'",
+            executor.getVerboseFailures(),
+            this);
+      }
+    }
+    maybeSaveUsedInputs();
+  }
+  /**
+   * If enabled, read and save the contents of the output '.jdeps' file for minimum classpath
+   * incremental compile on the next call to execute.
+   */
+  @VisibleForTesting
+  void maybeSaveUsedInputs() {
+    minCommandLine = null;
+    usedInputs = null;
+    if (minimumCommandLineBase != null) {
+      // Find the actually needed dependencies from the just-generated jdeps file.
+      // TODO(b/30902566): Cache jdeps md5, avoid rereading and recreating command when unchanged.
+      Set<String> usedInputJars = readJdeps(outputJar);
+      if (usedInputJars == null) {
+        return;
+      }
+      List<Artifact> orderedClasspath = classpathEntries.toList();
+      // Only continue if we anticipate the added work and memory will pay off.
+      if (orderedClasspath.size() - usedInputJars.size() <= MINIMUM_REDUCTION_TO_SAVE_USED_INPUTS) {
+        return;
+      }
+      ImmutableList.Builder<Artifact> minInputsBuilder = ImmutableList.builder();
+      for (Artifact artifact : orderedClasspath) {
+        if (usedInputJars.contains(artifact.getExecPathString())) {
+          minInputsBuilder.add(artifact);
+        }
+      }
+      final ImmutableList<Artifact> minimumInputs = minInputsBuilder.build();
+      // The two things needed to enable minimum incremental classpath compile - command & inputs
+      minCommandLine =
+          CustomCommandLine.builder()
+              .add(minimumCommandLineBase.arguments())
+              .add("--classpath")
+              .add(getClasspathArg(minimumInputs, classDirectory, pathDelimiter))
+              .add("--strict_java_deps")
+              .add(strictJavaDeps.toString())
+              .add(new JarsToTargetsArgv(minimumInputs, directJars))
+              .build();
+
+      // Keep in sync with inputs in constructor call to 'super', except do not
+      // include compileTimeDependencyArtifacts or paramFile, which are unneeded here.
+      usedInputs = Iterables.concat(minimumInputs, baseInputs, getTools());
     }
   }
 
@@ -378,6 +477,7 @@ public final class JavaCompileAction extends AbstractAction {
   protected String computeKey() {
     Fingerprint f = new Fingerprint();
     f.addString(GUID);
+    f.addBoolean(minimumCommandLineBase != null);
     f.addStrings(commandLine.arguments());
     return f.hexDigestAndReset();
   }
@@ -643,7 +743,7 @@ public final class JavaCompileAction extends AbstractAction {
       result.add(new JarsToTargetsArgv(classpath, directJars));
 
       if (configuration.getFragment(JavaConfiguration.class).getReduceJavaClasspath()
-          == JavaClasspathMode.JAVABUILDER) {
+          != JavaClasspathMode.OFF) {
         result.add("--reduce_classpath");
 
         if (!compileTimeDependencyArtifacts.isEmpty()) {
@@ -764,6 +864,31 @@ public final class JavaCompileAction extends AbstractAction {
     };
   }
 
+  @VisibleForTesting
+  static Set<String> readJdeps(Artifact outputJar) {
+    Set<String> jdeps = new HashSet<>();
+    Path jdepsFile = FileSystemUtils.replaceExtension(outputJar.getPath(), ".jdeps");
+    String label = outputJar.getOwnerLabel().toShorthandString();
+    if (!jdepsFile.exists() || !jdepsFile.isFile()) {
+      logger.warning("Jdeps file missing for " + label);
+      return null;
+    }
+    try (InputStream bis = new BufferedInputStream(jdepsFile.getInputStream())) {
+      Deps.Dependencies deps = Deps.Dependencies.parseFrom(bis);
+      if (!deps.hasSuccess() || !deps.getSuccess() || !deps.hasRuleLabel()) {
+        logger.warning("Cannot use jdeps file for " + label);
+        return null;
+      }
+      for (Deps.Dependency dep : deps.getDependencyList()) {
+        jdeps.add(dep.getPath());
+      }
+    } catch (IOException e) {
+      logger.warning("Failed to read jdeps file for " + label);
+      return null;
+    }
+    return jdeps;
+  }
+
   /**
    * Tells {@link Builder} how to create new artifacts. Is there so that {@link Builder} can be
    * exercised in tests without creating a full {@link RuleContext}.
@@ -982,6 +1107,16 @@ public final class JavaCompileAction extends AbstractAction {
               .addAll(instrumentationJars)
               .build();
 
+      // Minimum compile command line null unless explicitly enabled via JavaClasspathMode.
+      CommandLine minimumCommandLineBase = null;
+      if (javaConfiguration.getReduceJavaClasspath() == JavaClasspathMode.EXPERIMENTAL_BLAZE) {
+        minimumCommandLineBase =
+            CustomCommandLine.builder()
+                .add(spawnCommandLineBase)
+                .add(commonJavaBuilderArgs)
+                .build();
+      }
+
       return new JavaCompileAction(
           owner,
           tools,
@@ -1005,6 +1140,8 @@ public final class JavaCompileAction extends AbstractAction {
           executionInfo,
           strictJavaDeps,
           compileTimeDependencyArtifacts,
+          minimumCommandLineBase,
+          pathSeparator,
           resources.size() + classpathResources.size() + translations.size());
     }
 
diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaConfiguration.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaConfiguration.java
index c7eec0495f..bbb5432517 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaConfiguration.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaConfiguration.java
@@ -48,7 +48,7 @@ public final class JavaConfiguration extends Fragment {
     /** JavaBuilder computes the reduced classpath before invoking javac. */
     JAVABUILDER,
     /** Blaze computes the reduced classpath before invoking JavaBuilder. */
-    BLAZE
+    EXPERIMENTAL_BLAZE
   }
 
   /**