Emit a 'manifest' file containing information about the compiled sources

For each compilation unit, JavaBuilder will record:

-source path
-package name
-a list of top-level class names
-whether the file was generated by an annotation processor

--
MOS_MIGRATED_REVID=96158093

7 files changed, 149 insertions, 119 deletions

diff --git a/src/java_tools/buildjar/BUILD b/src/java_tools/buildjar/BUILD
index 073272b16a..28773711e8 100644
--- a/src/java_tools/buildjar/BUILD
+++ b/src/java_tools/buildjar/BUILD
@@ -6,6 +6,7 @@ java_binary(
     main_class = "com.google.devtools.build.buildjar.BazelJavaBuilder",
     deps = [
         "//src/main/protobuf:proto_deps",
+        "//src/main/protobuf:proto_java_compilation",
         "//src/main/protobuf:proto_worker_protocol",
         "//third_party:error_prone",
         "//third_party:guava",
@@ -194,6 +195,7 @@ bootstrap_java_library(
     srcjars = [
         "//src/main/protobuf:proto_deps_srcjar",
         "//src/main/protobuf:proto_worker_protocol_srcjar",
+        "//src/main/protobuf:proto_java_compilation_srcjar",
     ],
 )
 
diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/AbstractJavaBuilder.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/AbstractJavaBuilder.java
index 766d1f55e7..035d3b3bbf 100644
--- a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/AbstractJavaBuilder.java
+++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/AbstractJavaBuilder.java
@@ -168,14 +168,12 @@ public abstract class AbstractJavaBuilder extends AbstractLibraryBuilder {
         if (build.getGeneratedSourcesOutputJar() != null) {
           buildGensrcJar(build, err);
         }
-        if (build.getGeneratedClassOutputJar() != null) {
-          buildGenclassJar(build);
-        }
       }
       successful = true;
     } finally {
       build.getDependencyModule().emitUsedClasspath(build.getClassPath());
       build.getDependencyModule().emitDependencyInformation(build.getClassPath(), successful);
+      build.getProcessingModule().emitManifestProto();
       shutdown(err);
     }
   }
diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/AbstractLibraryBuilder.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/AbstractLibraryBuilder.java
index c402cc07a9..cf1c985b0f 100644
--- a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/AbstractLibraryBuilder.java
+++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/AbstractLibraryBuilder.java
@@ -22,12 +22,6 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
-import java.nio.file.FileVisitResult;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.SimpleFileVisitor;
-import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Enumeration;
@@ -84,30 +78,6 @@ public abstract class AbstractLibraryBuilder extends CommonJavaLibraryProcessor
   }
 
   /**
-   * Build a jar file containing class files that were generated by an annotation processor.
-   */
-  public void buildGenclassJar(final JavaLibraryBuildRequest build) throws IOException {
-    final JarCreator jar = new JarCreator(build.getGeneratedClassOutputJar());
-    jar.setNormalize(true);
-    jar.setCompression(build.compressJar());
-    final Path classDir = Paths.get(build.getClassDir()).toAbsolutePath();
-    Files.walkFileTree(
-        classDir,
-        new SimpleFileVisitor<Path>() {
-          @Override
-          public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
-              throws IOException {
-            if (build.getProcessingModule().isGeneratedClass(file)) {
-              file = file.toAbsolutePath();
-              jar.addEntry(classDir.relativize(file).toString(), file.toString());
-            }
-            return FileVisitResult.CONTINUE;
-          }
-        });
-    jar.execute();
-  }
-
-  /**
    * Adds a collection of resource entries. Each entry is a string composed of a
    * pair of parts separated by a colon ':'. The name of the resource comes from
    * the second part, and the path to the resource comes from the whole string
diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/JavaLibraryBuildRequest.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/JavaLibraryBuildRequest.java
index 4d14071d93..6fc97bbde4 100644
--- a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/JavaLibraryBuildRequest.java
+++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/JavaLibraryBuildRequest.java
@@ -141,9 +141,10 @@ public final class JavaLibraryBuildRequest {
     if (optionsParser.getSourceGenDir() != null) {
       processingBuilder.setSourceGenDir(Paths.get(optionsParser.getSourceGenDir()));
     }
-    if (optionsParser.getClassDir() != null) {
-      processingBuilder.setClassDir(Paths.get(optionsParser.getClassDir()));
+    if (optionsParser.getManifestProtoPath() != null) {
+      processingBuilder.setManifestProtoPath(Paths.get(optionsParser.getManifestProtoPath()));
     }
+    processingBuilder.addAllSourceRoots(optionsParser.getSourceRoots());
     this.processingModule = processingBuilder.build();
 
     ImmutableList.Builder<BlazeJavaCompilerPlugin> pluginsBuilder =
@@ -184,7 +185,7 @@ public final class JavaLibraryBuildRequest {
     this.javacOpts = ImmutableList.copyOf(optionsParser.getJavacOpts());
     this.sourceGenDir = optionsParser.getSourceGenDir();
     this.generatedSourcesOutputJar = optionsParser.getGeneratedSourcesOutputJar();
-    this.generatedClassOutputJar = optionsParser.getGeneratedClassOutputJar();
+    this.generatedClassOutputJar = optionsParser.getManifestProtoPath();
   }
 
   public ImmutableList<String> getJavacOpts() {
diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/OptionsParser.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/OptionsParser.java
index 93c6c94576..2e18449a23 100644
--- a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/OptionsParser.java
+++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/OptionsParser.java
@@ -51,7 +51,8 @@ public final class OptionsParser {
 
   private String sourceGenDir;
   private String generatedSourcesOutputJar;
-  private String generatedClassOutputJar;
+  private String manifestProtoPath;
+  private final Set<String> sourceRoots = new HashSet<>();
 
   private final List<String> sourceFiles = new ArrayList<>();
   private final List<String> sourceJars = new ArrayList<>();
@@ -95,8 +96,7 @@ public final class OptionsParser {
    *
    * @throws InvalidCommandLineException on an invalid option being passed.
    */
-  private void processCommandlineArgs(Deque<String> argQueue)
-      throws InvalidCommandLineException {
+  private void processCommandlineArgs(Deque<String> argQueue) throws InvalidCommandLineException {
     for (String arg = argQueue.pollFirst(); arg != null; arg = argQueue.pollFirst()) {
       switch (arg) {
         case "--javacopts":
@@ -106,18 +106,20 @@ public final class OptionsParser {
           // terminator to the passed arguments.
           collectFlagArguments(javacOpts, argQueue, "--");
           break;
-        case "--direct_dependency": {
-          String jar = getArgument(argQueue, arg);
-          String target = getArgument(argQueue, arg);
-          directJarsToTargets.put(jar, target);
-          break;
-        }
-        case "--indirect_dependency": {
-          String jar = getArgument(argQueue, arg);
-          String target = getArgument(argQueue, arg);
-          indirectJarsToTargets.put(jar, target);
-          break;
-        }
+        case "--direct_dependency":
+          {
+            String jar = getArgument(argQueue, arg);
+            String target = getArgument(argQueue, arg);
+            directJarsToTargets.put(jar, target);
+            break;
+          }
+        case "--indirect_dependency":
+          {
+            String jar = getArgument(argQueue, arg);
+            String target = getArgument(argQueue, arg);
+            indirectJarsToTargets.put(jar, target);
+            break;
+          }
         case "--strict_java_deps":
           strictJavaDeps = getArgument(argQueue, arg);
           break;
@@ -139,8 +141,11 @@ public final class OptionsParser {
         case "--generated_sources_output":
           generatedSourcesOutputJar = getArgument(argQueue, arg);
           break;
-        case "--classes_from_generated_sources_output":
-          generatedClassOutputJar = getArgument(argQueue, arg);
+        case "--output_manifest_proto":
+          manifestProtoPath = getArgument(argQueue, arg);
+          break;
+        case "--source_roots":
+          collectFlagArguments(sourceRoots, argQueue, "-");
           break;
         case "--sources":
           collectFlagArguments(sourceFiles, argQueue, "-");
@@ -227,8 +232,7 @@ public final class OptionsParser {
    * @param arg the argument to pre-process.
    * @throws java.io.IOException if one of the files containing options cannot be read.
    */
-  private static void expandArgument(Deque<String> expanded, String arg)
-      throws IOException {
+  private static void expandArgument(Deque<String> expanded, String arg) throws IOException {
     if (arg.startsWith("@") && !arg.startsWith("@@")) {
       for (String line : Files.readAllLines(Paths.get(arg.substring(1)), UTF_8)) {
         if (line.length() > 0) {
@@ -248,8 +252,8 @@ public final class OptionsParser {
    * @param args
    * @param terminatorPrefix the terminator prefix to stop collecting of argument flags.
    */
-  private static void collectFlagArguments(Collection<String> output, Deque<String> args,
-      String terminatorPrefix) {
+  private static void collectFlagArguments(
+      Collection<String> output, Deque<String> args, String terminatorPrefix) {
     for (String arg = args.pollFirst(); arg != null; arg = args.pollFirst()) {
       if (arg.startsWith(terminatorPrefix)) {
         args.addFirst(arg);
@@ -275,8 +279,7 @@ public final class OptionsParser {
         break;
       }
       if (arg.contains(",")) {
-        throw new InvalidCommandLineException("processor argument may not contain commas: "
-            + arg);
+        throw new InvalidCommandLineException("processor argument may not contain commas: " + arg);
       }
       output.add(arg);
     }
@@ -339,8 +342,12 @@ public final class OptionsParser {
     return generatedSourcesOutputJar;
   }
 
-  public String getGeneratedClassOutputJar() {
-    return generatedClassOutputJar;
+  public String getManifestProtoPath() {
+    return manifestProtoPath;
+  }
+
+  public Set<String> getSourceRoots() {
+    return sourceRoots;
   }
 
   public List<String> getSourceFiles() {
diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/processing/AnnotationProcessingModule.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/processing/AnnotationProcessingModule.java
index e9d4c1832a..65a831f1e5 100644
--- a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/processing/AnnotationProcessingModule.java
+++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/processing/AnnotationProcessingModule.java
@@ -15,50 +15,108 @@
 package com.google.devtools.build.buildjar.javac.plugins.processing;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import com.google.devtools.build.buildjar.javac.plugins.BlazeJavaCompilerPlugin;
+import com.google.devtools.build.buildjar.proto.JavaCompilation.CompilationUnit;
+import com.google.devtools.build.buildjar.proto.JavaCompilation.Manifest;
 
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.HashSet;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
 /**
  * A module for information about the compilation's annotation processing.
  */
 public class AnnotationProcessingModule {
-  
+
   /**
    * A builder for {@link AnnotationProcessingModule}s.
    */
   public static class Builder {
     private Path sourceGenDir;
-    private Path classDir;
+    private Path manifestProto;
+    private ImmutableSet.Builder<Path> sourceRoots = ImmutableSet.builder();
 
     private Builder() {}
 
     public AnnotationProcessingModule build() {
-      return new AnnotationProcessingModule(sourceGenDir, classDir);
+      return new AnnotationProcessingModule(
+          sourceGenDir, manifestProto, validateSourceRoots(sourceRoots.build()));
+    }
+
+    /**
+     * Verify that source roots do not contain other source roots.
+     *
+     * <p>If one source root is an ancestor of another, the source path to
+     * use in the manifest will be ambiguous.
+     */
+    private ImmutableSet<Path> validateSourceRoots(ImmutableSet<Path> roots) {
+      // It's sad that this is quadratic, but the number of source roots
+      // should be <= 2.
+      for (Path a : roots) {
+        for (Path b : roots) {
+          if (a.equals(b) || b.getNameCount() == 0) {
+            continue;
+          }
+          if (a.startsWith(b)) {
+            throw new IllegalArgumentException(
+                String.format("Source root %s is a parent of %s", b, a));
+          }
+        }
+      }
+      return roots;
     }
 
     public void setSourceGenDir(Path sourceGenDir) {
-      this.sourceGenDir = sourceGenDir.toAbsolutePath();
+      this.sourceGenDir = sourceGenDir;
+    }
+
+    public void setManifestProtoPath(Path manifestProto) {
+      this.manifestProto = manifestProto.toAbsolutePath();
     }
 
-    public void setClassDir(Path classDir) {
-      this.classDir = classDir.toAbsolutePath();
+    public void addAllSourceRoots(Set<String> sourceRoots) {
+      for (String root : sourceRoots) {
+        this.sourceRoots.add(Paths.get(root));
+      }
     }
   }
 
   private final boolean enabled;
   private final Path sourceGenDir;
-  private final Path classDir;
+  private final Path manifestProto;
+  private final ImmutableSet<Path> sourceRoots;
+
+  public boolean isGenerated(Path path) {
+    return path.startsWith(sourceGenDir);
+  }
 
-  public Path sourceGenDir() {
-    return sourceGenDir;
+  public Path stripSourceRoot(Path path) {
+    if (path.startsWith(sourceGenDir)) {
+      return sourceGenDir.relativize(path);
+    }
+    for (Path sourceRoot : sourceRoots) {
+      if (path.startsWith(sourceRoot)) {
+        return sourceRoot.relativize(path);
+      }
+    }
+    return path;
   }
 
-  private AnnotationProcessingModule(Path sourceGenDir, Path classDir) {
+  private AnnotationProcessingModule(
+      Path sourceGenDir, Path manifestProto, ImmutableSet<Path> sourceRoots) {
     this.sourceGenDir = sourceGenDir;
-    this.classDir = classDir;
-    this.enabled = sourceGenDir != null && classDir != null;
+    this.manifestProto = manifestProto;
+    this.sourceRoots = sourceRoots;
+    this.enabled = sourceGenDir != null && manifestProto != null;
   }
 
   public static Builder builder() {
@@ -71,38 +129,33 @@ public class AnnotationProcessingModule {
     }
   }
 
-  /**
-   * The set of prefixes of generated class files.
-   */
-  private final HashSet<String> pathPrefixes = new HashSet<>();
+  private final Map<String, CompilationUnit> units = new HashMap<>();
 
-  /**
-   * Record the prefix of a group of generated class files.
-   *
-   * <p>Prefixes are used to handle generated inner classes. Since
-   * e.g. j/c/g/Foo.class and j/c/g/Foo$Inner.class both correspond to the
-   * same generated source file, only the prefix "j/c/g/Foo" is recorded.
-   */
-  void recordPrefix(String pathPrefix) {
-    pathPrefixes.add(pathPrefix);
+  public void recordUnit(CompilationUnit unit) {
+    units.put(unit.getPath(), unit);
   }
 
-  /** Returns true if the given path is to a generated source file. */
-  public boolean isGeneratedSource(Path sourcePath) {
-    return sourcePath.toAbsolutePath().startsWith(sourceGenDir);
+  private Manifest buildManifestProto() {
+    Manifest.Builder builder = Manifest.newBuilder();
+
+    List<String> keys = new ArrayList<>(units.keySet());
+    Collections.sort(keys);
+    for (String key : keys) {
+      CompilationUnit unit = units.get(key);
+      builder.addCompilationUnit(unit);
+    }
+
+    return builder.build();
   }
 
-  /** Returns true if the given path is to a generated class file. */
-  public boolean isGeneratedClass(Path path) {
-    if (!path.getFileName().toString().endsWith(".class")) {
-      return false;
+  public void emitManifestProto() throws IOException {
+    if (!enabled) {
+      return;
     }
-    String prefix = classDir.relativize(path.toAbsolutePath()).toString();
-    prefix = prefix.substring(0, prefix.length() - ".class".length());
-    int idx = prefix.lastIndexOf('$');
-    if (idx > 0) {
-      prefix = prefix.substring(0, idx);
+    try (OutputStream out = Files.newOutputStream(manifestProto)) {
+      buildManifestProto().writeTo(out);
+    } catch (IOException ex) {
+      throw new IOException("Cannot write manifest to " + manifestProto, ex);
     }
-    return pathPrefixes.contains(prefix);
   }
 }
diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/processing/AnnotationProcessingPlugin.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/processing/AnnotationProcessingPlugin.java
index 315e6fed88..ad8b0d9ce0 100644
--- a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/processing/AnnotationProcessingPlugin.java
+++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/javac/plugins/processing/AnnotationProcessingPlugin.java
@@ -15,6 +15,7 @@
 package com.google.devtools.build.buildjar.javac.plugins.processing;
 
 import com.google.devtools.build.buildjar.javac.plugins.BlazeJavaCompilerPlugin;
+import com.google.devtools.build.buildjar.proto.JavaCompilation.CompilationUnit;
 
 import com.sun.tools.javac.comp.AttrContext;
 import com.sun.tools.javac.comp.Env;
@@ -42,34 +43,32 @@ public class AnnotationProcessingPlugin extends BlazeJavaCompilerPlugin {
   @Override
   public void postAttribute(Env<AttrContext> env) {
     if (toplevels.add(env.toplevel)) {
-      recordSymbols(env.toplevel);
+      recordInfo(env.toplevel);
     }
   }
 
-  /**
-   * For each top-level type, record the path prefixes of that type's class,
-   * and all inner and anonymous classes declared inside that type.
-   *
-   * <p>e.g. for j.c.g.Foo we record the prefix j/c/g/Foo, which will match
-   * j/c/g/Foo.class, j/c/g/Foo$Inner.class, j/c/g/Foo$1.class, etc.
-   */
-  private void recordSymbols(JCCompilationUnit toplevel) {
-    if (toplevel.sourcefile == null) {
-      return;
-    }
-    Path sourcePath = Paths.get(toplevel.sourcefile.toUri());
-    if (!processingModule.isGeneratedSource(sourcePath)) {
-      return;
+  private void recordInfo(JCCompilationUnit toplevel) {
+    CompilationUnit.Builder builder = CompilationUnit.newBuilder();
+
+    if (toplevel.sourcefile != null) {
+      // FileObject#getName() returns the original exec root-relative path of
+      // the source file, which is want we want.
+      // Paths.get(sourcefile.toUri()) would absolutize the path.
+      Path path = Paths.get(toplevel.sourcefile.getName());
+      builder.setPath(processingModule.stripSourceRoot(path).toString());
+      builder.setGeneratedByAnnotationProcessor(processingModule.isGenerated(path));
     }
-    String packageBase = "";
+
     if (toplevel.getPackageName() != null) {
-      packageBase = toplevel.getPackageName().toString().replace('.', '/') + "/";
+      builder.setPkg(toplevel.getPackageName().toString());
     }
+
     for (JCTree decl : toplevel.defs) {
       if (decl instanceof JCClassDecl) {
-        String pathPrefix = packageBase + ((JCClassDecl) decl).getSimpleName();
-        processingModule.recordPrefix(pathPrefix);
+        builder.addTopLevel(((JCClassDecl) decl).getSimpleName().toString());
       }
     }
+
+    processingModule.recordUnit(builder.build());
   }
 }