Filter out non-.flat files while linking.

RELNOTES: None
PiperOrigin-RevId: 188121571

6 files changed, 120 insertions, 60 deletions

diff --git a/src/tools/android/java/com/google/devtools/build/android/Aapt2ResourcePackagingAction.java b/src/tools/android/java/com/google/devtools/build/android/Aapt2ResourcePackagingAction.java
index 1d72254640..6ca6782638 100644
--- a/src/tools/android/java/com/google/devtools/build/android/Aapt2ResourcePackagingAction.java
+++ b/src/tools/android/java/com/google/devtools/build/android/Aapt2ResourcePackagingAction.java
@@ -163,24 +163,24 @@ public class Aapt2ResourcePackagingAction {
                 .collect(toList());
         assetDirs.addAll(options.primaryData.assetDirs);
 
-        final PackagedResources packagedResources =
-            ResourceLinker.create(aaptConfigOptions.aapt2, linkedOut)
-                .profileUsing(profiler)
-                .customPackage(options.packageForR)
-                .outputAsProto(aaptConfigOptions.resourceTableAsProto)
-                .dependencies(ImmutableList.of(StaticLibrary.from(aaptConfigOptions.androidJar)))
-                .include(compiledResourceDeps)
-                .withAssets(assetDirs)
-                .buildVersion(aaptConfigOptions.buildToolsVersion)
-                .conditionalKeepRules(aaptConfigOptions.conditionalKeepRules == TriState.YES)
-                .filterToDensity(options.densities)
-                .includeOnlyConfigs(aaptConfigOptions.resourceConfigs)
-                .link(compiled)
-                .copyPackageTo(options.packagePath)
-                .copyProguardTo(options.proguardOutput)
-                .copyMainDexProguardTo(options.mainDexProguardOutput)
-                .createSourceJar(options.srcJarOutput)
-                .copyRTxtTo(options.rOutput);
+      final PackagedResources packagedResources =
+          ResourceLinker.create(aaptConfigOptions.aapt2, executorService, linkedOut)
+              .profileUsing(profiler)
+              .customPackage(options.packageForR)
+              .outputAsProto(aaptConfigOptions.resourceTableAsProto)
+              .dependencies(ImmutableList.of(StaticLibrary.from(aaptConfigOptions.androidJar)))
+              .include(compiledResourceDeps)
+              .withAssets(assetDirs)
+              .buildVersion(aaptConfigOptions.buildToolsVersion)
+              .conditionalKeepRules(aaptConfigOptions.conditionalKeepRules == TriState.YES)
+              .filterToDensity(options.densities)
+              .includeOnlyConfigs(aaptConfigOptions.resourceConfigs)
+              .link(compiled)
+              .copyPackageTo(options.packagePath)
+              .copyProguardTo(options.proguardOutput)
+              .copyMainDexProguardTo(options.mainDexProguardOutput)
+              .createSourceJar(options.srcJarOutput)
+              .copyRTxtTo(options.rOutput);
         profiler.recordEndOf("link");
         if (options.resourcesOutput != null) {
           profiler.startTask("package");
diff --git a/src/tools/android/java/com/google/devtools/build/android/Aapt2ResourceShrinkingAction.java b/src/tools/android/java/com/google/devtools/build/android/Aapt2ResourceShrinkingAction.java
index a72ffe9015..cac026a543 100644
--- a/src/tools/android/java/com/google/devtools/build/android/Aapt2ResourceShrinkingAction.java
+++ b/src/tools/android/java/com/google/devtools/build/android/Aapt2ResourceShrinkingAction.java
@@ -105,7 +105,8 @@ public class Aapt2ResourceShrinkingAction {
               .compile(resourceCompiler, workingResourcesDirectory);
       profiler.recordEndOf("compile");
 
-      ResourceLinker.create(aapt2ConfigOptions.aapt2, scopedTmp.subDirectoryOf("linking"))
+      ResourceLinker.create(
+              aapt2ConfigOptions.aapt2, executorService, scopedTmp.subDirectoryOf("linking"))
           .profileUsing(profiler)
           .dependencies(ImmutableList.of(StaticLibrary.from(aapt2ConfigOptions.androidJar)))
           .profileUsing(profiler)
diff --git a/src/tools/android/java/com/google/devtools/build/android/BUILD b/src/tools/android/java/com/google/devtools/build/android/BUILD
index 771226edc6..e11eec0260 100644
--- a/src/tools/android/java/com/google/devtools/build/android/BUILD
+++ b/src/tools/android/java/com/google/devtools/build/android/BUILD
@@ -64,6 +64,7 @@ java_library(
         "//src/tools/android/java/com/google/devtools/build/android/proto:resources_java_proto",
         "//src/tools/android/java/com/google/devtools/build/android/proto:serialize_format_java_pb",
         "//src/tools/android/java/com/google/devtools/build/android/resources",
+        "//src/tools/android/java/com/google/devtools/build/android/ziputils:ziputils_lib",
         "//third_party:android_common_25_0_0",
         "//third_party:asm",
         "//third_party:guava",
diff --git a/src/tools/android/java/com/google/devtools/build/android/CompileLibraryResourcesAction.java b/src/tools/android/java/com/google/devtools/build/android/CompileLibraryResourcesAction.java
index 3c9151748a..ea0306ba40 100644
--- a/src/tools/android/java/com/google/devtools/build/android/CompileLibraryResourcesAction.java
+++ b/src/tools/android/java/com/google/devtools/build/android/CompileLibraryResourcesAction.java
@@ -16,7 +16,6 @@ package com.google.devtools.build.android;
 
 import com.android.repository.Revision;
 import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.ListeningExecutorService;
 import com.google.devtools.build.android.Converters.ExistingPathConverter;
 import com.google.devtools.build.android.Converters.PathConverter;
 import com.google.devtools.build.android.Converters.RevisionConverter;
@@ -28,7 +27,6 @@ import com.google.devtools.common.options.OptionEffectTag;
 import com.google.devtools.common.options.OptionsBase;
 import com.google.devtools.common.options.OptionsParser;
 import com.google.devtools.common.options.ShellQuotedParamsFilePreProcessor;
-import java.io.Closeable;
 import java.nio.file.FileSystems;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -136,8 +134,7 @@ public class CompileLibraryResourcesAction {
     Preconditions.checkNotNull(options.output);
     Preconditions.checkNotNull(options.aapt2);
 
-    final ListeningExecutorService defaultService = ExecutorServiceCloser.createDefaultService();
-    try (Closeable serviceCloser = ExecutorServiceCloser.createWith(defaultService);
+    try (ExecutorServiceCloser executorService = ExecutorServiceCloser.createWithFixedPoolOf(15);
         ScopedTemporaryDirectory scopedTmp =
             new ScopedTemporaryDirectory("android_resources_tmp")) {
       final Path tmp = scopedTmp.getPath();
@@ -147,7 +144,7 @@ public class CompileLibraryResourcesAction {
 
       final ResourceCompiler compiler =
           ResourceCompiler.create(
-              defaultService, compiledResources, options.aapt2, options.buildToolsVersion);
+              executorService, compiledResources, options.aapt2, options.buildToolsVersion);
       options
           .resources
           .toData(options.manifest)
diff --git a/src/tools/android/java/com/google/devtools/build/android/ValidateAndLinkResourcesAction.java b/src/tools/android/java/com/google/devtools/build/android/ValidateAndLinkResourcesAction.java
index 5af2c67a59..cd8befc7b8 100644
--- a/src/tools/android/java/com/google/devtools/build/android/ValidateAndLinkResourcesAction.java
+++ b/src/tools/android/java/com/google/devtools/build/android/ValidateAndLinkResourcesAction.java
@@ -167,7 +167,8 @@ public class ValidateAndLinkResourcesAction {
     final Profiler profiler = LoggingProfiler.createAndStart("manifest");
 
     try (ScopedTemporaryDirectory scopedTmp =
-        new ScopedTemporaryDirectory("android_resources_tmp")) {
+            new ScopedTemporaryDirectory("android_resources_tmp");
+        ExecutorServiceCloser executorService = ExecutorServiceCloser.createWithFixedPoolOf(15)) {
       CompiledResources resources =
           // TODO(b/64570523): Remove when the flags are standardized.
           Optional.ofNullable(options.resources)
@@ -184,7 +185,7 @@ public class ValidateAndLinkResourcesAction {
                           scopedTmp.getPath().resolve("manifest-aapt-dummy/AndroidManifest.xml"),
                           options.packageForR));
       profiler.recordEndOf("manifest").startTask("link");
-      ResourceLinker.create(aapt2Options.aapt2, scopedTmp.getPath())
+      ResourceLinker.create(aapt2Options.aapt2, executorService, scopedTmp.getPath())
           .profileUsing(profiler)
           .dependencies(Optional.ofNullable(options.deprecatedLibraries).orElse(options.libraries))
           .include(
diff --git a/src/tools/android/java/com/google/devtools/build/android/aapt2/ResourceLinker.java b/src/tools/android/java/com/google/devtools/build/android/aapt2/ResourceLinker.java
index c230ddccc5..7c4949f9ea 100644
--- a/src/tools/android/java/com/google/devtools/build/android/aapt2/ResourceLinker.java
+++ b/src/tools/android/java/com/google/devtools/build/android/aapt2/ResourceLinker.java
@@ -22,25 +22,31 @@ import com.google.common.base.MoreObjects;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Streams;
+import com.google.common.util.concurrent.ListeningExecutorService;
 import com.google.devtools.build.android.AaptCommandBuilder;
 import com.google.devtools.build.android.AndroidResourceOutputs;
 import com.google.devtools.build.android.Profiler;
+import com.google.devtools.build.android.ziputils.ZipIn;
+import com.google.devtools.build.android.ziputils.ZipOut;
 import java.io.IOException;
+import java.nio.channels.FileChannel;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
 import java.util.Collection;
 import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.function.Function;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /** Performs linking of {@link CompiledResources} using aapt2. */
 public class ResourceLinker {
-
-  private String customPackage;
-  private boolean outputAsProto;
-
   /** Represents errors thrown during linking. */
   public static class LinkError extends Aapt2Exception {
 
@@ -48,7 +54,7 @@ public class ResourceLinker {
       super(e);
     }
 
-    public static LinkError of(IOException e) {
+    public static LinkError of(Throwable e) {
       return new LinkError(e);
     }
   }
@@ -57,10 +63,14 @@ public class ResourceLinker {
 
   private final Path aapt2;
 
+  private final ListeningExecutorService executorService;
   private final Path workingDirectory;
 
   private List<StaticLibrary> linkAgainst = ImmutableList.of();
 
+  private String customPackage;
+  private boolean outputAsProto;
+
   private Revision buildToolsVersion;
   private List<String> densities = ImmutableList.of();
   private Path androidJar;
@@ -72,14 +82,17 @@ public class ResourceLinker {
   private List<Path> assetDirs = ImmutableList.of();
   private boolean conditionalKeepRules = false;
 
-  private ResourceLinker(Path aapt2, Path workingDirectory) {
+  private ResourceLinker(
+      Path aapt2, ListeningExecutorService executorService, Path workingDirectory) {
     this.aapt2 = aapt2;
+    this.executorService = executorService;
     this.workingDirectory = workingDirectory;
   }
 
-  public static ResourceLinker create(Path aapt2, Path workingDirectory) {
+  public static ResourceLinker create(
+      Path aapt2, ListeningExecutorService executorService, Path workingDirectory) {
     Preconditions.checkArgument(Files.exists(workingDirectory));
-    return new ResourceLinker(aapt2, workingDirectory);
+    return new ResourceLinker(aapt2, executorService, workingDirectory);
   }
 
   public ResourceLinker profileUsing(Profiler profiler) {
@@ -140,26 +153,20 @@ public class ResourceLinker {
    *
    * @throws IOException
    */
-  public StaticLibrary linkStatically(CompiledResources resources) {
-    final Path outPath = workingDirectory.resolve("lib.apk");
-    final Path rTxt = workingDirectory.resolve("R.txt");
-    final Path sourceJar = workingDirectory.resolve("r.srcjar");
-    Path javaSourceDirectory = workingDirectory.resolve("java");
-
+  public StaticLibrary linkStatically(CompiledResources compiled) {
     try {
+      final Path outPath = workingDirectory.resolve("lib.apk");
+      final Path rTxt = workingDirectory.resolve("R.txt");
+      final Path sourceJar = workingDirectory.resolve("r.srcjar");
+      Path javaSourceDirectory = workingDirectory.resolve("java");
       profiler.startTask("linkstatic");
-      final List<String> compiledResourcePaths =
-          include
-              .stream()
-              .map(compiledResources -> compiledResources.getZip().toString())
-              .collect(toList());
       final Collection<String> pathsToLinkAgainst = StaticLibrary.toPathStrings(linkAgainst);
-      logger.fine(
+      logger.finer(
           new AaptCommandBuilder(aapt2)
               .forBuildToolsVersion(buildToolsVersion)
               .forVariantType(VariantType.LIBRARY)
               .add("link")
-              .add("--manifest", resources.getManifest())
+              .add("--manifest", compiled.getManifest())
               .add("--static-lib")
               .add("--no-static-lib-packages")
               .add("--custom-package", customPackage)
@@ -167,8 +174,7 @@ public class ResourceLinker {
               .thenAdd("--no-version-vectors")
               .when(outputAsProto)
               .thenAdd("--proto-format")
-              .add("-R", resources.getZip())
-              .addRepeated("-R", compiledResourcePaths)
+              .addParameterableRepeated("-R", compiledResourcesToPaths(compiled), workingDirectory)
               .addRepeated("-I", pathsToLinkAgainst)
               .add("--auto-add-overlay")
               .add("-o", outPath)
@@ -176,17 +182,17 @@ public class ResourceLinker {
               .thenAdd("--java", javaSourceDirectory)
               .when(linkAgainst.size() == 1) // If using all compiled resources, generates R.txt
               .thenAdd("--output-text-symbols", rTxt)
-              .execute(String.format("Statically linking %s", resources)));
+              .execute(String.format("Statically linking %s", compiled)));
       profiler.recordEndOf("linkstatic");
       // working around aapt2 not producing transitive R.txt and R.java
       if (linkAgainst.size() > 1) {
         profiler.startTask("rfix");
-        logger.info(
+        logger.finer(
             new AaptCommandBuilder(aapt2)
                 .forBuildToolsVersion(buildToolsVersion)
                 .forVariantType(VariantType.LIBRARY)
                 .add("link")
-                .add("--manifest", resources.getManifest())
+                .add("--manifest", compiled.getManifest())
                 .add("--no-static-lib-packages")
                 .whenVersionIsAtLeast(new Revision(23))
                 .thenAdd("--no-version-vectors")
@@ -205,7 +211,7 @@ public class ResourceLinker {
                 .add("-o", outPath.resolveSibling("transitive.apk"))
                 .add("--java", javaSourceDirectory)
                 .add("--output-text-symbols", rTxt)
-                .execute(String.format("Generating R files %s", resources)));
+                .execute(String.format("Generating R files %s", compiled)));
         profiler.recordEndOf("rfix");
       }
 
@@ -218,6 +224,66 @@ public class ResourceLinker {
     }
   }
 
+  private List<String> compiledResourcesToPaths(CompiledResources compiled) throws IOException {
+    // Using sequential streams to maintain the overlay order for aapt2.
+    return Stream.concat(include.stream(), Stream.of(compiled))
+        .sequential()
+        .map(CompiledResources::getZip)
+        .map(z -> executorService.submit(() -> filterZip(z)))
+        .map(rethrowLinkError(Future::get))
+        // the process will always take as long as the longest Future
+        .map(Path::toString)
+        .collect(toList());
+  }
+
+  private Path filterZip(Path path) throws IOException {
+    Path outPath =
+        workingDirectory
+            .resolve("filtered")
+            // make absolute paths relative so that resolve will make a new path.
+            .resolve(path.isAbsolute() ? path.subpath(1, path.getNameCount()) : path);
+    // TODO(74258184): How can this path already exist?
+    if (Files.exists(outPath)) {
+      return outPath;
+    }
+    Files.createDirectories(outPath.getParent());
+    try (FileChannel inChannel = FileChannel.open(path, StandardOpenOption.READ);
+        FileChannel outChannel =
+            FileChannel.open(outPath, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {
+      final ZipIn zipIn = new ZipIn(inChannel, path.toString());
+      final ZipOut zipOut = new ZipOut(outChannel, outPath.toString());
+      zipIn.scanEntries(
+          (in, header, dirEntry, data) -> {
+            if (header.getFilename().endsWith(".flat")) {
+              zipOut.nextEntry(dirEntry);
+              zipOut.write(header);
+              zipOut.write(data);
+            }
+          });
+      zipOut.close();
+    }
+    return outPath;
+  }
+
+  private static <T, R> Function<T, R> rethrowLinkError(CheckedFunction<T, R> checked) {
+    return (T arg) -> {
+      try {
+        return checked.apply(arg);
+      } catch (ExecutionException e) {
+        throw LinkError.of(Optional.ofNullable(e.getCause()).orElse(e)); // unwrap
+      } catch (IOException e) {
+        throw LinkError.of(e);
+      } catch (Throwable e) { // unexpected error, rethrow for debugging.
+        throw new RuntimeException(e);
+      }
+    };
+  }
+
+  @FunctionalInterface
+  private interface CheckedFunction<T, R> {
+    R apply(T arg) throws Throwable;
+  }
+
   public PackagedResources link(CompiledResources compiled) {
     try {
       final Path outPath = workingDirectory.resolve("bin.apk");
@@ -255,13 +321,7 @@ public class ResourceLinker {
                           compiled.getAssetsStrings().stream())
                       .collect(toList()))
               .addRepeated("-I", StaticLibrary.toPathStrings(linkAgainst))
-              .addRepeated(
-                  "-R",
-                  include
-                      .stream()
-                      .map(compiledResources -> compiledResources.getZip().toString())
-                      .collect(toList()))
-              .add("-R", compiled.getZip())
+              .addParameterableRepeated("-R", compiledResourcesToPaths(compiled), workingDirectory)
               // Never compress apks.
               .add("-0", "apk")
               // Add custom no-compress extensions.