diff options
author | corysmith <corysmith@google.com> | 2018-03-06 20:08:56 -0800 |
---|---|---|
committer | Copybara-Service <copybara-piper@google.com> | 2018-03-06 20:10:55 -0800 |
commit | 59455a58d5203401af785f6229c3d30c94b57fec (patch) | |
tree | 32336ca0a8ba783599ecb022115a316b9ad69df8 /src/tools | |
parent | 67627d3034de3b66431abfb0d178a0c27408a30b (diff) |
Filter out non-.flat files while linking.
RELNOTES: None
PiperOrigin-RevId: 188121571
Diffstat (limited to 'src/tools')
6 files changed, 120 insertions, 60 deletions
diff --git a/src/tools/android/java/com/google/devtools/build/android/Aapt2ResourcePackagingAction.java b/src/tools/android/java/com/google/devtools/build/android/Aapt2ResourcePackagingAction.java index 1d72254640..6ca6782638 100644 --- a/src/tools/android/java/com/google/devtools/build/android/Aapt2ResourcePackagingAction.java +++ b/src/tools/android/java/com/google/devtools/build/android/Aapt2ResourcePackagingAction.java @@ -163,24 +163,24 @@ public class Aapt2ResourcePackagingAction { .collect(toList()); assetDirs.addAll(options.primaryData.assetDirs); - final PackagedResources packagedResources = - ResourceLinker.create(aaptConfigOptions.aapt2, linkedOut) - .profileUsing(profiler) - .customPackage(options.packageForR) - .outputAsProto(aaptConfigOptions.resourceTableAsProto) - .dependencies(ImmutableList.of(StaticLibrary.from(aaptConfigOptions.androidJar))) - .include(compiledResourceDeps) - .withAssets(assetDirs) - .buildVersion(aaptConfigOptions.buildToolsVersion) - .conditionalKeepRules(aaptConfigOptions.conditionalKeepRules == TriState.YES) - .filterToDensity(options.densities) - .includeOnlyConfigs(aaptConfigOptions.resourceConfigs) - .link(compiled) - .copyPackageTo(options.packagePath) - .copyProguardTo(options.proguardOutput) - .copyMainDexProguardTo(options.mainDexProguardOutput) - .createSourceJar(options.srcJarOutput) - .copyRTxtTo(options.rOutput); + final PackagedResources packagedResources = + ResourceLinker.create(aaptConfigOptions.aapt2, executorService, linkedOut) + .profileUsing(profiler) + .customPackage(options.packageForR) + .outputAsProto(aaptConfigOptions.resourceTableAsProto) + .dependencies(ImmutableList.of(StaticLibrary.from(aaptConfigOptions.androidJar))) + .include(compiledResourceDeps) + .withAssets(assetDirs) + .buildVersion(aaptConfigOptions.buildToolsVersion) + .conditionalKeepRules(aaptConfigOptions.conditionalKeepRules == TriState.YES) + .filterToDensity(options.densities) + .includeOnlyConfigs(aaptConfigOptions.resourceConfigs) + .link(compiled) + .copyPackageTo(options.packagePath) + .copyProguardTo(options.proguardOutput) + .copyMainDexProguardTo(options.mainDexProguardOutput) + .createSourceJar(options.srcJarOutput) + .copyRTxtTo(options.rOutput); profiler.recordEndOf("link"); if (options.resourcesOutput != null) { profiler.startTask("package"); diff --git a/src/tools/android/java/com/google/devtools/build/android/Aapt2ResourceShrinkingAction.java b/src/tools/android/java/com/google/devtools/build/android/Aapt2ResourceShrinkingAction.java index a72ffe9015..cac026a543 100644 --- a/src/tools/android/java/com/google/devtools/build/android/Aapt2ResourceShrinkingAction.java +++ b/src/tools/android/java/com/google/devtools/build/android/Aapt2ResourceShrinkingAction.java @@ -105,7 +105,8 @@ public class Aapt2ResourceShrinkingAction { .compile(resourceCompiler, workingResourcesDirectory); profiler.recordEndOf("compile"); - ResourceLinker.create(aapt2ConfigOptions.aapt2, scopedTmp.subDirectoryOf("linking")) + ResourceLinker.create( + aapt2ConfigOptions.aapt2, executorService, scopedTmp.subDirectoryOf("linking")) .profileUsing(profiler) .dependencies(ImmutableList.of(StaticLibrary.from(aapt2ConfigOptions.androidJar))) .profileUsing(profiler) diff --git a/src/tools/android/java/com/google/devtools/build/android/BUILD b/src/tools/android/java/com/google/devtools/build/android/BUILD index 771226edc6..e11eec0260 100644 --- a/src/tools/android/java/com/google/devtools/build/android/BUILD +++ b/src/tools/android/java/com/google/devtools/build/android/BUILD @@ -64,6 +64,7 @@ java_library( "//src/tools/android/java/com/google/devtools/build/android/proto:resources_java_proto", "//src/tools/android/java/com/google/devtools/build/android/proto:serialize_format_java_pb", "//src/tools/android/java/com/google/devtools/build/android/resources", + "//src/tools/android/java/com/google/devtools/build/android/ziputils:ziputils_lib", "//third_party:android_common_25_0_0", "//third_party:asm", "//third_party:guava", diff --git a/src/tools/android/java/com/google/devtools/build/android/CompileLibraryResourcesAction.java b/src/tools/android/java/com/google/devtools/build/android/CompileLibraryResourcesAction.java index 3c9151748a..ea0306ba40 100644 --- a/src/tools/android/java/com/google/devtools/build/android/CompileLibraryResourcesAction.java +++ b/src/tools/android/java/com/google/devtools/build/android/CompileLibraryResourcesAction.java @@ -16,7 +16,6 @@ package com.google.devtools.build.android; import com.android.repository.Revision; import com.google.common.base.Preconditions; -import com.google.common.util.concurrent.ListeningExecutorService; import com.google.devtools.build.android.Converters.ExistingPathConverter; import com.google.devtools.build.android.Converters.PathConverter; import com.google.devtools.build.android.Converters.RevisionConverter; @@ -28,7 +27,6 @@ import com.google.devtools.common.options.OptionEffectTag; import com.google.devtools.common.options.OptionsBase; import com.google.devtools.common.options.OptionsParser; import com.google.devtools.common.options.ShellQuotedParamsFilePreProcessor; -import java.io.Closeable; import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; @@ -136,8 +134,7 @@ public class CompileLibraryResourcesAction { Preconditions.checkNotNull(options.output); Preconditions.checkNotNull(options.aapt2); - final ListeningExecutorService defaultService = ExecutorServiceCloser.createDefaultService(); - try (Closeable serviceCloser = ExecutorServiceCloser.createWith(defaultService); + try (ExecutorServiceCloser executorService = ExecutorServiceCloser.createWithFixedPoolOf(15); ScopedTemporaryDirectory scopedTmp = new ScopedTemporaryDirectory("android_resources_tmp")) { final Path tmp = scopedTmp.getPath(); @@ -147,7 +144,7 @@ public class CompileLibraryResourcesAction { final ResourceCompiler compiler = ResourceCompiler.create( - defaultService, compiledResources, options.aapt2, options.buildToolsVersion); + executorService, compiledResources, options.aapt2, options.buildToolsVersion); options .resources .toData(options.manifest) diff --git a/src/tools/android/java/com/google/devtools/build/android/ValidateAndLinkResourcesAction.java b/src/tools/android/java/com/google/devtools/build/android/ValidateAndLinkResourcesAction.java index 5af2c67a59..cd8befc7b8 100644 --- a/src/tools/android/java/com/google/devtools/build/android/ValidateAndLinkResourcesAction.java +++ b/src/tools/android/java/com/google/devtools/build/android/ValidateAndLinkResourcesAction.java @@ -167,7 +167,8 @@ public class ValidateAndLinkResourcesAction { final Profiler profiler = LoggingProfiler.createAndStart("manifest"); try (ScopedTemporaryDirectory scopedTmp = - new ScopedTemporaryDirectory("android_resources_tmp")) { + new ScopedTemporaryDirectory("android_resources_tmp"); + ExecutorServiceCloser executorService = ExecutorServiceCloser.createWithFixedPoolOf(15)) { CompiledResources resources = // TODO(b/64570523): Remove when the flags are standardized. Optional.ofNullable(options.resources) @@ -184,7 +185,7 @@ public class ValidateAndLinkResourcesAction { scopedTmp.getPath().resolve("manifest-aapt-dummy/AndroidManifest.xml"), options.packageForR)); profiler.recordEndOf("manifest").startTask("link"); - ResourceLinker.create(aapt2Options.aapt2, scopedTmp.getPath()) + ResourceLinker.create(aapt2Options.aapt2, executorService, scopedTmp.getPath()) .profileUsing(profiler) .dependencies(Optional.ofNullable(options.deprecatedLibraries).orElse(options.libraries)) .include( diff --git a/src/tools/android/java/com/google/devtools/build/android/aapt2/ResourceLinker.java b/src/tools/android/java/com/google/devtools/build/android/aapt2/ResourceLinker.java index c230ddccc5..7c4949f9ea 100644 --- a/src/tools/android/java/com/google/devtools/build/android/aapt2/ResourceLinker.java +++ b/src/tools/android/java/com/google/devtools/build/android/aapt2/ResourceLinker.java @@ -22,25 +22,31 @@ import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.Streams; +import com.google.common.util.concurrent.ListeningExecutorService; import com.google.devtools.build.android.AaptCommandBuilder; import com.google.devtools.build.android.AndroidResourceOutputs; import com.google.devtools.build.android.Profiler; +import com.google.devtools.build.android.ziputils.ZipIn; +import com.google.devtools.build.android.ziputils.ZipOut; import java.io.IOException; +import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.Collection; import java.util.List; import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.function.Function; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import java.util.stream.Stream; /** Performs linking of {@link CompiledResources} using aapt2. */ public class ResourceLinker { - - private String customPackage; - private boolean outputAsProto; - /** Represents errors thrown during linking. */ public static class LinkError extends Aapt2Exception { @@ -48,7 +54,7 @@ public class ResourceLinker { super(e); } - public static LinkError of(IOException e) { + public static LinkError of(Throwable e) { return new LinkError(e); } } @@ -57,10 +63,14 @@ public class ResourceLinker { private final Path aapt2; + private final ListeningExecutorService executorService; private final Path workingDirectory; private List<StaticLibrary> linkAgainst = ImmutableList.of(); + private String customPackage; + private boolean outputAsProto; + private Revision buildToolsVersion; private List<String> densities = ImmutableList.of(); private Path androidJar; @@ -72,14 +82,17 @@ public class ResourceLinker { private List<Path> assetDirs = ImmutableList.of(); private boolean conditionalKeepRules = false; - private ResourceLinker(Path aapt2, Path workingDirectory) { + private ResourceLinker( + Path aapt2, ListeningExecutorService executorService, Path workingDirectory) { this.aapt2 = aapt2; + this.executorService = executorService; this.workingDirectory = workingDirectory; } - public static ResourceLinker create(Path aapt2, Path workingDirectory) { + public static ResourceLinker create( + Path aapt2, ListeningExecutorService executorService, Path workingDirectory) { Preconditions.checkArgument(Files.exists(workingDirectory)); - return new ResourceLinker(aapt2, workingDirectory); + return new ResourceLinker(aapt2, executorService, workingDirectory); } public ResourceLinker profileUsing(Profiler profiler) { @@ -140,26 +153,20 @@ public class ResourceLinker { * * @throws IOException */ - public StaticLibrary linkStatically(CompiledResources resources) { - final Path outPath = workingDirectory.resolve("lib.apk"); - final Path rTxt = workingDirectory.resolve("R.txt"); - final Path sourceJar = workingDirectory.resolve("r.srcjar"); - Path javaSourceDirectory = workingDirectory.resolve("java"); - + public StaticLibrary linkStatically(CompiledResources compiled) { try { + final Path outPath = workingDirectory.resolve("lib.apk"); + final Path rTxt = workingDirectory.resolve("R.txt"); + final Path sourceJar = workingDirectory.resolve("r.srcjar"); + Path javaSourceDirectory = workingDirectory.resolve("java"); profiler.startTask("linkstatic"); - final List<String> compiledResourcePaths = - include - .stream() - .map(compiledResources -> compiledResources.getZip().toString()) - .collect(toList()); final Collection<String> pathsToLinkAgainst = StaticLibrary.toPathStrings(linkAgainst); - logger.fine( + logger.finer( new AaptCommandBuilder(aapt2) .forBuildToolsVersion(buildToolsVersion) .forVariantType(VariantType.LIBRARY) .add("link") - .add("--manifest", resources.getManifest()) + .add("--manifest", compiled.getManifest()) .add("--static-lib") .add("--no-static-lib-packages") .add("--custom-package", customPackage) @@ -167,8 +174,7 @@ public class ResourceLinker { .thenAdd("--no-version-vectors") .when(outputAsProto) .thenAdd("--proto-format") - .add("-R", resources.getZip()) - .addRepeated("-R", compiledResourcePaths) + .addParameterableRepeated("-R", compiledResourcesToPaths(compiled), workingDirectory) .addRepeated("-I", pathsToLinkAgainst) .add("--auto-add-overlay") .add("-o", outPath) @@ -176,17 +182,17 @@ public class ResourceLinker { .thenAdd("--java", javaSourceDirectory) .when(linkAgainst.size() == 1) // If using all compiled resources, generates R.txt .thenAdd("--output-text-symbols", rTxt) - .execute(String.format("Statically linking %s", resources))); + .execute(String.format("Statically linking %s", compiled))); profiler.recordEndOf("linkstatic"); // working around aapt2 not producing transitive R.txt and R.java if (linkAgainst.size() > 1) { profiler.startTask("rfix"); - logger.info( + logger.finer( new AaptCommandBuilder(aapt2) .forBuildToolsVersion(buildToolsVersion) .forVariantType(VariantType.LIBRARY) .add("link") - .add("--manifest", resources.getManifest()) + .add("--manifest", compiled.getManifest()) .add("--no-static-lib-packages") .whenVersionIsAtLeast(new Revision(23)) .thenAdd("--no-version-vectors") @@ -205,7 +211,7 @@ public class ResourceLinker { .add("-o", outPath.resolveSibling("transitive.apk")) .add("--java", javaSourceDirectory) .add("--output-text-symbols", rTxt) - .execute(String.format("Generating R files %s", resources))); + .execute(String.format("Generating R files %s", compiled))); profiler.recordEndOf("rfix"); } @@ -218,6 +224,66 @@ public class ResourceLinker { } } + private List<String> compiledResourcesToPaths(CompiledResources compiled) throws IOException { + // Using sequential streams to maintain the overlay order for aapt2. + return Stream.concat(include.stream(), Stream.of(compiled)) + .sequential() + .map(CompiledResources::getZip) + .map(z -> executorService.submit(() -> filterZip(z))) + .map(rethrowLinkError(Future::get)) + // the process will always take as long as the longest Future + .map(Path::toString) + .collect(toList()); + } + + private Path filterZip(Path path) throws IOException { + Path outPath = + workingDirectory + .resolve("filtered") + // make absolute paths relative so that resolve will make a new path. + .resolve(path.isAbsolute() ? path.subpath(1, path.getNameCount()) : path); + // TODO(74258184): How can this path already exist? + if (Files.exists(outPath)) { + return outPath; + } + Files.createDirectories(outPath.getParent()); + try (FileChannel inChannel = FileChannel.open(path, StandardOpenOption.READ); + FileChannel outChannel = + FileChannel.open(outPath, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) { + final ZipIn zipIn = new ZipIn(inChannel, path.toString()); + final ZipOut zipOut = new ZipOut(outChannel, outPath.toString()); + zipIn.scanEntries( + (in, header, dirEntry, data) -> { + if (header.getFilename().endsWith(".flat")) { + zipOut.nextEntry(dirEntry); + zipOut.write(header); + zipOut.write(data); + } + }); + zipOut.close(); + } + return outPath; + } + + private static <T, R> Function<T, R> rethrowLinkError(CheckedFunction<T, R> checked) { + return (T arg) -> { + try { + return checked.apply(arg); + } catch (ExecutionException e) { + throw LinkError.of(Optional.ofNullable(e.getCause()).orElse(e)); // unwrap + } catch (IOException e) { + throw LinkError.of(e); + } catch (Throwable e) { // unexpected error, rethrow for debugging. + throw new RuntimeException(e); + } + }; + } + + @FunctionalInterface + private interface CheckedFunction<T, R> { + R apply(T arg) throws Throwable; + } + public PackagedResources link(CompiledResources compiled) { try { final Path outPath = workingDirectory.resolve("bin.apk"); @@ -255,13 +321,7 @@ public class ResourceLinker { compiled.getAssetsStrings().stream()) .collect(toList())) .addRepeated("-I", StaticLibrary.toPathStrings(linkAgainst)) - .addRepeated( - "-R", - include - .stream() - .map(compiledResources -> compiledResources.getZip().toString()) - .collect(toList())) - .add("-R", compiled.getZip()) + .addParameterableRepeated("-R", compiledResourcesToPaths(compiled), workingDirectory) // Never compress apks. .add("-0", "apk") // Add custom no-compress extensions. |