// Copyright 2017 The Bazel Authors. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package com.google.devtools.build.android.aapt2; import static com.google.devtools.build.android.ziputils.DataDescriptor.EXTCRC; import static com.google.devtools.build.android.ziputils.DataDescriptor.EXTLEN; import static com.google.devtools.build.android.ziputils.DataDescriptor.EXTSIZ; import static com.google.devtools.build.android.ziputils.DirectoryEntry.CENCRC; import static com.google.devtools.build.android.ziputils.DirectoryEntry.CENLEN; import static com.google.devtools.build.android.ziputils.DirectoryEntry.CENSIZ; import static com.google.devtools.build.android.ziputils.DirectoryEntry.CENTIM; import static com.google.devtools.build.android.ziputils.LocalFileHeader.LOCFLG; import static com.google.devtools.build.android.ziputils.LocalFileHeader.LOCTIM; import static java.util.stream.Collectors.toList; import com.android.builder.core.VariantType; import com.android.repository.Revision; import com.google.common.base.Joiner; import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Streams; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.devtools.build.android.AaptCommandBuilder; import com.google.devtools.build.android.AndroidCompiledDataDeserializer; import com.google.devtools.build.android.AndroidResourceOutputs; import com.google.devtools.build.android.FullyQualifiedName; import com.google.devtools.build.android.Profiler; import com.google.devtools.build.android.aapt2.ResourceCompiler.CompiledType; import com.google.devtools.build.android.ziputils.DataDescriptor; import com.google.devtools.build.android.ziputils.DirectoryEntry; import com.google.devtools.build.android.ziputils.DosTime; import com.google.devtools.build.android.ziputils.EntryHandler; import com.google.devtools.build.android.ziputils.LocalFileHeader; import com.google.devtools.build.android.ziputils.ZipIn; import com.google.devtools.build.android.ziputils.ZipOut; import java.io.IOException; import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.function.Function; import java.util.function.Predicate; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; /** Performs linking of {@link CompiledResources} using aapt2. */ public class ResourceLinker { private static final Predicate IS_JAR = s -> s.endsWith(".jar"); private static final String PROTO_EXTENSION = "-pb.apk"; private static final String BINARY_EXTENSION = "apk"; private boolean debug; private static final Predicate IS_FLAT_FILE = h -> h.getFilename().endsWith(".flat"); private static final Predicate COMMENT_ABSENT = h -> Strings.isNullOrEmpty(h.getComment()); private static final Predicate USE_GENERATED = COMMENT_ABSENT.or( h -> ResourceCompiler.getCompiledType(h.getFilename()) == CompiledType.GENERATED); private static final Predicate USE_DEFAULT = COMMENT_ABSENT.or( h -> ResourceCompiler.getCompiledType(h.getComment()) != CompiledType.GENERATED); private static final ImmutableSet PSEUDO_LOCALE_FILTERS = ImmutableSet.of("en_XA", "ar_XB"); /** Represents errors thrown during linking. */ public static class LinkError extends Aapt2Exception { private LinkError(Throwable e) { super(e); } public static LinkError of(Throwable e) { return new LinkError(e); } } private boolean generatePseudoLocale; private static Logger logger = Logger.getLogger(ResourceLinker.class.getName()); private final Path aapt2; private final ListeningExecutorService executorService; private final Path workingDirectory; private List linkAgainst = ImmutableList.of(); private String customPackage; private boolean outputAsProto; private Revision buildToolsVersion; private List densities = ImmutableList.of(); private Path androidJar; private Profiler profiler = Profiler.empty(); private List uncompressedExtensions = ImmutableList.of(); private List resourceConfigs = ImmutableList.of(); private Path baseApk; private List include = ImmutableList.of(); private List assetDirs = ImmutableList.of(); private boolean conditionalKeepRules = false; private ResourceLinker( Path aapt2, ListeningExecutorService executorService, Path workingDirectory) { this.aapt2 = aapt2; this.executorService = executorService; this.workingDirectory = workingDirectory; } public static ResourceLinker create( Path aapt2, ListeningExecutorService executorService, Path workingDirectory) { Preconditions.checkArgument(Files.exists(workingDirectory)); return new ResourceLinker(aapt2, executorService, workingDirectory); } public ResourceLinker includeGeneratedLocales(boolean generatePseudoLocale) { this.generatePseudoLocale = generatePseudoLocale; return this; } public ResourceLinker profileUsing(Profiler profiler) { this.profiler = profiler; return this; } /** Dependent static libraries to be linked to. */ public ResourceLinker dependencies(List libraries) { this.linkAgainst = libraries; return this; } /** Dependent compiled resources to be included in the binary. */ public ResourceLinker include(List include) { this.include = include; return this; } public ResourceLinker withAssets(List assetDirs) { this.assetDirs = assetDirs; return this; } public ResourceLinker buildVersion(Revision buildToolsVersion) { this.buildToolsVersion = buildToolsVersion; return this; } public ResourceLinker debug(boolean debug) { this.debug = debug; return this; } public ResourceLinker conditionalKeepRules(boolean conditionalKeepRules) { this.conditionalKeepRules = conditionalKeepRules; return this; } public ResourceLinker baseApkToLinkAgainst(Path baseApk) { this.baseApk = baseApk; return this; } public ResourceLinker customPackage(String customPackage) { this.customPackage = customPackage; return this; } public ResourceLinker filterToDensity(List densities) { this.densities = densities; return this; } public ResourceLinker outputAsProto(boolean outputAsProto) { this.outputAsProto = outputAsProto; return this; } /** * Statically links the {@link CompiledResources} with the dependencies to produce a {@link * StaticLibrary}. */ public StaticLibrary linkStatically(CompiledResources compiled) { try { final Path outPath = workingDirectory.resolve("lib.apk"); final Path rTxt = workingDirectory.resolve("R.txt"); final Path sourceJar = workingDirectory.resolve("r.srcjar"); Path javaSourceDirectory = workingDirectory.resolve("java"); profiler.startTask("linkstatic"); final Collection pathsToLinkAgainst = StaticLibrary.toPathStrings(linkAgainst); logger.finer( new AaptCommandBuilder(aapt2) .forBuildToolsVersion(buildToolsVersion) .forVariantType(VariantType.LIBRARY) .add("link") .when(outputAsProto) // Used for testing: aapt2 does not output static libraries in // proto format. .thenAdd("--proto-format") .when(!outputAsProto) .thenAdd("--static-lib") .add("--manifest", compiled.getManifest()) .add("--no-static-lib-packages") .add("--custom-package", customPackage) .whenVersionIsAtLeast(new Revision(23)) .thenAdd("--no-version-vectors") .addParameterableRepeated( "-R", compiledResourcesToPaths(compiled, IS_FLAT_FILE), workingDirectory) .addRepeated("-I", pathsToLinkAgainst) .add("--auto-add-overlay") .add("-o", outPath) .when(linkAgainst.size() == 1) // If using all compiled resources, generates sources .thenAdd("--java", javaSourceDirectory) .when(linkAgainst.size() == 1) // If using all compiled resources, generates R.txt .thenAdd("--output-text-symbols", rTxt) .execute(String.format("Statically linking %s", compiled))); profiler.recordEndOf("linkstatic"); // working around aapt2 not producing transitive R.txt and R.java if (linkAgainst.size() > 1) { profiler.startTask("rfix"); logger.finer( new AaptCommandBuilder(aapt2) .forBuildToolsVersion(buildToolsVersion) .forVariantType(VariantType.LIBRARY) .add("link") .add("--manifest", compiled.getManifest()) .add("--no-static-lib-packages") .whenVersionIsAtLeast(new Revision(23)) .thenAdd("--no-version-vectors") .when(outputAsProto) .thenAdd("--proto-format") // only link against jars .addRepeated("-I", pathsToLinkAgainst.stream().filter(IS_JAR).collect(toList())) .add("-R", outPath) // only include non-jars .addRepeated( "-R", pathsToLinkAgainst.stream().filter(IS_JAR.negate()).collect(toList())) .add("--auto-add-overlay") .add("-o", outPath.resolveSibling("transitive.apk")) .add("--java", javaSourceDirectory) .add("--output-text-symbols", rTxt) .execute(String.format("Generating R files %s", compiled))); profiler.recordEndOf("rfix"); } profiler.startTask("sourcejar"); AndroidResourceOutputs.createSrcJar(javaSourceDirectory, sourceJar, /* staticIds= */ true); profiler.recordEndOf("sourcejar"); return StaticLibrary.from(outPath, rTxt, ImmutableList.of(), sourceJar); } catch (IOException e) { throw LinkError.of(e); } } private List compiledResourcesToPaths( CompiledResources compiled, Predicate shouldKeep) { // Using sequential streams to maintain the overlay order for aapt2. return Stream.concat(include.stream(), Stream.of(compiled)) .sequential() .map(CompiledResources::getZip) .map(z -> executorService.submit(() -> filterZip(z, shouldKeep))) .map(rethrowLinkError(Future::get)) // the process will always take as long as the longest Future .map(Path::toString) .collect(toList()); } private Path filterZip(Path path, Predicate shouldKeep) throws IOException { Path outPath = workingDirectory .resolve("filtered") // make absolute paths relative so that resolve will make a new path. .resolve(path.isAbsolute() ? path.subpath(1, path.getNameCount()) : path); // TODO(74258184): How can this path already exist? if (Files.exists(outPath)) { return outPath; } Files.createDirectories(outPath.getParent()); try (FileChannel inChannel = FileChannel.open(path, StandardOpenOption.READ); FileChannel outChannel = FileChannel.open(outPath, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) { final ZipIn zipIn = new ZipIn(inChannel, path.toString()); final ZipOut zipOut = new ZipOut(outChannel, outPath.toString()); zipIn.scanEntries( (in, header, dirEntry, data) -> { if (shouldKeep.test(dirEntry)) { zipOut.nextEntry(dirEntry); zipOut.write(header); zipOut.write(data); } }); zipOut.close(); } return outPath; } private static Function rethrowLinkError(CheckedFunction checked) { return (T arg) -> { try { return checked.apply(arg); } catch (ExecutionException e) { throw LinkError.of(Optional.ofNullable(e.getCause()).orElse(e)); // unwrap } catch (IOException e) { throw LinkError.of(e); } catch (Throwable e) { // unexpected error, rethrow for debugging. throw new RuntimeException(e); } }; } @FunctionalInterface private interface CheckedFunction { R apply(T arg) throws Throwable; } private String replaceExtension(String fileName, String newExtension) { int lastIndex = fileName.lastIndexOf('.'); if (lastIndex == -1) { return fileName.concat(".").concat(newExtension); } return fileName.substring(0, lastIndex).concat(".").concat(newExtension); } private ProtoApk linkProtoApk( CompiledResources compiled, Path rTxt, Path proguardConfig, Path mainDexProguard, Path javaSourceDirectory, Path resourceIds) throws IOException { profiler.startTask("fulllink"); final Path linked = workingDirectory.resolve("bin." + PROTO_EXTENSION); logger.fine( new AaptCommandBuilder(aapt2) .forBuildToolsVersion(buildToolsVersion) .forVariantType(VariantType.DEFAULT) .add("link") .whenVersionIsAtLeast(new Revision(23)) .thenAdd("--no-version-vectors") // Turn off namespaced resources .add("--no-static-lib-packages") .when(Objects.equals(logger.getLevel(), Level.FINE)) .thenAdd("-v") .add("--manifest", compiled.getManifest()) // Enables resource redefinition and merging .add("--auto-add-overlay") // Always link to proto, as resource shrinking needs the extra information. .add("--proto-format") .when(debug) .thenAdd("--debug-mode") .add("--custom-package", customPackage) .when(densities.size() == 1) .thenAddRepeated("--preferred-density", densities) .add("--stable-ids", compiled.getStableIds()) .addRepeated( "-A", Streams.concat( assetDirs.stream().map(Path::toString), compiled.getAssetsStrings().stream()) .collect(toList())) .addRepeated("-I", StaticLibrary.toPathStrings(linkAgainst)) .addParameterableRepeated( "-R", compiledResourcesToPaths( compiled, generatePseudoLocale && resourceConfigs.stream().anyMatch(PSEUDO_LOCALE_FILTERS::contains) ? IS_FLAT_FILE.and(USE_GENERATED) : IS_FLAT_FILE.and(USE_DEFAULT)), workingDirectory) // Never compress apks. .add("-0", "apk") // Add custom no-compress extensions. .addRepeated("-0", uncompressedExtensions) // Filter by resource configuration type. .when(!resourceConfigs.isEmpty()) .thenAdd("-c", Joiner.on(',').join(resourceConfigs)) .add("--output-text-symbols", rTxt) .add("--emit-ids", resourceIds) .add("--java", javaSourceDirectory) .add("--proguard", proguardConfig) .add("--proguard-main-dex", mainDexProguard) .when(conditionalKeepRules) .thenAdd("--proguard-conditional-keep-rules") .add("-o", linked) .execute(String.format("Linking %s", compiled.getManifest()))); profiler.recordEndOf("fulllink"); return ProtoApk.readFrom( densities.size() < 2 ? linked : optimizeForDensities(compiled, linked)); } private Path combineApks(Path protoApk, Path binaryApk, Path workingDirectory) throws IOException { // Linking against apk as a static library elides assets, amoung other things. // So, copy the missing details to the new apk. profiler.startTask("combine"); final Path combined = workingDirectory.resolve("combined.apk"); try (FileChannel nonResourceChannel = FileChannel.open(protoApk, StandardOpenOption.READ); FileChannel resourceChannel = FileChannel.open(binaryApk, StandardOpenOption.READ); FileChannel outChannel = FileChannel.open(combined, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) { final ZipIn resourcesIn = new ZipIn(resourceChannel, binaryApk.toString()); final ZipIn nonResourcesIn = new ZipIn(nonResourceChannel, protoApk.toString()); final ZipOut zipOut = new ZipOut(outChannel, combined.toString()); Set skip = new HashSet<>(); skip.add("resources.pb"); final EntryHandler entryHandler = (in, header, dirEntry, data) -> { final String filename = dirEntry.getFilename(); // Make sure we aren't copying the same entry twice. if (!skip.contains(filename)) { skip.add(filename); String comment = dirEntry.getComment(); byte[] extra = dirEntry.getExtraData(); zipOut.nextEntry( dirEntry.clone(filename, extra, comment).set(CENTIM, DosTime.EPOCH.time)); zipOut.write(header.clone(filename, extra).set(LOCTIM, DosTime.EPOCH.time)); zipOut.write(data); if ((header.get(LOCFLG) & LocalFileHeader.SIZE_MASKED_FLAG) != 0) { DataDescriptor desc = DataDescriptor.allocate() .set(EXTCRC, dirEntry.get(CENCRC)) .set(EXTSIZ, dirEntry.get(CENSIZ)) .set(EXTLEN, dirEntry.get(CENLEN)); zipOut.write(desc); } } }; resourcesIn.scanEntries(entryHandler); nonResourcesIn.scanEntries(entryHandler); zipOut.close(); return combined; } finally { profiler.recordEndOf("combine"); } } private Path extractAttributes(CompiledResources compiled) throws IOException { profiler.startTask("attributes"); Path attributes = workingDirectory.resolve("tool.attributes"); // extract tool annotations from the compile resources. final SdkToolAttributeWriter writer = new SdkToolAttributeWriter(attributes); Stream.concat(include.stream(), Stream.of(compiled)) .parallel() .map(AndroidCompiledDataDeserializer.create()::readAttributes) .map(Map::entrySet) .flatMap(Set::stream) .distinct() .forEach(e -> e.getValue().writeResource((FullyQualifiedName) e.getKey(), writer)); writer.flush(); profiler.recordEndOf("attributes"); return attributes; } private Path optimizeForDensities(CompiledResources compiled, Path binary) throws IOException { profiler.startTask("optimize"); final Path optimized = workingDirectory.resolve("optimized." + PROTO_EXTENSION); logger.fine( new AaptCommandBuilder(aapt2) .forBuildToolsVersion(buildToolsVersion) .forVariantType(VariantType.DEFAULT) .add("optimize") .when(Objects.equals(logger.getLevel(), Level.FINE)) .thenAdd("-v") .add("--target-densities", densities.stream().collect(Collectors.joining(","))) .add("-o", optimized) .add(binary.toString()) .execute(String.format("Optimizing %s", compiled.getManifest()))); profiler.recordEndOf("optimize"); return optimized; } /** Links compiled resources into an apk */ public PackagedResources link(CompiledResources compiled) { try { Path rTxt = workingDirectory.resolve("R.txt"); Path proguardConfig = workingDirectory.resolve("proguard.cfg"); Path mainDexProguard = workingDirectory.resolve("proguard.maindex.cfg"); Path javaSourceDirectory = Files.createDirectories(workingDirectory.resolve("java")); Path resourceIds = workingDirectory.resolve("ids.txt"); try (ProtoApk protoApk = linkProtoApk( compiled, rTxt, proguardConfig, mainDexProguard, javaSourceDirectory, resourceIds)) { return PackagedResources.of( outputAsProto ? protoApk.asApkPath() : link(protoApk, resourceIds), // convert proto to binary protoApk.asApkPath(), rTxt, proguardConfig, mainDexProguard, javaSourceDirectory, resourceIds, extractAttributes(compiled)); } } catch (IOException e) { throw new LinkError(e); } } /** Link a proto apk to produce an apk. */ public Path link(ProtoApk protoApk, Path resourceIds) { try { final Path protoApkPath = protoApk.asApkPath(); final Path working = workingDirectory .resolve("link-proto") .resolve(replaceExtension(protoApkPath.getFileName().toString(), "working")); final Path manifest = protoApk.writeManifestAsXmlTo(working); final Path apk = working.resolve("binary.apk"); logger.fine( new AaptCommandBuilder(aapt2) .forBuildToolsVersion(buildToolsVersion) .forVariantType(VariantType.DEFAULT) .add("link") .when(Objects.equals(logger.getLevel(), Level.FINE)) .thenAdd("-v") .whenVersionIsAtLeast(new Revision(23)) .thenAdd("--no-version-vectors") .add("--stable-ids", resourceIds) .add("--manifest", manifest) .addRepeated("-I", StaticLibrary.toPathStrings(linkAgainst)) .add("-R", protoApk.asApkPath()) .add("-o", apk.toString()) .execute(String.format("Re-linking %s", protoApkPath))); return combineApks(protoApkPath, apk, working); } catch (IOException e) { throw new LinkError(e); } } public ResourceLinker storeUncompressed(List uncompressedExtensions) { this.uncompressedExtensions = uncompressedExtensions; return this; } public ResourceLinker includeOnlyConfigs(List resourceConfigs) { this.resourceConfigs = resourceConfigs; return this; } public ResourceLinker using(Path androidJar) { this.androidJar = androidJar; return this; } @Override public String toString() { return MoreObjects.toStringHelper(this) .add("aapt2", aapt2) .add("linkAgainst", linkAgainst) .add("buildToolsVersion", buildToolsVersion) .add("workingDirectory", workingDirectory) .add("densities", densities) .add("androidJar", androidJar) .add("uncompressedExtensions", uncompressedExtensions) .add("resourceConfigs", resourceConfigs) .add("baseApk", baseApk) .toString(); } }