From 8fe0f45852a620a078013310989396caed273342 Mon Sep 17 00:00:00 2001 From: corysmith Date: Tue, 31 Jul 2018 08:30:55 -0700 Subject: Add apk converted to proto and all attributes from CompiledResources to ResourcesZip. Add new proto format for tool attributes stored in the AndroidDataXml for storing them in the resources.zip. RELNOTES:None PiperOrigin-RevId: 206765679 --- .../android/Aapt2ResourcePackagingAction.java | 85 +++++----- .../android/Aapt2ResourceShrinkingAction.java | 127 +++++++++------ .../android/AndroidCompiledDataDeserializer.java | 53 ++++++- .../devtools/build/android/ResourcesZip.java | 96 +++++++++++- .../build/android/aapt2/PackagedResources.java | 45 +++++- .../build/android/aapt2/ResourceLinker.java | 171 +++++++++++++++++++-- .../build/android/proto/serialize_format.proto | 8 + 7 files changed, 465 insertions(+), 120 deletions(-) (limited to 'src/tools') diff --git a/src/tools/android/java/com/google/devtools/build/android/Aapt2ResourcePackagingAction.java b/src/tools/android/java/com/google/devtools/build/android/Aapt2ResourcePackagingAction.java index 6fc5e76419..ac89b36ee3 100644 --- a/src/tools/android/java/com/google/devtools/build/android/Aapt2ResourcePackagingAction.java +++ b/src/tools/android/java/com/google/devtools/build/android/Aapt2ResourcePackagingAction.java @@ -119,10 +119,7 @@ public class Aapt2ResourcePackagingAction { options.densities, filteredResources, mergedResources), new DensitySpecificManifestProcessor(options.densities, densityManifest)); - profiler.recordEndOf("merging"); - - - profiler.startTask("compile"); + profiler.recordEndOf("merging").startTask("compile"); final ResourceCompiler compiler = ResourceCompiler.create( executorService, @@ -131,37 +128,37 @@ public class Aapt2ResourcePackagingAction { aaptConfigOptions.buildToolsVersion, aaptConfigOptions.generatePseudoLocale); - CompiledResources compiled = - options - .primaryData - .processDataBindings( - options.dataBindingInfoOut, options.packageForR, databindingResourcesRoot) - .compile(compiler, compiledResources) - .processManifest( - manifest -> - AndroidManifestProcessor.with(STD_LOGGER) - .processManifest( - options.applicationId, - options.versionCode, - options.versionName, - manifest, - processedManifest)) - .processManifest( - manifest -> - new DensitySpecificManifestProcessor(options.densities, densityManifest) - .process(manifest)); - profiler.recordEndOf("compile").startTask("link"); - // Write manifestOutput now before the dummy manifest is created. - if (options.manifestOutput != null) { - AndroidResourceOutputs.copyManifestToOutput(compiled, options.manifestOutput); - } - - List compiledResourceDeps = - // Last defined dependencies will overwrite previous one, so always place direct - // after transitive. - concat(options.transitiveData.stream(), options.directData.stream()) - .map(DependencyAndroidData::getCompiledSymbols) - .collect(toList()); + CompiledResources compiled = + options + .primaryData + .processDataBindings( + options.dataBindingInfoOut, options.packageForR, databindingResourcesRoot) + .compile(compiler, compiledResources) + .processManifest( + manifest -> + AndroidManifestProcessor.with(STD_LOGGER) + .processManifest( + options.applicationId, + options.versionCode, + options.versionName, + manifest, + processedManifest)) + .processManifest( + manifest -> + new DensitySpecificManifestProcessor(options.densities, densityManifest) + .process(manifest)); + profiler.recordEndOf("compile").startTask("link"); + // Write manifestOutput now before the dummy manifest is created. + if (options.manifestOutput != null) { + AndroidResourceOutputs.copyManifestToOutput(compiled, options.manifestOutput); + } + + List compiledResourceDeps = + // Last defined dependencies will overwrite previous one, so always place direct + // after transitive. + concat(options.transitiveData.stream(), options.directData.stream()) + .map(DependencyAndroidData::getCompiledSymbols) + .collect(toList()); List assetDirs = concat( @@ -171,7 +168,7 @@ public class Aapt2ResourcePackagingAction { options.directAssets.stream()) .flatMap(dep -> dep.assetDirs.stream()) .collect(toList()); - assetDirs.addAll(options.primaryData.assetDirs); + assetDirs.addAll(options.primaryData.assetDirs); final PackagedResources packagedResources = ResourceLinker.create(aaptConfigOptions.aapt2, executorService, linkedOut) @@ -193,18 +190,14 @@ public class Aapt2ResourcePackagingAction { .copyMainDexProguardTo(options.mainDexProguardOutput) .createSourceJar(options.srcJarOutput) .copyRTxtTo(options.rOutput); - profiler.recordEndOf("link"); - if (options.resourcesOutput != null) { - profiler.startTask("package"); + profiler.recordEndOf("link"); + if (options.resourcesOutput != null) { // The compiled resources and the merged resources should be the same. // TODO(corysmith): Decompile or otherwise provide the exact resources in the apk. - ResourcesZip.fromApk( - mergedAndroidData.getResourceDir(), - packagedResources.getApk(), - packagedResources.getResourceIds()) - .writeTo(options.resourcesOutput, /* compress= */ false); - profiler.recordEndOf("package"); - } + packagedResources + .packageWith(mergedAndroidData.getResourceDir()) + .writeTo(options.resourcesOutput, false); } } } +} diff --git a/src/tools/android/java/com/google/devtools/build/android/Aapt2ResourceShrinkingAction.java b/src/tools/android/java/com/google/devtools/build/android/Aapt2ResourceShrinkingAction.java index af2a98ab2f..8d1be26dab 100644 --- a/src/tools/android/java/com/google/devtools/build/android/Aapt2ResourceShrinkingAction.java +++ b/src/tools/android/java/com/google/devtools/build/android/Aapt2ResourceShrinkingAction.java @@ -25,12 +25,16 @@ import com.google.devtools.build.android.aapt2.CompiledResources; import com.google.devtools.build.android.aapt2.ResourceCompiler; import com.google.devtools.build.android.aapt2.ResourceLinker; import com.google.devtools.build.android.aapt2.StaticLibrary; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionDocumentationCategory; +import com.google.devtools.common.options.OptionEffectTag; +import com.google.devtools.common.options.OptionsBase; import com.google.devtools.common.options.OptionsParser; import com.google.devtools.common.options.ShellQuotedParamsFilePreProcessor; -import java.io.Closeable; import java.io.File; import java.nio.file.FileSystems; import java.nio.file.Path; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.function.Function; @@ -53,68 +57,101 @@ import java.util.function.Function; */ public class Aapt2ResourceShrinkingAction { + /** Aapt2 shrinking specific options */ + public static final class Aapt2ShrinkOptions extends OptionsBase { + @Option( + name = "useProtoApk", + defaultValue = "false", + category = "config", + documentationCategory = OptionDocumentationCategory.UNCATEGORIZED, + effectTags = {OptionEffectTag.UNKNOWN}, + help = "Path to the shrunk jar from a Proguard run with shrinking enabled.") + public boolean useProtoApk; + } + public static void main(String[] args) throws Exception { final Profiler profiler = LoggingProfiler.createAndStart("shrink").startTask("flags"); // Parse arguments. OptionsParser optionsParser = - OptionsParser.newOptionsParser(Options.class, Aapt2ConfigOptions.class); + OptionsParser.newOptionsParser( + ImmutableList.of(Options.class, Aapt2ConfigOptions.class, Aapt2ShrinkOptions.class)); optionsParser.enableParamsFileSupport( new ShellQuotedParamsFilePreProcessor(FileSystems.getDefault())); optionsParser.parseAndExitUponError(args); Aapt2ConfigOptions aapt2ConfigOptions = optionsParser.getOptions(Aapt2ConfigOptions.class); Options options = optionsParser.getOptions(Options.class); + Aapt2ShrinkOptions aapt2ShrinkOptions = optionsParser.getOptions(Aapt2ShrinkOptions.class); profiler.recordEndOf("flags").startTask("setup"); - final ListeningExecutorService executorService = ExecutorServiceCloser.createDefaultService(); try (ScopedTemporaryDirectory scopedTmp = new ScopedTemporaryDirectory("android_resources_tmp"); - Closeable closer = ExecutorServiceCloser.createWith(executorService)) { - - Path workingResourcesDirectory = scopedTmp.subDirectoryOf("resources"); - final ResourceCompiler resourceCompiler = - ResourceCompiler.create( - executorService, - workingResourcesDirectory, - aapt2ConfigOptions.aapt2, - aapt2ConfigOptions.buildToolsVersion, - aapt2ConfigOptions.generatePseudoLocale); - profiler.recordEndOf("setup").startTask("compile"); + ExecutorServiceCloser executorService = ExecutorServiceCloser.createWithFixedPoolOf(15)) { final ResourcesZip resourcesZip = ResourcesZip.createFrom( options.resourcesZip, scopedTmp.subDirectoryOf("merged-resources")); - final CompiledResources compiled = - resourcesZip - .shrink( - options - .dependencyManifests - .stream() - .map(Path::toFile) - .map(manifestToPackageUsing(executorService)) - .map(futureToString()) - .collect(toSet()), - options.rTxt, - options.shrunkJar, - options.primaryManifest, - options.proguardMapping, - options.log, - scopedTmp.subDirectoryOf("shrunk-resources")) - .writeArchiveTo(options.shrunkResources, false) - .compile(resourceCompiler, workingResourcesDirectory); - profiler.recordEndOf("compile"); + Path workingResourcesDirectory = scopedTmp.subDirectoryOf("resources"); + final ResourceLinker linker = + ResourceLinker.create( + aapt2ConfigOptions.aapt2, executorService, scopedTmp.subDirectoryOf("linking")) + .profileUsing(profiler); + + final Set packages = + options + .dependencyManifests + .stream() + .map(Path::toFile) + .map(manifestToPackageUsing(executorService)) + .map(futureToString()) + .collect(toSet()); - ResourceLinker.create( - aapt2ConfigOptions.aapt2, executorService, scopedTmp.subDirectoryOf("linking")) - .profileUsing(profiler) - .dependencies(ImmutableList.of(StaticLibrary.from(aapt2ConfigOptions.androidJar))) - .profileUsing(profiler) - .outputAsProto(aapt2ConfigOptions.resourceTableAsProto) - .buildVersion(aapt2ConfigOptions.buildToolsVersion) - .includeOnlyConfigs(aapt2ConfigOptions.resourceConfigs) - .debug(aapt2ConfigOptions.debug) - .link(compiled) - .copyPackageTo(options.shrunkApk) - .copyRTxtTo(options.rTxtOutput); + if (aapt2ShrinkOptions.useProtoApk) { + resourcesZip + .shrinkUsingProto( + packages, + options.rTxt, + options.shrunkJar, + options.primaryManifest, + options.proguardMapping, + options.log, + scopedTmp.subDirectoryOf("shrunk-resources")) + .writeBinaryTo(linker, options.shrunkApk) + .writeReportTo(options.log) + .writeResourceToZip(options.shrunkResources); + } else { + final ResourceCompiler resourceCompiler = + ResourceCompiler.create( + executorService, + workingResourcesDirectory, + aapt2ConfigOptions.aapt2, + aapt2ConfigOptions.buildToolsVersion, + aapt2ConfigOptions.generatePseudoLocale); + profiler.recordEndOf("setup").startTask("compile"); + + final CompiledResources compiled = + resourcesZip + .shrink( + packages, + options.rTxt, + options.shrunkJar, + options.primaryManifest, + options.proguardMapping, + options.log, + scopedTmp.subDirectoryOf("shrunk-resources")) + .writeArchiveTo(options.shrunkResources, false) + .compile(resourceCompiler, workingResourcesDirectory); + profiler.recordEndOf("compile"); + linker + .dependencies(ImmutableList.of(StaticLibrary.from(aapt2ConfigOptions.androidJar))) + .profileUsing(profiler) + .outputAsProto(aapt2ConfigOptions.resourceTableAsProto) + .buildVersion(aapt2ConfigOptions.buildToolsVersion) + .includeOnlyConfigs(aapt2ConfigOptions.resourceConfigs) + .debug(aapt2ConfigOptions.debug) + .link(compiled) + .copyPackageTo(options.shrunkApk) + .copyRTxtTo(options.rTxtOutput); + } profiler.recordEndOf("shrink"); } } diff --git a/src/tools/android/java/com/google/devtools/build/android/AndroidCompiledDataDeserializer.java b/src/tools/android/java/com/google/devtools/build/android/AndroidCompiledDataDeserializer.java index b4ba3761fe..1af171ff53 100644 --- a/src/tools/android/java/com/google/devtools/build/android/AndroidCompiledDataDeserializer.java +++ b/src/tools/android/java/com/google/devtools/build/android/AndroidCompiledDataDeserializer.java @@ -16,6 +16,7 @@ package com.google.devtools.build.android; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Predicates.not; import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toMap; import android.aapt.pb.internal.ResourcesInternal.CompiledFile; import com.android.SdkConstants; @@ -78,6 +79,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.io.LittleEndianDataInputStream; import com.google.devtools.build.android.FullyQualifiedName.Factory; +import com.google.devtools.build.android.aapt2.CompiledResources; import com.google.devtools.build.android.proto.SerializeFormat; import com.google.devtools.build.android.proto.SerializeFormat.Header; import com.google.devtools.build.android.xml.ResourcesAttribute.AttributeType; @@ -88,9 +90,11 @@ import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.file.FileSystem; +import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.AbstractMap.SimpleImmutableEntry; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -99,9 +103,12 @@ import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Map.Entry; import java.util.Optional; import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; import java.util.logging.Logger; +import java.util.stream.Stream; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import javax.annotation.concurrent.NotThreadSafe; @@ -574,11 +581,14 @@ public class AndroidCompiledDataDeserializer implements AndroidDataDeserializer } private void readAttributesFile( - InputStream resourceFileStream, FileSystem fileSystem, KeyValueConsumers consumers) + InputStream resourceFileStream, + FileSystem fileSystem, + BiConsumer combine, + BiConsumer overwrite) throws IOException { Header header = Header.parseDelimitedFrom(resourceFileStream); - List fullyQualifiedNames = new ArrayList<>(); + List fullyQualifiedNames = new ArrayList<>(); for (int i = 0; i < header.getEntryCount(); i++) { SerializeFormat.DataKey protoKey = SerializeFormat.DataKey.parseDelimitedFrom(resourceFileStream); @@ -587,7 +597,7 @@ public class AndroidCompiledDataDeserializer implements AndroidDataDeserializer DataSourceTable sourceTable = DataSourceTable.read(resourceFileStream, fileSystem, header); - for (DataKey fullyQualifiedName : fullyQualifiedNames) { + for (FullyQualifiedName fullyQualifiedName : fullyQualifiedNames) { SerializeFormat.DataValue protoValue = SerializeFormat.DataValue.parseDelimitedFrom(resourceFileStream); DataSource source = sourceTable.sourceFromId(protoValue.getSourceId()); @@ -595,13 +605,40 @@ public class AndroidCompiledDataDeserializer implements AndroidDataDeserializer AttributeType attributeType = AttributeType.valueOf(protoValue.getXmlValue().getValueType()); if (attributeType.isCombining()) { - consumers.combiningConsumer.accept(fullyQualifiedName, dataResourceXml); + combine.accept(fullyQualifiedName, dataResourceXml); } else { - consumers.overwritingConsumer.accept(fullyQualifiedName, dataResourceXml); + overwrite.accept(fullyQualifiedName, dataResourceXml); } } } + public Map readAttributes(CompiledResources resources) { + try (ZipFile zipFile = new ZipFile(resources.getZip().toFile())) { + return zipFile + .stream() + .filter(e -> e.getName().endsWith(".attributes")) + .flatMap( + entry -> { + try { + final Stream.Builder> builder = Stream.builder(); + final BiConsumer consumeToStream = + (k, v) -> builder.add(new SimpleImmutableEntry<>(k, v)); + readAttributesFile( + zipFile.getInputStream(entry), + FileSystems.getDefault(), + consumeToStream, + consumeToStream); + return builder.build(); + } catch (IOException e) { + throw new DeserializationException(e); + } + }) + .collect(toMap(Entry::getKey, Entry::getValue)); + } catch (IOException e) { + throw new DeserializationException(e); + } + } + public void readTable(InputStream in, KeyValueConsumers consumers) throws IOException { final ResourceTable resourceTable = ResourceTable.parseFrom(in); readPackages(consumers, resourceTable); @@ -640,7 +677,11 @@ public class AndroidCompiledDataDeserializer implements AndroidDataDeserializer Factory fqnFactory = Factory.fromDirectoryName(dirNameAndQualifiers); if (fileZipPath.endsWith(".attributes")) { - readAttributesFile(resourceFileStream, inPath.getFileSystem(), consumers); + readAttributesFile( + resourceFileStream, + inPath.getFileSystem(), + consumers.combiningConsumer, + consumers.overwritingConsumer); } else { LittleEndianDataInputStream dataInputStream = new LittleEndianDataInputStream(resourceFileStream); diff --git a/src/tools/android/java/com/google/devtools/build/android/ResourcesZip.java b/src/tools/android/java/com/google/devtools/build/android/ResourcesZip.java index dd3cb099f3..cf482ff030 100644 --- a/src/tools/android/java/com/google/devtools/build/android/ResourcesZip.java +++ b/src/tools/android/java/com/google/devtools/build/android/ResourcesZip.java @@ -22,13 +22,16 @@ import com.google.devtools.build.android.AndroidResourceOutputs.ZipBuilder; import com.google.devtools.build.android.AndroidResourceOutputs.ZipBuilderVisitorWithDirectories; import com.google.devtools.build.android.aapt2.CompiledResources; import com.google.devtools.build.android.aapt2.ResourceCompiler; +import com.google.devtools.build.android.aapt2.ResourceLinker; import java.io.FileOutputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.util.Set; import java.util.concurrent.ExecutionException; +import java.util.logging.Logger; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import javax.annotation.Nullable; @@ -38,17 +41,28 @@ import org.xml.sax.SAXException; /** Represents a collection of raw, merged resources with an optional id list. */ public class ResourcesZip { - private final Path resourcesRoot; - private final Path assetsRoot; + static final Logger logger = Logger.getLogger(ResourcesZip.class.toString()); + + @Nullable private final Path resourcesRoot; + @Nullable private final Path assetsRoot; @Nullable private final Path apkWithAssets; + @Nullable private final Path proto; + @Nullable private final Path attributes; @Nullable private final Path ids; private ResourcesZip( - Path resourcesRoot, Path assetsRoot, @Nullable Path ids, @Nullable Path apkWithAssets) { + @Nullable Path resourcesRoot, + @Nullable Path assetsRoot, + @Nullable Path ids, + @Nullable Path apkWithAssets, + @Nullable Path proto, + @Nullable Path attributes) { this.resourcesRoot = resourcesRoot; this.assetsRoot = assetsRoot; this.ids = ids; this.apkWithAssets = apkWithAssets; + this.proto = proto; + this.attributes = attributes; } /** @@ -56,7 +70,7 @@ public class ResourcesZip { * @param assetsRoot The root of the raw assets. */ public static ResourcesZip from(Path resourcesRoot, Path assetsRoot) { - return new ResourcesZip(resourcesRoot, assetsRoot, null, null); + return new ResourcesZip(resourcesRoot, assetsRoot, null, null, null, null); } /** @@ -69,6 +83,8 @@ public class ResourcesZip { resourcesRoot, assetsRoot, resourceIds != null && Files.exists(resourceIds) ? resourceIds : null, + null, + null, null); } @@ -82,7 +98,27 @@ public class ResourcesZip { resourcesRoot, /* assetsRoot= */ null, resourceIds != null && Files.exists(resourceIds) ? resourceIds : null, - apkWithAssets); + apkWithAssets, + null, + null); + } + + /** + * @param proto apk in proto format. + * @param attributes Tooling attributes. + * @param resourcesRoot The root of the raw resources. + * @param apkWithAssets The apk containing assets. + * @param resourceIds Optional path to a file containing the resource ids. + */ + public static ResourcesZip fromApkWithProto( + Path proto, Path attributes, Path resourcesRoot, Path apkWithAssets, Path resourceIds) { + return new ResourcesZip( + resourcesRoot, + /* assetsRoot= */ null, + resourceIds != null && Files.exists(resourceIds) ? resourceIds : null, + apkWithAssets, + proto, + attributes); } /** Creates a ResourcesZip from an archive by expanding into the workingDirectory. */ @@ -160,6 +196,15 @@ public class ResourcesZip { if (ids != null) { zip.addEntry("ids.txt", Files.readAllBytes(ids), ZipEntry.STORED); } + + if (proto != null && Files.exists(proto)) { + zip.addEntry("apk.pb", Files.readAllBytes(proto), ZipEntry.STORED); + } + + if (attributes != null && Files.exists(attributes)) { + zip.addEntry("tools.attributes.pb", Files.readAllBytes(attributes), ZipEntry.STORED); + } + } catch (IOException e) { throw new RuntimeException(e); } @@ -181,11 +226,50 @@ public class ResourcesZip { packages, rTxt, classJar, manifest, proguardMapping, resourcesRoot, logFile) .shrink(workingDirectory); return ShrunkResources.of( - new ResourcesZip(workingDirectory, assetsRoot, ids, null), + new ResourcesZip(workingDirectory, assetsRoot, ids, null, null, attributes), new UnvalidatedAndroidData( ImmutableList.of(workingDirectory), ImmutableList.of(assetsRoot), manifest)); } + public ShrunkProtoApk shrinkUsingProto( + Set packages, + Path rTxt, + Path classJar, + Path primaryManifest, + Path proguardMapping, + Path logFile, + Path workingDirectory) + throws ParserConfigurationException { + throw new UnsupportedOperationException(); + } + + static class ShrunkProtoApk { + private final Path apk; + private final Path report; + + ShrunkProtoApk(Path apk, Path report) { + this.apk = apk; + this.report = report; + } + + ShrunkProtoApk writeBinaryTo(ResourceLinker linker, Path binaryOut) throws IOException { + Files.copy(linker.convertToBinary(apk), binaryOut, StandardCopyOption.REPLACE_EXISTING); + return this; + } + + ShrunkProtoApk writeReportTo(Path reportOut) throws IOException { + Files.copy(report, reportOut); + return this; + } + + ShrunkProtoApk writeResourceToZip(Path resourcesZip) throws IOException { + try (final ZipBuilder zip = ZipBuilder.createFor(resourcesZip)) { + zip.addEntry("apk.pb", Files.readAllBytes(apk), ZipEntry.STORED); + } + return this; + } + } + static class ShrunkResources { private ResourcesZip resourcesZip; diff --git a/src/tools/android/java/com/google/devtools/build/android/aapt2/PackagedResources.java b/src/tools/android/java/com/google/devtools/build/android/aapt2/PackagedResources.java index faad2cd868..2d92b4e071 100644 --- a/src/tools/android/java/com/google/devtools/build/android/aapt2/PackagedResources.java +++ b/src/tools/android/java/com/google/devtools/build/android/aapt2/PackagedResources.java @@ -14,6 +14,7 @@ package com.google.devtools.build.android.aapt2; import com.google.devtools.build.android.AndroidResourceOutputs; +import com.google.devtools.build.android.ResourcesZip; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -23,47 +24,64 @@ import javax.annotation.Nullable; public class PackagedResources { private final Path apk; + private final Path proto; private final Path rTxt; private final Path proguardConfig; private final Path mainDexProguard; private final Path javaSourceDirectory; private final Path resourceIds; + private final Path attributes; private PackagedResources( Path apk, + Path proto, Path rTxt, Path proguardConfig, Path mainDexProguard, Path javaSourceDirectory, - Path resourceIds) { + Path resourceIds, + Path attributes) { this.apk = apk; + this.proto = proto; this.rTxt = rTxt; this.proguardConfig = proguardConfig; this.mainDexProguard = mainDexProguard; this.javaSourceDirectory = javaSourceDirectory; this.resourceIds = resourceIds; + this.attributes = attributes; } public static PackagedResources of( Path outPath, + Path protoPath, Path rTxt, Path proguardConfig, Path mainDexProguard, Path javaSourceDirectory, - Path resourceIds) + Path resourceIds, + Path attributes) throws IOException { return new PackagedResources( - outPath, rTxt, proguardConfig, mainDexProguard, javaSourceDirectory, resourceIds); + outPath, + protoPath, + rTxt, + proguardConfig, + mainDexProguard, + javaSourceDirectory, + resourceIds, + attributes); } public PackagedResources copyPackageTo(Path packagePath) throws IOException { return of( copy(apk, packagePath), + proto, rTxt, proguardConfig, mainDexProguard, javaSourceDirectory, - resourceIds); + resourceIds, + attributes); } public PackagedResources copyRTxtTo(Path rOutput) throws IOException { @@ -72,11 +90,13 @@ public class PackagedResources { } return new PackagedResources( apk, + proto, copy(rTxt, rOutput), proguardConfig, mainDexProguard, javaSourceDirectory, - resourceIds); + resourceIds, + attributes); } private Path copy(Path from, Path out) throws IOException { @@ -91,11 +111,13 @@ public class PackagedResources { } return of( apk, + proto, rTxt, copy(proguardConfig, proguardOut), mainDexProguard, javaSourceDirectory, - resourceIds); + resourceIds, + attributes); } public PackagedResources copyMainDexProguardTo(Path mainDexProguardOut) throws IOException { @@ -104,11 +126,13 @@ public class PackagedResources { } return of( apk, + proto, rTxt, proguardConfig, copy(mainDexProguard, mainDexProguardOut), javaSourceDirectory, - resourceIds); + resourceIds, + attributes); } public PackagedResources createSourceJar(@Nullable Path sourceJarPath) throws IOException { @@ -116,7 +140,12 @@ public class PackagedResources { return this; } AndroidResourceOutputs.createSrcJar(javaSourceDirectory, sourceJarPath, false); - return of(apk, rTxt, proguardConfig, mainDexProguard, sourceJarPath, resourceIds); + return of( + apk, proto, rTxt, proguardConfig, mainDexProguard, sourceJarPath, resourceIds, attributes); + } + + public ResourcesZip packageWith(Path resourceRoot) { + return ResourcesZip.fromApkWithProto(proto, attributes, resourceRoot, apk, resourceIds); } public Path getResourceIds() { diff --git a/src/tools/android/java/com/google/devtools/build/android/aapt2/ResourceLinker.java b/src/tools/android/java/com/google/devtools/build/android/aapt2/ResourceLinker.java index 3839a5d94b..9ea3703d42 100644 --- a/src/tools/android/java/com/google/devtools/build/android/aapt2/ResourceLinker.java +++ b/src/tools/android/java/com/google/devtools/build/android/aapt2/ResourceLinker.java @@ -21,26 +21,39 @@ import com.google.common.base.Joiner; import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Multimap; import com.google.common.collect.Streams; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.devtools.build.android.AaptCommandBuilder; +import com.google.devtools.build.android.AndroidCompiledDataDeserializer; +import com.google.devtools.build.android.AndroidDataWritingVisitor; +import com.google.devtools.build.android.AndroidResourceMerger.MergingException; import com.google.devtools.build.android.AndroidResourceOutputs; +import com.google.devtools.build.android.FullyQualifiedName; import com.google.devtools.build.android.Profiler; import com.google.devtools.build.android.aapt2.ResourceCompiler.CompiledType; +import com.google.devtools.build.android.proto.SerializeFormat.ToolAttributes; +import com.google.devtools.build.android.xml.Namespaces; import com.google.devtools.build.android.ziputils.DirectoryEntry; import com.google.devtools.build.android.ziputils.ZipIn; import com.google.devtools.build.android.ziputils.ZipOut; +import java.io.BufferedOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.function.Function; @@ -263,7 +276,7 @@ public class ResourceLinker { } private List compiledResourcesToPaths( - CompiledResources compiled, Predicate shouldKeep) throws IOException { + CompiledResources compiled, Predicate shouldKeep) { // Using sequential streams to maintain the overlay order for aapt2. return Stream.concat(include.stream(), Stream.of(compiled)) .sequential() @@ -323,9 +336,61 @@ public class ResourceLinker { R apply(T arg) throws Throwable; } + private String replaceExtension(String fileName, String newExtension) { + int lastIndex = fileName.lastIndexOf('.'); + if (lastIndex == -1) { + return fileName.concat(".").concat(newExtension); + } + return fileName.substring(0, lastIndex).concat(".").concat(newExtension); + } + + public Path convertToBinary(Path protoApk) { + try { + profiler.startTask("convertToBinary"); + final Path outPath = + workingDirectory.resolveSibling( + replaceExtension(protoApk.getFileName().toString(), "apk")); + logger.fine( + new AaptCommandBuilder(aapt2) + .add("convert") + .add("-o", outPath) + .add("--output-format", "binary") + .add(protoApk.toString()) + .execute("Converting " + protoApk)); + profiler.recordEndOf("convertToBinary"); + return outPath; + } catch (IOException e) { + throw new LinkError(e); + } + } + + public Path optimizeApk(Path apk) { + try { + profiler.startTask("optimizeApk"); + final Path outPath = + workingDirectory.resolveSibling( + replaceExtension(apk.getFileName().toString(), ".optimized.apk")); + logger.fine( + new AaptCommandBuilder(aapt2) + .forBuildToolsVersion(buildToolsVersion) + .forVariantType(VariantType.DEFAULT) + .add("optimize") + .when(Objects.equals(logger.getLevel(), Level.FINE)) + .thenAdd("-v") + .add("-o", outPath) + .add(apk.toString()) + .execute(String.format("Optimizing %s", apk))); + return outPath; + } catch (IOException e) { + throw new LinkError(e); + } finally { + profiler.recordEndOf("optimizeApk"); + } + } + public PackagedResources link(CompiledResources compiled) { try { - final Path outPath = workingDirectory.resolve("bin.apk"); + final Path outPath = workingDirectory.resolve("bin.pb"); Path rTxt = workingDirectory.resolve("R.txt"); Path proguardConfig = workingDirectory.resolve("proguard.cfg"); Path mainDexProguard = workingDirectory.resolve("proguard.maindex.cfg"); @@ -342,8 +407,7 @@ public class ResourceLinker { .thenAdd("--no-version-vectors") // Turn off namespaced resources .add("--no-static-lib-packages") - .when(outputAsProto) - .thenAdd("--proto-format") + .add("--proto-format") .when(Objects.equals(logger.getLevel(), Level.FINE)) .thenAdd("-v") .add("--manifest", compiled.getManifest()) @@ -387,13 +451,35 @@ public class ResourceLinker { .thenAdd("--proguard-conditional-keep-rules") .add("-o", outPath) .execute(String.format("Linking %s", compiled.getManifest()))); - profiler.recordEndOf("fulllink"); - profiler.startTask("optimize"); + profiler.recordEndOf("fulllink").startTask("attributes"); + + final Path attributes = workingDirectory.resolve("tool.attributes"); + // extract tool annotations from the compile resources. + final ToolProtoWriter writer = new ToolProtoWriter(attributes); + Stream.concat(include.stream(), Stream.of(compiled)) + .parallel() + .map(AndroidCompiledDataDeserializer.create()::readAttributes) + .map(Map::entrySet) + .flatMap(Set::stream) + .distinct() + .forEach(e -> e.getValue().writeResource((FullyQualifiedName) e.getKey(), writer)); + writer.flush(); + + profiler.recordEndOf("attributes"); if (densities.size() < 2) { return PackagedResources.of( - outPath, rTxt, proguardConfig, mainDexProguard, javaSourceDirectory, resourceIds); + outputAsProto ? outPath : convertToBinary(outPath), // convert proto to apk + outPath, + rTxt, + proguardConfig, + mainDexProguard, + javaSourceDirectory, + resourceIds, + attributes); } - final Path optimized = workingDirectory.resolve("optimized.apk"); + + profiler.startTask("optimize"); + final Path optimized = workingDirectory.resolve("optimized.pb"); logger.fine( new AaptCommandBuilder(aapt2) .forBuildToolsVersion(buildToolsVersion) @@ -406,8 +492,16 @@ public class ResourceLinker { .add(outPath.toString()) .execute(String.format("Optimizing %s", compiled.getManifest()))); profiler.recordEndOf("optimize"); + return PackagedResources.of( - optimized, rTxt, proguardConfig, mainDexProguard, javaSourceDirectory, resourceIds); + outputAsProto ? optimized : convertToBinary(optimized), + optimized, + rTxt, + proguardConfig, + mainDexProguard, + javaSourceDirectory, + resourceIds, + attributes); } catch (IOException e) { throw new LinkError(e); } @@ -442,4 +536,63 @@ public class ResourceLinker { .add("baseApk", baseApk) .toString(); } + + private static class ToolProtoWriter implements AndroidDataWritingVisitor { + + final Multimap attributes = HashMultimap.create(); + private final Path out; + + ToolProtoWriter(Path out) { + this.out = out; + } + + @Override + public void flush() throws IOException { + ToolAttributes.Builder builder = ToolAttributes.newBuilder(); + for (Entry> entry : attributes.asMap().entrySet()) { + builder.putAttributes( + entry.getKey(), + ToolAttributes.ToolAttributeValues.newBuilder().addAllValues(entry.getValue()).build()); + } + try (OutputStream stream = new BufferedOutputStream(Files.newOutputStream(out))) { + builder.build().writeTo(stream); + } + } + + @Override + public Path copyManifest(Path sourceManifest) { + throw new UnsupportedOperationException(); + } + + @Override + public void copyAsset(Path source, String relativeDestinationPath) { + throw new UnsupportedOperationException(); + } + + @Override + public void copyResource(Path source, String relativeDestinationPath) throws MergingException { + throw new UnsupportedOperationException(); + } + + @Override + public void defineAttribute(FullyQualifiedName fqn, String name, String value) { + attributes.put(removeNamespace(name), value); + } + + private String removeNamespace(String qualifiedName) { + int indexColon = qualifiedName.indexOf(':'); + if (indexColon == -1) { + return qualifiedName; + } + return qualifiedName.substring(indexColon); + } + + @Override + public void defineNamespacesFor(FullyQualifiedName fqn, Namespaces namespaces) {} + + @Override + public ValueResourceDefinitionMetadata define(FullyQualifiedName fqn) { + throw new UnsupportedOperationException(); + } + } } diff --git a/src/tools/android/java/com/google/devtools/build/android/proto/serialize_format.proto b/src/tools/android/java/com/google/devtools/build/android/proto/serialize_format.proto index b87d5fee83..a7f913057d 100644 --- a/src/tools/android/java/com/google/devtools/build/android/proto/serialize_format.proto +++ b/src/tools/android/java/com/google/devtools/build/android/proto/serialize_format.proto @@ -87,3 +87,11 @@ message DataValueXml { map attribute = 8; map namespace = 9; } + +// Container for serialized attributes. +message ToolAttributes { + message ToolAttributeValues { + repeated string values = 1; + } + map attributes = 1; +} -- cgit v1.2.3