diff options
Diffstat (limited to 'src')
10 files changed, 136 insertions, 821 deletions
diff --git a/src/test/java/com/google/devtools/build/android/dexer/DexFileMergerTest.java b/src/test/java/com/google/devtools/build/android/dexer/DexFileMergerTest.java index c772e427e8..d4fb9a33f2 100644 --- a/src/test/java/com/google/devtools/build/android/dexer/DexFileMergerTest.java +++ b/src/test/java/com/google/devtools/build/android/dexer/DexFileMergerTest.java @@ -1,16 +1,16 @@ // Copyright 2017 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// // +// // Licensed under the Apache License, Version 2.0 (the "License"); +// // you may not use this file except in compliance with the License. +// // You may obtain a copy of the License at +// // +// // http://www.apache.org/licenses/LICENSE-2.0 +// // +// // Unless required by applicable law or agreed to in writing, software +// // distributed under the License is distributed on an "AS IS" BASIS, +// // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// // See the License for the specific language governing permissions and +// // limitations under the License. package com.google.devtools.build.android.dexer; import static com.google.common.truth.Truth.assertThat; @@ -91,7 +91,7 @@ public class DexFileMergerTest { @Test public void testMergeDexArchive_multidex() throws Exception { Path dexArchive = buildDexArchive(); - Path outputArchive = runDexFileMerger(dexArchive, 200, "multidex_from_dex_archive.dex.zip"); + Path outputArchive = runDexFileMerger(dexArchive, 20, "multidex_from_dex_archive.dex.zip"); int expectedClassCount = matchingFileCount(dexArchive, ".*\\.class.dex$"); assertMultidexOutput(expectedClassCount, outputArchive, ImmutableSet.<String>of()); @@ -228,7 +228,7 @@ public class DexFileMergerTest { Set<String> shard = dexFiles.get(expectedDexFileName(i)); for (String c1 : prev) { for (String c2 : shard) { - assertThat(ZipEntryComparator.compareClassNames(c2, c1)) + assertThat(DexFileMerger.compareClassNames(c2, c1)) .named(c2 + " in shard " + i + " should compare as larger than " + c1 + "; list of all shards for reference: " + dexFiles) .isGreaterThan(0); @@ -251,7 +251,7 @@ public class DexFileMergerTest { } Multimap<String, String> dexFiles = assertMultidexOutput(expectedClassCount, outputArchive, mainDexList); - assertThat(dexFiles.keySet().size()).isAtLeast(2); + assertThat(dexFiles.keySet()).hasSize(2); if (minimalMainDex) { assertThat(dexFiles.get("classes.dex")).containsExactlyElementsIn(mainDexList); } else { diff --git a/src/test/java/com/google/devtools/build/android/dexer/DexFileSplitterTest.java b/src/test/java/com/google/devtools/build/android/dexer/DexFileSplitterTest.java deleted file mode 100644 index c2354ca134..0000000000 --- a/src/test/java/com/google/devtools/build/android/dexer/DexFileSplitterTest.java +++ /dev/null @@ -1,245 +0,0 @@ -// Copyright 2017 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package com.google.devtools.build.android.dexer; - -import static com.google.common.truth.Truth.assertThat; -import static org.junit.Assert.fail; - -import com.android.dx.command.dexer.DxContext; -import com.android.dx.dex.code.PositionList; -import com.google.common.base.Function; -import com.google.common.base.Predicates; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; -import com.google.common.collect.Iterators; -import java.io.IOException; -import java.nio.file.FileSystems; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.HashSet; -import java.util.Set; -import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; -import javax.annotation.Nullable; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** Tests for {@link DexFileSplitter}. */ -@RunWith(JUnit4.class) -public class DexFileSplitterTest { - - private static final Path WORKING_DIR = Paths.get(System.getProperty("user.dir")); - private static final Path INPUT_JAR = WORKING_DIR.resolve(System.getProperty("testinputjar")); - private static final Path INPUT_JAR2 = WORKING_DIR.resolve(System.getProperty("testinputjar2")); - private static final Path MAIN_DEX_LIST_FILE = - WORKING_DIR.resolve(System.getProperty("testmaindexlist")); - static final String DEX_PREFIX = "classes"; - - @Test - public void testSingleInputSingleOutput() throws Exception { - Path dexArchive = buildDexArchive(); - ImmutableList<Path> outputArchives = runDexSplitter(256 * 256, "from_single", dexArchive); - assertThat(outputArchives).hasSize(1); - - ImmutableSet<String> expectedFiles = dexEntries(dexArchive); - assertThat(dexEntries(outputArchives.get(0))).containsExactlyElementsIn(expectedFiles); - } - - @Test - public void testDuplicateInputIgnored() throws Exception { - Path dexArchive = buildDexArchive(); - ImmutableList<Path> outputArchives = - runDexSplitter(256 * 256, "from_duplicate", dexArchive, dexArchive); - assertThat(outputArchives).hasSize(1); - - ImmutableSet<String> expectedFiles = dexEntries(dexArchive); - assertThat(dexEntries(outputArchives.get(0))).containsExactlyElementsIn(expectedFiles); - } - - @Test - public void testSingleInputMultidexOutput() throws Exception { - Path dexArchive = buildDexArchive(); - ImmutableList<Path> outputArchives = runDexSplitter(200, "multidex_from_single", dexArchive); - assertThat(outputArchives.size()).isGreaterThan(1); // test sanity - - ImmutableSet<String> expectedEntries = dexEntries(dexArchive); - assertExpectedEntries(outputArchives, expectedEntries); - } - - @Test - public void testMultipleInputsMultidexOutput() throws Exception { - Path dexArchive = buildDexArchive(); - Path dexArchive2 = buildDexArchive(INPUT_JAR2, "jar2.dex.zip"); - ImmutableList<Path> outputArchives = runDexSplitter(200, "multidex", dexArchive, dexArchive2); - assertThat(outputArchives.size()).isGreaterThan(1); // test sanity - - HashSet<String> expectedEntries = new HashSet<>(); - expectedEntries.addAll(dexEntries(dexArchive)); - expectedEntries.addAll(dexEntries(dexArchive2)); - assertExpectedEntries(outputArchives, expectedEntries); - } - - @Test - public void testMainDexList() throws Exception { - Path dexArchive = buildDexArchive(); - ImmutableList<Path> outputArchives = - runDexSplitter( - 200, - "main_dex_list", - MAIN_DEX_LIST_FILE, - /*minimalMainDex=*/ false, - dexArchive); - - ImmutableSet<String> expectedEntries = dexEntries(dexArchive); - assertThat(outputArchives.size()).isGreaterThan(1); // test sanity - assertThat(dexEntries(outputArchives.get(0))) - .containsAllIn(expectedMainDexEntries()); - assertExpectedEntries(outputArchives, expectedEntries); - } - - @Test - public void testMinimalMainDex() throws Exception { - Path dexArchive = buildDexArchive(); - ImmutableList<Path> outputArchives = - runDexSplitter( - 256 * 256, - "minimal_main_dex", - MAIN_DEX_LIST_FILE, - /*minimalMainDex=*/ true, - dexArchive); - - ImmutableSet<String> expectedEntries = dexEntries(dexArchive); - assertThat(outputArchives.size()).isGreaterThan(1); // test sanity - assertThat(dexEntries(outputArchives.get(0))) - .containsExactlyElementsIn(expectedMainDexEntries()); - assertExpectedEntries(outputArchives, expectedEntries); - } - - private static Iterable<String> expectedMainDexEntries() throws IOException { - return Iterables.transform( - Files.readAllLines(MAIN_DEX_LIST_FILE), - new Function<String, String>() { - @Override - public String apply(String input) { - return input + ".dex"; - } - }); - } - - @Test - public void testMultidexOffWithMultidexFlags() throws Exception { - Path dexArchive = buildDexArchive(); - try { - runDexSplitter( - 200, - "should_fail", - /*mainDexList=*/ null, - /*minimalMainDex=*/ true, - dexArchive); - fail("Expected IllegalArgumentException"); - } catch (IllegalArgumentException e) { - assertThat(e) - .hasMessageThat() - .isEqualTo("--minimal-main-dex not allowed without --main-dex-list"); - } - } - - private void assertExpectedEntries( - ImmutableList<Path> outputArchives, Set<String> expectedEntries) throws IOException { - ImmutableSet.Builder<String> actualFiles = ImmutableSet.builder(); - for (Path outputArchive : outputArchives) { - actualFiles.addAll(dexEntries(outputArchive)); - } - // ImmutableSet.Builder.build would fail if there were duplicates. Additionally we make sure - // all expected files are here - assertThat(actualFiles.build()).containsExactlyElementsIn(expectedEntries); - } - - private ImmutableSet<String> dexEntries(Path dexArchive) throws IOException { - try (ZipFile input = new ZipFile(dexArchive.toFile())) { - ImmutableSet<String> result = ImmutableSet.copyOf(Iterators.filter( - Iterators.transform(Iterators.forEnumeration(input.entries()), ZipEntryName.INSTANCE), - Predicates.containsPattern(".*\\.class.dex$"))); - assertThat(result).isNotEmpty(); // test sanity - return result; - } - } - - private ImmutableList<Path> runDexSplitter(int maxNumberOfIdxPerDex, String outputRoot, - Path... dexArchives) throws IOException { - return runDexSplitter( - maxNumberOfIdxPerDex, - outputRoot, - /*mainDexList=*/ null, - /*minimalMainDex=*/ false, - dexArchives); - } - - private ImmutableList<Path> runDexSplitter( - int maxNumberOfIdxPerDex, - String outputRoot, - @Nullable Path mainDexList, - boolean minimalMainDex, - Path... dexArchives) - throws IOException { - DexFileSplitter.Options options = new DexFileSplitter.Options(); - options.inputArchives = ImmutableList.copyOf(dexArchives); - options.outputDirectory = - FileSystems.getDefault().getPath(System.getenv("TEST_TMPDIR"), outputRoot); - options.maxNumberOfIdxPerDex = maxNumberOfIdxPerDex; - options.mainDexListFile = mainDexList; - options.minimalMainDex = minimalMainDex; - DexFileSplitter.splitIntoShards(options); - assertThat(options.outputDirectory.toFile().exists()).isTrue(); - ImmutableSet<Path> files = - ImmutableSet.copyOf(Files.newDirectoryStream(options.outputDirectory, "*.zip")); - ImmutableList.Builder<Path> result = ImmutableList.builder(); - for (int i = 1; i <= files.size(); ++i) { - Path path = options.outputDirectory.resolve(i + ".shard.zip"); - assertThat(files).contains(path); - result.add(path); - } - return result.build(); // return expected files in sorted order - } - - private Path buildDexArchive() throws Exception { - return buildDexArchive(INPUT_JAR, "libtests.dex.zip"); - } - - private Path buildDexArchive(Path inputJar, String outputZip) throws Exception { - DexBuilder.Options options = new DexBuilder.Options(); - // Use Jar file that has this test in it as the input Jar - options.inputJar = inputJar; - options.outputZip = - FileSystems.getDefault().getPath(System.getenv("TEST_TMPDIR"), outputZip); - options.maxThreads = 1; - Dexing.DexingOptions dexingOptions = new Dexing.DexingOptions(); - dexingOptions.optimize = true; - dexingOptions.positionInfo = PositionList.LINES; - DexBuilder.buildDexArchive(options, new Dexing(new DxContext(), dexingOptions)); - return options.outputZip; - } - - // Can't use lambda for Java 7 compatibility so we can run this Jar through dx without desugaring. - private enum ZipEntryName implements Function<ZipEntry, String> { - INSTANCE; - @Override - public String apply(ZipEntry input) { - return input.getName(); - } - } -} diff --git a/src/test/java/com/google/devtools/build/android/dexer/DexLimitTrackerTest.java b/src/test/java/com/google/devtools/build/android/dexer/DexLimitTrackerTest.java deleted file mode 100644 index a1e6668ac9..0000000000 --- a/src/test/java/com/google/devtools/build/android/dexer/DexLimitTrackerTest.java +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2017 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package com.google.devtools.build.android.dexer; - -import static com.google.common.truth.Truth.assertThat; - -import com.android.dex.Dex; -import com.android.dx.command.dexer.DxContext; -import com.android.dx.dex.DexOptions; -import com.android.dx.dex.cf.CfOptions; -import com.android.dx.dex.file.DexFile; -import com.google.common.io.ByteStreams; -import java.io.IOException; -import java.io.InputStream; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** Tests for {@link DexLimitTracker}. */ -@RunWith(JUnit4.class) -public class DexLimitTrackerTest { - - private Dex dex; - - @Before - public void setUp() throws IOException { - dex = DexFiles.toDex(convertClass(DexLimitTrackerTest.class)); - } - - @Test - public void testUnderLimit() { - DexLimitTracker tracker = - new DexLimitTracker(Math.max(dex.methodIds().size(), dex.fieldIds().size())); - assertThat(tracker.track(dex)).isFalse(); - } - - @Test - public void testOverLimit() throws IOException { - DexLimitTracker tracker = - new DexLimitTracker(Math.max(dex.methodIds().size(), dex.fieldIds().size()) - 1); - assertThat(tracker.track(dex)).isTrue(); - assertThat(tracker.track(dex)).isTrue(); - assertThat(tracker.track(DexFiles.toDex(convertClass(DexLimitTracker.class)))).isTrue(); - } - - @Test - public void testRepeatedReferencesDeduped() throws IOException { - DexLimitTracker tracker = - new DexLimitTracker(Math.max(dex.methodIds().size(), dex.fieldIds().size())); - assertThat(tracker.track(dex)).isFalse(); - assertThat(tracker.track(dex)).isFalse(); - assertThat(tracker.track(dex)).isFalse(); - assertThat(tracker.track(dex)).isFalse(); - assertThat(tracker.track(DexFiles.toDex(convertClass(DexLimitTracker.class)))).isTrue(); - assertThat(tracker.track(dex)).isTrue(); - } - - @Test - public void testGoOverLimit() throws IOException { - DexLimitTracker tracker = - new DexLimitTracker(Math.max(dex.methodIds().size(), dex.fieldIds().size())); - assertThat(tracker.track(dex)).isFalse(); - assertThat(tracker.track(DexFiles.toDex(convertClass(DexLimitTracker.class)))).isTrue(); - } - - @Test - public void testClear() throws IOException { - DexLimitTracker tracker = - new DexLimitTracker(Math.max(dex.methodIds().size(), dex.fieldIds().size())); - assertThat(tracker.track(dex)).isFalse(); - assertThat(tracker.track(DexFiles.toDex(convertClass(DexLimitTracker.class)))).isTrue(); - tracker.clear(); - assertThat(tracker.track(dex)).isFalse(); - } - - private static DexFile convertClass(Class<?> clazz) throws IOException { - String path = clazz.getName().replace('.', '/') + ".class"; - try (InputStream in = - Thread.currentThread().getContextClassLoader().getResourceAsStream(path)) { - return new DexConverter(new Dexing(new DxContext(), new DexOptions(), new CfOptions())) - .toDexFile(ByteStreams.toByteArray(in), path); - } - } -} diff --git a/src/tools/android/java/com/google/devtools/build/android/dexer/BUILD.tools b/src/tools/android/java/com/google/devtools/build/android/dexer/BUILD.tools index 51beb3df39..dd5be15715 100644 --- a/src/tools/android/java/com/google/devtools/build/android/dexer/BUILD.tools +++ b/src/tools/android/java/com/google/devtools/build/android/dexer/BUILD.tools @@ -29,10 +29,3 @@ java_binary( visibility = ["//tools/android:__subpackages__"], runtime_deps = [":dexer"], ) - -java_binary( - name = "DexFileSplitter", - main_class = "com.google.devtools.build.android.dexer.DexFileSplitter", - visibility = ["//tools/android:__subpackages__"], - runtime_deps = [":dexer"], -) diff --git a/src/tools/android/java/com/google/devtools/build/android/dexer/DexFileAggregator.java b/src/tools/android/java/com/google/devtools/build/android/dexer/DexFileAggregator.java index dcb0dec15f..aff159ec49 100644 --- a/src/tools/android/java/com/google/devtools/build/android/dexer/DexFileAggregator.java +++ b/src/tools/android/java/com/google/devtools/build/android/dexer/DexFileAggregator.java @@ -17,10 +17,16 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import com.android.dex.Dex; +import com.android.dex.FieldId; +import com.android.dex.MethodId; +import com.android.dex.ProtoId; +import com.android.dex.TypeList; import com.android.dx.command.dexer.DxContext; import com.android.dx.merge.CollisionPolicy; import com.android.dx.merge.DexMerger; +import com.google.auto.value.AutoValue; import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; @@ -29,6 +35,7 @@ import java.io.IOException; import java.nio.BufferOverflowException; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.zip.ZipEntry; @@ -45,14 +52,16 @@ class DexFileAggregator implements Closeable { private static final String DEX_EXTENSION = ".dex"; private final ArrayList<Dex> currentShard = new ArrayList<>(); + private final HashSet<FieldDescriptor> fieldsInCurrentShard = new HashSet<>(); + private final HashSet<MethodDescriptor> methodsInCurrentShard = new HashSet<>(); private final boolean forceJumbo; + private final int maxNumberOfIdxPerDex; private final int wasteThresholdPerDex; private final MultidexStrategy multidex; private final DxContext context; private final ListeningExecutorService executor; private final DexFileArchive dest; private final String dexPrefix; - private final DexLimitTracker tracker; private int nextDexFileIndex = 0; private ListenableFuture<Void> lastWriter = Futures.<Void>immediateFuture(null); @@ -71,27 +80,44 @@ class DexFileAggregator implements Closeable { this.executor = executor; this.multidex = multidex; this.forceJumbo = forceJumbo; + this.maxNumberOfIdxPerDex = maxNumberOfIdxPerDex; this.wasteThresholdPerDex = wasteThresholdPerDex; this.dexPrefix = dexPrefix; - tracker = new DexLimitTracker(maxNumberOfIdxPerDex); } public DexFileAggregator add(Dex dexFile) { if (multidex.isMultidexAllowed()) { // To determine whether currentShard is "full" we track unique field and method signatures, // which predicts precisely the number of field and method indices. - if (tracker.track(dexFile) && !currentShard.isEmpty()) { + // Update xxxInCurrentShard first, then check if we overflowed. + // This can yield slightly larger .dex files than checking first, at the price of having to + // process the class that put us over the edge twice. + trackFieldsAndMethods(dexFile); + if (!currentShard.isEmpty() + && (fieldsInCurrentShard.size() > maxNumberOfIdxPerDex + || methodsInCurrentShard.size() > maxNumberOfIdxPerDex)) { // For simplicity just start a new shard to fit the given file. // Don't bother with waiting for a later file that might fit the old shard as in the extreme // we'd have to wait until the end to write all shards. rotateDexFile(); - tracker.track(dexFile); + trackFieldsAndMethods(dexFile); } } currentShard.add(dexFile); return this; } + private void trackFieldsAndMethods(Dex dexFile) { + int fieldCount = dexFile.fieldIds().size(); + for (int fieldIndex = 0; fieldIndex < fieldCount; ++fieldIndex) { + fieldsInCurrentShard.add(FieldDescriptor.fromDex(dexFile, fieldIndex)); + } + int methodCount = dexFile.methodIds().size(); + for (int methodIndex = 0; methodIndex < methodCount; ++methodIndex) { + methodsInCurrentShard.add(MethodDescriptor.fromDex(dexFile, methodIndex)); + } + } + @Override public void close() throws IOException { try { @@ -125,7 +151,8 @@ class DexFileAggregator implements Closeable { private void rotateDexFile() { writeMergedFile(currentShard.toArray(/* apparently faster than pre-sized array */ new Dex[0])); currentShard.clear(); - tracker.clear(); + fieldsInCurrentShard.clear(); + methodsInCurrentShard.clear(); } private void writeMergedFile(Dex... dexes) { @@ -184,6 +211,47 @@ class DexFileAggregator implements Closeable { return dexPrefix + (i == 0 ? "" : i + 1) + DEX_EXTENSION; } + private static String typeName(Dex dex, int typeIndex) { + return dex.typeNames().get(typeIndex); + } + + @AutoValue + abstract static class FieldDescriptor { + static FieldDescriptor fromDex(Dex dex, int fieldIndex) { + FieldId field = dex.fieldIds().get(fieldIndex); + String name = dex.strings().get(field.getNameIndex()); + String declaringClass = typeName(dex, field.getDeclaringClassIndex()); + String type = typeName(dex, field.getTypeIndex()); + return new AutoValue_DexFileAggregator_FieldDescriptor(declaringClass, name, type); + } + + abstract String declaringClass(); + abstract String fieldName(); + abstract String fieldType(); + } + + @AutoValue + abstract static class MethodDescriptor { + static MethodDescriptor fromDex(Dex dex, int methodIndex) { + MethodId method = dex.methodIds().get(methodIndex); + ProtoId proto = dex.protoIds().get(method.getProtoIndex()); + String name = dex.strings().get(method.getNameIndex()); + String declaringClass = typeName(dex, method.getDeclaringClassIndex()); + String returnType = typeName(dex, proto.getReturnTypeIndex()); + TypeList parameterTypeIndices = dex.readTypeList(proto.getParametersOffset()); + ImmutableList.Builder<String> parameterTypes = ImmutableList.builder(); + for (short parameterTypeIndex : parameterTypeIndices.getTypes()) { + parameterTypes.add(typeName(dex, parameterTypeIndex & 0xFFFF)); + } + return new AutoValue_DexFileAggregator_MethodDescriptor( + declaringClass, name, parameterTypes.build(), returnType); + } + + abstract String declaringClass(); + abstract String methodName(); + abstract ImmutableList<String> parameterTypes(); + abstract String returnType(); + } private class RunDexMerger implements Callable<Dex> { @@ -211,7 +279,7 @@ class DexFileAggregator implements Closeable { private final ListenableFuture<Dex> dex; private final String filename; - @SuppressWarnings ("hiding") private final DexFileArchive dest; + private final DexFileArchive dest; public WriteFile(String filename, ListenableFuture<Dex> dex, DexFileArchive dest) { this.filename = filename; diff --git a/src/tools/android/java/com/google/devtools/build/android/dexer/DexFileMerger.java b/src/tools/android/java/com/google/devtools/build/android/dexer/DexFileMerger.java index e892f7a94a..658f95f46f 100644 --- a/src/tools/android/java/com/google/devtools/build/android/dexer/DexFileMerger.java +++ b/src/tools/android/java/com/google/devtools/build/android/dexer/DexFileMerger.java @@ -47,6 +47,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.concurrent.Executors; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; @@ -152,7 +153,7 @@ class DexFileMerger { // Undocumented dx option for testing multidex logic @Option( name = "set-max-idx-number", - defaultValue = "" + DexFormat.MAX_MEMBER_IDX, + defaultValue = "" + (DexFormat.MAX_MEMBER_IDX + 1), documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, effectTags = {OptionEffectTag.UNKNOWN}, help = "Limit on fields and methods in a single dex file." @@ -301,6 +302,48 @@ class DexFileMerger { return MoreExecutors.listeningDecorator(Executors.newCachedThreadPool()); } + /** + * Sorts java class names such that outer classes preceed their inner + * classes and "package-info" preceeds all other classes in its package. + * + * @param a {@code non-null;} first class name + * @param b {@code non-null;} second class name + * @return {@code compareTo()}-style result + */ + // Copied from com.android.dx.cf.direct.ClassPathOpener + @VisibleForTesting + static int compareClassNames(String a, String b) { + // Ensure inner classes sort second + a = a.replace('$', '0'); + b = b.replace('$', '0'); + + /* + * Assuming "package-info" only occurs at the end, ensures package-info + * sorts first. + */ + a = a.replace("package-info", ""); + b = b.replace("package-info", ""); + + return a.compareTo(b); + } + + /** + * Comparator that orders {@link ZipEntry ZipEntries} {@link #LIKE_DX like Android's dx tool}. + */ + private static enum ZipEntryComparator implements Comparator<ZipEntry> { + /** + * Comparator to order more or less order alphabetically by file name. See + * {@link DexFileMerger#compareClassNames} for the exact name comparison. + */ + LIKE_DX; + + @Override + // Copied from com.android.dx.cf.direct.ClassPathOpener + public int compare(ZipEntry a, ZipEntry b) { + return compareClassNames(a.getName(), b.getName()); + } + } + private DexFileMerger() { } } diff --git a/src/tools/android/java/com/google/devtools/build/android/dexer/DexFileSplitter.java b/src/tools/android/java/com/google/devtools/build/android/dexer/DexFileSplitter.java deleted file mode 100644 index a7eb651b6b..0000000000 --- a/src/tools/android/java/com/google/devtools/build/android/dexer/DexFileSplitter.java +++ /dev/null @@ -1,266 +0,0 @@ -// Copyright 2017 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package com.google.devtools.build.android.dexer; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; -import static java.nio.charset.StandardCharsets.UTF_8; - -import com.android.dex.Dex; -import com.android.dex.DexFormat; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Predicate; -import com.google.common.base.Predicates; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.io.ByteStreams; -import com.google.common.io.Closer; -import com.google.devtools.build.android.Converters.ExistingPathConverter; -import com.google.devtools.build.android.Converters.PathConverter; -import com.google.devtools.common.options.Option; -import com.google.devtools.common.options.OptionDocumentationCategory; -import com.google.devtools.common.options.OptionEffectTag; -import com.google.devtools.common.options.OptionsBase; -import com.google.devtools.common.options.OptionsParser; -import java.io.BufferedOutputStream; -import java.io.Closeable; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.Comparator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; -import java.util.zip.ZipOutputStream; - -/** - * Shuffles .class.dex files from input archives into 1 or more archives each to be merged into a - * single final .dex file by {@link DexFileMerger}, respecting main dex list and other constraints - * similar to how dx would process these files if they were in a single input archive. - */ -class DexFileSplitter implements Closeable { - - /** - * Commandline options. - */ - public static class Options extends OptionsBase { - @Option( - name = "input", - allowMultiple = true, - defaultValue = "", - category = "input", - documentationCategory = OptionDocumentationCategory.UNCATEGORIZED, - effectTags = {OptionEffectTag.UNKNOWN}, - converter = ExistingPathConverter.class, - abbrev = 'i', - help = "Input dex archive." - ) - public List<Path> inputArchives; - - @Option( - name = "output", - defaultValue = ".", - category = "output", - documentationCategory = OptionDocumentationCategory.UNCATEGORIZED, - effectTags = {OptionEffectTag.UNKNOWN}, - converter = PathConverter.class, - abbrev = 'o', - help = "Directory to write dex archives to merge." - ) - public Path outputDirectory; - - @Option( - name = "main-dex-list", - defaultValue = "null", - category = "multidex", - documentationCategory = OptionDocumentationCategory.UNCATEGORIZED, - effectTags = {OptionEffectTag.UNKNOWN}, - converter = ExistingPathConverter.class, - help = "List of classes to be placed into \"main\" classes.dex file." - ) - public Path mainDexListFile; - - @Option( - name = "minimal-main-dex", - defaultValue = "false", - category = "multidex", - documentationCategory = OptionDocumentationCategory.UNCATEGORIZED, - effectTags = {OptionEffectTag.UNKNOWN}, - help = - "If true, *only* classes listed in --main_dex_list file are placed into \"main\" " - + "classes.dex file." - ) - public boolean minimalMainDex; - - // Undocumented dx option for testing multidex logic - @Option( - name = "set-max-idx-number", - defaultValue = "" + DexFormat.MAX_MEMBER_IDX, - documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, - effectTags = {OptionEffectTag.UNKNOWN}, - help = "Limit on fields and methods in a single dex file." - ) - public int maxNumberOfIdxPerDex; - } - - public static void main(String[] args) throws Exception { - OptionsParser optionsParser = - OptionsParser.newOptionsParser(Options.class); - optionsParser.setAllowResidue(false); - optionsParser.parseAndExitUponError(args); - - splitIntoShards(optionsParser.getOptions(Options.class)); - } - - @VisibleForTesting - static void splitIntoShards(Options options) throws IOException { - checkArgument( - !options.minimalMainDex || options.mainDexListFile != null, - "--minimal-main-dex not allowed without --main-dex-list"); - - if (!Files.exists(options.outputDirectory)) { - Files.createDirectories(options.outputDirectory); - } - - ImmutableSet<String> classesInMainDex = options.mainDexListFile != null - ? ImmutableSet.copyOf(Files.readAllLines(options.mainDexListFile, UTF_8)) - : null; - try (Closer closer = Closer.create(); - DexFileSplitter out = - new DexFileSplitter(options.outputDirectory, options.maxNumberOfIdxPerDex)) { - // 1. Scan inputs in order and keep first occurrence of each class, keeping all zips open. - // We don't process anything yet so we can shard in sorted order, which is what dx would do - // if presented with a single jar containing all the given inputs. - // TODO(kmb): Abandon alphabetic sorting to process each input fully before moving on (still - // requires scanning inputs twice for main dex list). - LinkedHashMap<String, ZipFile> deduped = new LinkedHashMap<>(); - for (Path inputArchive : options.inputArchives) { - ZipFile zip = closer.register(new ZipFile(inputArchive.toFile())); - zip.stream() - .filter(ZipEntryPredicates.suffixes(".dex", ".class")) - .forEach(e -> deduped.putIfAbsent(e.getName(), zip)); - } - ImmutableList<Map.Entry<String, ZipFile>> files = - deduped - .entrySet() - .stream() - .sorted(Comparator.comparing(e -> e.getKey(), ZipEntryComparator::compareClassNames)) - .collect(ImmutableList.toImmutableList()); - - // 2. Process each class in desired order, rolling from shard to shard as needed. - if (classesInMainDex == null || classesInMainDex.isEmpty()) { - out.processDexFiles(files, Predicates.alwaysTrue()); - } else { - // To honor --main_dex_list make two passes: - // 1. process only the classes listed in the given file - // 2. process the remaining files - Predicate<String> mainDexFilter = ZipEntryPredicates.classFileNameFilter(classesInMainDex); - out.processDexFiles(files, mainDexFilter); - // Fail if main_dex_list is too big, following dx's example - checkState(out.shardsWritten() == 0, "Too many classes listed in main dex list file " - + "%s, main dex capacity exceeded", options.mainDexListFile); - if (options.minimalMainDex) { - out.nextShard(); // Start new .dex file if requested - } - out.processDexFiles(files, Predicates.not(mainDexFilter)); - } - } - } - - private final int maxNumberOfIdxPerDex; - private final Path outputDirectory; - - private int curShard = 0; - private ZipOutputStream out; - private DexLimitTracker tracker; - - private DexFileSplitter(Path outputDirectory, int maxNumberOfIdxPerDex) throws IOException { - checkArgument(!Files.isRegularFile(outputDirectory), "Must be a directory: ", outputDirectory); - this.maxNumberOfIdxPerDex = maxNumberOfIdxPerDex; - this.outputDirectory = outputDirectory; - startShard(); - } - - private void nextShard() throws IOException { - out.close(); // will NPE if called after close() - ++curShard; - startShard(); - } - - private void startShard() throws IOException { - tracker = new DexLimitTracker(maxNumberOfIdxPerDex); - out = - new ZipOutputStream( - new BufferedOutputStream( - Files.newOutputStream( - outputDirectory.resolve((curShard + 1) + ".shard.zip"), - StandardOpenOption.CREATE_NEW, - StandardOpenOption.WRITE))); - } - - private int shardsWritten() { - return curShard; - } - - @Override - public void close() throws IOException { - if (out != null) { - out.close(); - out = null; - ++curShard; - } - } - - private void processDexFiles( - ImmutableList<Map.Entry<String, ZipFile>> filesToProcess, Predicate<String> filter) - throws IOException { - for (Map.Entry<String, ZipFile> entry : filesToProcess) { - String filename = entry.getKey(); - if (filter.apply(filename)) { - ZipFile zipFile = entry.getValue(); - processDexEntry(zipFile, zipFile.getEntry(filename)); - } - } - } - - private void processDexEntry(ZipFile zip, ZipEntry entry) throws IOException { - String filename = entry.getName(); - checkState(filename.endsWith(".class.dex"), - "%s isn't a dex archive: %s", zip.getName(), filename); - checkState(entry.getMethod() == ZipEntry.STORED, "Expect to process STORED: %s", filename); - try (InputStream entryStream = zip.getInputStream(entry)) { - // We don't want to use the Dex(InputStream) constructor because it closes the stream, - // which will break the for loop, and it has its own bespoke way of reading the file into - // a byte buffer before effectively calling Dex(byte[]) anyway. - // TODO(kmb) since entry is stored, mmap content and give to Dex(ByteBuffer) and output zip - byte[] content = new byte[(int) entry.getSize()]; - ByteStreams.readFully(entryStream, content); // throws if file is smaller than expected - checkState(entryStream.read() == -1, - "Too many bytes in jar entry %s, expected %s", entry, entry.getSize()); - - Dex dexFile = new Dex(content); - if (tracker.track(dexFile)) { - nextShard(); - tracker.track(dexFile); - } - out.putNextEntry(entry); - out.write(content); - out.closeEntry(); - } - } -} diff --git a/src/tools/android/java/com/google/devtools/build/android/dexer/DexLimitTracker.java b/src/tools/android/java/com/google/devtools/build/android/dexer/DexLimitTracker.java deleted file mode 100644 index 5c7631eaf0..0000000000 --- a/src/tools/android/java/com/google/devtools/build/android/dexer/DexLimitTracker.java +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2017 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package com.google.devtools.build.android.dexer; - -import com.android.dex.Dex; -import com.android.dex.FieldId; -import com.android.dex.MethodId; -import com.android.dex.ProtoId; -import com.android.dex.TypeList; -import com.google.auto.value.AutoValue; -import com.google.auto.value.extension.memoized.Memoized; -import com.google.common.collect.ImmutableList; -import java.util.HashSet; - -/** - * Helper to track how many unique field and method references we've seen in a given set of .dex - * files. - */ -class DexLimitTracker { - - private final HashSet<FieldDescriptor> fieldsSeen = new HashSet<>(); - private final HashSet<MethodDescriptor> methodsSeen = new HashSet<>(); - private final int maxNumberOfIdxPerDex; - - public DexLimitTracker(int maxNumberOfIdxPerDex) { - this.maxNumberOfIdxPerDex = maxNumberOfIdxPerDex; - } - - /** - * Tracks the field and method references in the given file and returns whether we're within - * limits. - * - * @return {@code true} if method or field references are outside limits, {@code false} both - * are within limits. - */ - public boolean track(Dex dexFile) { - trackFieldsAndMethods(dexFile); - return fieldsSeen.size() > maxNumberOfIdxPerDex - || methodsSeen.size() > maxNumberOfIdxPerDex; - } - - public void clear() { - fieldsSeen.clear(); - methodsSeen.clear(); - } - - private void trackFieldsAndMethods(Dex dexFile) { - int fieldCount = dexFile.fieldIds().size(); - for (int fieldIndex = 0; fieldIndex < fieldCount; ++fieldIndex) { - fieldsSeen.add(FieldDescriptor.fromDex(dexFile, fieldIndex)); - } - int methodCount = dexFile.methodIds().size(); - for (int methodIndex = 0; methodIndex < methodCount; ++methodIndex) { - methodsSeen.add(MethodDescriptor.fromDex(dexFile, methodIndex)); - } - } - - private static String typeName(Dex dex, int typeIndex) { - return dex.typeNames().get(typeIndex); - } - - @AutoValue - abstract static class FieldDescriptor { - static FieldDescriptor fromDex(Dex dex, int fieldIndex) { - FieldId field = dex.fieldIds().get(fieldIndex); - String name = dex.strings().get(field.getNameIndex()); - String declaringClass = typeName(dex, field.getDeclaringClassIndex()); - String type = typeName(dex, field.getTypeIndex()); - return new AutoValue_DexLimitTracker_FieldDescriptor(declaringClass, name, type); - } - - abstract String declaringClass(); - abstract String fieldName(); - abstract String fieldType(); - - @Override - @Memoized - public abstract int hashCode(); - } - - @AutoValue - abstract static class MethodDescriptor { - static MethodDescriptor fromDex(Dex dex, int methodIndex) { - MethodId method = dex.methodIds().get(methodIndex); - ProtoId proto = dex.protoIds().get(method.getProtoIndex()); - String name = dex.strings().get(method.getNameIndex()); - String declaringClass = typeName(dex, method.getDeclaringClassIndex()); - String returnType = typeName(dex, proto.getReturnTypeIndex()); - TypeList parameterTypeIndices = dex.readTypeList(proto.getParametersOffset()); - ImmutableList.Builder<String> parameterTypes = ImmutableList.builder(); - for (short parameterTypeIndex : parameterTypeIndices.getTypes()) { - parameterTypes.add(typeName(dex, parameterTypeIndex & 0xFFFF)); - } - return new AutoValue_DexLimitTracker_MethodDescriptor( - declaringClass, name, parameterTypes.build(), returnType); - } - - abstract String declaringClass(); - abstract String methodName(); - abstract ImmutableList<String> parameterTypes(); - abstract String returnType(); - - @Override - @Memoized - public abstract int hashCode(); - } -} diff --git a/src/tools/android/java/com/google/devtools/build/android/dexer/ZipEntryComparator.java b/src/tools/android/java/com/google/devtools/build/android/dexer/ZipEntryComparator.java deleted file mode 100644 index 1cd9b84321..0000000000 --- a/src/tools/android/java/com/google/devtools/build/android/dexer/ZipEntryComparator.java +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2017 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package com.google.devtools.build.android.dexer; - -import com.google.common.annotations.VisibleForTesting; -import java.util.Comparator; -import java.util.zip.ZipEntry; - -/** - * Comparator that orders {@link ZipEntry ZipEntries} {@link #LIKE_DX like Android's dx tool}. - */ -enum ZipEntryComparator implements Comparator<ZipEntry> { - /** - * Comparator to order more or less order alphabetically by file name. See - * {@link #compareClassNames} for the exact name comparison. - */ - LIKE_DX; - - @Override - // Copied from com.android.dx.cf.direct.ClassPathOpener - public int compare(ZipEntry a, ZipEntry b) { - return compareClassNames(a.getName(), b.getName()); - } - - /** - * Sorts java class names such that outer classes preceed their inner - * classes and "package-info" preceeds all other classes in its package. - * - * @param a {@code non-null;} first class name - * @param b {@code non-null;} second class name - * @return {@code compareTo()}-style result - */ - // Copied from com.android.dx.cf.direct.ClassPathOpener - @VisibleForTesting - static int compareClassNames(String a, String b) { - // Ensure inner classes sort second - a = a.replace('$', '0'); - b = b.replace('$', '0'); - - /* - * Assuming "package-info" only occurs at the end, ensures package-info - * sorts first. - */ - a = a.replace("package-info", ""); - b = b.replace("package-info", ""); - - return a.compareTo(b); - } -} diff --git a/src/tools/android/java/com/google/devtools/build/android/dexer/ZipEntryPredicates.java b/src/tools/android/java/com/google/devtools/build/android/dexer/ZipEntryPredicates.java index 5dd51d4edf..08e77bf4f2 100644 --- a/src/tools/android/java/com/google/devtools/build/android/dexer/ZipEntryPredicates.java +++ b/src/tools/android/java/com/google/devtools/build/android/dexer/ZipEntryPredicates.java @@ -14,7 +14,6 @@ package com.google.devtools.build.android.dexer; import com.google.common.base.Predicate; -import com.google.common.base.Predicates; import com.google.common.collect.ImmutableSet; import java.util.zip.ZipEntry; @@ -45,13 +44,10 @@ class ZipEntryPredicates { } public static Predicate<ZipEntry> classFileFilter(final ImmutableSet<String> classFileNames) { - return Predicates.compose(classFileNameFilter(classFileNames), zipEntry -> zipEntry.getName()); - } - - public static Predicate<String> classFileNameFilter(final ImmutableSet<String> classFileNames) { - return new Predicate<String>() { + return new Predicate<ZipEntry>() { @Override - public boolean apply(String filename) { + public boolean apply(ZipEntry input) { + String filename = input.getName(); if (filename.endsWith(".class.dex")) { // Chop off file suffix generated by DexBuilder filename = filename.substring(0, filename.length() - ".dex".length()); |