diff options
author | Lukacs Berki <lberki@google.com> | 2015-06-05 09:57:19 +0000 |
---|---|---|
committer | Florian Weikert <fwe@google.com> | 2015-06-05 11:34:33 +0000 |
commit | 92e945ad89759610839f8855f19d32d375fd3da7 (patch) | |
tree | 4d7d96e873007e02aace60cab460d7be0cfb81c2 /src/tools/android/java/com/google/devtools/build/android/ziputils | |
parent | 14d905b5cce9a1bbc2911331809b03679b23dad1 (diff) |
Move the parallel dexing tools to the Bazel tree.
--
MOS_MIGRATED_REVID=95278949
Diffstat (limited to 'src/tools/android/java/com/google/devtools/build/android/ziputils')
18 files changed, 3572 insertions, 0 deletions
diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/BUILD b/src/tools/android/java/com/google/devtools/build/android/ziputils/BUILD new file mode 100644 index 0000000000..f99ea5e701 --- /dev/null +++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/BUILD @@ -0,0 +1,65 @@ +# Low level zip archive processing library. + +package( + default_visibility = ["//visibility:public"], +) + +java_library( + name = "ziputils_lib", + srcs = glob( + ["*.java"], + exclude = [ + "DexMapper.java", + "DexReducer.java", + "SplitZip.java", + "Splitter.java", + ], + ), + visibility = ["//visibility:public"], + deps = [ + "//third_party:guava", + "//third_party:jsr305", + ], +) + +java_library( + name = "splitter_lib", + srcs = glob( + [ + "SplitZip.java", + "Splitter.java", + ], + ), + visibility = ["//visibility:public"], + deps = [ + ":ziputils_lib", + "//third_party:guava", + "//third_party:jsr305", + ], +) + +java_binary( + name = "mapper", + srcs = [ + "DexMapper.java", + ], + main_class = "com.google.devtools.build.android.ziputils.DexMapper", + visibility = ["//visibility:public"], + deps = [ + ":splitter_lib", + "//src/main/java:options", + ], +) + +java_binary( + name = "reducer", + srcs = [ + "DexReducer.java", + ], + main_class = "com.google.devtools.build.android.ziputils.DexReducer", + visibility = ["//visibility:public"], + deps = [ + ":ziputils_lib", + "//src/main/java:options", + ], +) diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/BufferedFile.java b/src/tools/android/java/com/google/devtools/build/android/ziputils/BufferedFile.java new file mode 100644 index 0000000000..19772540f3 --- /dev/null +++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/BufferedFile.java @@ -0,0 +1,163 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.android.ziputils; + +import com.google.common.base.Preconditions; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; + +/** + * An API for reading big files through a direct byte buffer spanning a region of the file. + * This object maintains an internal buffer, which may store all or some of the file content. + * When a request for data is made ({@link #getBuffer(long, int) }, the implementation will + * first determine if the requested data range is within the region specified at time of + * construction. If it is, it checks to see if the request is within the capacity range of + * the current internal buffer. If not, the buffer is reallocated, based at the requested offset. + * Then the implementation checks to see if the requested data falls within the current fill limit + * of the internal buffer. If not additional data is read from the file. Finally, a slice of + * the internal buffer is returned, with the requested data. + * + * <p>This is optimized for forward scanning of files. Random access is supported, but will likely + * be inefficient, especially if the entire file doesn't fit in the internal buffer. + * + * <p>Clients of this API should take care not to keep references to returned buffers indefinitely, + * as this would prevent collection of buffers discarded by the {@code BufferedFile} object. + */ +public class BufferedFile { + + private int maxAlloc; + private long offset; + private long limit; + private FileChannel channel; + private ByteBuffer current; + private long currOff; + + /** + * Same as {@code BufferedFile(channel, 0, channel.size(), blockSize)}. + * + * @param channel file channel opened for reading. + * @param blockSize maximum buffer allocation. + * @throws NullPointerException if {@code channel} is {@code null}. + * @throws IllegalArgumentException if {@code maxAlloc}, {@code off}, or {@code len} are negative + * or if {@code off + len > channel.size()}. + * @throws IOException + */ + public BufferedFile(FileChannel channel, int blockSize) throws IOException { + this(channel, 0, channel.size(), blockSize); + } + + /** + * Allocates a buffered file. + * + * @param channel file channel opened for reading. + * @param off the first byte that can be read through this object. + * @param len the max number of bytes that can be read through this object. + * @param blockSize default max buffer allocation size is {@code Math.min(blockSize, len)}. + * @throws NullPointerException if {@code channel} is {@code null}. + * @throws IllegalArgumentException if {@code blockSize}, {@code off}, or {@code len} are negative + * or if {@code off + len > channel.size()}. + * @throws IOException if thrown by the underlying file channel. + */ + public BufferedFile(FileChannel channel, long off, long len, int blockSize) throws IOException { + Preconditions.checkNotNull(channel); + Preconditions.checkArgument(blockSize >= 0); + Preconditions.checkArgument(off >= 0); + Preconditions.checkArgument(len >= 0); + Preconditions.checkArgument(off + len <= channel.size()); + this.maxAlloc = (int) Math.min(blockSize, len); + this.offset = off; + this.limit = off + len; + this.channel = channel; + this.current = null; + currOff = -1; + } + + /** + * Returns the offset of the first byte beyond the readable region. + * @return the file offset just beyond the readable region. + */ + public long limit() { + return limit; + } + + /** + * Returns a byte buffer for reading {@code len} bytes from the {@code off} position + * in the file. If the requested bytes are already loaded in the internal buffer, a slice is + * returned, with position 0 and limit set to {@code len}. The slice may have a capacity greater + * than its limit, if more bytes are already available in the internal buffer. If the requested + * bytes are not available, but can fit in the current internal buffer, then more data is read, + * before a slice is created as described above. If the requested data falls outside the range + * that can be fitted into the current internal buffer, then a new internal buffer is allocated. + * The prior internal buffer (if any), is no longer referenced by this object (but it may still + * be referenced by the client, holding references to byte buffers returned from prior call to + * this method). The new internal buffer will be based at {@code off} file position, and have a + * capacity equal to the maximum of the {@code blockSize} of this buffer and {@code len}, except + * that it will never exceed the the number of bytes from {@code off} to the end of the readable + * region of the file (min-max rule). + * + * @param off + * @param len + * @return a slice of the internal byte buffer containing the requested data. Except, if the + * client request data beyond the readable region of the file, the {@code len} value is reduced + * to the maximum number of bytes available from the given {@code off}. + * @throws IllegalArgumentException if {@code len} is less than 0, or {@code off} is outside the + * readable region specified when constructing this object. + * @throws IOException if thrown by the underlying file channel. + */ + public synchronized ByteBuffer getBuffer(long off, int len) throws IOException { + Preconditions.checkArgument(off >= offset); + Preconditions.checkArgument(len >= 0); + Preconditions.checkArgument(off < limit || (off == limit && len == 0)); + if (limit - off < len) { // never return data beyond limit + len = (int) (limit - off); + } + Preconditions.checkState(off + len <= limit); + if (current == null || off < currOff || off + len > currOff + current.capacity()) { + allocate(off, len); + Preconditions.checkState(current != null && off == currOff + && off + len <= currOff + current.capacity()); + } + Preconditions.checkState(current != null && off >= currOff + && off + len <= currOff + current.capacity()); + if (off - currOff + len > current.limit()) { + readMore((int) (off - currOff) + len); + } + Preconditions.checkState(current != null && off >= currOff + && off + len <= currOff + current.limit()); + current.position((int) (off - currOff)); + return (ByteBuffer) current.slice().limit(len); + } + + private void readMore(int newMin) throws IOException { + channel.position(currOff + current.limit()); + current.position(current.limit()); + current.limit(current.capacity()); + do { + channel.read(current); + } while(current.position() < newMin); + current.limit(current.position()).position(0); + } + + private void allocate(long off, int len) { + current = ByteBuffer.allocateDirect(bufferSize(off, len)); + current.limit(0); + currOff = off; + } + + private int bufferSize(long off, int len) { + return (int) Math.min(Math.max(len, maxAlloc), limit - off); + } +} diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/CentralDirectory.java b/src/tools/android/java/com/google/devtools/build/android/ziputils/CentralDirectory.java new file mode 100644 index 0000000000..c7c2ff649d --- /dev/null +++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/CentralDirectory.java @@ -0,0 +1,165 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.android.ziputils; + +import static com.google.devtools.build.android.ziputils.DirectoryEntry.CENOFF; + +import com.google.common.base.Preconditions; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.NavigableMap; +import java.util.TreeMap; + +/** + * Provides a view of a zip file's central directory. For reading, a single memory mapped view is + * used. For writing, the central directory is stored as one or more views, each backed by a direct + * byte buffer. + */ +public class CentralDirectory extends View<CentralDirectory> { + + // Cached map from entry name to directory entry. + private NavigableMap<String, DirectoryEntry> mapByNameSorted; + // Cached map from entry file offset to directory entry. + private NavigableMap<Integer, DirectoryEntry> mapByOffsetSorted; + // Number of directory entries in this view. + private int count; + // Parsed or added entries + private List<DirectoryEntry> entries; + + /** + * Gets the number of directory entries in this view. + */ + public int getCount() { + return count; + } + + /** + * Returns a list of directory entries, in the order they occur in the central directory. + * This will typically also be the order of entries in the zip file, although that's not + * guaranteed. + */ + public List<DirectoryEntry> list() { + return entries; + } + + /** + * Returns a navigable map of directory entries, by zip entry file offset. + */ + public NavigableMap<Integer, DirectoryEntry> mapByOffset() { + if (entries == null) { + return null; + } + return mapEntriesByOffset(); + } + + /** + * Returns a navigable map of directory entries, by entry filename. + */ + public NavigableMap<String, DirectoryEntry> mapByFilename() { + if (entries == null) { + return null; + } + return mapEntriesByName(); + } + + /** + * Returns a {@code CentralDirectory} of the given buffer. This may be a full or a partial + * central directory. This method assumes ownership of the underlying buffer. Unlike most + * "view-of" methods, this method doesn't slice the argument buffer, and rather than advancing + * the buffer position, it sets it to 0. + * + * @param buffer containing data of a central directory. + * @return a {@code CentralDirectory} of the data at the current position of the given byte + * buffer. + */ + public static CentralDirectory viewOf(ByteBuffer buffer) { + buffer.position(0); + return new CentralDirectory(buffer); + } + + private CentralDirectory(ByteBuffer buffer) { + super(buffer); + count = -1; + } + + /** + * Parses this central directory, and maps the contained entries with {@link DirectoryEntry}s. + * + * @return this central directory view + * @throws IllegalStateException if the file offset is not set prior to parsing + */ + public CentralDirectory parse() throws IllegalStateException { + Preconditions.checkState(fileOffset != -1, "File offset not set prior to parsing"); + count = 0; + clearMaps(); + int relPos = 0; + buffer.position(0); + while (buffer.hasRemaining() && buffer.getInt(buffer.position()) == DirectoryEntry.SIGNATURE) { + count++; + DirectoryEntry entry = DirectoryEntry.viewOf(buffer).at(fileOffset + relPos); + entries.add(entry); + relPos += entry.getSize(); + buffer.position(relPos); + } + return this; + } + + /** + * Creates a new directory entry for output. The given entry is copied into the buffer of this + * central directory, and a view of the copied data is returned. + * + * @param entry prototype directory entry, typically an entry read from another zip file, for + * an entry being copied. + * @return a directory entry view of the copied entry. + */ + public DirectoryEntry nextEntry(DirectoryEntry entry) { + DirectoryEntry clone = entry.copy(buffer); + if (count == -1) { + clearMaps(); + count = 1; + } else { + count++; + } + entries.add(clone); + return clone; + } + + private NavigableMap<String, DirectoryEntry> mapEntriesByName() { + if (mapByNameSorted == null) { + mapByNameSorted = new TreeMap<>(); + for (DirectoryEntry entry : entries) { + mapByNameSorted.put(entry.getFilename(), entry); + } + } + return mapByNameSorted; + } + + private NavigableMap<Integer, DirectoryEntry> mapEntriesByOffset() { + if (mapByOffsetSorted == null) { + mapByOffsetSorted = new TreeMap<>(); + for (DirectoryEntry entry : entries) { + mapByOffsetSorted.put(entry.get(CENOFF), entry); + } + } + return mapByOffsetSorted; + } + + private void clearMaps() { + entries = new ArrayList<>(); + mapByOffsetSorted = null; + mapByNameSorted = null; + } +} diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/DataDescriptor.java b/src/tools/android/java/com/google/devtools/build/android/ziputils/DataDescriptor.java new file mode 100644 index 0000000000..61f39077bc --- /dev/null +++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/DataDescriptor.java @@ -0,0 +1,162 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.android.ziputils; + +import static java.nio.ByteOrder.LITTLE_ENDIAN; + +import java.nio.ByteBuffer; + +/** + * Provides a view of a data descriptor record, for a zip file entry. + */ +public class DataDescriptor extends View<DataDescriptor> { + private boolean hasMarker; + + /** + * Returns a {@code DataDescriptor} view of the given buffer. The buffer is assumed to contain a + * valid "data descriptor" record beginning at the buffers current position. + * + * @param buffer containing the data of a data descriptor record. + * @return a {@code DataDescriptor} of the data at the current position of the given byte + * buffer. + */ + public static DataDescriptor viewOf(ByteBuffer buffer) { + DataDescriptor view = new DataDescriptor(buffer.slice()); + int size = view.getSize(); + view.buffer.position(0).limit(size); + buffer.position(buffer.position() + size); + return view; + } + + private DataDescriptor(ByteBuffer buffer) { + super(buffer); + hasMarker = buffer.getInt(0) == SIGNATURE; + } + + /** + * Creates a {@code DataDescriptor} with a heap allocated buffer. + * + * @return a {@code DataDescriptor} with a heap allocated buffer. + */ + public static DataDescriptor allocate() { + ByteBuffer buffer = ByteBuffer.allocate(SIZE).order(LITTLE_ENDIAN); + return new DataDescriptor(buffer).init(); + } + + /** + * Creates a {@code DataDescriptor} view over a writable buffer. The given buffers position is + * advanced by the number of bytes consumed by the view. + * + * @param buffer buffer to hold data for the "data descriptor" record. + * @return a {@code DataDescriptor} with a heap allocated buffer. + */ + public static DataDescriptor view(ByteBuffer buffer) { + DataDescriptor view = new DataDescriptor(buffer.slice()).init(); + buffer.position(buffer.position() + SIZE); + return view; + } + + /** + * Copies this {@code DataDescriptor} into a writable buffer. The copy is made at the + * buffer's current position, and the position is advanced, by the size of the copy. + * + * @param buffer buffer to hold data for the copy. + * @return a {@code DataDescriptor} backed by the given buffer. + */ + public DataDescriptor copy(ByteBuffer buffer) { + int size = getSize(); + DataDescriptor view = new DataDescriptor(buffer.slice()); + this.buffer.rewind(); + view.buffer.put(this.buffer).flip(); + buffer.position(buffer.position() + size); + this.buffer.rewind(); + return view; + } + + private DataDescriptor init() { + buffer.putInt(0, SIGNATURE); + hasMarker = true; + buffer.limit(SIZE); + return this; + } + + /** + * Data descriptor signature. + */ + public static final int SIGNATURE = 0x08074b50; // 134695760L + + /** + * Data descriptor size (including optional signature). + */ + public static final int SIZE = 16; + + /** + * For accessing the data descriptor signature, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)} + * methods. + */ + public static final IntFieldId<DataDescriptor> EXTSIG = new IntFieldId<>(0); + + /** + * For accessing the "crc" data descriptor field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)} + * methods. + */ + public static final IntFieldId<DataDescriptor> EXTCRC = new IntFieldId<>(4); + + /** + * For accessing the "compressed size" data descriptor field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)} + * methods. + */ + public static final IntFieldId<DataDescriptor> EXTSIZ = new IntFieldId<>(8); + + /** + * For accessing the "uncompressed size" data descriptor field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)} + * methods. + */ + public static final IntFieldId<DataDescriptor> EXTLEN = new IntFieldId<>(12); + + + /** + * Overrides the generic field getter, to handle optionality of signature. + * @see View#get(com.google.devtools.build.android.ziputils.View.IntFieldId). + */ + @Override @SuppressWarnings("unchecked") // safe by specification (FieldId.type()). + public int get(IntFieldId<? extends DataDescriptor> item) { + int address = hasMarker ? item.address() : (item.address() - 4); + return address < 0 ? -1 : buffer.getInt(address); + } + + /** + * Returns whether this data descriptor has the optional signature. + * @return {@code true} if this data descriptor has a signature, {@code false} otherwise. + */ + public final boolean hasMarker() { + return hasMarker; + } + + /** + * Returns the size of this data descriptor. + * @return 12, or 16, depending on whether or not this data descriptor has a signature. + */ + public final int getSize() { + return hasMarker ? SIZE : SIZE - 4; + } +} diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/DexMapper.java b/src/tools/android/java/com/google/devtools/build/android/ziputils/DexMapper.java new file mode 100644 index 0000000000..3e0ec47228 --- /dev/null +++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/DexMapper.java @@ -0,0 +1,97 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.android.ziputils; + +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParser; + +import java.util.List; + +/** + * Command-line entry point for dex mapper utility, that maps an applications classes (given in + * one or more input jar files), to one or more output jars, that may then be compiled separately. + * This class performs command line parsing and delegates to an {@link SplitZip} instance. + * + * <p>The result of compiling each output archive can be consolidated using the dex reducer utility. + */ +public class DexMapper { + + /** + * @param args the command line arguments + */ + public static void main(String[] args) { + OptionsParser optionsParser = OptionsParser.newOptionsParser(Options.class); + optionsParser.parseAndExitUponError(args); + Options options = optionsParser.getOptions(Options.class); + List<String> inputs = options.inputJars; + List<String> outputs = options.outputJars; + String filterFile = options.mainDexFilter; + String resourceFile = options.outputResources; + + try { + new SplitZip() + .setVerbose(false) + .useDefaultEntryDate() + .addInputs(inputs) + .addOutputs(outputs) + .setMainClassListFile(filterFile) + .setResourceFile(resourceFile) + .run() + .close(); + } catch (Exception ex) { + System.err.println("Caught exception" + ex.getMessage()); + ex.printStackTrace(System.out); + System.exit(1); + } + } + + /** + * Commandline options. + */ + public static class Options extends OptionsBase { + @Option(name = "input_jar", + defaultValue = "null", + category = "input", + allowMultiple = true, + abbrev = 'i', + help = "Input file to read classes and jars from. Classes in " + + " earlier files override those in later ones.") + public List<String> inputJars; + + @Option(name = "output_jar", + defaultValue = "null", + category = "output", + allowMultiple = true, + abbrev = 'o', + help = "Output file to write. Each argument is one shard. " + + "Output files are filled in the order specified.") + public List<String> outputJars; + + @Option(name = "main_dex_filter", + defaultValue = "null", + category = "input", + abbrev = 'f', + help = "List of classes to include in the first output file.") + public String mainDexFilter; + + @Option(name = "output_resources", + defaultValue = "null", + category = "output", + abbrev = 'r', + help = "File to write the Java resources to.") + public String outputResources; + } +} diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/DexReducer.java b/src/tools/android/java/com/google/devtools/build/android/ziputils/DexReducer.java new file mode 100644 index 0000000000..e814fdf05f --- /dev/null +++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/DexReducer.java @@ -0,0 +1,133 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.android.ziputils; + +import static com.google.devtools.build.android.ziputils.DataDescriptor.EXTCRC; +import static com.google.devtools.build.android.ziputils.DataDescriptor.EXTLEN; +import static com.google.devtools.build.android.ziputils.DataDescriptor.EXTSIZ; +import static com.google.devtools.build.android.ziputils.DirectoryEntry.CENCRC; +import static com.google.devtools.build.android.ziputils.DirectoryEntry.CENLEN; +import static com.google.devtools.build.android.ziputils.DirectoryEntry.CENSIZ; +import static com.google.devtools.build.android.ziputils.DirectoryEntry.CENTIM; +import static com.google.devtools.build.android.ziputils.LocalFileHeader.LOCFLG; +import static com.google.devtools.build.android.ziputils.LocalFileHeader.LOCTIM; + +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParser; + +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +/** + * Command-line entry point for the dex reducer utility. This utility extracts .dex files + * from one or more archives, and packaging them in a single output archive, renaming entries + * to: classes.dex, classes2.dex, ... + * + * <p>This utility is intended used to consolidate the result of compiling the output produced + * by the dex mapper utility.</p> + */ +public class DexReducer implements EntryHandler { + private static final String SUFFIX = ".dex"; + private static final String BASENAME = "classes"; + private ZipOut out; + private int count = 0; + private String outFile; + private List<String> paths; + + DexReducer() { + outFile = null; + paths = new ArrayList<>(); + } + + /** + * Command-line entry point. + * @param args + */ + public static void main(String[] args) { + try { + DexReducer dexDexReducer = new DexReducer(); + dexDexReducer.parseArguments(args); + dexDexReducer.run(); + } catch (Exception ex) { + System.err.println("DexReducer threw exception: " + ex.getMessage()); + System.exit(1); + } + } + + private void parseArguments(String[] args) { + OptionsParser optionsParser = OptionsParser.newOptionsParser(Options.class); + optionsParser.parseAndExitUponError(args); + Options options = optionsParser.getOptions(Options.class); + paths = options.inputZips; + outFile = options.outputZip; + } + + private void run() throws IOException { + out = new ZipOut(new FileOutputStream(outFile, false).getChannel(), outFile); + for (String filename : paths) { + ZipIn in = new ZipIn(new FileInputStream(filename).getChannel(), filename); + in.scanEntries(this); + } + out.close(); + } + + @Override + public void handle(ZipIn in, LocalFileHeader header, DirectoryEntry dirEntry, ByteBuffer data) + throws IOException { + String path = header.getFilename(); + if (!path.endsWith(".dex")) { + return; + } + count++; + String filename = BASENAME + (count == 1 ? "" : Integer.toString(count)) + SUFFIX; + String comment = dirEntry.getComment(); + byte[] extra = dirEntry.getExtraData(); + out.nextEntry(dirEntry.clone(filename, extra, comment).set(CENTIM, DosTime.EPOCH.time)); + out.write(header.clone(filename, extra).set(LOCTIM, DosTime.EPOCH.time)); + out.write(data); + if ((header.get(LOCFLG) & LocalFileHeader.SIZE_MASKED_FLAG) != 0) { + DataDescriptor desc = DataDescriptor.allocate() + .set(EXTCRC, dirEntry.get(CENCRC)) + .set(EXTSIZ, dirEntry.get(CENSIZ)) + .set(EXTLEN, dirEntry.get(CENLEN)); + out.write(desc); + } + } + + /** + * Commandline options. + */ + public static class Options extends OptionsBase { + @Option(name = "input_zip", + defaultValue = "null", + category = "input", + allowMultiple = true, + abbrev = 'i', + help = "Input zip file containing entries to collect and enumerate.") + public List<String> inputZips; + + @Option(name = "output_zip", + defaultValue = "null", + category = "output", + abbrev = 'o', + help = "Output zip file, containing enumerated entries.") + public String outputZip; + } +} diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/DirectoryEntry.java b/src/tools/android/java/com/google/devtools/build/android/ziputils/DirectoryEntry.java new file mode 100644 index 0000000000..203354054f --- /dev/null +++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/DirectoryEntry.java @@ -0,0 +1,316 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.android.ziputils; + +import static java.nio.ByteOrder.LITTLE_ENDIAN; +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.nio.ByteBuffer; + +/** + * Provides a view of a central directory entry. + */ +public class DirectoryEntry extends View<DirectoryEntry> { + + /** + * Returns a {@code DirectoryEntry of the given buffer. The buffer is assumed to contain a + * valid "central directory entry" record beginning at the buffers current position. + * + * @param buffer containing the data of a "central directory entry" record. + * @return a {@code DirectoryEntry} of the data at the current position of the given byte + * buffer. + */ + public static DirectoryEntry viewOf(ByteBuffer buffer) { + DirectoryEntry view = new DirectoryEntry(buffer.slice()); + int size = view.getSize(); + buffer.position(buffer.position() + size); + view.buffer.position(0).limit(size); + return view; + } + + private DirectoryEntry(ByteBuffer buffer) { + super(buffer); + } + + /** + * Creates a {@code DirectoryEntry} with a heap allocated buffer. Apart from the signature, + * and provided extra data and comment data, the returned object is uninitialized. + * + * @param name entry file name. Cannot be {@code null}. + * @param extraData extra data, or {@code null} + * @param comment zip file comment, or {@code null}. + * @return a {@code DirectoryEntry} with a heap allocated buffer. + */ + public static DirectoryEntry allocate(String name, byte[] extraData, String comment) { + byte[] nameData = name.getBytes(UTF_8); + byte[] commentData = comment != null ? comment.getBytes(UTF_8) : EMPTY; + if (extraData == null) { + extraData = EMPTY; + } + int size = SIZE + nameData.length + extraData.length + commentData.length; + ByteBuffer buffer = ByteBuffer.allocate(size).order(LITTLE_ENDIAN); + return new DirectoryEntry(buffer).init(nameData, extraData, commentData, size); + } + + /** + * Creates a {@code DirectoryEntry} over a writable buffer. The given buffers position is + * advanced by the number of bytes consumed by the view. Apart from the signature, and + * provided extra data and comment data, the returned view is uninitialized. + * + * @param buffer buffer to hold data for the "central directory entry" record. + * @param name entry file name. Cannot be {@code null}. + * @param extraData extra data, or {@code null} + * @param comment zip file global comment, or {@code null}. + * @return a {@code DirectoryEntry} with a heap allocated buffer. + */ + public static DirectoryEntry view(ByteBuffer buffer, String name, byte[] extraData, + String comment) { + byte[] nameData = name.getBytes(UTF_8); + byte[] commentData = comment != null ? comment.getBytes(UTF_8) : EMPTY; + if (extraData == null) { + extraData = EMPTY; + } + int size = SIZE + nameData.length + extraData.length + commentData.length; + DirectoryEntry view = new DirectoryEntry(buffer.slice()).init(nameData, extraData, + commentData, size); + buffer.position(buffer.position() + size); + return view; + } + + /** + * Copies this {@code DirectoryEntry} into a heap allocated buffer, overwriting path name, + * extra data and comment with the given values. + * + * @param name entry file name. Cannot be {@code null}. + * @param extraData extra data, or {@code null} + * @param comment zip file global comment, or {@code null}. + * @return a {@code DirectoryEntry} with a heap allocated buffer, initialized with the + * given values, and otherwise as a copy of this object. + */ + public DirectoryEntry clone(String name, byte[] extraData, String comment) { + return DirectoryEntry.allocate(name, extraData, comment).copy(this, CENOFF, CENCRC, CENSIZ, + CENLEN, CENFLG, CENHOW, CENTIM, CENVER, CENVER, CENDSK, CENATX, CENATT); + } + + /** + * Copies this {@code DirectoryEntry} into a writable buffer. The copy is made at the + * buffer's current position, and the position is advanced, by the size of the copy. + * + * @param buffer buffer to hold data for the copy. + * @return a {@code DirectoryEntry} backed by the given buffer. + */ + public DirectoryEntry copy(ByteBuffer buffer) { + int size = getSize(); + DirectoryEntry view = new DirectoryEntry(buffer.slice()); + this.buffer.rewind(); + view.buffer.put(this.buffer).flip(); + buffer.position(buffer.position() + size); + this.buffer.rewind().limit(size); + return view; + } + + private DirectoryEntry init(byte[] name, byte[] extra, byte[] comment, int size) { + buffer.putInt(0, SIGNATURE); + set(CENNAM, (short) name.length); + set(CENEXT, (short) extra.length); + set(CENCOM, (short) comment.length); + buffer.position(SIZE); + buffer.put(name); + if (extra.length > 0) { + buffer.put(extra); + } + if (comment.length > 0) { + buffer.put(comment); + } + buffer.position(0).limit(size); + return this; + } + + public final int getSize() { + return SIZE + get(CENNAM) + get(CENEXT) + get(CENCOM); + } + + /** + * Directory entry signature. + */ + public static final int SIGNATURE = 0x02014b50; // 33639248L + + /** + * Size of directory entry, not including variable data. + * Also the offset of the entry filename. + */ + public static final int SIZE = 46; + + /** + * For accessing the directory entry signature, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)} + * methods. + */ + public static final IntFieldId<DirectoryEntry> CENSIG = new IntFieldId<>(0); + + /** + * For accessing the "made by version" directory entry field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)} + * methods. + */ + public static final ShortFieldId<DirectoryEntry> CENVEM = new ShortFieldId<>(4); + + /** + * For accessing the "version needed" directory entry field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)} + * methods. + */ + public static final ShortFieldId<DirectoryEntry> CENVER = new ShortFieldId<>(6); + + /** + * For accessing the "flags" directory entry field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)} + * methods. + */ + public static final ShortFieldId<DirectoryEntry> CENFLG = new ShortFieldId<>(8); + + /** + * For accessing the "method" directory entry field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)} + * methods. + */ + public static final ShortFieldId<DirectoryEntry> CENHOW = new ShortFieldId<>(10); + + /** + * For accessing the "modified time" directory entry field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)} + * methods. + */ + public static final IntFieldId<DirectoryEntry> CENTIM = new IntFieldId<>(12); + + /** + * For accessing the "crc" directory entry field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)} + * methods. + */ + public static final IntFieldId<DirectoryEntry> CENCRC = new IntFieldId<>(16); + + /** + * For accessing the "compressed size" directory entry field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)} + * methods. + */ + public static final IntFieldId<DirectoryEntry> CENSIZ = new IntFieldId<>(20); + + /** + * For accessing the "uncompressed size" directory entry field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)} + * methods. + */ + public static final IntFieldId<DirectoryEntry> CENLEN = new IntFieldId<>(24); + + /** + * For accessing the "filename length" directory entry field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)} + * methods. + */ + public static final ShortFieldId<DirectoryEntry> CENNAM = new ShortFieldId<>(28); + + /** + * For accessing the "extra data length" directory entry field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)} + * methods. + */ + public static final ShortFieldId<DirectoryEntry> CENEXT = new ShortFieldId<>(30); + + /** + * For accessing the "file comment length" directory entry field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)} + * methods. + */ + public static final ShortFieldId<DirectoryEntry> CENCOM = new ShortFieldId<>(32); + + /** + * For accessing the "disk number" directory entry field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)} + * methods. + */ + public static final ShortFieldId<DirectoryEntry> CENDSK = new ShortFieldId<>(34); + + /** + * For accessing the "internal attributes" directory entry field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)} + * methods. + */ + public static final ShortFieldId<DirectoryEntry> CENATT = new ShortFieldId<>(36); + + /** + * For accessing the "external attributes" directory entry field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)} + * methods. + */ + public static final IntFieldId<DirectoryEntry> CENATX = new IntFieldId<>(38); + + /** + * For accessing the "local file header offset" directory entry field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)} + * methods. + */ + public static final IntFieldId<DirectoryEntry> CENOFF = new IntFieldId<>(42); + + /** + * Returns the filename of this entry. + */ + public final String getFilename() { + return getString(SIZE, get(CENNAM)); + } + + /** + * Returns the extra data of this entry. + * @return a byte array with 0 or more bytes. + */ + public final byte[] getExtraData() { + return getBytes(SIZE + get(CENNAM), get(CENEXT)); + } + + /** + * Return the comment of this entry. + * @return a string with 0 or more characters. + */ + public final String getComment() { + return getString(SIZE + get(CENNAM) + get(CENEXT), get(CENCOM)); + } + + /** + * Returns entry data size, based on directory entry information. For a valid zip file, this will + * be the correct size of of the entry data. + * @return if the {@link #CENHOW} field is 0, returns the value of the + * {@link #CENLEN} field (uncompressed size), otherwise returns the value of + * the {@link #CENSIZ} field (compressed size). + */ + public int dataSize() { + return get(CENHOW) == 0 ? get(CENLEN) : get(CENSIZ); + } +} diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/DosTime.java b/src/tools/android/java/com/google/devtools/build/android/ziputils/DosTime.java new file mode 100644 index 0000000000..0d228aabf6 --- /dev/null +++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/DosTime.java @@ -0,0 +1,69 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.android.ziputils; + +import java.util.Calendar; +import java.util.Date; +import java.util.GregorianCalendar; + +/** + * This class supports conversion from a {@code java.util.Date} object, to a + * 4 bytes DOS date and time representation. + */ +public final class DosTime { + + /** DOS representation of DOS epoch (midnight, jan 1, 1980) */ + public static final DosTime EPOCH; + /** {@code java.util.Date} for DOS epoch */ + public static final Date DOS_EPOCH; + private static final Calendar calendar; + + /** + * DOS representation of date passed to constructor. + * Time is lower the 16 bit, date the upper 16 bit. + */ + public final int time; + + /** + * Creates a DOS representation of the given date. + * @param date date to represent in DOS format. + */ + public DosTime(Date date) { + this.time = dateToDosTime(date); + } + + static { + calendar = new GregorianCalendar(1980, 0, 1, 0, 0, 0); + DOS_EPOCH = calendar.getTime(); + EPOCH = new DosTime(DOS_EPOCH); + } + + private static synchronized int dateToDosTime(Date date) { + calendar.setTime(date); + int year = calendar.get(Calendar.YEAR); + if (year < 1980) { + throw new IllegalArgumentException("date must be in or after 1980"); + } + if (year > 2107) { + throw new IllegalArgumentException("date must be before 2107"); + } + int month = calendar.get(Calendar.MONTH) + 1; + int day = calendar.get(Calendar.DAY_OF_MONTH); + int hour = calendar.get(Calendar.HOUR_OF_DAY); + int minute = calendar.get(Calendar.MINUTE); + int second = calendar.get(Calendar.SECOND); + return ((year - 1980) << 25) | (month << 21) | (day << 16) + | (hour << 11) | (minute << 5) | (second >> 1); + } +} diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/EndOfCentralDirectory.java b/src/tools/android/java/com/google/devtools/build/android/ziputils/EndOfCentralDirectory.java new file mode 100644 index 0000000000..efb6bdf9aa --- /dev/null +++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/EndOfCentralDirectory.java @@ -0,0 +1,188 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.android.ziputils; + +import static java.nio.ByteOrder.LITTLE_ENDIAN; +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.nio.ByteBuffer; + +/** + * Provides a view of a zip files "end of central directory" record. + */ +public class EndOfCentralDirectory extends View<EndOfCentralDirectory> { + + /** + * Returns a {@code EndOfCentralDirectory} of the given buffer. + * + * @param buffer containing the data of a "end of central directory" record. + * @return a {@code EndOfCentralDirectory} of the data at the current position of the given + * byte buffer. + */ + public static EndOfCentralDirectory viewOf(ByteBuffer buffer) { + EndOfCentralDirectory view = new EndOfCentralDirectory(buffer.slice()); + view.buffer.position(0).limit(view.getSize()); + buffer.position(buffer.position() + view.buffer.remaining()); + return view; + } + + private EndOfCentralDirectory(ByteBuffer buffer) { + super(buffer); + } + + /** + * Creates a {@code EndOfCentralDirectory} with a heap allocated buffer. + * + * @param comment zip file comment, or {@code null}. + * @return a {@code EndOfCentralDirectory} with a heap allocated buffer. + */ + public static EndOfCentralDirectory allocate(String comment) { + byte[] commentData = comment != null ? comment.getBytes(UTF_8) : EMPTY; + int size = SIZE + commentData.length; + ByteBuffer buffer = ByteBuffer.allocate(size).order(LITTLE_ENDIAN); + return new EndOfCentralDirectory(buffer).init(commentData); + } + + /** + * Creates a {@code EndOfCentralDirectory} over a writable buffer. + * + * @param buffer buffer to hold data for the "end of central directory" record. + * @param comment zip file global comment, or {@code null}. + * @return a {@code EndOfCentralDirectory} with a heap allocated buffer. + */ + public static EndOfCentralDirectory view(ByteBuffer buffer, String comment) { + byte[] commentData = comment != null ? comment.getBytes(UTF_8) : EMPTY; + int size = SIZE + commentData.length; + EndOfCentralDirectory view = new EndOfCentralDirectory(buffer.slice()).init(commentData); + buffer.position(buffer.position() + size); + return view; + } + + /** + * Copies this {@code EndOfCentralDirectory} over a writable buffer. + * + * @param buffer writable byte buffer to hold the data of the copy. + */ + public EndOfCentralDirectory copy(ByteBuffer buffer) { + EndOfCentralDirectory view = new EndOfCentralDirectory(buffer.slice()); + this.buffer.rewind(); + view.buffer.put(this.buffer).flip(); + this.buffer.rewind(); + buffer.position(buffer.position() + this.buffer.remaining()); + return view; + } + + private EndOfCentralDirectory init(byte[] comment) { + buffer.putInt(0, SIGNATURE); + set(ENDCOM, (short) comment.length); + if (comment.length > 0) { + buffer.position(SIZE); + buffer.put(comment); + } + buffer.position(0).limit(SIZE + comment.length); + return this; + } + + /** + * Signature of end of directory record. + */ + public static final int SIGNATURE = 0x06054b50; //101010256L + + /** + * Size of end of directory record, not including variable data. + * Also the offset of the file comment, if any. + */ + public static final int SIZE = 22; + + /** + * For accessing the end of central directory signature, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)} + * methods. + */ + public static final IntFieldId<EndOfCentralDirectory> ENDSIG = new IntFieldId<>(0); + + /** + * For accessing the "this disk number" end of central directory field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)} + * methods. + */ + public static final ShortFieldId<EndOfCentralDirectory> ENDDSK = new ShortFieldId<>(4); + + + /** + * For accessing the "central directory start disk" end of central directory field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)} + * methods. + */ + public static final ShortFieldId<EndOfCentralDirectory> ENDDCD = new ShortFieldId<>(6); + + /** + * For accessing the "central directory local records" end of central directory field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)} + * methods. + */ + public static final ShortFieldId<EndOfCentralDirectory> ENDSUB = new ShortFieldId<>(8); + + /** + * For accessing the "central directory total records" end of central directory field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)} + * methods. + */ + public static final ShortFieldId<EndOfCentralDirectory> ENDTOT = new ShortFieldId<>(10); + + /** + * For accessing the "central directory size" end of central directory field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)} + * methods. + */ + public static final IntFieldId<EndOfCentralDirectory> ENDSIZ = new IntFieldId<>(12); + + /** + * For accessing the "central directory offset" end of central directory field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)} + * methods. + */ + public static final IntFieldId<EndOfCentralDirectory> ENDOFF = new IntFieldId<>(16); + + /** + * For accessing the "file comment length" end of central directory field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)} + * methods. + */ + public static final ShortFieldId<EndOfCentralDirectory> ENDCOM = new ShortFieldId<>(20); + + /** + * Returns the file comment. + * @return the file comment, or an empty string. + */ + public final String getComment() { + return getString(SIZE, get(ENDCOM)); + } + + /** + * Returns the total size of the end of directory record, including file comment, if any. + * @return the total size of the end of directory record. + */ + public int getSize() { + return SIZE + get(ENDCOM); + } +} diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/EntryHandler.java b/src/tools/android/java/com/google/devtools/build/android/ziputils/EntryHandler.java new file mode 100644 index 0000000000..9f55dfb6aa --- /dev/null +++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/EntryHandler.java @@ -0,0 +1,41 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.android.ziputils; + +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * Entry handler to pass to {@link ZipIn#scanEntries(EntryHandler)}. Implementations, + * typically perform actions such as selecting, copying, renaming, merging, entries, + * and writing entries to one or more {@link ZipOut}. + */ +public interface EntryHandler { + + /** + * Handles a zip file entry. Called by the {@code ZipIn} scanner, for each entry. + * + * @param in The {@code ZipIn} from which we're called. + * @param header The header for the entry to handle. + * @param dirEntry The directory entry corresponding to this entry. + * @param data byte buffer containing the data of the entry. If the data size cannot be + * determine from the header or directory entry (zip error), then the provided buffer + * may not contain all of the data. + * @throws IOException implementations may thrown an IOException to signal that an error occurred + * handling an entry. An IOException may also be generated by certain methods that the + * may invoke on the arguments passed to this method. + */ + void handle(ZipIn in, LocalFileHeader header, DirectoryEntry dirEntry, ByteBuffer data) + throws IOException; +} diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/LocalFileHeader.java b/src/tools/android/java/com/google/devtools/build/android/ziputils/LocalFileHeader.java new file mode 100644 index 0000000000..be7ce3a6d0 --- /dev/null +++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/LocalFileHeader.java @@ -0,0 +1,262 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.android.ziputils; + +import static java.nio.ByteOrder.LITTLE_ENDIAN; +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.nio.ByteBuffer; + +/** + * Provides a view of a local file header for a zip file entry. + */ +public class LocalFileHeader extends View<LocalFileHeader> { + + /** + * Returns a {@code LocalFileHeader} view of the given buffer. The buffer is assumed to contain a + * valid "central directory entry" record beginning at the buffers current position. + * + * @param buffer containing the data of a "central directory entry" record. + * @return a {@code LocalFileHeader} of the data at the current position of the given byte + * buffer. + */ + public static LocalFileHeader viewOf(ByteBuffer buffer) { + LocalFileHeader view = new LocalFileHeader(buffer.slice()); + view.buffer.limit(view.getSize()); + buffer.position(buffer.position() + view.buffer.remaining()); + return view; + } + + private LocalFileHeader(ByteBuffer buffer) { + super(buffer); + } + + /** + * Creates a {@code LocalFileHeader} with a heap allocated buffer. Apart from the signature + * and extra data data (if any), the returned object is uninitialized. + * + * @param name entry file name. Cannot be {@code null}. + * @param extraData extra data, or {@code null} + * @return a {@code LocalFileHeader} with a heap allocated buffer. + */ + public static LocalFileHeader allocate(String name, byte[] extraData) { + byte[] nameData = name.getBytes(UTF_8); + if (extraData == null) { + extraData = EMPTY; + } + int size = SIZE + nameData.length + extraData.length; + ByteBuffer buffer = ByteBuffer.allocate(size).order(LITTLE_ENDIAN); + return new LocalFileHeader(buffer).init(nameData, extraData, size); + } + + /** + * Creates a {@code LocalFileHeader} over a writable buffer. The given buffer's position is + * advanced by the number of bytes consumed by the view. Apart from the signature and extra data + * (if any), the returned view is uninitialized. + * + * @param buffer buffer to hold data for the "central directory entry" record. + * @param name entry file name. Cannot be {@code null}. + * @param extraData extra data, or {@code null} + * @return a {@code DirectoryEntry} with a heap allocated buffer. + */ + public static LocalFileHeader view(ByteBuffer buffer, String name, byte[] extraData) { + byte[] nameData = name.getBytes(UTF_8); + if (extraData == null) { + extraData = EMPTY; + } + int size = SIZE + nameData.length + extraData.length; + LocalFileHeader view = new LocalFileHeader(buffer.slice()).init(nameData, extraData, size); + buffer.position(buffer.position() + size); + return view; + } + + /** + * Copies this {@code LocalFileHeader} into a heap allocated buffer, overwriting the current + * path name and extra data with the given values. + * + * @param name entry file name. Cannot be {@code null}. + * @param extraData extra data, or {@code null} + * @return a {@code LocalFileHeader} with a heap allocated buffer, initialized with the given + * name and extra data, and otherwise a copy of this object. + */ + public LocalFileHeader clone(String name, byte[] extraData) { + return LocalFileHeader.allocate(name, extraData).copy(this, LOCCRC, LOCSIZ, LOCLEN, LOCFLG, + LOCHOW, LOCTIM, LOCVER); + } + + /** + * Copies this {@code LocalFileHeader} into a writable buffer. The copy is made at the + * buffer's current position, and the position is advanced, by the size of the copy. + * + * @param buffer buffer to hold data for the copy. + * @return a {@code LocalFileHeader} backed by the given buffer. + */ + public LocalFileHeader copy(ByteBuffer buffer) { + int size = getSize(); + LocalFileHeader view = new LocalFileHeader(buffer.slice()); + this.buffer.rewind(); + view.buffer.put(this.buffer).flip(); + buffer.position(buffer.position() + size); + this.buffer.rewind(); + return view; + } + + private LocalFileHeader init(byte[] name, byte[] extra, int size) { + buffer.putInt(0, SIGNATURE); + set(LOCNAM, (short) name.length); + set(LOCEXT, (short) extra.length); + buffer.position(SIZE); + buffer.put(name); + if (extra.length > 0) { + buffer.put(extra); + } + buffer.position(0).limit(size); + return this; + } + + /** + * Flag used to mark a compressed entry, for which the size is unknown at the time + * of writing the header. + */ + public static final short SIZE_MASKED_FLAG = 0x8; + + /** + * Signature of local file header. + */ + public static final int SIGNATURE = 0x04034b50; // 67324752L + + /** + * Size of local file header, not including variable data. + * Also the offset of the filename. + */ + public static final int SIZE = 30; + + /** + * For accessing the local header signature, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)} + * methods. + */ + public static final IntFieldId<LocalFileHeader> LOCSIG = new IntFieldId<>(0); + + /** + * For accessing the "needed version" local header field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)} + * methods. + */ + public static final ShortFieldId<LocalFileHeader> LOCVER = new ShortFieldId<>(4); + + /** + * For accessing the "flags" local header field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)} + * methods. + */ + public static final ShortFieldId<LocalFileHeader> LOCFLG = new ShortFieldId<>(6); + + /** + * For accessing the "method" local header field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)} + * methods. + */ + public static final ShortFieldId<LocalFileHeader> LOCHOW = new ShortFieldId<>(8); + + /** + * For accessing the "modified time" local header field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)} + * methods. + */ + public static final IntFieldId<LocalFileHeader> LOCTIM = new IntFieldId<>(10); + + /** + * For accessing the "crc" local header field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)} + * methods. + */ + public static final IntFieldId<LocalFileHeader> LOCCRC = new IntFieldId<>(14); + + /** + * For accessing the "compressed size" local header field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)} + * methods. + */ + public static final IntFieldId<LocalFileHeader> LOCSIZ = new IntFieldId<>(18); + + /** + * For accessing the "uncompressed size" local header field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.IntFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.IntFieldId, int)} + * methods. + */ + public static final IntFieldId<LocalFileHeader> LOCLEN = new IntFieldId<>(22); + + /** + * For accessing the "filename length" local header field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)} + * methods. + */ + public static final ShortFieldId<LocalFileHeader> LOCNAM = new ShortFieldId<>(26); + + /** + * For accessing the "extra data length" local header field, with the + * {@link View#get(com.google.devtools.build.android.ziputils.View.ShortFieldId)} + * and {@link View#set(com.google.devtools.build.android.ziputils.View.ShortFieldId, short)} + * methods. + */ + public static final ShortFieldId<LocalFileHeader> LOCEXT = new ShortFieldId<>(28); + + /** + * Returns the filename for this entry. + */ + public final String getFilename() { + return getString(SIZE, get(LOCNAM)); + } + + /** + * Returns the extra data for this entry. + * @return an array of 0 or more bytes. + */ + public final byte[] getExtraData() { + return getBytes(SIZE + get(LOCNAM), get(LOCEXT)); + } + + /** + * Returns the size of this header, including filename, and extra data (if any). + */ + public final int getSize() { + return SIZE + get(LOCNAM) + get(LOCEXT); + } + + /** + * Returns entry data size, based on directory entry information. For a valid zip file, this will + * be the correct size of the entry data, or -1, if the size cannot be determined from the + * header. Notice, if ths method returns 0, it may be because the writer of the zip file forgot + * to set the {@link #SIZE_MASKED_FLAG}. + * + * @return if the {@link #LOCHOW} field is 0, returns the value of the + * {@link #LOCLEN} field (uncompressed size). If {@link #LOCHOW} is not 0, and the + * {@link #SIZE_MASKED_FLAG} is not set, returns the value of the {@link #LOCSIZ} field + * (compressed size). Otherwise return -1 (size unknown). + */ + public int dataSize() { + return get(LOCHOW) == 0 ? get(LOCLEN) + : (get(LOCFLG) & SIZE_MASKED_FLAG) == 0 ? get(LOCSIZ) : -1; + } +} diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/README b/src/tools/android/java/com/google/devtools/build/android/ziputils/README new file mode 100644 index 0000000000..25891bda78 --- /dev/null +++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/README @@ -0,0 +1,26 @@ +This directory contains utilities needed to support parallel dex compilation +of Android applications. + +The DexMapper utility maps an applications classes (given in one or more input jar files), +to one or more output jars, that may then be compiled separately. + +The DexReducer utility extracts .dex files from one or more archives, and package them in a +single output archive, renaming entries to: classes.dex, classes2.dex, ... + +These utilities uses a low-level zip-file manipulation library contained in this package. +The library is optimized for serial parsing of zip archives and tasks such as copying entries +to one or more output archives.It employs direct byte buffers, to avoid copying file data to +the java heap, and allowing fast copies from input to output. Output is asynchronous, for optimal +performance when running on non-memory-based file systems. + +WARNING: This library was designed for creating build tools needed to support parallel +dex compilation of Android applications, related functions. While the library provides +fairly general facilities for processing zip files, the API is still immature, and +subject to change. + +Defined targets +--------------------------------------------------- +blaze build java/com/google/devtools/build/android/ziputils:ziputils_lib +blaze build java/com/google/devtools/build/android/ziputils:mapper +blaze build java/com/google/devtools/build/android/ziputils:reducer +blaze test javatests/com/google/devtools/build/android/ziputils:all_tests diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/ScanUtil.java b/src/tools/android/java/com/google/devtools/build/android/ziputils/ScanUtil.java new file mode 100644 index 0000000000..185838f718 --- /dev/null +++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/ScanUtil.java @@ -0,0 +1,104 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.android.ziputils; + +import com.google.common.base.Preconditions; + +import java.nio.ByteBuffer; + +/** + * Helper class for finding byte patterns in a byte buffer, scanning forwards or backwards. + * This is used to located the "end of central directory" marker, and in other instances + * where zip file elements must be located by scanning for known signatures. + * For instance, when it's necessary to scan for zip entries, without + * relying on the central directory to provide the exact location, or when scanning for + * data descriptor at the end of an entry of unknown size. + */ +public class ScanUtil { + + /** + * Finds the next previous position of a given byte sequence in a given byte buffer, + * starting from the buffer's current position (excluded). + * + * @param target byte sequence to search for. + * @param buffer byte buffer in which to search. + * @return position of the last location of the target sequence, at a position + * strictly smaller than the current position, or -1 if not found. + * @throws IllegalArgumentException if either {@code target} or {@code buffer} is {@code null}. + */ + static int scanBackwardsTo(byte[] target, ByteBuffer buffer) { + Preconditions.checkNotNull(target); + Preconditions.checkNotNull(buffer); + if (target.length == 0) { + return buffer.position() - 1; + } + int pos = buffer.position() - target.length; + if (pos < 0) { + return -1; + } + scan: + while (true) { + while (pos >= 0 && buffer.get(pos) != target[0]) { + pos--; + } + if (pos < 0) { + return -1; + } + for (int i = 1; i < target.length; i++) { + if (buffer.get(pos + i) != target[i]) { + pos--; + continue scan; + } + } + return pos; + } + } + + /** + * Finds the next position of a given byte sequence in a given byte buffer, starting from the + * buffer's current position (included). + * + * @param target byte sequence to search for. + * @param buffer byte buffer in which to search. + * @return position of the first location of the target sequence, or -1 if not found. + * @throws IllegalArgumentException if either {@code target} or {@code buffer} is {@code null}. + */ + static int scanTo(byte[] target, ByteBuffer buffer) { + Preconditions.checkNotNull(target); + Preconditions.checkNotNull(buffer); + if (!buffer.hasRemaining()) { + return -1; + } + int pos = buffer.position(); + if (target.length == 0) { + return pos; + } + scan: + while (true) { + while (pos <= buffer.limit() - target.length && buffer.get(pos) != target[0]) { + pos++; + } + if (pos > buffer.limit() - target.length + 1) { + return -1; + } + for (int i = 1; i < target.length; i++) { + if (buffer.get(pos + i) != target[i]) { + pos++; + continue scan; + } + } + return pos; + } + } +} diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/SplitZip.java b/src/tools/android/java/com/google/devtools/build/android/ziputils/SplitZip.java new file mode 100644 index 0000000000..3d39b4731a --- /dev/null +++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/SplitZip.java @@ -0,0 +1,452 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.android.ziputils; + +import static com.google.devtools.build.android.ziputils.DataDescriptor.EXTCRC; +import static com.google.devtools.build.android.ziputils.DataDescriptor.EXTLEN; +import static com.google.devtools.build.android.ziputils.DataDescriptor.EXTSIZ; +import static com.google.devtools.build.android.ziputils.DirectoryEntry.CENCRC; +import static com.google.devtools.build.android.ziputils.DirectoryEntry.CENLEN; +import static com.google.devtools.build.android.ziputils.DirectoryEntry.CENSIZ; +import static com.google.devtools.build.android.ziputils.DirectoryEntry.CENTIM; +import static com.google.devtools.build.android.ziputils.LocalFileHeader.LOCFLG; +import static com.google.devtools.build.android.ziputils.LocalFileHeader.LOCTIM; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.base.Preconditions; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + +/** + * Extracts entries from a set of input archives, and copies them to N output archive of + * approximately equal size, while attempting to split archives on package (directory) boundaries. + * Optionally, accept a list of entries to be added to the first output archive, splitting + * remaining entries by package boundaries. + */ +public class SplitZip implements EntryHandler { + private boolean verbose = false; + private final List<ZipIn> inputs; + private final List<ZipOut> outputs; + private String filterFile; + private InputStream filterInputStream; + private String resourceFile; + private Date date; + private DosTime dosTime; + // Internal state variables: + private boolean finished = false; + private Set<String> filter; + private ZipOut[] zipOuts; + private ZipOut resourceOut; + private final Map<String, ZipOut> assignments = new HashMap<>(); + private final Map<String, CentralDirectory> centralDirectories; + private final Set<String> classes = new TreeSet<>(); + + /** + * Creates an un-configured {@code SplitZip} instance. + */ + public SplitZip() { + inputs = new ArrayList<>(); + outputs = new ArrayList<>(); + centralDirectories = new HashMap<>(); + } + + /** + * Configures a resource file. By default, resources are output in the initial shard. + * If a resource file is specified, resources are written to this instead. + * @param resourceFile in not {@code null}, the name of a file in which to output resources. + * @return this object. + */ + public SplitZip setResourceFile(String resourceFile) { + this.resourceFile = resourceFile; + return this; + } + + // Package private for testing with mock file + SplitZip setResourceFile(ZipOut resOut) { + resourceOut = resOut; + return this; + } + + /** + * Gets the name of the resource output file. If no resource output file is configured, resources + * are output in the initial shard. + * @return the name of the resource output file, or {@code null} if no file has been configured. + */ + public String getResourceFile() { + return resourceFile; + } + + /** + * Configures a file containing a list of files to be included in the first output archive. + * + * @param clFile path of class file list. + * @return this object + */ + public SplitZip setMainClassListFile(String clFile) { + filterFile = clFile; + return this; + } + + // Package private for testing with mock file + SplitZip setMainClassListFile(InputStream clInputStream) { + filterInputStream = clInputStream; + return this; + } + + /** + * Gets the path of the file listing the content of the initial shard. + * @return return path of file list file, or {@code null} if not set. + */ + public String getMainClassListFile() { + return filterFile; + } + + /** + * Configures verbose mode. + * + * @param flag set to {@code true} to turn on verbose mode. + * @return this object + */ + public SplitZip setVerbose(boolean flag) { + verbose = flag; + return this; + } + + /** + * Gets the verbosity mode.. + * @return {@code true} iff verbose mode is enabled + */ + public boolean isVerbose() { + return verbose; + } + + /** + * Sets date to overwrite timestamp of copied entries. Setting the date to {@code null} means + * using the date and time information in the input file. Set an explicit date to override. + * + * @param date modified date and time to set for entries in output. + * @return this object. + */ + public SplitZip setEntryDate(Date date) { + this.date = date; + this.dosTime = date == null ? null : new DosTime(date); + return this; + } + + /** + * Sets date to {@link DosTime#DOS_EPOCH}. + * @return this object. + */ + public SplitZip useDefaultEntryDate() { + this.date = DosTime.DOS_EPOCH; + this.dosTime = DosTime.EPOCH; + return this; + } + + /** + * Gets the entry modified date. + */ + public Date getEntryDate() { + return date; + } + + /** + * Configures multiple input file locations. + * + * @param inputs list of input locations. + * @return this object + * @throws java.io.IOException + */ + public SplitZip addInputs(Iterable<String> inputs) throws IOException { + for (String i : inputs) { + addInput(i); + } + return this; + } + + /** + * Configures an input location. An input file must be a zip archive. + * + * @param filename path for an input location. + * @return this object + * @throws java.io.IOException + */ + public SplitZip addInput(String filename) throws IOException { + if (filename != null) { + inputs.add(new ZipIn(new FileInputStream(filename).getChannel(), filename)); + } + return this; + } + + // Package private, for testing using mock file system. + SplitZip addInput(ZipIn in) throws IOException { + Preconditions.checkNotNull(in); + inputs.add(in); + return this; + } + + /** + * Configures multiple output file locations. + * + * @param outputs list of output files. + * @return this object + * @throws java.io.IOException + */ + public SplitZip addOutputs(Iterable<String> outputs) throws IOException { + for (String o : outputs) { + addOutput(o); + } + return this; + } + + /** + * Configures an output location. + * + * @param output path for an output location. + * @return this object + * @throws java.io.IOException + */ + public SplitZip addOutput(String output) throws IOException { + Preconditions.checkNotNull(output); + outputs.add(new ZipOut(new FileOutputStream(output, false).getChannel(), output)); + return this; + } + + // Package private for testing with mock file + SplitZip addOutput(ZipOut output) throws IOException { + Preconditions.checkNotNull(output); + outputs.add(output); + return this; + } + + /** + * Executes this {@code SplitZip}, reading content from the configured input locations, creating + * the specified number of archives, in the configured output directory. + * + * @return this object + * @throws java.io.IOException + */ + public SplitZip run() throws IOException { + verbose("SplitZip: Splitting in: " + outputs.size()); + verbose("SplitZip: with filter: " + filterFile); + checkConfig(); + // Prepare output files + zipOuts = outputs.toArray(new ZipOut[outputs.size()]); + if (resourceFile != null) { + resourceOut = new ZipOut(new FileOutputStream(resourceFile, false).getChannel(), + resourceFile); + } else if (resourceOut == null) { // may have been set for testing + resourceOut = zipOuts[0]; + } + + // Read directories of input files + for (ZipIn zip : inputs) { + zip.endOfCentralDirectory(); + centralDirectories.put(zip.getFilename(), zip.centralDirectory()); + zip.centralDirectory(); + } + // Assign input entries to output files + split(); + // Copy entries to the assigned output files + for (ZipIn zip : inputs) { + zip.scanEntries(this); + } + return this; + } + + /** + * Copies an entry to the assigned output files. Called for each entry in the input files. + * @param in + * @param header + * @param dirEntry + * @param data + * @throws IOException + */ + @Override + public void handle(ZipIn in, LocalFileHeader header, DirectoryEntry dirEntry, + ByteBuffer data) throws IOException { + String localFilename = header.getFilename(); + ZipOut out = assignments.remove(localFilename); + if (out == null) { + // Skip unassigned file; + return; + } + if (dirEntry == null) { + // Shouldn't get here, as there should be no assignment. + System.out.println("Warning: no directory entry"); + return; + } + // Clone directory entry + DirectoryEntry entryOut = out.nextEntry(dirEntry); + if (dosTime != null) { + // Overwrite time stamp + header.set(LOCTIM, dosTime.time); + entryOut.set(CENTIM, dosTime.time); + } + out.write(header); + out.write(data); + if ((header.get(LOCFLG) & LocalFileHeader.SIZE_MASKED_FLAG) != 0) { + // Instead of this, we could fix the header with the size information + // from the directory entry. For now, keep the entry encoded as-is. + DataDescriptor desc = DataDescriptor.allocate() + .set(EXTCRC, dirEntry.get(CENCRC)) + .set(EXTSIZ, dirEntry.get(CENSIZ)) + .set(EXTLEN, dirEntry.get(CENLEN)); + out.write(desc); + } + } + + /** + * Writes any remaining output data to the output stream. + * + * @throws IOException if the output stream or the filter throws an IOException + * @throws IllegalStateException if this method was already called earlier + */ + public void finish() throws IOException { + checkNotFinished(); + finished = true; + if (resourceOut != null) { + resourceOut.finish(); + } + for (ZipOut zo : zipOuts) { + zo.finish(); + } + } + + /** + * Writes any remaining output data to the output stream and closes it. + * + * @throws IOException if the output stream or the filter throws an IOException + */ + public void close() throws IOException { + if (!finished) { + finish(); + } + if (resourceOut != null) { + resourceOut.close(); + } + for (ZipOut zo : zipOuts) { + zo.close(); + } + } + + private void checkNotFinished() { + if (finished) { + throw new IllegalStateException(); + } + } + + /** + * Validates configuration before execution. + */ + private void checkConfig() throws IOException { + if (outputs.size() < 1) { + throw new IllegalStateException("Require at least one output file"); + } + filter = filterFile == null && filterInputStream == null ? null : readPaths(filterFile); + } + + /** + * Parses the entries and assign each entry to an output file. + */ + private void split() { + for (ZipIn in : inputs) { + CentralDirectory cdir = centralDirectories.get(in.getFilename()); + for (DirectoryEntry entry : cdir.list()) { + String filename = entry.getFilename(); + if (filename.endsWith(".class")) { + // Only pass classes to the splitter, so that it can do the best job + // possible distributing them across output files. + classes.add(filename); + } else if (!filename.endsWith("/")) { + // Non class files (resources) are either assigned to the first + // output file, or to a specified resource output file. + assignments.put(filename, resourceOut); + } + } + } + Splitter entryFilter = new Splitter(outputs.size(), classes.size()); + if (filter != null) { + // Assign files in the filter to the first output file. + entryFilter.assign(filter); + entryFilter.nextShard(); // minimal initial shard + } + for (String path : classes) { + int assignment = entryFilter.assign(path); + Preconditions.checkState(assignment >= 0 && assignment < zipOuts.length); + assignments.put(path, zipOuts[assignment]); + } + } + + /** + * Reads paths of classes required in first shard. For testing purposes, this relies + * on the file system configured for the {@code Zip} library class. + */ + private Set<String> readPaths(String fileName) throws IOException { + Set<String> paths = new HashSet<>(); + BufferedReader reader = null; + try { + if (filterInputStream == null) { + filterInputStream = new FileInputStream(fileName); + } + reader = new BufferedReader(new InputStreamReader(filterInputStream, UTF_8)); + String line; + while (null != (line = reader.readLine())) { + paths.add(fixPath(line)); + } + return paths; + } finally { + if (reader != null) { + reader.close(); + } + } + } + + // TODO(bazel-team): Got this from 'dx'. I'm not sure we need this part. Keep it for now, + // to make sure we read the main dex list the exact same way that dx would. + private String fixPath(String path) { + if (File.separatorChar == '\\') { + path = path.replace('\\', '/'); + } + int index = path.lastIndexOf("/./"); + if (index != -1) { + return path.substring(index + 3); + } + if (path.startsWith("./")) { + return path.substring(2); + } + return path; + } + + private void verbose(String msg) { + if (verbose) { + System.out.println(msg); + } + } +} diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/Splitter.java b/src/tools/android/java/com/google/devtools/build/android/ziputils/Splitter.java new file mode 100644 index 0000000000..f701bb1287 --- /dev/null +++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/Splitter.java @@ -0,0 +1,136 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.android.ziputils; + +import com.google.common.base.Preconditions; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +class Splitter { + + private static final String ARCHIVE_FILE_SEPARATOR = "/"; + + private final int numberOfShards; + private final Map<String, Integer> assigned; + private int size = 0; + private int shard = 0; + private String currPath = null; + private int remaining; + private int idealSize; + private int almostFull; + + /** + * Creates a splitter for splitting an expected number of entries into + * a given number of shards. The current shard is shard 0. + */ + public Splitter(int numberOfShards, int expectedSize) { + this.numberOfShards = numberOfShards; + this.remaining = expectedSize; + this.assigned = new HashMap<>(); + idealSize = remaining / (numberOfShards - shard); + // Before you change this, please do the math. + // It's not always perfect, but designed to keep shards reasonably balanced in most cases. + int limit = Math.min(Math.min(10, (idealSize + 3) / 4), (int) Math.log(numberOfShards)); + almostFull = idealSize - limit; + } + + /** + * Forces mapping of the given entries to be that of the current shard. + * The estimated number of remaining entries to process is adjusted, + * by subtracting the number of as-of-yet unassigned entries from the + * filter. + */ + public void assign(Collection<String> filter) { + if (filter != null) { + for (String s : filter) { + if (!assigned.containsKey(s)) { + remaining--; + } + assigned.put(s, shard); + } + size = filter.size(); + } + } + + /** + * Forces increment of the current shard. May be called externally. + * Typically right after calling {@link #assign(java.util.Collection)}. + */ + public void nextShard() { + if (shard < numberOfShards - 1) { + shard++; + size = 0; + addEntries(0); + } + } + + /** + * Adjusts the number of estimated entries to be process by the given count. + */ + public void addEntries(int count) { + this.remaining += count; + idealSize = numberOfShards > shard ? remaining / (numberOfShards - shard) : remaining; + // Before you change this, please do the math. + // It's not always perfect, but designed to keep shards reasonably balanced in most cases. + int limit = Math.min(Math.min(10, (idealSize + 3) / 4), (int) Math.log(numberOfShards)); + almostFull = idealSize - limit; + } + + /** + * Assigns the given entry to an output file. + */ + public int assign(String path) { + Preconditions.checkState(shard < numberOfShards, "Too many shards!"); + Integer assignment = assigned.get(path); + if (assignment != null) { + return assignment; + } + remaining--; + + // last shard, no choice + if (shard == numberOfShards - 1) { + size++; + assigned.put(currPath, shard); + return shard; + } + + // Forced split to try to avoid empty shards + if (remaining < numberOfShards - shard - 1) { + if (size > 0) { + nextShard(); + } + size++; + assigned.put(currPath, shard); + return shard; + } + + String prevPath = currPath; + currPath = path; + // If current shard is at least "almost full", check for package boundary? + if (prevPath != null && size >= almostFull) { + int i = currPath.lastIndexOf(ARCHIVE_FILE_SEPARATOR); + String dir = i > 0 ? currPath.substring(0, i) : "."; + i = prevPath.lastIndexOf(ARCHIVE_FILE_SEPARATOR); + String prevDir = i > 0 ? prevPath.substring(0, i) : "."; + if (!dir.equals(prevDir)) { + nextShard(); + } + } + assigned.put(currPath, shard); + size++; + return shard; + } +} diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/View.java b/src/tools/android/java/com/google/devtools/build/android/ziputils/View.java new file mode 100644 index 0000000000..cc3535f010 --- /dev/null +++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/View.java @@ -0,0 +1,271 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.android.ziputils; + +import static java.nio.ByteOrder.LITTLE_ENDIAN; +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.nio.ByteBuffer; + +/** + * A {@code View} represents a range of a larger sequence of bytes (e.g. part of a file). + * It consist of an internal byte buffer providing access to the data range, and an + * offset of the first byte in the buffer within the larger sequence. + * Subclasses will typically assign a specific interpretations of the data in a view + * (e.g. records of a specific type). + * + * <p>Instances of subclasses may generally be "allocated", or created "of" or "over" a + * byte buffer provided at creation time. + * + * <p>An "allocated" view gets a new heap allocated byte buffer. Allocation methods + * typically defines parameters for variable sized part of the object they create a + * view of. Once allocated, the variable size parts (e.g. filename) cannot be changed. + * + * <p>A view created "of" an existing byte buffer, expects the buffer to contain an + * appropriate record at its current position. The buffers limit is set at the + * end of the object being viewed. + * + * <p>A view created "over" an existing byte buffer, reserves space in the buffer for the + * object being viewed. The variable sized parts of the object are initialized, but + * otherwise the existing buffer data are left as-is, and can be manipulated through the + * view. + * + * <p> An view can also be copied into an existing byte buffer. This is like creating a + * view "over" the buffer, but initializing the new view as an exact copy of the one + * being copied. + * + * @param <SUB> to be type-safe, a subclass {@code S} must extend {@code View<S>}. It must not + * extend {@code View<S2>}, where {@code S2} is another subclass, which is not also a superclass + * of {@code S}. To maintain this guarantee, this class is declared abstract and package private. + * Unchecked warnings are suppressed as per this specification constraint. + */ +abstract class View<SUB extends View<?>> { + + /** Zero length byte array */ + protected static final byte[] EMPTY = {}; + + /** {@code ByteBuffer} backing this view. */ + protected final ByteBuffer buffer; + + /** + * Offset of first byte covered by this view. For input views, this is the file offset where the + * item occur. For output, it's the offset at which we expect to write the item (may be -1, for + * unknown). + */ + protected long fileOffset; + + /** + * Creates a view backed by the given {@code ByteBuffer}. Sets the buffer's byte order to + * little endian, and sets the file offset to -1 (unknown). + * + * @param buffer backing byte buffer. + */ + protected View(ByteBuffer buffer) { + buffer.order(LITTLE_ENDIAN); + this.buffer = buffer; + this.fileOffset = -1; + } + + /** + * Sets the file offset of the data item covered by this view, + * + * @param fileOffset + * @return this object. + */ + @SuppressWarnings("unchecked") // safe by specification + public SUB at(long fileOffset) { + this.fileOffset = fileOffset; + return (SUB) this; + } + + /** + * Gets the fileOffset of this view. + * + * @return the location of the viewed object within the underlying file. + */ + public long fileOffset() { + return fileOffset; + } + + /** + * Returns an array with data copied from the backing byte buffer, at the given offset, relative + * to the beginning of this view. This method does not perform boundary checks of offset or + * length. This method may temporarily changes the position of the backing buffer. However, after + * the call, the position will be unchanged. + * + * @param off offset relative to this view. + * @param len number of bytes to return. + * @return Newly allocated array with copy of requested data. + * @throws IndexOutOfBoundsException in case of illegal arguments. + */ + protected byte[] getBytes(int off, int len) { + if (len == 0) { + return EMPTY; + } + byte[] bytes; + try { + bytes = new byte[len]; + int currPos = buffer.position(); + buffer.position(off); + buffer.get(bytes); + buffer.position(currPos); + } catch (Exception ex) { + throw new IndexOutOfBoundsException(); + } + return bytes; + } + + /** + * Returns a String representation of {@code len} bytes starting at offset {@code off} in this + * view. This method may temporarily changes the position of the backing buffer. However, after + * the call, the position will be unchanged. + * + * @param off offset relative to backing buffer. + * @param len number of bytes to return. This method may throw an + * @return Newly allocated String created by interpreting the specified bytes as UTF-8 data. + * @throws IndexOutOfBoundsException in case of illegal arguments. + */ + protected String getString(int off, int len) { + if (len == 0) { + return ""; + } + if (buffer.hasArray()) { + return new String(buffer.array(), buffer.arrayOffset() + off, len, UTF_8); + } else { + return new String(getBytes(off, len), UTF_8); + } + } + + /** + * Gets the value of an identified integer field. + * + * @param id field identifier + * @return the value of the field identified by {@code id}. + */ + public int get(IntFieldId<? extends SUB> id) { + return buffer.getInt(id.address()); + } + + /** + * Gets the value of an identified short field. + * + * @param id field identifier + * @return the value of the field identified by {@code id}. + */ + public short get(ShortFieldId<? extends SUB> id) { + return buffer.getShort(id.address()); + } + + /** + * Sets the value of an identified integer field. + * + * @param id field identifier + * @param value value to set for the field identified by {@code id}. + * @return this object. + */ + @SuppressWarnings("unchecked") // safe by specification + public SUB set(IntFieldId<? extends SUB> id, int value) { + buffer.putInt(id.address(), value); + return (SUB) this; + } + + /** + * Sets the value of an identified short field. + * + * @param id field identifier + * @param value value to set for the field identified by {@code id}. + * @return this object. + */ + @SuppressWarnings("unchecked") // safe by specification + public SUB set(ShortFieldId<? extends SUB> id, short value) { + buffer.putShort(id.address(), value); + return (SUB) this; + } + + /** + * Copies the values of one or more identified fields from another view to this view. + * + * @param from The view from which to copy field values. + * @param ids field identifiers for fields to copy. + * @return this object. + */ + @SuppressWarnings("unchecked") // safe by specification + public SUB copy(View<SUB> from, FieldId<? extends SUB, ?>... ids) { + for (FieldId<? extends SUB, ?> id : ids) { + int address = id.address; + buffer.put(address, from.buffer.get(address++)); + buffer.put(address, from.buffer.get(address++)); + if (id.type() == Integer.TYPE) { + buffer.put(address, from.buffer.get(address++)); + buffer.put(address, from.buffer.get(address)); + } + } + return (SUB) this; + } + + /** + * Base class for data field descriptors. Describes a data field's type and address in a view. + * This base class allows the + * {@link #copy(View, com.google.devtools.build.android.ziputils.View.FieldId[])} method + * to operate of fields of mixed types. + * + * @param <T> {@code Integer.TYPE} or {@code Short.TYPE}. + * @param <V> subclass of {@code View} for which a field id is defined. + */ + protected abstract static class FieldId<V extends View<?>, T> { + private final int address; + private final Class<T> type; + + protected FieldId(int address, Class<T> type) { + this.address = address; + this.type = type; + } + + /** + * Returns the class of the field type, {@code Class<T>}. + */ + public Class<T> type() { + return type; + } + + /** + * Returns the field address, within a record of type {@code V} + */ + public int address() { + return address; + } + } + + /** + * Describes an integer fields for a view. + * + * @param <V> subclass of {@code View} for which a field id is defined. + */ + protected static class IntFieldId<V extends View<?>> extends FieldId<V, Integer> { + protected IntFieldId(int address) { + super(address, Integer.TYPE); + } + } + + /** + * Describes a short field for a view. + * + * @param <V> subclass of {@code View} for which a field id is defined. + */ + protected static class ShortFieldId<V extends View<?>> extends FieldId<V, Short> { + protected ShortFieldId(int address) { + super(address, Short.TYPE); + } + } +} diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/ZipIn.java b/src/tools/android/java/com/google/devtools/build/android/ziputils/ZipIn.java new file mode 100644 index 0000000000..afda2132dc --- /dev/null +++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/ZipIn.java @@ -0,0 +1,695 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.android.ziputils; + +import static com.google.devtools.build.android.ziputils.DataDescriptor.EXTLEN; +import static com.google.devtools.build.android.ziputils.DataDescriptor.EXTSIZ; +import static com.google.devtools.build.android.ziputils.DirectoryEntry.CENLEN; +import static com.google.devtools.build.android.ziputils.DirectoryEntry.CENOFF; +import static com.google.devtools.build.android.ziputils.DirectoryEntry.CENSIZ; +import static com.google.devtools.build.android.ziputils.EndOfCentralDirectory.ENDOFF; +import static com.google.devtools.build.android.ziputils.EndOfCentralDirectory.ENDSUB; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.FileChannel; +import java.util.Map.Entry; + +/** + * API for reading a zip file. This does not perform decompression of entry data, but provides + * a raw view of the content of a zip archive. + */ +public class ZipIn { + + private static final byte[] EOCD_SIG = {0x50, 0x4b, 0x05, 0x06}; + private static final byte[] HEADER_SIG = {0x50, 0x4b, 0x03, 0x04}; + private static final byte[] DATA_DESC_SIG = {0x50, 0x4b, 0x07, 0x08}; + + + /** + * Max end-of-central-directory size, including variable length file comment.. + */ + private static final int MAX_EOCD_SIZE = 1024; + + /** + * Max local file header size, including long filename. + */ + private static final int MAX_HEADER_SIZE = 64 * 1024; + + /** + * Default size of direct byte buffer used for reading content. Actual allocation will not + * exceed the archive content size, and may be at least as big as the largest entry. + */ + private static final int READ_BLOCK_SIZE = 20 * 1024 * 1024; + + private final String filename; // filename or nickname. + private final FileChannel fileChannel; // input file. + private BufferedFile bufferedFile; + private CentralDirectory cdir = null; + private EndOfCentralDirectory eocd = null; + private final boolean useDirectory; + private final boolean ignoreDeleted; + private final boolean verbose = false; + + /** + * Creates a {@code ZipIn} view of a file, with a (nick)name. + * + * @param channel File channel open for reading. + * @param filename filename or nickname. + */ + public ZipIn(FileChannel channel, String filename) { + this.fileChannel = channel; + this.filename = filename; + this.useDirectory = true; + this.ignoreDeleted = useDirectory; + } + + /** + * Gets the file name for this zip input file. + * @return the filename set at time of construction. + */ + public String getFilename() { + return filename; + } + + /** + * Returns a view of the "end of central directory" record expected at (or towards) the end of a + * zip file. + * + * @return A read-only, {@link EndOfCentralDirectory}. + * @throws IOException + */ + public EndOfCentralDirectory endOfCentralDirectory() throws IOException { + if (eocd == null) { + loadEndOfCentralDirectory(); + } + return eocd; + } + + /** + * Returns a memory mapped view of the central directory. + * + * @return A read-only, {@link CentralDirectory} of the central directory. + * @throws IOException + */ + public CentralDirectory centralDirectory() throws IOException { + if (cdir == null) { + loadCentralDirectory(); + } + return cdir; + } + + /** + * Scans all entries in the zip file and invokes the given {@link EntryHandler} on each. + * + * @param handler handler to invoke for each file entry. + * @throws IOException + */ + public void scanEntries(EntryHandler handler) throws IOException { + centralDirectory(); + ZipEntry zipEntry = nextFrom(null); + while (zipEntry.getCode() != ZipEntry.Status.ENTRY_NOT_FOUND) { + if (zipEntry.getCode() != ZipEntry.Status.ENTRY_OK) { + throw new IOException(zipEntry.getCode().toString()); + } + handler.handle(this, zipEntry.getHeader(), zipEntry.getDirEntry(), zipEntry.getContent()); + if (useDirectory && ignoreDeleted) { + zipEntry = ZipIn.this.nextFrom(zipEntry.getDirEntry()); + } else { + zipEntry = nextFrom(zipEntry.limit()); + } + } + } + + /** + * Finds the next header, by scanning for a local header signature starting + * at {@code offset}. This method will find headers for deleted or updated entries that + * are not listed in the central directory, and may pickup false positive (e.g. entries + * of an embedded zip file stored without compression). This method is primarily intended + * for applications trying to recover data from corrupt archives. + * + * @param offset offset where to start the search. + * @return the next local header at or beyond {@code offset}, or {@code null} if no + * header is found. + * @throws IOException + */ + public LocalFileHeader nextHeaderFrom(long offset) throws IOException { + int skipped = 0; + for (ByteBuffer buffer = getData(offset + skipped, MAX_HEADER_SIZE); + buffer.limit() >= LocalFileHeader.SIZE; + buffer = getData(offset + skipped, MAX_HEADER_SIZE)) { + int markerOffset = ScanUtil.scanTo(HEADER_SIG, buffer); + if (markerOffset < 0) { + skipped += buffer.limit() - 3; + } else { + skipped += markerOffset; + LocalFileHeader header = markerOffset == 0 ? localHeaderIn(buffer, offset + skipped) + : localHeaderAt(offset + skipped); + if (header != null) { + if (skipped > 0) { + System.out.println("Warning: local header search: skipped " + skipped + " bytes"); + } + return header; + } + // If localHeaderIn or localHeaderAt decided it is not a header location, + // we continue the search. + skipped += 4; + } + } + return null; + } + + /** + * Finds the header at the next higher offset listed in the central directory as containing + * a local file header, starting from the offset of the given {@code dirEntry}. This method will + * bypass any deleted or updated entries not listed in the directory, and also any entries from + * embedded zip files, or random instance of the header signature. This is the preferred method + * for sequentially reading the entries of a valid zip file. + * + * @param dirEntry directory entry for the "current entry", providing the start point + * for searching the central directory for the entry with the next higher offset. + * @return the next header according to the central directory, or {@code null} if there are no + * more headers. + * @throws IOException + */ + public LocalFileHeader nextHeaderFrom(DirectoryEntry dirEntry) throws IOException { + Integer nextOffset = dirEntry == null ? -1 : dirEntry.get(CENOFF); + while ((nextOffset = cdir.mapByOffset().higherKey(nextOffset)) != null) { + LocalFileHeader header = localHeaderAt(nextOffset); + if (header != null) { + return header; + } + System.out.println("Warning: no header for file listed in directory " + + dirEntry.getFilename()); + // The file is corrupt! Continue to see how bad it is. + } + return null; + } + + /** + * Provides a {@code LocalFileHeader} view of a local header located at the offset indicated + * by the given {@code dirEntry}. + * + * @param dirEntry the directory entry referring to the headers location. + * @return the requested header, or {@code null} if the given location can't possibly contain a + * valid file header (e.g. missing header signature), or if {@code dirEntry} is {@code null}. + * @throws IOException + */ + public LocalFileHeader localHeaderFor(DirectoryEntry dirEntry) throws IOException { + return dirEntry == null ? null : localHeaderAt(dirEntry.get(CENOFF)); + } + + /** + * Provides a {@code LocalFileHeader} view of a local header located at the offset indicated + * by the given {@code dirEntry}. + * + * @param offset offset a which the a header is presumed to exist. + * @return the requested header, or {@code null} if the given location can't possibly contain a + * valid file header (e.g. missing header signature). + * @throws IOException + */ + public LocalFileHeader localHeaderAt(long offset) throws IOException { + return localHeaderIn(getData(offset, MAX_HEADER_SIZE), offset); + } + + /** + * Finds the next zip file entry, by scanning for a local header using the + * {@link #nextHeaderFrom(long) }method. + * + * @param offset offset where to start the search. + * @return a {@code ZipEntry} object with the result of the search. + * @throws IOException + */ + public ZipEntry nextFrom(long offset) throws IOException { + LocalFileHeader header = ZipIn.this.nextHeaderFrom(offset); + return entryWith(header); + } + + /** + * Finds the next zip file entry, by first invoking + * {@link #nextHeaderFrom(com.google.devtools.build.android.ziputils.DirectoryEntry) } + * to find its header. + * + * @param entry the directory entry for the "current" zip entry, or {@code null} to get + * the first entry. + * @return a {@code ZipEntry} object with the result of the search. + * @throws IOException + */ + public ZipEntry nextFrom(DirectoryEntry entry) throws IOException { + int offset = entry == null ? -1 : entry.get(CENOFF); + Entry<Integer, DirectoryEntry> mapEntry = cdir.mapByOffset().higherEntry(offset); + if (mapEntry == null) { + return entryWith(null); + } + LocalFileHeader header = localHeaderAt(mapEntry.getKey()); + return entryWith(header, mapEntry.getValue()); + } + + /** + * Finds the zip file entry, for a given directory entry. + * + * @param entry the directory entry for which a zip entry is requested. + * @return a {@code ZipEntry} object with the result of the search. + * @throws IOException + */ + public ZipEntry entryFor(DirectoryEntry entry) throws IOException { + return entryWith(localHeaderFor(entry), entry); + } + + /** + * Returns the zip file entry at the given offset. + * + * @param offset presumed location of local file header. + * @return a {@link ZipEntry} for the given location. + * @throws IOException + */ + public ZipEntry entryAt(long offset) throws IOException { + LocalFileHeader header = localHeaderAt(offset); + return entryWith(header); + } + + /** + * Constructs a {@link ZipEntry} view of the entry at the location of the given header. + * + * @param header a previously located header. If (@code useDirectory} is set, this will + * attempt to lookup a corresponding directory entry. If there is none, and {@code ignoreDeleted} + * is also set, the return value will flag this entry with a + * {@code ZipEntry.Status.ENTRY_NOT_FOUND} status code. + * + * @return {@link ZipEntry} for the given location. + * @throws IOException + */ + public ZipEntry entryWith(LocalFileHeader header) throws IOException { + if (header == null) { + return new ZipEntry().withCode(ZipEntry.Status.ENTRY_NOT_FOUND); + } + // header != null + long offset = header.fileOffset(); + DirectoryEntry dirEntry = null; + if (useDirectory) { + dirEntry = cdir.mapByOffset().get((int) offset); + if (dirEntry == null && ignoreDeleted) { + return new ZipEntry().withCode(ZipEntry.Status.ENTRY_DELETED); + } + } + return entryWith(header, dirEntry); + } + + /** + * Scans for a data descriptor from a given offset. + * + * @param offset position where to start the search. + * @param dirEntry directory entry for validation, or {@code null}. + * @return A data descriptor view for the next position containing the data descriptor signature. + * @throws IOException + */ + public DataDescriptor descriptorFrom(final long offset, final DirectoryEntry dirEntry) + throws IOException { + int skipped = 0; + for (ByteBuffer buffer = getData(offset + skipped, MAX_HEADER_SIZE); + buffer.limit() >= 16; buffer = getData(offset + skipped, MAX_HEADER_SIZE)) { + int markerOffset = ScanUtil.scanTo(DATA_DESC_SIG, buffer); + if (markerOffset < 0) { + skipped += buffer.limit() - 3; + } else { + skipped += markerOffset; + return markerOffset == 0 ? descriptorIn(buffer, offset + skipped, dirEntry) + : descriptorAt(offset + skipped, dirEntry); + } + } + return null; + } + + /** + * Creates a data descriptor view at a given offset. + * + * @param offset presumed location of data descriptor. + * @param dirEntry directory entry to use for validation, or {@code null}. + * @return a data descriptor view over the given file offset. + * @throws IOException + */ + public DataDescriptor descriptorAt(long offset, DirectoryEntry dirEntry) throws IOException { + return descriptorIn(getData(offset, 16), offset, dirEntry); + } + + /** + * Constructs a zip entry object for the location of the given header, with the corresponding + * directory entry. + * + * @param header local file header for the entry. + * @param dirEntry corresponding directory entry, or {@code null} if not available. + * @return a zip entry with the given header and directory entry. + * @throws IOException + */ + private ZipEntry entryWith(LocalFileHeader header, DirectoryEntry dirEntry) throws IOException { + ZipEntry zipEntry = new ZipEntry().withHeader(header).withEntry(dirEntry); + int offset = (int) (header.fileOffset() + header.getSize()); + // !useDirectory || dirEntry != null || !ignoreDeleted + String entryName = header.getFilename(); + if (dirEntry != null && !entryName.equals(dirEntry.getFilename())) { + return zipEntry.withEntry(dirEntry).withCode(ZipEntry.Status.FILENAME_ERROR); + } + int sizeByHeader = header.dataSize(); + int sizeByDir = dirEntry != null ? dirEntry.dataSize() : -1; + ByteBuffer content; + if (sizeByDir == sizeByHeader && sizeByDir >= 0) { + // Ideal case, header and directory in agreement + content = getData(offset, sizeByHeader); + if (content.limit() == sizeByHeader) { + return zipEntry.withContent(content).withCode(ZipEntry.Status.ENTRY_OK); + } else { + return zipEntry.withContent(content).withCode(ZipEntry.Status.NOT_ENOUGH_DATA); + } + } + if (sizeByDir >= 0) { + // If file is correct, we get here because of a 0x8 flag, and we expect + // data to be followed by a data descriptor. + content = getData(offset, sizeByDir); + DataDescriptor dataDesc = descriptorAt(offset + sizeByDir, dirEntry); + if (dataDesc != null) { + return zipEntry.withContent(content).withDescriptor(dataDesc).withCode( + ZipEntry.Status.ENTRY_OK); + } + return zipEntry.withContent(content).withCode(ZipEntry.Status.NO_DATA_DESC); + } + if (!ignoreDeleted) { + if (sizeByHeader >= 0) { + content = getData(offset, sizeByHeader); + if (content.limit() == sizeByHeader) { + return zipEntry.withContent(content).withCode(ZipEntry.Status.ENTRY_OK); + } + return zipEntry.withContent(content).withCode(ZipEntry.Status.NOT_ENOUGH_DATA); + } else { + + DataDescriptor dataDesc = descriptorFrom(offset, dirEntry); + if (dataDesc == null) { + // Only way now would be to decompress + return zipEntry.withCode(ZipEntry.Status.UNKNOWN_SIZE); + } + int sizeByDesc = dataDesc.get(EXTSIZ); + if (sizeByDesc != dataDesc.fileOffset() - offset) { + // That just can't be the right + return zipEntry.withDescriptor(dataDesc).withCode(ZipEntry.Status.UNKNOWN_SIZE); + } + content = getData(offset, sizeByDesc); + return zipEntry.withContent(content).withDescriptor(dataDesc).withCode( + ZipEntry.Status.ENTRY_OK); + } + } + return zipEntry.withCode(ZipEntry.Status.UNKNOWN_SIZE); + } + + /** + * Constructs a local header view over a give byte buffer. + * + * @param buffer byte buffer with local header data. + * @param offset file offset at which the buffer is based. + * @return a local header view. + */ + private LocalFileHeader localHeaderIn(ByteBuffer buffer, long offset) { + return buffer.limit() < LocalFileHeader.SIZE + || buffer.getInt(0) != LocalFileHeader.SIGNATURE + ? null : LocalFileHeader.viewOf(buffer).at(offset); + } + + /** + * Constructs a data descriptor view over a given byte buffer. + * + * @param buf byte buffer with data descriptor data. + * @param offset file offset at which the buffer is based. + * @param dirEntry directory entry with presumed reliable content size information. + * @return a data descriptor + */ + private DataDescriptor descriptorIn(ByteBuffer buf, long offset, DirectoryEntry dirEntry) { + if (buf.limit() < 12) { + return null; + } + DataDescriptor desc = DataDescriptor.viewOf(buf).at(offset); + if (desc.hasMarker() || (dirEntry != null + && desc.get(EXTSIZ) == dirEntry.get(CENSIZ) + && desc.get(EXTLEN) == dirEntry.get(CENLEN))) { + return desc; + } + return null; + } + + /** + * Obtains a byte buffer at a given offset. + */ + private ByteBuffer getData(long offset, int size) throws IOException { + return bufferedFile.getBuffer(offset, size).order(ByteOrder.LITTLE_ENDIAN); + } + + /** + * Locates the "end of central directory" record, expected located at the end of the file, and + * reads it into a byte buffer. Called on the first invocation of + * {@link #endOfCentralDirectory() }. + * + * @throws IOException + */ + protected void loadEndOfCentralDirectory() throws IOException { + cdir = null; + long size = fileChannel.size(); + verbose("Loading ZipIn: " + filename); + verbose("-- size: " + size); + int cap = (int) Math.min(size, MAX_EOCD_SIZE); + ByteBuffer buffer = ByteBuffer.allocate(cap).order(ByteOrder.LITTLE_ENDIAN); + long offset = size - cap; + while (true) { + fileChannel.position(offset); + while (buffer.hasRemaining()) { + fileChannel.read(buffer, offset); + } + // scan to find it... + int endOfDirOffset = ScanUtil.scanBackwardsTo(EOCD_SIG, buffer); + if (endOfDirOffset < 0) { + if (offset == 0) { + if (useDirectory) { + throw new IllegalStateException("No end of central directory marker"); + } else { + break; + } + } + offset = Math.max(offset - 1000, 0); + buffer.clear(); + continue; + } + long eocdFileOffset = offset + endOfDirOffset; + verbose("-- EOCD: " + eocdFileOffset + " size: " + (size - eocdFileOffset)); + buffer.position(endOfDirOffset); + eocd = EndOfCentralDirectory.viewOf(buffer).at(offset + endOfDirOffset); + // TODO (bazel-team): check that the end of central directory, points to a valid + // first directory entry. If not, assume we happened to find the signature inside + // a file comment, and resume the search. + break; + } + + if (eocd != null) { + bufferedFile = new BufferedFile(fileChannel, 0, eocd.get(ENDOFF), + READ_BLOCK_SIZE); + } else { + bufferedFile = new BufferedFile(fileChannel, READ_BLOCK_SIZE); + } + } + + /** + * Maps the central directory to memory. Called on the first invocation of + * {@link #centralDirectory() }. + * + * @throws IOException + */ + protected void loadCentralDirectory() throws IOException { + if (eocd == null) { + loadEndOfCentralDirectory(); + } + if (eocd == null) { + return; + } + long cdOffset = eocd.get(ENDOFF); + long len = eocd.fileOffset() - cdOffset; + verbose("-- CDIR: " + cdOffset + " size: " + len + " count: " + eocd.get(ENDSUB)); + // Read directory to buffer. + // TODO(bazel-team): we currently assume the directory fits in memory (and int). + ByteBuffer buffer = ByteBuffer.allocateDirect((int) len); + while (len > 0) { + int read = fileChannel.read(buffer, cdOffset); + len -= read; + cdOffset += read; + } + buffer.rewind(); + cdir = CentralDirectory.viewOf(buffer).at(cdOffset).parse(); + cdir.buffer.flip(); + } + + /** + * Zip file entry container class, for use with the low-level scanning operations of this + * API, supporting zip file scanner construction. + */ + public static class ZipEntry { + + private LocalFileHeader header; + private DataDescriptor descriptor; + private ByteBuffer content; + private DirectoryEntry entry; + private Status code; + + /** + * Creates a zip entry, setting the initial status to not found. + */ + public ZipEntry() { + code = Status.ENTRY_NOT_FOUND; + } + + /** + * Gets the header of this zip entry. + */ + public LocalFileHeader getHeader() { + return header; + } + + /** + * Sets the header of this zip entry. + * @return this object. + */ + public ZipEntry withHeader(LocalFileHeader header) { + this.header = header; + return this; + } + + /** + * Gets the data descriptor of this zip entry, if any. + */ + public DataDescriptor getDescriptor() { + return descriptor; + } + + /** + * Sets the data descriptor of this zip entry. + * @return this object. + */ + public ZipEntry withDescriptor(DataDescriptor descriptor) { + this.descriptor = descriptor; + return this; + } + + /** + * Gets a byte buffer for accessing the raw content of this zip entry. + */ + public ByteBuffer getContent() { + return content; + } + + /** + * Sets the byte buffer providing access to the raw content of this zip entry. + * @return this object + */ + public ZipEntry withContent(ByteBuffer content) { + this.content = content; + return this; + } + + /** + * Gets the central directory entry for this zip entry, if any. + */ + public DirectoryEntry getDirEntry() { + return entry; + } + + /** + * Sets the central directory entry for this zip entry. + * @return this object. + */ + public ZipEntry withEntry(DirectoryEntry entry) { + this.entry = entry; + return this; + } + + /** + * Gets the status code for parsing this zip entry. + */ + public Status getCode() { + return code; + } + + /** + * Sets the status code for this zip entry. + * @return this object. + */ + public ZipEntry withCode(Status code) { + this.code = code; + return this; + } + + /** + * Calculates, best-effort, the file offset just past this zip entry. + */ + public long limit() { + if (header == null) { + return 0; + } + if (descriptor != null) { + return descriptor.fileOffset() + descriptor.getSize(); + } + long offset = header.fileOffset() + header.dataSize(); + if (content != null) { + offset += content.limit(); + } + return offset; + } + + /** + * Zip entry parsing status codes. + */ + public enum Status { + /** + * This zip entry contains valid header and data + */ + ENTRY_OK, + /** + * No header at the given location + */ + ENTRY_NOT_FOUND, + /** + * The given location contains a header that is not listed in the central directory + */ + ENTRY_DELETED, + /** + * The header in the given location has a different filename than the + * directory entry for this location. + */ + FILENAME_ERROR, + /** + * The given location has the header signature, but the remaining data is insufficient + * to constitute a complete entry. + */ + NOT_ENOUGH_DATA, + /** + * The entry appears to be missing an expected data descriptor. + */ + NO_DATA_DESC, + /** + * The implementation was unable to determine the size of the content of the entry. + * The client will have to either parse using the central directory, or if all else + * fails, attempt to decompress the entry. + */ + UNKNOWN_SIZE, + } + } + + private void verbose(String msg) { + if (verbose) { + System.out.println(msg); + } + } +} diff --git a/src/tools/android/java/com/google/devtools/build/android/ziputils/ZipOut.java b/src/tools/android/java/com/google/devtools/build/android/ziputils/ZipOut.java new file mode 100644 index 0000000000..c1954d4dea --- /dev/null +++ b/src/tools/android/java/com/google/devtools/build/android/ziputils/ZipOut.java @@ -0,0 +1,227 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.android.ziputils; + +import static com.google.devtools.build.android.ziputils.DirectoryEntry.CENOFF; +import static com.google.devtools.build.android.ziputils.EndOfCentralDirectory.ENDOFF; +import static com.google.devtools.build.android.ziputils.EndOfCentralDirectory.ENDSIZ; +import static com.google.devtools.build.android.ziputils.EndOfCentralDirectory.ENDSUB; +import static com.google.devtools.build.android.ziputils.EndOfCentralDirectory.ENDTOT; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +/** + * API for writing to a zip archive. This does not currently perform compression, + * but merely provides the facilities for creating a zip archive. The client must + * ensure that file content is written conforming to the created headers. + */ +public class ZipOut { + /** + * Central directory output buffer block size + */ + private static final int DIR_BLOCK_SIZE = 1024 * 1024; + + private final String filename; + // To ensure good performance when writing to a remote file system + // we need to use asynchronous output. However, because we may want this + // to run on and off devices, we can't depend on java.nio.AsynchronousFileChannel (JDK 1.7). + // Instead, we use a FileChannel (JDK 1.4), and use a single-thread pool, + // to execute writes serially, but non-blocking from our client's + // point-of-view. All data written, are assumed to remain unchanged until the write is complete. + // ZipIn is designed to not reuse internal buffers, to make direct data transfer safe. + // This optimizes the common cases where input is processed serially. + private final FileChannel fileChannel; + private final ExecutorService executor; + private final List<Future<?>> futures; + private final List<CentralDirectory> centralDirectory; + private CentralDirectory current; + private int fileOffset = 0; + private int entryCount = 0; + private boolean finished = false; + private final boolean verbose = false; + + /** + * Creates a {@code ZipOut} for writing to file, with the given (nick)name. + * + * @param channel File channel open for output. + * @param filename File name or nickname. + * @throws java.io.IOException + */ + public ZipOut(FileChannel channel, String filename) throws IOException { + this.executor = Executors.newSingleThreadExecutor(); + this.futures = new ArrayList<>(); + this.fileChannel = channel; + this.filename = filename; + centralDirectory = new ArrayList<>(); + fileOffset = (int) fileChannel.position(); + } + + /** + * Returns a writable copy of the given + * {@link com.google.devtools.build.android.ziputils.DirectoryEntry}, backed by an internal + * direct byte buffer, allocated over storage for this files central directory. The file offset + * is set, and must not be changed. The variable entry data (filename, extra data and comment), + * must not be changed (in a way that changes the total size of the directory entry). + * @param entry directory entry to copy. + * @return a writable directory entry view, over the provided byte buffer. + */ + public DirectoryEntry nextEntry(DirectoryEntry entry) { + entryCount++; + int size = entry.getSize(); + if (current == null || current.buffer.remaining() < size) { + ByteBuffer buffer = ByteBuffer.allocateDirect(DIR_BLOCK_SIZE); + current = CentralDirectory.viewOf(buffer); + centralDirectory.add(current); + } + return current.nextEntry(entry).set(CENOFF, fileOffset); + } + + /** + * Writes content to the current entry. Content is written as-is, and the client is responsible + * for compression, consistent with the storage method set in the current directory entry. The + * client must first write an appropriate local file header, and if necessary, complete the entry + * with a data descriptor. If the header indicates that the content is compressed, the client is + * responsible for compressing the data before writing. + * + * <p>Data is written serially, but asynchronously, to the output file. The client must not change + * the underlying data after it has been scheduled for writing. Usually, a client will release + * any references to the data, so that storage may be eligible for GC, once the write operation + * has completed. It's safe to pass references to views of data obtained from a {#link ZipIn}, + * object, because {@code ZipIn} doesn't reuse internal buffers. + * + * @param content + */ + public synchronized void write(ByteBuffer content) { + fileOffset += content.remaining(); + futures.add(executor.submit(new OutputTask(content))); + } + + /** + * Writes a {@link com.google.devtools.build.android.ziputils.View} to the current entry. + * Used to write a {@link com.google.devtools.build.android.ziputils.LocalFileHeader} + * before the content, and if needed, a + * {@link com.google.devtools.build.android.ziputils.DataDescriptor} after the content. + * <P> + * See also {@link #write(java.nio.ByteBuffer)}. + * </P> + * + * @param view the view to write as part of the current entry. + * @throws java.io.IOException + */ + public void write(View<?> view) throws IOException { + view.at(fileOffset).buffer.rewind(); + write(view.buffer); + } + + /** + * Returns the file position for the next write operation. Because writes are asynchronous, this + * may not be the actual position of the underlying file channel. + * @return the file position position for the next write operation. + */ + public int fileOffset() { + return fileOffset; + } + + /** + * Writes out the central directory. This doesn't close the output file. + */ + public void finish() throws IOException { + if (finished) { + return; + } + finished = true; + int cdOffset = fileOffset; + for (CentralDirectory cd : centralDirectory) { + //cd.finish().buffer.rewind(); + cd.buffer.flip(); + write(cd.buffer); + } + int size = fileOffset - cdOffset; + verbose("ZipOut finishing: " + filename); + verbose("-- CDIR: " + cdOffset + " count: " + entryCount); + verbose("-- EOCD: " + fileOffset + " size: " + size); + EndOfCentralDirectory eocd = EndOfCentralDirectory.allocate(null) + .set(ENDSUB, (short) entryCount) + .set(ENDTOT, (short) entryCount) + .set(ENDSIZ, size) + .set(ENDOFF, cdOffset) + .at(fileOffset); + eocd.buffer.rewind(); + write(eocd.buffer); + verbose("-- size: " + fileOffset); + } + + /** + * Closes the output file. If this object has not been finished yet, this method will call + * {@link #finish()} before closing the output channel. + * + * @throws java.io.IOException + */ + public void close() throws IOException { + if (!finished) { + finish(); + } + try { + executor.shutdown(); + executor.awaitTermination(30, TimeUnit.SECONDS); + for (Future<?> f : futures) { + try { + f.get(); + } catch (ExecutionException ex) { + throw new IOException(ex.getCause().getMessage()); + } + } + } catch (InterruptedException ex) { + executor.shutdownNow(); + Thread.currentThread().interrupt(); + } + fileChannel.close(); + } + + /** + * Helper class to write asynchronously to the output channel. + */ + private class OutputTask implements Runnable { + + final ByteBuffer buffer; + + public OutputTask(ByteBuffer buffer) { + this.buffer = buffer; + } + + @Override + public void run() { + try { + fileChannel.write(buffer); + } catch (IOException ex) { + throw new IllegalStateException("Unexpected IOException writing to output channel"); + } + } + } + + private void verbose(String msg) { + if (verbose) { + System.out.println(msg); + } + } +} |