diff options
author | Googler <noreply@google.com> | 2015-04-02 15:06:46 +0000 |
---|---|---|
committer | Kristina Chodorow <kchodorow@google.com> | 2015-04-03 20:36:27 +0000 |
commit | 61a8f1d87b4439599c89e5dcdd83b4efcc6a8b97 (patch) | |
tree | 6d4d44acda4df05bdd3678ec3de654194ecb0b21 /src/java_tools/singlejar | |
parent | d10e352331b4ca95e5fb86856b60deae9af29b3d (diff) |
Description redacted.
--
MOS_MIGRATED_REVID=90162183
Diffstat (limited to 'src/java_tools/singlejar')
21 files changed, 3548 insertions, 1874 deletions
diff --git a/src/java_tools/singlejar/BUILD b/src/java_tools/singlejar/BUILD index bc33f28df9..ab0eae6a4f 100644 --- a/src/java_tools/singlejar/BUILD +++ b/src/java_tools/singlejar/BUILD @@ -2,8 +2,9 @@ package(default_visibility = ["//src:__subpackages__"]) java_library( name = "libSingleJar", - srcs = glob(["java/**/*.java"]), + srcs = glob(["java/**/singlejar/**/*.java"]), deps = [ + ":zip", "//src/main/java:shell", "//third_party:guava", "//third_party:jsr305", @@ -18,10 +19,11 @@ java_binary( java_test( name = "tests", - srcs = glob(["javatests/**/*.java"]), + srcs = glob(["javatests/**/singlejar/**/*.java"]), args = ["com.google.devtools.build.singlejar.SingleJarTests"], deps = [ ":libSingleJar", + ":zip", "//src/main/java:shell", "//src/test/java:testutil", "//third_party:guava", @@ -30,3 +32,24 @@ java_test( "//third_party:truth", ], ) + +java_library( + name = "zip", + srcs = glob(["java/**/zip/**/*.java"]), + deps = [ + "//third_party:jsr305", + ], +) + +java_test( + name = "zipTests", + srcs = glob(["javatests/**/zip/**/*.java"]), + args = ["com.google.devtools.build.zip.ZipTests"], + deps = [ + ":zip", + "//src/test/java:testutil", + "//third_party:guava", + "//third_party:junit4", + "//third_party:truth", + ], +) diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/JarUtils.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/JarUtils.java index a9c8ee3f65..a21f58307d 100644 --- a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/JarUtils.java +++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/JarUtils.java @@ -14,6 +14,8 @@ package com.google.devtools.build.singlejar; +import com.google.devtools.build.zip.ExtraData; + import java.io.IOException; import java.util.Date; diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/JavaIoFileSystem.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/JavaIoFileSystem.java index 0da6e33040..f66f23dba2 100644 --- a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/JavaIoFileSystem.java +++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/JavaIoFileSystem.java @@ -37,6 +37,11 @@ public final class JavaIoFileSystem implements SimpleFileSystem { } @Override + public File getFile(String filename) throws IOException { + return new File(filename); + } + + @Override public boolean delete(String filename) { return new File(filename).delete(); } diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/SimpleFileSystem.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/SimpleFileSystem.java index 844f12b714..1bacc3c529 100644 --- a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/SimpleFileSystem.java +++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/SimpleFileSystem.java @@ -16,6 +16,7 @@ package com.google.devtools.build.singlejar; import com.google.devtools.build.singlejar.OptionFileExpander.OptionFileProvider; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -36,6 +37,11 @@ public interface SimpleFileSystem extends OptionFileProvider { OutputStream getOutputStream(String filename) throws IOException; /** + * Returns the File object in the file system for this filename. + */ + File getFile(String filename) throws IOException; + + /** * Delete the file with the given name and return whether deleting it was * successfull. */ diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/SingleJar.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/SingleJar.java index 4551fd1813..8fd677e1b7 100644 --- a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/SingleJar.java +++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/SingleJar.java @@ -19,6 +19,7 @@ import com.google.devtools.build.singlejar.ZipCombiner.OutputMode; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; @@ -80,10 +81,10 @@ public class SingleJar { protected boolean includeBuildData = true; /** List of build information properties files */ - protected List<String> buildInformationFiles = new ArrayList<String>(); + protected List<String> buildInformationFiles = new ArrayList<>(); /** Extraneous build informations (key=value) */ - protected List<String> buildInformations = new ArrayList<String>(); + protected List<String> buildInformations = new ArrayList<>(); /** The (optional) native executable that will be prepended to this JAR. */ private String launcherBin = null; @@ -223,21 +224,8 @@ public class SingleJar { // Copy the jars into the jar file. for (String inputJar : inputJars) { - InputStream in = fileSystem.getInputStream(inputJar); - try { - combiner.addZip(inputJar, in); - InputStream inToClose = in; - in = null; - inToClose.close(); - } finally { - if (in != null) { - try { - in.close(); - } catch (IOException e) { - // Preserve original exception. - } - } - } + File jar = fileSystem.getFile(inputJar); + combiner.addZip(jar); } // Close the output file. If something goes wrong here, delete the file. diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ZipCombiner.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ZipCombiner.java index d38c6d4493..75dacb92ef 100644 --- a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ZipCombiner.java +++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ZipCombiner.java @@ -14,44 +14,43 @@ package com.google.devtools.build.singlejar; -import static java.nio.charset.StandardCharsets.ISO_8859_1; import static java.nio.charset.StandardCharsets.UTF_8; import com.google.devtools.build.singlejar.ZipEntryFilter.CustomMergeStrategy; import com.google.devtools.build.singlejar.ZipEntryFilter.StrategyCallback; +import com.google.devtools.build.zip.ExtraData; +import com.google.devtools.build.zip.ZipFileEntry; +import com.google.devtools.build.zip.ZipFileEntry.Compression; +import com.google.devtools.build.zip.ZipReader; +import com.google.devtools.build.zip.ZipUtil; +import com.google.devtools.build.zip.ZipWriter; import java.io.BufferedOutputStream; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.EOFException; -import java.io.FilterOutputStream; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Calendar; import java.util.Date; -import java.util.GregorianCalendar; import java.util.HashMap; -import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.concurrent.atomic.AtomicBoolean; import java.util.zip.CRC32; -import java.util.zip.DataFormatException; import java.util.zip.Deflater; +import java.util.zip.DeflaterInputStream; import java.util.zip.Inflater; +import java.util.zip.InflaterInputStream; import javax.annotation.Nullable; -import javax.annotation.concurrent.NotThreadSafe; /** * An object that combines multiple ZIP files into a single file. It only * supports a subset of the ZIP format, specifically: * <ul> * <li>It only supports STORE and DEFLATE storage methods.</li> - * <li>There may be no data before the first file or between files.</li> - * <li>It ignores any data after the last file.</li> + * <li>It only supports 32-bit ZIP files.</li> * </ul> * * <p>These restrictions are also present in the JDK implementations @@ -68,1551 +67,538 @@ import javax.annotation.concurrent.NotThreadSafe; * <p>Also see: * <a href="http://www.pkware.com/documents/casestudies/APPNOTE.TXT">ZIP format</a> */ -@NotThreadSafe -public final class ZipCombiner implements AutoCloseable { - - /** - * A Date set to the 1/1/1980, 00:00:00, the minimum value that can be stored - * in a ZIP file. - */ - public static final Date DOS_EPOCH = new GregorianCalendar(1980, 0, 1, 0, 0, 0).getTime(); - - private static final int DEFAULT_CENTRAL_DIRECTORY_BLOCK_SIZE = 1048576; // 1 MB for each block - - // The following constants are ZIP-specific. - private static final int LOCAL_FILE_HEADER_MARKER = 0x04034b50; - private static final int DATA_DESCRIPTOR_MARKER = 0x08074b50; - private static final int CENTRAL_DIRECTORY_MARKER = 0x02014b50; - private static final int END_OF_CENTRAL_DIRECTORY_MARKER = 0x06054b50; - - private static final int FILE_HEADER_BUFFER_SIZE = 30; - - private static final int VERSION_TO_EXTRACT_OFFSET = 4; - private static final int GENERAL_PURPOSE_FLAGS_OFFSET = 6; - private static final int COMPRESSION_METHOD_OFFSET = 8; - private static final int MTIME_OFFSET = 10; - private static final int MDATE_OFFSET = 12; - private static final int CRC32_OFFSET = 14; - private static final int COMPRESSED_SIZE_OFFSET = 18; - private static final int UNCOMPRESSED_SIZE_OFFSET = 22; - private static final int FILENAME_LENGTH_OFFSET = 26; - private static final int EXTRA_LENGTH_OFFSET = 28; - - private static final int DIRECTORY_ENTRY_BUFFER_SIZE = 46; - - // Set if the size, compressed size and CRC are set to zero, and present in - // the data descriptor after the data. - private static final int SIZE_MASKED_FLAG = 1 << 3; - - private static final int STORED_METHOD = 0; - private static final int DEFLATE_METHOD = 8; - - private static final int VERSION_STORED = 10; // Version 1.0 - private static final int VERSION_DEFLATE = 20; // Version 2.0 - - private static final long MAXIMUM_DATA_SIZE = 0xffffffffL; - - // This class relies on the buffer to have sufficient space for a complete - // file name. 2^16 is the maximum number of bytes in a file name. - private static final int BUFFER_SIZE = 65536; - - /** An empty entry used to skip files that have already been copied (or skipped). */ - private static final FileEntry COPIED_FILE_ENTRY = new FileEntry(null, null, 0); - - /** An empty entry used to mark files that have already been renamed. */ - private static final FileEntry RENAMED_FILE_ENTRY = new FileEntry(null, null, 0); - - /** A zero length array of ExtraData. */ - public static final ExtraData[] NO_EXTRA_ENTRIES = new ExtraData[0]; - +public class ZipCombiner implements AutoCloseable { + public static final Date DOS_EPOCH = new Date(ZipUtil.DOS_EPOCH); /** * Whether to compress or decompress entries. */ public enum OutputMode { - /** * Output entries using any method. */ DONT_CARE, - /** - * Output all entries using DEFLATE method, except directory entries. It is - * always more efficient to store directory entries uncompressed. + * Output all entries using DEFLATE method, except directory entries. It is always more + * efficient to store directory entries uncompressed. */ FORCE_DEFLATE, - /** * Output all entries using STORED method. */ - FORCE_STORED; + FORCE_STORED, } - // A two-element enum for copyOrSkip type methods. - private static enum SkipMode { - + /** + * The type of action to take for a ZIP file entry. + */ + private enum ActionType { /** - * Copy the read data to the output stream. + * Skip the entry. + */ + SKIP, + /** + * Copy the entry. */ COPY, - /** - * Do not write anything to the output stream. + * Rename the entry. */ - SKIP; - } - - /** - * Stores internal information about merges or skips. - */ - private static final class FileEntry { - - /** If null, the file should be skipped. Otherwise, it should be merged. */ - private final CustomMergeStrategy mergeStrategy; - private final ByteArrayOutputStream outputBuffer; - private final int dosTime; - - private FileEntry(CustomMergeStrategy mergeStrategy, ByteArrayOutputStream outputBuffer, - int dosTime) { - this.mergeStrategy = mergeStrategy; - this.outputBuffer = outputBuffer; - this.dosTime = dosTime; - } + RENAME, + /** + * Merge the entry. + */ + MERGE; } /** - * The directory entry info used for files whose extra directory entry info is not given - * explicitly. It uses {@code -1} for {@link DirectoryEntryInfo#withMadeByVersion(short)}, which - * indicates it will be set to the same version as "needed to extract." - * - * <p>The {@link DirectoryEntryInfo#withExternalFileAttribute(int)} value is set to {@code 0}, - * whose meaning depends on the value of {@code madeByVersion}, but is usually a reasonable - * default. - */ - public static final DirectoryEntryInfo DEFAULT_DIRECTORY_ENTRY_INFO = - new DirectoryEntryInfo((short) -1, 0); - - /** - * Contains information related to a zip entry that is stored in the central directory record. - * This does not contain all the information stored in the central directory record, only the - * information that can be customized and is not automatically calculated or detected. + * The action to take for a ZIP file entry. */ - public static final class DirectoryEntryInfo { - private final short madeByVersion; - private final int externalFileAttribute; - - private DirectoryEntryInfo(short madeByVersion, int externalFileAttribute) { - this.madeByVersion = madeByVersion; - this.externalFileAttribute = externalFileAttribute; - } + private static final class EntryAction { + private final ActionType type; + @Nullable private final Date date; + @Nullable private final String newName; + @Nullable private final CustomMergeStrategy strategy; + @Nullable private final ByteArrayOutputStream mergeBuffer; /** - * This will be written as "made by" version in the central directory. - * If -1 (default) then "made by" will be the same to version "needed to extract". + * Create an action of the specified type with no extra details. + * + * @param type the type of action */ - public DirectoryEntryInfo withMadeByVersion(short madeByVersion) { - return new DirectoryEntryInfo(madeByVersion, externalFileAttribute); + public EntryAction(ActionType type) { + this(type, null, null, null, null); } /** - * This will be written as external file attribute. The meaning of this depends upon the value - * set with {@link #withMadeByVersion(short)}. If that value indicates a Unix source, then this - * value has the file mode and permission bits in the upper two bytes (e.g. possibly - * {@code 0100644} for a regular file). + * Create a duplicate action with a different {@link ActionType}. + * + * @param type the type of action + * @param action the action to copy */ - public DirectoryEntryInfo withExternalFileAttribute(int externalFileAttribute) { - return new DirectoryEntryInfo(madeByVersion, externalFileAttribute); - } - } - - /** - * The central directory, which is grown as required; instead of using a single large buffer, we - * store a sequence of smaller buffers. With a single large buffer, whenever we grow the buffer by - * 2x, we end up requiring 3x the memory temporarily, which can lead to OOM problems even if there - * would still be enough memory. - * - * <p>The invariants for the fields are as follows: - * <ul> - * <li>All blocks must have the same size. - * <li>The list of blocks must contain all blocks, including the current block (even if empty). - * <li>The current block offset must apply to the last block in the list, which is - * simultaneously the current block. - * <li>The current block may only be {@code null} if the list is empty. - * </ul> - */ - private static final class CentralDirectory { - private final int blockSize; // We allow this to be overridden for testing. - private List<byte[]> blockList = new ArrayList<>(); - private byte[] currentBlock; - private int currentBlockOffset = 0; - private int size = 0; - - CentralDirectory(int centralDirectoryBlockSize) { - this.blockSize = centralDirectoryBlockSize; + public EntryAction(ActionType type, EntryAction action) { + this(type, action.getDate(), action.getNewName(), action.getStrategy(), + action.getMergeBuffer()); } /** - * Appends the given data to the central directory and returns the start - * offset within the central directory to allow back-patching. + * Create an action of the specified type and details. + * + * @param type the type of action + * @param date the custom date to set on the entry + * @param newName the custom name to create the entry as + * @param strategy the {@link CustomMergeStrategy} to use for merging this entry + * @param mergeBuffer the output stream to use for merge results */ - int writeToCentralDirectory(byte[] b, int off, int len) { - checkArgument(len >= 0); - int offsetStarted = size; - while (len > 0) { - if (currentBlock == null - || currentBlockOffset >= currentBlock.length) { - currentBlock = new byte[blockSize]; - currentBlockOffset = 0; - blockList.add(currentBlock); - } - int maxCopy = Math.min(blockSize - currentBlockOffset, len); - System.arraycopy(b, off, currentBlock, currentBlockOffset, maxCopy); - off += maxCopy; - len -= maxCopy; - size += maxCopy; - currentBlockOffset += maxCopy; - } - return offsetStarted; + public EntryAction(ActionType type, Date date, String newName, CustomMergeStrategy strategy, + ByteArrayOutputStream mergeBuffer) { + checkArgument(!(type == ActionType.RENAME && newName == null), + "NewName must not be null if the ActionType is RENAME."); + checkArgument(!(type == ActionType.MERGE && strategy == null), + "Strategy must not be null if the ActionType is MERGE."); + checkArgument(!(type == ActionType.MERGE && mergeBuffer == null), + "MergeBuffer must not be null if the ActionType is MERGE."); + this.type = type; + this.date = date; + this.newName = newName; + this.strategy = strategy; + this.mergeBuffer = mergeBuffer; } - /** Calls through to {@link #writeToCentralDirectory(byte[], int, int)}. */ - int writeToCentralDirectory(byte[] b) { - return writeToCentralDirectory(b, 0, b.length); + /** + * @return the type + */ + public ActionType getType() { + return type; } - /** - * Writes an unsigned int in little-endian byte order to the central directory at the - * given offset. Does not perform range checking. + * @return the date */ - void setUnsignedInt(int offset, int value) { - blockList.get(cdIndex(offset + 0))[cdOffset(offset + 0)] = (byte) (value & 0xff); - blockList.get(cdIndex(offset + 1))[cdOffset(offset + 1)] = (byte) ((value >> 8) & 0xff); - blockList.get(cdIndex(offset + 2))[cdOffset(offset + 2)] = (byte) ((value >> 16) & 0xff); - blockList.get(cdIndex(offset + 3))[cdOffset(offset + 3)] = (byte) ((value >> 24) & 0xff); + public Date getDate() { + return date; } - - private int cdIndex(int offset) { - return offset / blockSize; + /** + * @return the new name + */ + public String getNewName() { + return newName; } - - private int cdOffset(int offset) { - return offset % blockSize; + /** + * @return the strategy + */ + public CustomMergeStrategy getStrategy() { + return strategy; } - /** - * Writes the central directory to the given output stream and returns the size, i.e., the - * number of bytes written. + * @return the mergeBuffer */ - int writeTo(OutputStream out) throws IOException { - for (int i = 0; i < blockList.size() - 1; i++) { - out.write(blockList.get(i)); - } - if (currentBlock != null) { - out.write(currentBlock, 0, currentBlockOffset); - } - return size; + public ByteArrayOutputStream getMergeBuffer() { + return mergeBuffer; } } - /** - * An output stream that counts how many bytes were written. - */ - private static final class ByteCountingOutputStream extends FilterOutputStream { - private long bytesWritten = 0L; + private final class FilterCallback implements StrategyCallback { + private String filename; + private final AtomicBoolean called = new AtomicBoolean(); - ByteCountingOutputStream(OutputStream out) { - super(out); + public void resetForFile(String filename) { + this.filename = filename; + this.called.set(false); } - @Override - public void write(byte[] b, int off, int len) throws IOException { - out.write(b, off, len); - bytesWritten += len; + @Override public void skip() throws IOException { + checkCall(); + actions.put(filename, new EntryAction(ActionType.SKIP)); } - @Override - public void write(int b) throws IOException { - out.write(b); - bytesWritten++; + @Override public void copy(Date date) throws IOException { + checkCall(); + actions.put(filename, new EntryAction(ActionType.COPY, date, null, null, null)); } - } - - private final OutputMode mode; - private final ZipEntryFilter entryFilter; - - private final ByteCountingOutputStream out; - - // An input buffer to allow reading blocks of data. Keeping it here avoids - // another copy operation that would be required by the BufferedInputStream. - // The valid data is between bufferOffset and bufferOffset+bufferLength (exclusive). - private final byte[] buffer = new byte[BUFFER_SIZE]; - private int bufferOffset = 0; - private int bufferLength = 0; - - private String currentInputFile; - - // An intermediate buffer for the file header data. Keeping it here avoids - // creating a new buffer for every entry. - private final byte[] headerBuffer = new byte[FILE_HEADER_BUFFER_SIZE]; - - // An intermediate buffer for a central directory entry. Keeping it here - // avoids creating a new buffer for every entry. - private final byte[] directoryEntryBuffer = new byte[DIRECTORY_ENTRY_BUFFER_SIZE]; - // The Inflater is a class member to avoid creating a new instance for every - // entry in the ZIP file. - private final Inflater inflater = new Inflater(true); - - // The contents of this buffer are never read. The Inflater is only used to - // determine the length of the compressed data, and the buffer is a throw- - // away buffer for the decompressed data. - private final byte[] inflaterBuffer = new byte[BUFFER_SIZE]; - - private final Map<String, FileEntry> fileNames = new HashMap<>(); - - private final CentralDirectory centralDirectory; - private int fileCount = 0; - - private boolean finished = false; - - // Package private for testing. - ZipCombiner(OutputMode mode, ZipEntryFilter entryFilter, OutputStream out, - int centralDirectoryBlockSize) { - this.mode = mode; - this.entryFilter = entryFilter; - this.out = new ByteCountingOutputStream(new BufferedOutputStream(out)); - this.centralDirectory = new CentralDirectory(centralDirectoryBlockSize); - } - - /** - * Creates a new instance with the given parameters. The {@code entryFilter} - * is called for every entry in the ZIP files and the combined ZIP file is - * written to {@code out}. The output mode determines whether entries must be - * written in compressed or decompressed form. Note that the result is - * invalid if an exception is thrown from any of the methods in this class, - * and before a call to {@link #close} or {@link #finish}. - */ - public ZipCombiner(OutputMode mode, ZipEntryFilter entryFilter, OutputStream out) { - this(mode, entryFilter, out, DEFAULT_CENTRAL_DIRECTORY_BLOCK_SIZE); - } - - /** - * Creates a new instance with the given parameters and the DONT_CARE mode. - */ - public ZipCombiner(ZipEntryFilter entryFilter, OutputStream out) { - this(OutputMode.DONT_CARE, entryFilter, out); - } - - /** - * Creates a new instance with the {@link CopyEntryFilter} as the filter and - * the given mode and output stream. - */ - public ZipCombiner(OutputMode mode, OutputStream out) { - this(mode, new CopyEntryFilter(), out); - } - - /** - * Creates a new instance with the {@link CopyEntryFilter} as the filter, the - * DONT_CARE mode and the given output stream. - */ - public ZipCombiner(OutputStream out) { - this(OutputMode.DONT_CARE, new CopyEntryFilter(), out); - } - - /** - * Returns whether the output zip already contains a file or directory with - * the given name. - */ - public boolean containsFile(String filename) { - return fileNames.containsKey(filename); - } - - /** - * Makes a write call to the output stream, and updates the current offset. - */ - private void write(byte[] b, int off, int len) throws IOException { - out.write(b, off, len); - } - - /** Calls through to {@link #write(byte[], int, int)}. */ - private void write(byte[] b) throws IOException { - write(b, 0, b.length); - } - - /** - * Reads at least one more byte into the internal buffer. This method must - * only be called when more data is necessary to correctly decode the ZIP - * format. - * - * <p>This method automatically compacts the existing data in the buffer by - * moving it to the beginning of the buffer. - * - * @throws EOFException if no more data is available from the input stream - * @throws IOException if the underlying stream throws one - */ - private void readMoreData(InputStream in) throws IOException { - if ((bufferLength > 0) && (bufferOffset > 0)) { - System.arraycopy(buffer, bufferOffset, buffer, 0, bufferLength); - } - if (bufferLength >= buffer.length) { - // The buffer size is specifically chosen to avoid this situation. - throw new AssertionError("Internal error: buffer overrun."); - } - bufferOffset = 0; - int bytesRead = in.read(buffer, bufferLength, buffer.length - bufferLength); - if (bytesRead <= 0) { - throw new EOFException(); + @Override public void rename(String newName, Date date) throws IOException { + checkCall(); + actions.put(filename, new EntryAction(ActionType.RENAME, date, newName, null, null)); } - bufferLength += bytesRead; - } - /** - * Reads data until the buffer is filled with at least {@code length} bytes. - * - * @throws IllegalArgumentException if not 0 <= length <= buffer.length - * @throws IOException if the underlying input stream throws one or the end - * of the input stream is reached before the required - * number of bytes is read - */ - private void readFully(InputStream in, int length) throws IOException { - checkArgument(length >= 0, "length too small: %s", length); - checkArgument(length <= buffer.length, "length too large: %s", length); - while (bufferLength < length) { - readMoreData(in); + @Override public void customMerge(Date date, CustomMergeStrategy strategy) throws IOException { + checkCall(); + actions.put(filename, new EntryAction(ActionType.MERGE, date, null, strategy, + new ByteArrayOutputStream())); } - } - - /** - * Reads an unsigned short in little-endian byte order from the buffer at the - * given offset. Does not perform range checking. - */ - private int getUnsignedShort(byte[] source, int offset) { - int a = source[offset + 0] & 0xff; - int b = source[offset + 1] & 0xff; - return (b << 8) | a; - } - - /** - * Reads an unsigned int in little-endian byte order from the buffer at the - * given offset. Does not perform range checking. - */ - private long getUnsignedInt(byte[] source, int offset) { - int a = source[offset + 0] & 0xff; - int b = source[offset + 1] & 0xff; - int c = source[offset + 2] & 0xff; - int d = source[offset + 3] & 0xff; - return ((d << 24) | (c << 16) | (b << 8) | a) & 0xffffffffL; - } - /** - * Writes an unsigned short in little-endian byte order to the buffer at the - * given offset. Does not perform range checking. - */ - private void setUnsignedShort(byte[] target, int offset, short value) { - target[offset + 0] = (byte) (value & 0xff); - target[offset + 1] = (byte) ((value >> 8) & 0xff); - } - - /** - * Writes an unsigned int in little-endian byte order to the buffer at the - * given offset. Does not perform range checking. - */ - private void setUnsignedInt(byte[] target, int offset, int value) { - target[offset + 0] = (byte) (value & 0xff); - target[offset + 1] = (byte) ((value >> 8) & 0xff); - target[offset + 2] = (byte) ((value >> 16) & 0xff); - target[offset + 3] = (byte) ((value >> 24) & 0xff); - } - - /** - * Copies or skips {@code length} amount of bytes from the input stream to the - * output stream. If the internal buffer is not empty, those bytes are copied - * first. When the method returns, there may be more bytes remaining in the - * buffer. - * - * @throws IOException if the underlying stream throws one - */ - private void copyOrSkipData(InputStream in, long length, SkipMode skip) throws IOException { - checkArgument(length >= 0); - while (length > 0) { - if (bufferLength == 0) { - readMoreData(in); - } - int bytesToWrite = (length < bufferLength) ? (int) length : bufferLength; - if (skip == SkipMode.COPY) { - write(buffer, bufferOffset, bytesToWrite); - } - bufferOffset += bytesToWrite; - bufferLength -= bytesToWrite; - length -= bytesToWrite; + private void checkCall() { + checkState(called.compareAndSet(false, true), "The callback was already called once."); } } - /** - * Copies or skips {@code length} amount of bytes from the input stream to the - * output stream. If the internal buffer is not empty, those bytes are copied - * first. When the method returns, there may be more bytes remaining in the - * buffer. In addition to writing to the output stream, it also writes to the - * central directory. - * - * @throws IOException if the underlying stream throws one - */ - private void forkOrSkipData(InputStream in, long length, SkipMode skip) throws IOException { - checkArgument(length >= 0); - while (length > 0) { - if (bufferLength == 0) { - readMoreData(in); - } - int bytesToWrite = (length < bufferLength) ? (int) length : bufferLength; - if (skip == SkipMode.COPY) { - write(buffer, bufferOffset, bytesToWrite); - centralDirectory.writeToCentralDirectory(buffer, bufferOffset, bytesToWrite); - } - bufferOffset += bytesToWrite; - bufferLength -= bytesToWrite; - length -= bytesToWrite; - } + /** Returns a {@link Deflater} for performing ZIP compression. */ + private static Deflater getDeflater() { + return new Deflater(Deflater.DEFAULT_COMPRESSION, true); } - /** - * A mutable integer reference value to allow returning two values from a - * method. - */ - private static class MutableInt { - - private int value; - - MutableInt(int initialValue) { - this.value = initialValue; - } - - public void setValue(int value) { - this.value = value; - } - - public int getValue() { - return value; - } + /** Returns a {@link Inflater} for performing ZIP decompression. */ + private static Inflater getInflater() { + return new Inflater(true); } - /** - * Uses the inflater to decompress some data into the given buffer. This - * method performs no error checking on the input parameters and also does - * not update the buffer parameters of the input buffer (such as bufferOffset - * and bufferLength). It's only here to avoid code duplication. - * - * <p>The Inflater may not be in the finished state when this method is - * called. - * - * <p>This method returns 0 if it read data and reached the end of the - * DEFLATE stream without producing output. In that case, {@link - * Inflater#finished} is guaranteed to return true. - * - * @throws IOException if the underlying stream throws an IOException or if - * illegal data is encountered - */ - private int inflateData(InputStream in, byte[] dest, int off, int len, MutableInt consumed) - throws IOException { - // Defend against Inflater.finished() returning true. - consumed.setValue(0); - int bytesProduced = 0; - int bytesConsumed = 0; - while ((bytesProduced == 0) && !inflater.finished()) { - inflater.setInput(buffer, bufferOffset + bytesConsumed, bufferLength - bytesConsumed); - int remainingBefore = inflater.getRemaining(); - try { - bytesProduced = inflater.inflate(dest, off, len); - } catch (DataFormatException e) { - throw new IOException("Invalid deflate stream in ZIP file.", e); - } - bytesConsumed += remainingBefore - inflater.getRemaining(); - consumed.setValue(bytesConsumed); - if (bytesProduced == 0) { - if (inflater.needsDictionary()) { - // The DEFLATE algorithm as used in the ZIP file format does not - // require an additional dictionary. - throw new AssertionError("Inflater unexpectedly requires a dictionary."); - } else if (inflater.needsInput()) { - readMoreData(in); - } else if (inflater.finished()) { - return 0; - } else { - // According to the Inflater specification, this cannot happen. - throw new AssertionError("Inflater unexpectedly produced no output."); - } - } + /** Copies all data from the input stream to the output stream. */ + private static long copyStream(InputStream from, OutputStream to) throws IOException { + byte[] buf = new byte[0x1000]; + long total = 0; + int r; + while ((r = from.read(buf)) != -1) { + to.write(buf, 0, r); + total += r; } - return bytesProduced; + return total; } - /** - * Copies or skips data from the input stream to the output stream. To - * determine the length of the data, the data is decompressed with the - * DEFLATE algorithm, which stores the length implicitly as part of the - * compressed data, using a combination of end markers and length indicators. - * - * @see <a href="http://www.ietf.org/rfc/rfc1951.txt">RFC 1951</a> - * - * @throws IOException if the underlying stream throws an IOException - */ - private long copyOrSkipDeflateData(InputStream in, SkipMode skip) throws IOException { - long bytesCopied = 0; - inflater.reset(); - MutableInt consumedBytes = new MutableInt(0); - while (!inflater.finished()) { - // Neither the uncompressed data nor the length of it is used. The - // decompression is only required to determine the correct length of the - // compressed data to copy. - inflateData(in, inflaterBuffer, 0, inflaterBuffer.length, consumedBytes); - int bytesRead = consumedBytes.getValue(); - if (skip == SkipMode.COPY) { - write(buffer, bufferOffset, bytesRead); - } - bufferOffset += bytesRead; - bufferLength -= bytesRead; - bytesCopied += bytesRead; - } - return bytesCopied; - } + private final OutputMode mode; + private final ZipEntryFilter entryFilter; + private final FilterCallback callback; + private final ZipWriter out; - /** - * Returns a 32-bit integer containing a ZIP-compatible encoding of the given - * date. Only dates between 1980 and 2107 (inclusive) are supported. - * - * <p>The upper 16 bits contain the year, month, and day. The lower 16 bits - * contain the hour, minute, and second. The resolution of the second field - * is only 4 bits, which means that the only even second values can be - * stored - this method rounds down to the nearest even value. - * - * @throws IllegalArgumentException if the given date is outside the - * supported range - */ - // Only visible for testing. - static int dateToDosTime(Date date) { - Calendar calendar = new GregorianCalendar(); - calendar.setTime(date); - int year = calendar.get(Calendar.YEAR); - if (year < 1980) { - throw new IllegalArgumentException("date must be in or after 1980"); - } - // The ZIP format only provides 7 bits for the year. - if (year > 2107) { - throw new IllegalArgumentException("date must before 2107"); - } - int month = calendar.get(Calendar.MONTH) + 1; // Months from Calendar are zero-based. - int day = calendar.get(Calendar.DAY_OF_MONTH); - int hour = calendar.get(Calendar.HOUR_OF_DAY); - int minute = calendar.get(Calendar.MINUTE); - int second = calendar.get(Calendar.SECOND); - return ((year - 1980) << 25) | (month << 21) | (day << 16) - | (hour << 11) | (minute << 5) | (second >> 1); - } + private final Map<String, ZipFileEntry> entries; + private final Map<String, EntryAction> actions; /** - * Fills the directory entry, using the information from the header buffer, - * and writes it to the central directory. It returns the offset into the - * central directory that can be used for patching the entry. Requires that - * the entire entry header is present in {@link #headerBuffer}. It also uses - * the {@link ByteCountingOutputStream#bytesWritten}, so it must be called - * just before the header is written to the output stream. + * Creates a {@link ZipCombiner} for combining ZIP files using the specified {@link OutputMode}, + * {@link ZipEntryFilter}, and destination {@link OutputStream}. * - * @throws IOException if the current offset is too large for the ZIP format - */ - private int fillDirectoryEntryBuffer( - DirectoryEntryInfo directoryEntryInfo) throws IOException { - // central file header signature - setUnsignedInt(directoryEntryBuffer, 0, CENTRAL_DIRECTORY_MARKER); - short version = (short) getUnsignedShort(headerBuffer, VERSION_TO_EXTRACT_OFFSET); - short curMadeMyVersion = (directoryEntryInfo.madeByVersion == -1) - ? version : directoryEntryInfo.madeByVersion; - setUnsignedShort(directoryEntryBuffer, 4, curMadeMyVersion); // version made by - // version needed to extract - setUnsignedShort(directoryEntryBuffer, 6, version); - // general purpose bit flag - setUnsignedShort(directoryEntryBuffer, 8, - (short) getUnsignedShort(headerBuffer, GENERAL_PURPOSE_FLAGS_OFFSET)); - // compression method - setUnsignedShort(directoryEntryBuffer, 10, - (short) getUnsignedShort(headerBuffer, COMPRESSION_METHOD_OFFSET)); - // last mod file time, last mod file date - setUnsignedShort(directoryEntryBuffer, 12, - (short) getUnsignedShort(headerBuffer, MTIME_OFFSET)); - setUnsignedShort(directoryEntryBuffer, 14, - (short) getUnsignedShort(headerBuffer, MDATE_OFFSET)); - // crc-32 - setUnsignedInt(directoryEntryBuffer, 16, (int) getUnsignedInt(headerBuffer, CRC32_OFFSET)); - // compressed size - setUnsignedInt(directoryEntryBuffer, 20, - (int) getUnsignedInt(headerBuffer, COMPRESSED_SIZE_OFFSET)); - // uncompressed size - setUnsignedInt(directoryEntryBuffer, 24, - (int) getUnsignedInt(headerBuffer, UNCOMPRESSED_SIZE_OFFSET)); - // file name length - setUnsignedShort(directoryEntryBuffer, 28, - (short) getUnsignedShort(headerBuffer, FILENAME_LENGTH_OFFSET)); - // extra field length - setUnsignedShort(directoryEntryBuffer, 30, - (short) getUnsignedShort(headerBuffer, EXTRA_LENGTH_OFFSET)); - setUnsignedShort(directoryEntryBuffer, 32, (short) 0); // file comment length - setUnsignedShort(directoryEntryBuffer, 34, (short) 0); // disk number start - setUnsignedShort(directoryEntryBuffer, 36, (short) 0); // internal file attributes - setUnsignedInt(directoryEntryBuffer, 38, directoryEntryInfo.externalFileAttribute); - if (out.bytesWritten >= MAXIMUM_DATA_SIZE) { - throw new IOException("Unable to handle files bigger than 2^32 bytes."); - } - // relative offset of local header - setUnsignedInt(directoryEntryBuffer, 42, (int) out.bytesWritten); - fileCount++; - return centralDirectory.writeToCentralDirectory(directoryEntryBuffer); - } - - /** - * Fix the directory entry with the correct crc32, compressed size, and - * uncompressed size. + * @param mode the compression preference for the output ZIP file + * @param entryFilter the filter to use when adding ZIP files to the combined output + * @param out the {@link OutputStream} for writing the combined ZIP file */ - private void fixDirectoryEntry(int offset, long crc32, long compressedSize, - long uncompressedSize) { - // The constants from the top don't apply here, because this is the central directory entry. - centralDirectory.setUnsignedInt(offset + 16, (int) crc32); // crc-32 - centralDirectory.setUnsignedInt(offset + 20, (int) compressedSize); // compressed size - centralDirectory.setUnsignedInt(offset + 24, (int) uncompressedSize); // uncompressed size + public ZipCombiner(OutputMode mode, ZipEntryFilter entryFilter, OutputStream out) { + this.mode = mode; + this.entryFilter = entryFilter; + this.callback = new FilterCallback(); + this.out = new ZipWriter(new BufferedOutputStream(out), UTF_8); + this.entries = new HashMap<>(); + this.actions = new HashMap<>(); } /** - * (Un)Compresses and copies the current ZIP file entry. Requires that the - * entire entry header is present in {@link #headerBuffer}. It currently - * drops the extra data in the process. + * Creates a {@link ZipCombiner} for combining ZIP files using the specified + * {@link ZipEntryFilter}, and destination {@link OutputStream}. Uses the DONT_CARE + * {@link OutputMode}. * - * @throws IOException if the underlying stream throws an IOException + * @param entryFilter the filter to use when adding ZIP files to the combined output + * @param out the {@link OutputStream} for writing the combined ZIP file */ - private void modifyAndCopyEntry(String filename, InputStream in, int dosTime) - throws IOException { - final int method = getUnsignedShort(headerBuffer, COMPRESSION_METHOD_OFFSET); - final int flags = getUnsignedShort(headerBuffer, GENERAL_PURPOSE_FLAGS_OFFSET); - final int fileNameLength = getUnsignedShort(headerBuffer, FILENAME_LENGTH_OFFSET); - final int extraFieldLength = getUnsignedShort(headerBuffer, EXTRA_LENGTH_OFFSET); - // TODO(bazel-team): Read and copy the extra data if present. - - forkOrSkipData(in, fileNameLength, SkipMode.SKIP); - forkOrSkipData(in, extraFieldLength, SkipMode.SKIP); - if (method == STORED_METHOD) { - long compressedSize = getUnsignedInt(headerBuffer, COMPRESSED_SIZE_OFFSET); - copyStreamToEntry(filename, new FixedLengthInputStream(in, compressedSize), dosTime, - NO_EXTRA_ENTRIES, true, DEFAULT_DIRECTORY_ENTRY_INFO); - } else if (method == DEFLATE_METHOD) { - inflater.reset(); - copyStreamToEntry(filename, new DeflateInputStream(in), dosTime, NO_EXTRA_ENTRIES, false, - DEFAULT_DIRECTORY_ENTRY_INFO); - if ((flags & SIZE_MASKED_FLAG) != 0) { - copyOrSkipData(in, 16, SkipMode.SKIP); - } - } else { - throw new AssertionError("This should have been checked in validateHeader()."); - } + public ZipCombiner(ZipEntryFilter entryFilter, OutputStream out) { + this(OutputMode.DONT_CARE, entryFilter, out); } /** - * Copies or skips the current ZIP file entry. Requires that the entire entry - * header is present in {@link #headerBuffer}. It uses the current mode to - * decide whether to compress or decompress the entry. + * Creates a {@link ZipCombiner} for combining ZIP files using the specified {@link OutputMode}, + * and destination {@link OutputStream}. Uses a {@link CopyEntryFilter} as the + * {@link ZipEntryFilter}. * - * @throws IOException if the underlying stream throws an IOException + * @param mode the compression preference for the output ZIP file + * @param out the {@link OutputStream} for writing the combined ZIP file */ - private void copyOrSkipEntry(String filename, InputStream in, SkipMode skip, Date date, - DirectoryEntryInfo directoryEntryInfo) throws IOException { - copyOrSkipEntry(filename, in, skip, date, directoryEntryInfo, false); + public ZipCombiner(OutputMode mode, OutputStream out) { + this(mode, new CopyEntryFilter(), out); } /** - * Renames and otherwise copies the current ZIP file entry. Requires that the entire - * entry header is present in {@link #headerBuffer}. It uses the current mode to - * decide whether to compress or decompress the entry. + * Creates a {@link ZipCombiner} for combining ZIP files using the specified destination + * {@link OutputStream}. Uses the DONT_CARE {@link OutputMode} and a {@link CopyEntryFilter} as + * the {@link ZipEntryFilter}. * - * @throws IOException if the underlying stream throws an IOException + * @param out the {@link OutputStream} for writing the combined ZIP file */ - private void renameEntry(String filename, InputStream in, Date date, - DirectoryEntryInfo directoryEntryInfo) throws IOException { - copyOrSkipEntry(filename, in, SkipMode.COPY, date, directoryEntryInfo, true); + public ZipCombiner(OutputStream out) { + this(OutputMode.DONT_CARE, new CopyEntryFilter(), out); } /** - * Copies or skips the current ZIP file entry. Requires that the entire entry - * header is present in {@link #headerBuffer}. It uses the current mode to - * decide whether to compress or decompress the entry. + * Write all contents from the {@link InputStream} as a prefix file for the combined ZIP file. * - * @throws IOException if the underlying stream throws an IOException + * @param in the {@link InputStream} containing the prefix file data + * @throws IOException if there is an error writing the prefix file */ - private void copyOrSkipEntry(String filename, InputStream in, SkipMode skip, Date date, - DirectoryEntryInfo directoryEntryInfo, boolean rename) throws IOException { - final int method = getUnsignedShort(headerBuffer, COMPRESSION_METHOD_OFFSET); - - // We can cast here, because the result is only treated as a bitmask. - int dosTime = date == null ? (int) getUnsignedInt(headerBuffer, MTIME_OFFSET) - : dateToDosTime(date); - if (skip == SkipMode.COPY) { - if ((mode == OutputMode.FORCE_DEFLATE) && (method == STORED_METHOD) - && !filename.endsWith("/")) { - modifyAndCopyEntry(filename, in, dosTime); - return; - } else if ((mode == OutputMode.FORCE_STORED) && (method == DEFLATE_METHOD)) { - modifyAndCopyEntry(filename, in, dosTime); - return; - } - } - - int directoryOffset = copyOrSkipEntryHeader(filename, in, date, directoryEntryInfo, - skip, rename); - - copyOrSkipEntryData(filename, in, skip, directoryOffset); + public void prependExecutable(InputStream in) throws IOException { + out.startPrefixFile(); + copyStream(in, out); + out.endPrefixFile(); } /** - * Copies or skips the header of an entry, including filename and extra data. - * Requires that the entire entry header is present in {@link #headerBuffer}. + * Adds a directory entry to the combined ZIP file using the specified filename and date. * - * @returns the enrty offset in the central directory - * @throws IOException if the underlying stream throws an IOException + * @param filename the name of the directory to create + * @param date the modified time to assign to the directory + * @throws IOException if there is an error writing the directory entry */ - private int copyOrSkipEntryHeader(String filename, InputStream in, Date date, - DirectoryEntryInfo directoryEntryInfo, SkipMode skip, boolean rename) - throws IOException { - final int fileNameLength = getUnsignedShort(headerBuffer, FILENAME_LENGTH_OFFSET); - final int extraFieldLength = getUnsignedShort(headerBuffer, EXTRA_LENGTH_OFFSET); - - byte[] fileNameAsBytes = null; - if (rename) { - // If the entry is renamed, we patch the filename length in the buffer - // before it's copied, and before writing to the central directory. - fileNameAsBytes = filename.getBytes(UTF_8); - checkArgument(fileNameAsBytes.length <= 65535, - "File name too long: %s bytes (max. 65535)", fileNameAsBytes.length); - setUnsignedShort(headerBuffer, FILENAME_LENGTH_OFFSET, (short) fileNameAsBytes.length); - } - - int directoryOffset = 0; - if (skip == SkipMode.COPY) { - if (date != null) { - int dosTime = dateToDosTime(date); - setUnsignedShort(headerBuffer, MTIME_OFFSET, (short) dosTime); // lower 16 bits - setUnsignedShort(headerBuffer, MDATE_OFFSET, (short) (dosTime >> 16)); // upper 16 bits - } - // Call this before writing the data out, so that we get the correct offset. - directoryOffset = fillDirectoryEntryBuffer(directoryEntryInfo); - write(headerBuffer, 0, FILE_HEADER_BUFFER_SIZE); - } - if (!rename) { - forkOrSkipData(in, fileNameLength, skip); - } else { - forkOrSkipData(in, fileNameLength, SkipMode.SKIP); - write(fileNameAsBytes); - centralDirectory.writeToCentralDirectory(fileNameAsBytes); - } - forkOrSkipData(in, extraFieldLength, skip); - return directoryOffset; + public void addDirectory(String filename, Date date) throws IOException { + addDirectory(filename, date, new ExtraData[0]); } /** - * Copy or skip the data of an entry. Requires that the - * entire entry header is present in {@link #headerBuffer}. + * Adds a directory entry to the combined ZIP file using the specified filename, date, and extra + * data. * - * @throws IOException if the underlying stream throws an IOException + * @param filename the name of the directory to create + * @param date the modified time to assign to the directory + * @param extra the extra field data to add to the directory entry + * @throws IOException if there is an error writing the directory entry */ - private void copyOrSkipEntryData(String filename, InputStream in, SkipMode skip, - int directoryOffset) throws IOException { - final int flags = getUnsignedShort(headerBuffer, GENERAL_PURPOSE_FLAGS_OFFSET); - final int method = getUnsignedShort(headerBuffer, COMPRESSION_METHOD_OFFSET); - if ((flags & SIZE_MASKED_FLAG) != 0) { - // The compressed data size is unknown. - if (method != DEFLATE_METHOD) { - throw new AssertionError("This should have been checked in validateHeader()."); - } - copyOrSkipDeflateData(in, skip); - // The flags indicate that a data descriptor must follow the data. - readFully(in, 16); - if (getUnsignedInt(buffer, bufferOffset) != DATA_DESCRIPTOR_MARKER) { - throw new IOException("Missing data descriptor for " + filename + " in " + currentInputFile - + "."); - } - long crc32 = getUnsignedInt(buffer, bufferOffset + 4); - long compressedSize = getUnsignedInt(buffer, bufferOffset + 8); - long uncompressedSize = getUnsignedInt(buffer, bufferOffset + 12); - if (skip == SkipMode.COPY) { - fixDirectoryEntry(directoryOffset, crc32, compressedSize, uncompressedSize); - } - copyOrSkipData(in, 16, skip); - } else { - // The size value is present in the header, so just copy that amount. - long compressedSize = getUnsignedInt(headerBuffer, COMPRESSED_SIZE_OFFSET); - copyOrSkipData(in, compressedSize, skip); - } - } - - /** - * An input stream that reads a fixed number of bytes from the given input - * stream before it returns end-of-input. It uses the local buffer, so it - * can't be static. - */ - private class FixedLengthInputStream extends InputStream { + public void addDirectory(String filename, Date date, ExtraData[] extra) throws IOException { + checkArgument(filename.endsWith("/"), "Directory names must end with a /"); + checkState(!entries.containsKey(filename), + "Zip already contains a directory named %s", filename); - private final InputStream in; - private long remainingBytes; - private final byte[] singleByteBuffer = new byte[1]; - - FixedLengthInputStream(InputStream in, long remainingBytes) { - this.in = in; - this.remainingBytes = remainingBytes; - } - - @Override - public int read() throws IOException { - int bytesRead = read(singleByteBuffer, 0, 1); - return (bytesRead == -1) ? -1 : singleByteBuffer[0]; - } - - @Override - public int read(byte b[], int off, int len) throws IOException { - checkArgument(len >= 0); - checkArgument(off >= 0); - checkArgument(off + len <= b.length); - if (remainingBytes == 0) { - return -1; - } - if (bufferLength == 0) { - readMoreData(in); - } - int bytesToCopy = len; - if (remainingBytes < bytesToCopy) { - bytesToCopy = (int) remainingBytes; - } - if (bufferLength < bytesToCopy) { - bytesToCopy = bufferLength; - } - System.arraycopy(buffer, bufferOffset, b, off, bytesToCopy); - bufferOffset += bytesToCopy; - bufferLength -= bytesToCopy; - remainingBytes -= bytesToCopy; - return bytesToCopy; - } + ZipFileEntry entry = new ZipFileEntry(filename); + entry.setMethod(Compression.STORED); + entry.setCrc(0); + entry.setSize(0); + entry.setCompressedSize(0); + entry.setTime(date != null ? date.getTime() : new Date().getTime()); + entry.setExtra(extra); + out.putNextEntry(entry); + out.closeEntry(); + entries.put(filename, entry); } /** - * An input stream that reads from a given input stream, decoding that data - * according to the DEFLATE algorithm. The DEFLATE data stream implicitly - * contains its own end-of-input marker. It uses the local buffer, so it - * can't be static. + * Adds a file with the specified name to the combined ZIP file. + * + * @param filename the name of the file to create + * @param in the {@link InputStream} containing the file data + * @throws IOException if there is an error writing the file entry + * @throws IllegalArgumentException if the combined ZIP file already contains a file of the same + * name. */ - private class DeflateInputStream extends InputStream { - - private final InputStream in; - private final byte[] singleByteBuffer = new byte[1]; - private final MutableInt consumedBytes = new MutableInt(0); - - DeflateInputStream(InputStream in) { - this.in = in; - } - - @Override - public int read() throws IOException { - int bytesRead = read(singleByteBuffer, 0, 1); - // Do an unsigned cast on the byte from the buffer if it exists. - return (bytesRead == -1) ? -1 : (singleByteBuffer[0] & 0xff); - } - - @Override - public int read(byte b[], int off, int len) throws IOException { - if (inflater.finished()) { - return -1; - } - int length = inflateData(in, b, off, len, consumedBytes); - int bytesRead = consumedBytes.getValue(); - bufferOffset += bytesRead; - bufferLength -= bytesRead; - return length == 0 ? -1 : length; - } + public void addFile(String filename, InputStream in) throws IOException { + addFile(filename, null, in); } /** - * Handles a custom merge operation with the given strategy. This method - * creates an appropriate input stream and hands it to the strategy for - * processing. Requires that the entire entry header is present in {@link - * #headerBuffer}. + * Adds a file with the specified name and date to the combined ZIP file. * - * @throws IOException if one of the underlying stream throws an IOException, - * if the ZIP entry data is inconsistent, or if the - * implementation cannot handle the compression method - * given in the ZIP entry + * @param filename the name of the file to create + * @param date the modified time to assign to the file + * @param in the {@link InputStream} containing the file data + * @throws IOException if there is an error writing the file entry + * @throws IllegalArgumentException if the combined ZIP file already contains a file of the same + * name. */ - private void handleCustomMerge(final InputStream in, CustomMergeStrategy mergeStrategy, - ByteArrayOutputStream outputBuffer) throws IOException { - final int flags = getUnsignedShort(headerBuffer, GENERAL_PURPOSE_FLAGS_OFFSET); - final int method = getUnsignedShort(headerBuffer, COMPRESSION_METHOD_OFFSET); - final long compressedSize = getUnsignedInt(headerBuffer, COMPRESSED_SIZE_OFFSET); - - final int fileNameLength = getUnsignedShort(headerBuffer, FILENAME_LENGTH_OFFSET); - final int extraFieldLength = getUnsignedShort(headerBuffer, EXTRA_LENGTH_OFFSET); - - copyOrSkipData(in, fileNameLength, SkipMode.SKIP); - copyOrSkipData(in, extraFieldLength, SkipMode.SKIP); - if (method == STORED_METHOD) { - mergeStrategy.merge(new FixedLengthInputStream(in, compressedSize), outputBuffer); - } else if (method == DEFLATE_METHOD) { - inflater.reset(); - // TODO(bazel-team): Defend against the mergeStrategy not reading the complete input. - mergeStrategy.merge(new DeflateInputStream(in), outputBuffer); - if ((flags & SIZE_MASKED_FLAG) != 0) { - copyOrSkipData(in, 16, SkipMode.SKIP); + public void addFile(String filename, Date date, InputStream in) throws IOException { + ZipFileEntry entry = new ZipFileEntry(filename); + entry.setTime(date != null ? date.getTime() : new Date().getTime()); + addFile(entry, in); + } + + /** + * Adds a file with attributes specified by the {@link ZipFileEntry} to the combined ZIP file. + * + * @param entry the {@link ZipFileEntry} containing the entry meta-data + * @param in the {@link InputStream} containing the file data + * @throws IOException if there is an error writing the file entry + * @throws IllegalArgumentException if the combined ZIP file already contains a file of the same + * name. + */ + public void addFile(ZipFileEntry entry, InputStream in) throws IOException { + checkNotNull(entry, "Zip entry must not be null."); + checkNotNull(in, "Input stream must not be null."); + checkArgument(!entries.containsKey(entry.getName()), "Zip already contains a file named '%s'.", + entry.getName()); + + ByteArrayOutputStream uncompressed = new ByteArrayOutputStream(); + copyStream(in, uncompressed); + + writeEntryFromBuffer(new ZipFileEntry(entry), uncompressed.toByteArray()); + } + + /** + * Adds the contents of a ZIP file to the combined ZIP file using the specified + * {@link ZipEntryFilter} to determine the appropriate action for each file. + * + * @param zipFile the ZIP file to add to the combined ZIP file + * @throws IOException if there is an error reading the ZIP file or writing entries to the + * combined ZIP file + */ + public void addZip(File zipFile) throws IOException { + try (ZipReader zip = new ZipReader(zipFile)) { + for (ZipFileEntry entry : zip.entries()) { + String filename = entry.getName(); + EntryAction action = getAction(filename); + switch (action.getType()) { + case SKIP: + break; + case COPY: + case RENAME: + writeEntry(zip, entry, action); + break; + case MERGE: + entries.put(filename, null); + InputStream in = zip.getRawInputStream(entry); + if (entry.getMethod() == Compression.DEFLATED) { + in = new InflaterInputStream(in, getInflater()); + } + action.getStrategy().merge(in, action.getMergeBuffer()); + break; + } } - } else { - throw new AssertionError("This should have been checked in validateHeader()."); } } - /** - * Implementation of the strategy callback. - */ - private class TheStrategyCallback implements StrategyCallback { - - private String filename; - private final InputStream in; - - // Use an atomic boolean to make sure that only a single call goes - // through, even if there are multiple concurrent calls. Paranoid - // defensive programming. - private final AtomicBoolean callDone = new AtomicBoolean(); - - TheStrategyCallback(String filename, InputStream in) { - this.filename = filename; - this.in = in; - } - - // Verify that this is the first call and throw an exception if not. - private void checkCall() { - checkState(callDone.compareAndSet(false, true), "The callback was already called once."); + /** Returns the action to take for a file of the given filename. */ + private EntryAction getAction(String filename) throws IOException { + // If this filename has not been encountered before (no entry for filename) or this filename + // has been renamed (RENAME entry for filename), the desired action should be recomputed. + if (!actions.containsKey(filename) || actions.get(filename).getType() == ActionType.RENAME) { + callback.resetForFile(filename); + entryFilter.accept(filename, callback); } + checkState(actions.containsKey(filename), + "Action for file '%s' should have been set by ZipEntryFilter.", filename); - @Override - public void copy(Date date) throws IOException { - checkCall(); - if (!containsFile(filename)) { - fileNames.put(filename, COPIED_FILE_ENTRY); - copyOrSkipEntry(filename, in, SkipMode.COPY, date, DEFAULT_DIRECTORY_ENTRY_INFO); - } else { // can't copy, name already used for renamed entry - copyOrSkipEntry(filename, in, SkipMode.SKIP, null, DEFAULT_DIRECTORY_ENTRY_INFO); - } + EntryAction action = actions.get(filename); + // Only copy if this is the first instance of filename. + if (action.getType() == ActionType.COPY && entries.containsKey(filename)) { + action = new EntryAction(ActionType.SKIP, action); + actions.put(filename, action); } - - @Override - public void rename(String newName, Date date) throws IOException { - checkCall(); - if (!containsFile(newName)) { - fileNames.put(newName, RENAMED_FILE_ENTRY); - renameEntry(newName, in, date, DEFAULT_DIRECTORY_ENTRY_INFO); - } else { - copyOrSkipEntry(filename, in, SkipMode.SKIP, null, DEFAULT_DIRECTORY_ENTRY_INFO); + // Only rename if there is not already an entry with filename or filename's action is SKIP. + if (action.getType() == ActionType.RENAME) { + if (actions.containsKey(action.getNewName()) + && actions.get(action.getNewName()).getType() == ActionType.SKIP) { + action = new EntryAction(ActionType.SKIP, action); } - filename = newName; - } - - @Override - public void skip() throws IOException { - checkCall(); - if (!containsFile(filename)) {// don't overwrite possible RENAMED_FILE_ENTRY value - fileNames.put(filename, COPIED_FILE_ENTRY); + if (entries.containsKey(action.getNewName())) { + action = new EntryAction(ActionType.SKIP, action); } - copyOrSkipEntry(filename, in, SkipMode.SKIP, null, DEFAULT_DIRECTORY_ENTRY_INFO); - } - - @Override - public void customMerge(Date date, CustomMergeStrategy strategy) throws IOException { - checkCall(); - ByteArrayOutputStream outputBuffer = new ByteArrayOutputStream(); - fileNames.put(filename, new FileEntry(strategy, outputBuffer, dateToDosTime(date))); - handleCustomMerge(in, strategy, outputBuffer); } + return action; } - /** - * Validates that the current entry obeys all the restrictions of this implementation. - * - * @throws IOException if the current entry doesn't obey the restrictions - */ - private void validateHeader() throws IOException { - // We only handle DEFLATE and STORED, like java.util.zip. - final int method = getUnsignedShort(headerBuffer, COMPRESSION_METHOD_OFFSET); - if ((method != DEFLATE_METHOD) && (method != STORED_METHOD)) { - throw new IOException("Unable to handle compression methods other than DEFLATE!"); - } - - // If the method is STORED, then the size must be available in the header. - final int flags = getUnsignedShort(headerBuffer, GENERAL_PURPOSE_FLAGS_OFFSET); - if ((method == STORED_METHOD) && ((flags & SIZE_MASKED_FLAG) != 0)) { - throw new IOException("If the method is STORED, then the size must be available in the" - + " header!"); - } + /** Writes an entry with the given name, date and external file attributes from the buffer. */ + private void writeEntryFromBuffer(ZipFileEntry entry, byte[] uncompressed) throws IOException { + CRC32 crc = new CRC32(); + crc.update(uncompressed); - // If the method is STORED, the compressed and uncompressed sizes must be equal. - final long compressedSize = getUnsignedInt(headerBuffer, COMPRESSED_SIZE_OFFSET); - final long uncompressedSize = getUnsignedInt(headerBuffer, UNCOMPRESSED_SIZE_OFFSET); - if ((method == STORED_METHOD) && (compressedSize != uncompressedSize)) { - throw new IOException("Compressed and uncompressed sizes for STORED entry differ!"); - } - - // The compressed or uncompressed size being set to 0xffffffff is a strong indicator that the - // ZIP file is in ZIP64 mode, which supports files larger than 2^32. - // TODO(bazel-team): Support the ZIP64 extension. - if ((compressedSize == MAXIMUM_DATA_SIZE) || (uncompressedSize == MAXIMUM_DATA_SIZE)) { - throw new IOException("Unable to handle ZIP64 compressed files."); - } - } - - /** - * Reads a file entry from the input stream, calls the entryFilter to - * determine what to do with the entry, and performs the requested operation. - * Returns true if the input stream contained another entry. - * - * @throws IOException if one of the underlying stream throws an IOException, - * if the ZIP contains unsupported, inconsistent or - * incomplete data or if the filter throws an IOException - */ - private boolean handleNextEntry(final InputStream in) throws IOException { - // Just try to read the complete header and fail if it didn't work. - try { - readFully(in, FILE_HEADER_BUFFER_SIZE); - } catch (EOFException e) { - return false; - } - - System.arraycopy(buffer, bufferOffset, headerBuffer, 0, FILE_HEADER_BUFFER_SIZE); - bufferOffset += FILE_HEADER_BUFFER_SIZE; - bufferLength -= FILE_HEADER_BUFFER_SIZE; - if (getUnsignedInt(headerBuffer, 0) != LOCAL_FILE_HEADER_MARKER) { - return false; - } - validateHeader(); - - final int fileNameLength = getUnsignedShort(headerBuffer, FILENAME_LENGTH_OFFSET); - readFully(in, fileNameLength); - // TODO(bazel-team): If I read the spec correctly, this should be UTF-8 rather than ISO-8859-1. - final String filename = new String(buffer, bufferOffset, fileNameLength, ISO_8859_1); - - FileEntry handler = fileNames.get(filename); - // The handler is null if this is the first time we see an entry with this filename, - // or if all previous entries with this name were renamed by the filter (and we can - // pretend we didn't encounter the name yet). - // If the handler is RENAMED_FILE_ENTRY, a previous entry was renamed as filename, - // in which case the filter should now be invoked for this name for the first time, - // giving the filter a chance to choose an unique name. - if (handler == null || handler == RENAMED_FILE_ENTRY) { - TheStrategyCallback callback = new TheStrategyCallback(filename, in); - entryFilter.accept(filename, callback); - if (fileNames.get(callback.filename) == null && fileNames.get(filename) == null) { - throw new IllegalStateException(); - } - } else if (handler.mergeStrategy == null) { - copyOrSkipEntry(filename, in, SkipMode.SKIP, null, DEFAULT_DIRECTORY_ENTRY_INFO); + entry.setCrc(crc.getValue()); + entry.setSize(uncompressed.length); + if (mode == OutputMode.FORCE_STORED) { + entry.setMethod(Compression.STORED); + entry.setCompressedSize(uncompressed.length); + writeEntry(entry, new ByteArrayInputStream(uncompressed)); } else { - handleCustomMerge(in, handler.mergeStrategy, handler.outputBuffer); + ByteArrayOutputStream compressed = new ByteArrayOutputStream(); + copyStream(new DeflaterInputStream(new ByteArrayInputStream(uncompressed), getDeflater()), + compressed); + entry.setMethod(Compression.DEFLATED); + entry.setCompressedSize(compressed.size()); + writeEntry(entry, new ByteArrayInputStream(compressed.toByteArray())); } - return true; } /** - * Clears the internal buffer. + * Writes an entry from the specified source {@link ZipReader} and {@link ZipFileEntry} using the + * specified {@link EntryAction}. + * + * <p>Writes the output entry from the input entry performing inflation or deflation as needed + * and applies any values from the {@link EntryAction} as needed. */ - private void clearBuffer() { - bufferOffset = 0; - bufferLength = 0; - } - - /** - * Copies another ZIP file into the output. If multiple entries with the same - * name are present, the first such entry is copied, but the others are - * ignored. This is also true for multiple invocations of this method. The - * {@code inputName} parameter is used to provide better error messages in the - * case of a failure to decode the ZIP file. - * - * @throws IOException if one of the underlying stream throws an IOException, - * if the ZIP contains unsupported, inconsistent or - * incomplete data or if the filter throws an IOException - */ - public void addZip(String inputName, InputStream in) throws IOException { - if (finished) { - throw new IllegalStateException(); - } - if (in == null) { - throw new NullPointerException(); - } - clearBuffer(); - currentInputFile = inputName; - while (handleNextEntry(in)) {/*handleNextEntry has side-effect.*/} - } - - public void addZip(InputStream in) throws IOException { - addZip(null, in); - } - - private void copyStreamToEntry(String filename, InputStream in, int dosTime, - ExtraData[] extraDataEntries, boolean compress, DirectoryEntryInfo directoryEntryInfo) + private void writeEntry(ZipReader zip, ZipFileEntry entry, EntryAction action) throws IOException { - fileNames.put(filename, COPIED_FILE_ENTRY); - - byte[] fileNameAsBytes = filename.getBytes(UTF_8); - checkArgument(fileNameAsBytes.length <= 65535, - "File name too long: %s bytes (max. 65535)", fileNameAsBytes.length); - - // Note: This method can be called with an input stream that uses the buffer field of this - // class. We use a local buffer here to avoid conflicts. - byte[] localBuffer = new byte[4096]; - - byte[] uncompressedData = null; - if (!compress) { - ByteArrayOutputStream temp = new ByteArrayOutputStream(); - int bytesRead; - while ((bytesRead = in.read(localBuffer)) != -1) { - temp.write(localBuffer, 0, bytesRead); - } - uncompressedData = temp.toByteArray(); - } - byte[] extraData = null; - if (extraDataEntries.length != 0) { - int totalLength = 0; - for (ExtraData extra : extraDataEntries) { - int length = extra.getData().length; - if (totalLength > 0xffff - 4 - length) { - throw new IOException("Total length of extra data too big."); - } - totalLength += length + 4; - } - extraData = new byte[totalLength]; - int position = 0; - for (ExtraData extra : extraDataEntries) { - byte[] data = extra.getData(); - setUnsignedShort(extraData, position + 0, extra.getId()); - setUnsignedShort(extraData, position + 2, (short) data.length); - System.arraycopy(data, 0, extraData, position + 4, data.length); - position += data.length + 4; - } - } - - // write header - Arrays.fill(headerBuffer, (byte) 0); - setUnsignedInt(headerBuffer, 0, LOCAL_FILE_HEADER_MARKER); // file header signature - if (compress) { - setUnsignedShort(headerBuffer, 4, (short) VERSION_DEFLATE); // version to extract - setUnsignedShort(headerBuffer, 6, (short) SIZE_MASKED_FLAG); // general purpose bit flag - setUnsignedShort(headerBuffer, 8, (short) DEFLATE_METHOD); // compression method - } else { - setUnsignedShort(headerBuffer, 4, (short) VERSION_STORED); // version to extract - setUnsignedShort(headerBuffer, 6, (short) 0); // general purpose bit flag - setUnsignedShort(headerBuffer, 8, (short) STORED_METHOD); // compression method - } - setUnsignedShort(headerBuffer, 10, (short) dosTime); // mtime - setUnsignedShort(headerBuffer, 12, (short) (dosTime >> 16)); // mdate - if (uncompressedData != null) { - CRC32 crc = new CRC32(); - crc.update(uncompressedData); - setUnsignedInt(headerBuffer, 14, (int) crc.getValue()); // crc32 - setUnsignedInt(headerBuffer, 18, uncompressedData.length); // compressed size - setUnsignedInt(headerBuffer, 22, uncompressedData.length); // uncompressed size + checkArgument(action.getType() != ActionType.SKIP, + "Cannot write a zip entry whose action is of type SKIP."); + + ZipFileEntry outEntry = new ZipFileEntry(entry); + if (action.getType() == ActionType.RENAME) { + checkNotNull(action.getNewName(), + "ZipEntryFilter actions of type RENAME must not have a null filename."); + outEntry.setName(action.getNewName()); + } + + if (action.getDate() != null) { + outEntry.setTime(action.getDate().getTime()); + } + + InputStream data; + if (mode == OutputMode.FORCE_DEFLATE && entry.getMethod() != Compression.DEFLATED) { + // The output mode is deflate, but the entry compression is not. Create a deflater stream + // from the raw file data and deflate to a temporary byte array to determine the deflated + // size. Then use this byte array as the input stream for writing the entry. + ByteArrayOutputStream tmp = new ByteArrayOutputStream(); + copyStream(new DeflaterInputStream(zip.getRawInputStream(entry), getDeflater()), tmp); + data = new ByteArrayInputStream(tmp.toByteArray()); + outEntry.setMethod(Compression.DEFLATED); + outEntry.setCompressedSize(tmp.size()); + } else if (mode == OutputMode.FORCE_STORED && entry.getMethod() != Compression.STORED) { + // The output mode is stored, but the entry compression is not; create an inflater stream + // from the raw file data. + data = new InflaterInputStream(zip.getRawInputStream(entry), getInflater()); + outEntry.setMethod(Compression.STORED); + outEntry.setCompressedSize(entry.getSize()); } else { - setUnsignedInt(headerBuffer, 14, 0); // crc32 - setUnsignedInt(headerBuffer, 18, 0); // compressed size - setUnsignedInt(headerBuffer, 22, 0); // uncompressed size - } - setUnsignedShort(headerBuffer, 26, (short) fileNameAsBytes.length); // file name length - if (extraData != null) { - setUnsignedShort(headerBuffer, 28, (short) extraData.length); // extra field length - } else { - setUnsignedShort(headerBuffer, 28, (short) 0); // extra field length - } - - // This call works for both compressed or uncompressed entries. - int directoryOffset = fillDirectoryEntryBuffer(directoryEntryInfo); - write(headerBuffer); - write(fileNameAsBytes); - centralDirectory.writeToCentralDirectory(fileNameAsBytes); - if (extraData != null) { - write(extraData); - centralDirectory.writeToCentralDirectory(extraData); - } - - // write data - if (uncompressedData != null) { - write(uncompressedData); - } else { - try (DeflaterOutputStream deflaterStream = new DeflaterOutputStream()) { - int bytesRead; - while ((bytesRead = in.read(localBuffer)) != -1) { - deflaterStream.write(localBuffer, 0, bytesRead); - } - deflaterStream.finish(); - - // write data descriptor - Arrays.fill(headerBuffer, (byte) 0); - setUnsignedInt(headerBuffer, 0, DATA_DESCRIPTOR_MARKER); - setUnsignedInt(headerBuffer, 4, deflaterStream.getCRC()); // crc32 - setUnsignedInt(headerBuffer, 8, deflaterStream.getCompressedSize()); // compressed size - setUnsignedInt(headerBuffer, 12, deflaterStream.getUncompressedSize()); // uncompressed size - write(headerBuffer, 0, 16); - fixDirectoryEntry(directoryOffset, deflaterStream.getCRC(), - deflaterStream.getCompressedSize(), deflaterStream.getUncompressedSize()); - } + // Entry compression agrees with output mode; use the raw file data as is. + data = zip.getRawInputStream(entry); } + writeEntry(outEntry, data); } /** - * Adds a new entry into the output, by reading the input stream until it - * returns end of stream. Equivalent to - * {@link #addFile(String, Date, InputStream, DirectoryEntryInfo)}, but uses - * {@link #DEFAULT_DIRECTORY_ENTRY_INFO} for the file's directory entry. + * Writes the specified {@link ZipFileEntry} using the data from the given {@link InputStream}. */ - public void addFile(String filename, Date date, InputStream in) throws IOException { - addFile(filename, date, in, DEFAULT_DIRECTORY_ENTRY_INFO); + private void writeEntry(ZipFileEntry entry, InputStream data) throws IOException { + out.putNextEntry(entry); + copyStream(data, out); + out.closeEntry(); + entries.put(entry.getName(), entry); } /** - * Adds a new entry into the output, by reading the input stream until it - * returns end of stream. This method does not call {@link - * ZipEntryFilter#accept}. + * Returns true if the combined ZIP file already contains a file of the specified file name. * - * @throws IOException if one of the underlying streams throws an IOException - * or if the input stream returns more data than - * supported by the ZIP format - * @throws IllegalStateException if an entry with the given name already - * exists - * @throws IllegalArgumentException if the given file name is longer than - * supported by the ZIP format + * @param filename the filename of the file whose presence in the combined ZIP file is to be + * tested + * @return true if the combined ZIP file contains the specified file */ - public void addFile(String filename, Date date, InputStream in, - DirectoryEntryInfo directoryEntryInfo) throws IOException { - checkNotFinished(); - if (in == null) { - throw new NullPointerException(); - } - if (filename == null) { - throw new NullPointerException(); - } - checkState(!fileNames.containsKey(filename), - "jar already contains a file named %s", filename); - int dosTime = dateToDosTime(date != null ? date : new Date()); - copyStreamToEntry(filename, in, dosTime, NO_EXTRA_ENTRIES, - mode != OutputMode.FORCE_STORED, // Always compress if we're allowed to. - directoryEntryInfo); - } - - /** - * Adds a new directory entry into the output. This method does not call - * {@link ZipEntryFilter#accept}. Uses {@link #DEFAULT_DIRECTORY_ENTRY_INFO} for the added - * directory entry. - * - * @throws IOException if one of the underlying streams throws an IOException - * @throws IllegalStateException if an entry with the given name already - * exists - * @throws IllegalArgumentException if the given file name is longer than - * supported by the ZIP format - */ - public void addDirectory(String filename, Date date, ExtraData[] extraDataEntries) - throws IOException { - checkNotFinished(); - checkArgument(filename.endsWith("/")); // Can also throw NPE. - checkState(!fileNames.containsKey(filename), - "jar already contains a directory named %s", filename); - int dosTime = dateToDosTime(date != null ? date : new Date()); - copyStreamToEntry(filename, new ByteArrayInputStream(new byte[0]), dosTime, extraDataEntries, - false, // Never compress directory entries. - DEFAULT_DIRECTORY_ENTRY_INFO); - } - - /** - * Adds a new directory entry into the output. This method does not call - * {@link ZipEntryFilter#accept}. - * - * @throws IOException if one of the underlying streams throws an IOException - * @throws IllegalStateException if an entry with the given name already - * exists - * @throws IllegalArgumentException if the given file name is longer than - * supported by the ZIP format - */ - public void addDirectory(String filename, Date date) - throws IOException { - addDirectory(filename, date, NO_EXTRA_ENTRIES); - } - - /** - * A deflater output stream that also counts uncompressed and compressed - * numbers of bytes and computes the CRC so that the data descriptor marker - * is written correctly. - * - * <p>Not static, so it can access the write() methods. - */ - private class DeflaterOutputStream extends OutputStream { - - private final Deflater deflater = new Deflater(Deflater.DEFAULT_COMPRESSION, true); - private final CRC32 crc = new CRC32(); - private final byte[] outputBuffer = new byte[4096]; - private long uncompressedBytes = 0; - private long compressedBytes = 0; - - @Override - public void write(int b) throws IOException { - byte[] buf = new byte[] { (byte) (b & 0xff) }; - write(buf, 0, buf.length); - } - - @Override - public void write(byte b[], int off, int len) throws IOException { - checkNotFinished(); - uncompressedBytes += len; - crc.update(b, off, len); - deflater.setInput(b, off, len); - while (!deflater.needsInput()) { - deflate(); - } - } - - @Override - public void close() throws IOException { - super.close(); - deflater.end(); - } - - /** - * Writes out the remaining buffered data without closing the output - * stream. - */ - public void finish() throws IOException { - checkNotFinished(); - deflater.finish(); - while (!deflater.finished()) { - deflate(); - } - if ((compressedBytes >= MAXIMUM_DATA_SIZE) || (uncompressedBytes >= MAXIMUM_DATA_SIZE)) { - throw new IOException("Too much data for ZIP entry."); - } - } - - private void deflate() throws IOException { - int length = deflater.deflate(outputBuffer); - ZipCombiner.this.write(outputBuffer, 0, length); - compressedBytes += length; - } - - public int getCRC() { - return (int) crc.getValue(); - } - - public int getCompressedSize() { - return (int) compressedBytes; - } - - public int getUncompressedSize() { - return (int) uncompressedBytes; - } - - private void checkNotFinished() { - if (deflater.finished()) { - throw new IllegalStateException(); - } - } + public boolean containsFile(String filename) { + // TODO(apell): may be slightly different behavior because v1 returns true on skipped names. + return entries.containsKey(filename); } /** - * Writes any remaining output data to the output stream and also creates the - * merged entries by calling the {@link CustomMergeStrategy} implementations - * given back from the ZIP entry filter. + * Writes any remaining output data to the output stream and also creates the merged entries by + * calling the {@link CustomMergeStrategy} implementations given back from the + * {@link ZipEntryFilter}. * - * @throws IOException if the output stream or the filter throws an - * IOException + * @throws IOException if the output stream or the filter throws an IOException * @throws IllegalStateException if this method was already called earlier */ public void finish() throws IOException { - checkNotFinished(); - finished = true; - for (Map.Entry<String, FileEntry> entry : fileNames.entrySet()) { + for (Entry<String, EntryAction> entry : actions.entrySet()) { String filename = entry.getKey(); - CustomMergeStrategy mergeStrategy = entry.getValue().mergeStrategy; - ByteArrayOutputStream outputBuffer = entry.getValue().outputBuffer; - int dosTime = entry.getValue().dosTime; - if (mergeStrategy == null) { - // Do nothing. - } else { - mergeStrategy.finish(outputBuffer); - copyStreamToEntry(filename, new ByteArrayInputStream(outputBuffer.toByteArray()), dosTime, - NO_EXTRA_ENTRIES, true, DEFAULT_DIRECTORY_ENTRY_INFO); - } - } - - // Write central directory. - if (out.bytesWritten >= MAXIMUM_DATA_SIZE) { - throw new IOException("Unable to handle files bigger than 2^32 bytes."); - } - int startOfCentralDirectory = (int) out.bytesWritten; - int centralDirectorySize = centralDirectory.writeTo(out); - - // end of central directory signature - setUnsignedInt(directoryEntryBuffer, 0, END_OF_CENTRAL_DIRECTORY_MARKER); - // number of this disk - setUnsignedShort(directoryEntryBuffer, 4, (short) 0); - // number of the disk with the start of the central directory - setUnsignedShort(directoryEntryBuffer, 6, (short) 0); - // total number of entries in the central directory on this disk - setUnsignedShort(directoryEntryBuffer, 8, (short) fileCount); - // total number of entries in the central directory - setUnsignedShort(directoryEntryBuffer, 10, (short) fileCount); - // size of the central directory - setUnsignedInt(directoryEntryBuffer, 12, centralDirectorySize); - // offset of start of central directory with respect to the starting disk number - setUnsignedInt(directoryEntryBuffer, 16, startOfCentralDirectory); - // .ZIP file comment length - setUnsignedShort(directoryEntryBuffer, 20, (short) 0); - write(directoryEntryBuffer, 0, 22); - - out.flush(); - } + EntryAction action = entry.getValue(); + if (action.getType() == ActionType.MERGE) { + ByteArrayOutputStream uncompressed = action.getMergeBuffer(); + action.getStrategy().finish(uncompressed); - private void checkNotFinished() { - if (finished) { - throw new IllegalStateException(); + ZipFileEntry e = new ZipFileEntry(filename); + e.setTime(action.getDate() != null ? action.getDate().getTime() : new Date().getTime()); + writeEntryFromBuffer(e, uncompressed.toByteArray()); + } } + out.finish(); } /** * Writes any remaining output data to the output stream and closes it. * - * @throws IOException if the output stream or the filter throws an - * IOException + * @throws IOException if the output stream or the filter throws an IOException */ - @Override - public void close() throws IOException { - if (!finished) { - finish(); - } + @Override public void close() throws IOException { + finish(); out.close(); } - /** - * Turns this JAR file into an executable JAR by prepending an executable. - * JAR files are placed at the end of a file, and executables are placed - * at the beginning, so a file can be both, if desired. - * - * @param launcherIn The InputStream, from which the launcher is read. - * @throws NullPointerException if launcherIn is null - * @throws IOException if reading from launcherIn or writing to the output - * stream throws an IOException. - */ - public void prependExecutable(InputStream launcherIn) throws IOException { - if (launcherIn == null) { - throw new NullPointerException("No launcher specified"); - } - byte[] buf = new byte[BUFFER_SIZE]; - int bytesRead; - while ((bytesRead = launcherIn.read(buf)) > 0) { - out.write(buf, 0, bytesRead); - } - } - - /** - * Ensures the truth of an expression involving one or more parameters to the calling method. - */ + /** Ensures the truth of an expression involving one or more parameters to the calling method. */ private static void checkArgument(boolean expression, @Nullable String errorMessageTemplate, @Nullable Object... errorMessageArgs) { @@ -1621,18 +607,18 @@ public final class ZipCombiner implements AutoCloseable { } } - /** - * Ensures the truth of an expression involving one or more parameters to the calling method. - */ - private static void checkArgument(boolean expression) { - if (!expression) { - throw new IllegalArgumentException(); + /** Ensures that an object reference passed as a parameter to the calling method is not null. */ + public static <T> T checkNotNull(T reference, + @Nullable String errorMessageTemplate, + @Nullable Object... errorMessageArgs) { + if (reference == null) { + // If either of these parameters is null, the right thing happens anyway + throw new NullPointerException(String.format(errorMessageTemplate, errorMessageArgs)); } + return reference; } - /** - * Ensures the truth of an expression involving state. - */ + /** Ensures the truth of an expression involving state. */ private static void checkState(boolean expression, @Nullable String errorMessageTemplate, @Nullable Object... errorMessageArgs) { diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/zip/CountingOutputStream.java b/src/java_tools/singlejar/java/com/google/devtools/build/zip/CountingOutputStream.java new file mode 100644 index 0000000000..a08159ff14 --- /dev/null +++ b/src/java_tools/singlejar/java/com/google/devtools/build/zip/CountingOutputStream.java @@ -0,0 +1,54 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.zip; + +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +/** An OutputStream that counts the number of bytes written. */ +final class CountingOutputStream extends FilterOutputStream { + + private long count; + + /** + * Wraps another output stream, counting the number of bytes written. + * + * @param out the output stream to be wrapped + */ + public CountingOutputStream(OutputStream out) { + super(out); + } + + /** Returns the number of bytes written. */ + public long getCount() { + return count; + } + + @Override public void write(int b) throws IOException { + out.write(b); + count++; + } + + @Override public void write(byte[] b) throws IOException { + out.write(b); + count += b.length; + } + + @Override public void write(byte[] b, int off, int len) throws IOException { + out.write(b, off, len); + count += len; + } +}
\ No newline at end of file diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ExtraData.java b/src/java_tools/singlejar/java/com/google/devtools/build/zip/ExtraData.java index 2e8cb75e02..7bbe0c3dd9 100644 --- a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ExtraData.java +++ b/src/java_tools/singlejar/java/com/google/devtools/build/zip/ExtraData.java @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package com.google.devtools.build.singlejar; +package com.google.devtools.build.zip; /** * A holder class for extra data in a ZIP entry. @@ -21,7 +21,7 @@ package com.google.devtools.build.singlejar; * byte array passed into this class or returned from this class may not be * modified. */ -final class ExtraData { +public final class ExtraData { private final short id; private final byte[] data; diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/zip/README b/src/java_tools/singlejar/java/com/google/devtools/build/zip/README new file mode 100644 index 0000000000..48d662c547 --- /dev/null +++ b/src/java_tools/singlejar/java/com/google/devtools/build/zip/README @@ -0,0 +1,2 @@ +Zip is a library for reading and writing zip files, allowing more advanced manipulation than the +JDK equivalents by providing detailed zip entry data and raw file access.
\ No newline at end of file diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/zip/ZipFileEntry.java b/src/java_tools/singlejar/java/com/google/devtools/build/zip/ZipFileEntry.java new file mode 100644 index 0000000000..2b7a410c1f --- /dev/null +++ b/src/java_tools/singlejar/java/com/google/devtools/build/zip/ZipFileEntry.java @@ -0,0 +1,478 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.zip; + +import javax.annotation.Nullable; + +/** + * A full representation of a ZIP file entry. + * + * <p>See <a href="http://www.pkware.com/documents/casestudies/APPNOTE.TXT">ZIP Format</a> for + * a description of the entry fields. (Section 4.3.7 and 4.4) + */ +public final class ZipFileEntry { + + /** Compression method for ZIP entries. */ + public enum Compression { + STORED((short) 0, (short) 0x0a), + DEFLATED((short) 8, (short) 0x14); + + public static Compression fromValue(int value) { + for (Compression c : Compression.values()) { + if (c.getValue() == value) { + return c; + } + } + return null; + } + + private short value; + private short minVersion; + + private Compression(short value, short minVersion) { + this.value = value; + this.minVersion = minVersion; + } + + public short getValue() { + return value; + } + + public short getMinVersion() { + return minVersion; + } + } + + /** General purpose bit flag for ZIP entries. */ + public enum Flag { + DATA_DESCRIPTOR(3); + + private int bit; + + private Flag(int bit) { + this.bit = bit; + } + + public int getBit() { + return bit; + } + } + + private String name; + private long time = -1; + private long crc = -1; + private long size = -1; + private long csize = -1; + private Compression method; + private short version = -1; + private short versionNeeded = -1; + private short flags; + private short internalAttributes; + private int externalAttributes; + private long localHeaderOffset = -1; + @Nullable private byte[] extra; + @Nullable private String comment; + + /** + * Creates a new zip entry with the specified name. + * + * @param name the entry name + * @throws NullPointerException if the entry name is null + */ + public ZipFileEntry(String name) { + setName(name); + } + + /** + * Creates a new zip entry with fields taken from the specified zip entry. + * + * @param e a zip entry object + */ + public ZipFileEntry(ZipFileEntry e) { + if (e == null) { + throw new NullPointerException(); + } + this.name = e.getName(); + this.time = e.getTime(); + this.crc = e.getCrc(); + this.size = e.getSize(); + this.csize = e.getCompressedSize(); + this.method = e.getMethod(); + this.version = e.getVersion(); + this.versionNeeded = e.getVersionNeeded(); + this.flags = e.getFlags(); + this.internalAttributes = e.getInternalAttributes(); + this.externalAttributes = e.getExternalAttributes(); + this.localHeaderOffset = e.getLocalHeaderOffset(); + this.extra = e.getExtra(); + this.comment = e.getComment(); + } + + /** + * Sets the name of the entry. + * + * @param name the name + */ + public void setName(String name) { + if (name == null) { + throw new NullPointerException(); + } + this.name = name; + } + + /** + * Returns the name of the entry. + * + * @return the name of the entry + */ + public String getName() { + return name; + } + + /** + * Sets the modification time of the entry. + * + * @param time the entry modification time in number of milliseconds since the epoch + */ + public void setTime(long time) { + this.time = time; + } + + /** + * Returns the modification time of the entry, or -1 if not specified. + * + * @return the modification time of the entry, or -1 if not specified + */ + public long getTime() { + return time; + } + + /** + * Sets the CRC-32 checksum of the uncompressed entry data. + * + * @param crc the CRC-32 value + * @throws IllegalArgumentException if the specified CRC-32 value is less than 0 or greater than + * 0xFFFFFFFF + */ + public void setCrc(long crc) { + if (crc < 0 || crc > 0xffffffffL) { + throw new IllegalArgumentException("invalid entry crc-32"); + } + this.crc = crc; + } + + /** + * Returns the CRC-32 checksum of the uncompressed entry data, or -1 if not known. + * + * @return the CRC-32 checksum of the uncompressed entry data, or -1 if not known + */ + public long getCrc() { + return crc; + } + + /** + * Sets the uncompressed size of the entry data. + * + * @param size the uncompressed size in bytes + * @throws IllegalArgumentException if the specified size is less than 0, is greater than + * 0xFFFFFFFF + */ + public void setSize(long size) { + if (size < 0 || size > 0xffffffffL) { + throw new IllegalArgumentException("invalid entry size"); + } + this.size = size; + } + + /** + * Returns the uncompressed size of the entry data, or -1 if not known. + * + * @return the uncompressed size of the entry data, or -1 if not known + */ + public long getSize() { + return size; + } + + /** + * Sets the size of the compressed entry data. + * + * @param csize the compressed size in bytes + * @throws IllegalArgumentException if the specified size is less than 0, is greater than + * 0xFFFFFFFF + */ + public void setCompressedSize(long csize) { + if (csize < 0 || csize > 0xffffffffL) { + throw new IllegalArgumentException("invalid entry size"); + } + this.csize = csize; + } + + /** + * Returns the size of the compressed entry data, or -1 if not known. In the case of a stored + * entry, the compressed size will be the same as the uncompressed size of the entry. + * + * @return the size of the compressed entry data, or -1 if not known + */ + public long getCompressedSize() { + return csize; + } + + /** + * Sets the compression method for the entry. Increases the version and version needed if the new + * compression method requires a higher version. + * + * @param method the compression method, either STORED or DEFLATED + */ + public void setMethod(Compression method) { + if (method == null) { + throw new NullPointerException(); + } + this.method = method; + short minVersion = method.getMinVersion(); + version = (short) Math.max(version, minVersion); + versionNeeded = (short) Math.max(versionNeeded, minVersion); + } + + /** + * Returns the compression method of the entry. + * + * @return the compression method of the entry + */ + public Compression getMethod() { + return method; + } + + /** + * Sets the made by version for the entry. + * + * @param version the made by version to set + * @throws IllegalArgumentException if the specified version is less than the required version for + * the specified compression method + */ + public void setVersion(short version) { + if (method != null && version < method.getMinVersion()) { + throw new IllegalArgumentException(String.format( + "The minimum allowable version for method %s is 0x%02x.", + method.name(), method.getMinVersion())); + } + this.version = version; + } + + /** + * Returns the made by version of the entry. + * + * @return the made by version of the entry + */ + public short getVersion() { + return version; + } + + /** + * Sets the version needed to extract the entry. + * + * @param versionNeeded the version needed to extract to set + * @throws IllegalArgumentException if the specified version is less than the required version for + * the specified compression method + */ + public void setVersionNeeded(short versionNeeded) { + if (method != null && versionNeeded < method.getMinVersion()) { + throw new IllegalArgumentException(String.format( + "The minimum allowable version for method %s is 0x%02x.", + method.name(), method.getMinVersion())); + } + this.versionNeeded = versionNeeded; + } + + /** + * Returns the version needed to extract the entry. + * + * @return the version needed to extract the entry + */ + public short getVersionNeeded() { + return versionNeeded; + } + + /** + * Sets the general purpose bit flags for the entry. + * + * @param flags the general purpose bit flags to set + */ + public void setFlags(short flags) { + this.flags = flags; + } + + /** + * Sets or clears the specified bit of the general purpose bit flags. + * + * @param flag the flag to set or clear + * @param set whether the flag is to be set or cleared + */ + public void setFlag(Flag flag, boolean set) { + short mask = 0x0000; + mask |= 1 << flag.getBit(); + if (set) { + flags |= mask; + } else { + flags &= ~mask; + } + } + + /** + * Returns the general purpose bit flags of the entry. + * + * <p>See <a href="http://www.pkware.com/documents/casestudies/APPNOTE.TXT">ZIP Format</a> + * section 4.4.4. + * + * @return the general purpose bit flags of the entry + */ + public short getFlags() { + return flags; + } + + /** + * Sets the internal file attributes of the entry. + * + * @param internalAttributes the internal file attributes to set + */ + public void setInternalAttributes(short internalAttributes) { + this.internalAttributes = internalAttributes; + } + + /** + * Returns the internal file attributes of the entry. + * + * @return the internal file attributes of the entry + */ + public short getInternalAttributes() { + return internalAttributes; + } + + /** + * Sets the external file attributes of the entry. + * + * @param externalAttributes the external file attributes to set + */ + public void setExternalAttributes(int externalAttributes) { + this.externalAttributes = externalAttributes; + } + + /** + * Returns the external file attributes of the entry. + * + * @return the external file attributes of the entry + */ + public int getExternalAttributes() { + return externalAttributes; + } + + /** + * Sets the file offset, in bytes, of the location of the local file header for the entry. + * + * <p>See <a href="http://www.pkware.com/documents/casestudies/APPNOTE.TXT">ZIP Format</a> + * section 4.4.16 + * + * @param localHeaderOffset the file offset of the local header to set + * @throws IllegalArgumentException if the specified local header offset is less than 0 or greater + * than 0xFFFFFFFF + */ + void setLocalHeaderOffset(long localHeaderOffset) { + if (localHeaderOffset < 0 || localHeaderOffset > 0xffffffffL) { + throw new IllegalArgumentException("invalid local header offset"); + } + this.localHeaderOffset = localHeaderOffset; + } + + /** + * Returns the file offset of the local header of the entry. + * + * @return the file offset of the local header of the entry + */ + public long getLocalHeaderOffset() { + return localHeaderOffset; + } + + /** + * Sets the optional extra field data for the entry. + * + * <p><em>NOTE:</em> This sets the extra field exactly as specified. Use + * {@link #setExtra(ExtraData[])} to guarantee well formed extra field entries. + * + * @param extra the extra field data bytes + * @throws IllegalArgumentException if the length of the specified extra field data is greater + * than 0xFFFF bytes + */ + public void setExtra(@Nullable byte[] extra) { + if (extra != null && extra.length > 0xffff) { + throw new IllegalArgumentException("invalid extra field length"); + } + this.extra = extra; + } + + /** + * Sets the optional extra field data from the provided {@link ExtraData} array. Performs the + * necessary conversion to the raw byte array. + * + * <p><em>NOTE:</em> This will guarantee well formed extra field entries, but cannot guarantee + * usable data if Id or Data is specified incorrectly in {@link ExtraData}. + * + * @param extra the extra field data + * @throws IllegalArgumentException if the length of the specified extra field data is greater + * than 0xFFFF bytes + */ + public void setExtra(ExtraData[] extra) { + int extraDataLength = 0; + for (ExtraData e : extra) { + extraDataLength += 4 + e.getData().length; + } + + byte[] rawExtra = new byte[extraDataLength]; + + int index = 0; + for (ExtraData e : extra) { + ZipUtil.shortToLittleEndian(rawExtra, index, e.getId()); + ZipUtil.shortToLittleEndian(rawExtra, index + 2, (short) (e.getData().length & 0xffff)); + System.arraycopy(e.getData(), 0, rawExtra, index + 4, e.getData().length); + index += 4 + e.getData().length; + } + setExtra(rawExtra); + } + + /** + * Returns the extra field data for the entry, or null if none. + * + * @return the extra field data for the entry, or null if none + */ + public byte[] getExtra() { + return extra; + } + + /** + * Sets the optional comment string for the entry. + * + * @param comment the comment string + */ + public void setComment(@Nullable String comment) { + this.comment = comment; + } + + /** + * Returns the comment string for the entry, or null if none. + * + * @return the comment string for the entry, or null if none + */ + public String getComment() { + return comment; + } +} diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/zip/ZipReader.java b/src/java_tools/singlejar/java/com/google/devtools/build/zip/ZipReader.java new file mode 100644 index 0000000000..66f948e05d --- /dev/null +++ b/src/java_tools/singlejar/java/com/google/devtools/build/zip/ZipReader.java @@ -0,0 +1,472 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.zip; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.devtools.build.zip.ZipFileEntry.Compression; +import com.google.devtools.build.zip.ZipUtil.CentralDirectoryFileHeader; +import com.google.devtools.build.zip.ZipUtil.EndOfCentralDirectoryRecord; +import com.google.devtools.build.zip.ZipUtil.LocalFileHeader; + +import java.io.BufferedInputStream; +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.RandomAccessFile; +import java.nio.channels.Channels; +import java.nio.charset.Charset; +import java.util.Arrays; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.zip.Inflater; +import java.util.zip.InflaterInputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipException; +import java.util.zip.ZipFile; + +/** + * A ZIP file reader. + * + * <p>This class provides entry data in the form of {@link ZipFileEntry}, which provides more detail + * about the entry than the JDK equivalent {@link ZipEntry}. In addition to providing + * {@link InputStream}s for entries, similar to JDK {@link ZipFile#getInputStream(ZipEntry)}, it + * also provides access to the raw byte entry data via {@link #getRawInputStream(ZipFileEntry)}. + * + * <p>Using the raw access capabilities allows for more efficient ZIP file processing, such as + * merging, by not requiring each entry's data to be decompressed when read. + * + * <p><em>NOTE:</em> The entries are read from the central directory. If the entry is not listed + * there, it will not be returned from {@link #entries()} or {@link #getEntry(String)}. + */ +public class ZipReader implements Closeable, AutoCloseable { + + /** An input stream for reading the file data of a ZIP file entry. */ + private class ZipEntryInputStream extends InputStream { + private InputStream stream; + private long rem; + + /** + * Opens an input stream for reading at the beginning of the ZIP file entry's content. + * + * @param zipEntry the ZIP file entry to open the input stream for + * @throws ZipException if a ZIP format error has occurred + * @throws IOException if an I/O error has occurred + */ + private ZipEntryInputStream(ZipFileEntry zipEntry) throws IOException { + stream = new BufferedInputStream(Channels.newInputStream( + in.getChannel().position(zipEntry.getLocalHeaderOffset()))); + + byte[] fileHeader = new byte[LocalFileHeader.FIXED_DATA_SIZE]; + stream.read(fileHeader); + + if (!ZipUtil.arrayStartsWith(fileHeader, + ZipUtil.intToLittleEndian(LocalFileHeader.SIGNATURE))) { + throw new ZipException(String.format("The file '%s' is not a correctly formatted zip file: " + + "Expected a File Header at file offset %d, but was not present.", + file.getName(), zipEntry.getLocalHeaderOffset())); + } + + int nameLength = ZipUtil.getUnsignedShort(fileHeader, + LocalFileHeader.FILENAME_LENGTH_OFFSET); + int extraFieldLength = ZipUtil.getUnsignedShort(fileHeader, + LocalFileHeader.EXTRA_FIELD_LENGTH_OFFSET); + stream.skip(nameLength + extraFieldLength); + rem = zipEntry.getSize(); + if (zipEntry.getMethod() == Compression.DEFLATED) { + stream = new InflaterInputStream(stream, new Inflater(true)); + } + } + + @Override public int available() throws IOException { + return rem > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) rem; + } + + @Override public void close() throws IOException { + } + + @Override public void mark(int readlimit) { + } + + @Override public boolean markSupported() { + return false; + } + + @Override public int read() throws IOException { + byte[] b = new byte[1]; + if (read(b, 0, 1) == 1) { + return b[0] & 0xff; + } else { + return -1; + } + } + + @Override public int read(byte[] b) throws IOException { + return read(b, 0, b.length); + } + + @Override public int read(byte[] b, int off, int len) throws IOException { + if (rem == 0) { + return -1; + } + if (len > rem) { + len = available(); + } + len = stream.read(b, off, len); + rem -= len; + return len; + } + + @Override public long skip(long n) throws IOException { + if (n > rem) { + n = rem; + } + n = stream.skip(n); + rem -= n; + return n; + } + + @Override public void reset() throws IOException { + throw new IOException("Reset is not supported on this type of stream."); + } + } + + /** An input stream for reading the raw file data of a ZIP file entry. */ + private class RawZipEntryInputStream extends InputStream { + private InputStream stream; + private long rem; + + /** + * Opens an input stream for reading at the beginning of the ZIP file entry's content. + * + * @param zipEntry the ZIP file entry to open the input stream for + * @throws ZipException if a ZIP format error has occurred + * @throws IOException if an I/O error has occurred + */ + private RawZipEntryInputStream(ZipFileEntry zipEntry) throws IOException { + stream = new BufferedInputStream(Channels.newInputStream( + in.getChannel().position(zipEntry.getLocalHeaderOffset()))); + + byte[] fileHeader = new byte[LocalFileHeader.FIXED_DATA_SIZE]; + stream.read(fileHeader); + + if (!ZipUtil.arrayStartsWith(fileHeader, + ZipUtil.intToLittleEndian(LocalFileHeader.SIGNATURE))) { + throw new ZipException(String.format("The file '%s' is not a correctly formatted zip file: " + + "Expected a File Header at file offset %d, but was not present.", + file.getName(), zipEntry.getLocalHeaderOffset())); + } + + int nameLength = ZipUtil.getUnsignedShort(fileHeader, + LocalFileHeader.FILENAME_LENGTH_OFFSET); + int extraFieldLength = ZipUtil.getUnsignedShort(fileHeader, + LocalFileHeader.EXTRA_FIELD_LENGTH_OFFSET); + stream.skip(nameLength + extraFieldLength); + rem = zipEntry.getCompressedSize(); + } + + @Override public int available() throws IOException { + return rem > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) rem; + } + + @Override public void close() throws IOException { + } + + @Override public void mark(int readlimit) { + } + + @Override public boolean markSupported() { + return false; + } + + @Override public int read() throws IOException { + byte[] b = new byte[1]; + if (read(b, 0, 1) == 1) { + return b[0] & 0xff; + } else { + return -1; + } + } + + @Override public int read(byte[] b) throws IOException { + return read(b, 0, b.length); + } + + @Override public int read(byte[] b, int off, int len) throws IOException { + if (rem == 0) { + return -1; + } + if (len > rem) { + len = available(); + } + len = stream.read(b, off, len); + rem -= len; + return len; + } + + @Override public long skip(long n) throws IOException { + if (n > rem) { + n = rem; + } + n = stream.skip(n); + rem -= n; + return n; + } + + @Override public void reset() throws IOException { + throw new IOException("Reset is not supported on this type of stream."); + } + } + + private File file; + private Charset charset; + private RandomAccessFile in; + private Map<String, ZipFileEntry> zipEntries; + private String comment; + + /** + * Opens a zip file for raw acceess. + * + * <p>The UTF-8 charset is used to decode the entry names and comments. + * + * @param file the zip file + * @throws ZipException if a ZIP format error has occurred + * @throws IOException if an I/O error has occurred + */ + public ZipReader(File file) throws IOException { + this(file, UTF_8); + } + + /** + * Opens a zip file for raw acceess. + * + * @param file the zip file + * @param charset the charset to use to decode the entry names and comments. + * @throws ZipException if a ZIP format error has occurred + * @throws IOException if an I/O error has occurred + */ + public ZipReader(File file, Charset charset) throws IOException { + if (file == null || charset == null) { + throw new NullPointerException(); + } + this.file = file; + this.charset = charset; + this.in = new RandomAccessFile(file, "r"); + this.zipEntries = readCentralDirectory(); + } + + /** + * Returns the ZIP file comment. + * + * @return the ZIP file comment + */ + public String getComment() { + return comment; + } + + /** + * Returns a collection of the ZIP file entries. + * + * @return a collection of the ZIP file entries + */ + public Collection<ZipFileEntry> entries() { + return zipEntries.values(); + } + + /** + * Returns the ZIP file entry for the specified name, or null if not found. + * + * @param name the name of the entry + * @return the ZIP file entry, or null if not found + */ + public ZipFileEntry getEntry(String name) { + if (zipEntries.containsKey(name)) { + return zipEntries.get(name); + } else { + return null; + } + } + + /** + * Returns an input stream for reading the contents of the specified ZIP file entry. + * + * <p>Closing this ZIP file will, in turn, close all input streams that have been returned by + * invocations of this method. + * + * @param entry the ZIP file entry + * @return the input stream for reading the contents of the specified zip file entry + * @throws ZipException if a ZIP format error has occurred + * @throws IOException if an I/O error has occurred + */ + public InputStream getInputStream(ZipFileEntry entry) throws IOException { + if (!zipEntries.get(entry.getName()).equals(entry)) { + throw new ZipException(String.format( + "Zip file '%s' does not contain the requested entry '%s'.", file.getName(), + entry.getName())); + } + return new ZipEntryInputStream(entry); + } + + /** + * Returns an input stream for reading the raw contents of the specified ZIP file entry. + * + * <p><em>NOTE:</em> No inflating will take place; The data read from the input stream will be + * the exact byte content of the ZIP file entry on disk. + * + * <p>Closing this ZIP file will, in turn, close all input streams that have been returned by + * invocations of this method. + * + * @param entry the ZIP file entry + * @return the input stream for reading the contents of the specified zip file entry + * @throws ZipException if a ZIP format error has occurred + * @throws IOException if an I/O error has occurred + */ + public InputStream getRawInputStream(ZipFileEntry entry) throws IOException { + if (!zipEntries.get(entry.getName()).equals(entry)) { + throw new ZipException(String.format( + "Zip file '%s' does not contain the requested entry '%s'.", file.getName(), + entry.getName())); + } + return new RawZipEntryInputStream(entry); + } + + /** + * Closes the ZIP file. + * + * <p>Closing this ZIP file will close all of the input streams previously returned by invocations + * of the {@link #getRawInputStream(ZipFileEntry)} method. + */ + @Override public void close() throws IOException { + in.close(); + } + + /** + * Finds, reads and parses ZIP file entries from the central directory. + * + * @return a map of all ZIP file entries read from the central directory and their names + * @throws ZipException if a ZIP format error has occurred + * @throws IOException if an I/O error has occurred + */ + private Map<String, ZipFileEntry> readCentralDirectory() throws IOException { + byte[] eocdRecord = readEndOfCentralDirectoryRecord(); + + int commentLength = ZipUtil.getUnsignedShort(eocdRecord, + EndOfCentralDirectoryRecord.COMMENT_LENGTH_OFFSET); + this.comment = new String(Arrays.copyOfRange(eocdRecord, + EndOfCentralDirectoryRecord.FIXED_DATA_SIZE, + EndOfCentralDirectoryRecord.FIXED_DATA_SIZE + commentLength), charset); + + int totalEntries = ZipUtil.getUnsignedShort(eocdRecord, + EndOfCentralDirectoryRecord.TOTAL_ENTRIES_OFFSET); + long cdOffset = ZipUtil.getUnsignedInt(eocdRecord, + EndOfCentralDirectoryRecord.CD_OFFSET_OFFSET); + + return readCentralDirectoryFileHeaders(totalEntries, cdOffset); + } + + /** + * Looks for the target sub array in the buffer scanning backwards starting at offset. Returns the + * index where the target is found or -1 if not found. + * + * @param target the sub array to find + * @param buffer the array to scan + * @param offset the index of where to begin scanning + * @return the index of target within buffer or -1 if not found + */ + private int scanBackwards(byte[] target, byte[] buffer, int offset) { + int start = Math.min(offset, buffer.length - target.length); + for (int i = start; i >= 0; i--) { + for (int j = 0; j < target.length; j++) { + if (buffer[i + j] != target[j]) { + break; + } else if (j == target.length - 1) { + return i; + } + } + } + return -1; + } + + /** + * Finds and returns the byte array of the end of central directory record. + * + * @return the byte array of the end of central directory record + * @throws ZipException if a ZIP format error has occurred + * @throws IOException if an I/O error has occurred + */ + private byte[] readEndOfCentralDirectoryRecord() throws IOException { + byte[] signature = ZipUtil.intToLittleEndian(EndOfCentralDirectoryRecord.SIGNATURE); + byte[] buffer = new byte[(int) Math.min(64, in.length())]; + + int bytesRead = 0; + while (true) { + in.seek(in.length() - buffer.length); + in.readFully(buffer, 0, buffer.length - bytesRead); + + int signatureLocation = scanBackwards(signature, buffer, buffer.length - bytesRead - 1); + while (signatureLocation != -1) { + int eocdSize = buffer.length - signatureLocation; + if (eocdSize >= EndOfCentralDirectoryRecord.FIXED_DATA_SIZE) { + int commentLength = ZipUtil.getUnsignedShort(buffer, signatureLocation + + EndOfCentralDirectoryRecord.COMMENT_LENGTH_OFFSET); + int readCommentLength = buffer.length - signatureLocation + - EndOfCentralDirectoryRecord.FIXED_DATA_SIZE; + if (commentLength == readCommentLength) { + byte[] eocdRecord = new byte[eocdSize]; + System.arraycopy(buffer, signatureLocation, eocdRecord, 0, eocdSize); + return eocdRecord; + } + } + signatureLocation = scanBackwards(signature, buffer, signatureLocation - 1); + } + // expand buffer + bytesRead = buffer.length; + int newLength = (int) Math.min(buffer.length * 2, in.length()); + if (newLength == buffer.length) { + break; + } + byte[] newBuf = new byte[newLength]; + System.arraycopy(buffer, 0, newBuf, newBuf.length - buffer.length, buffer.length); + buffer = newBuf; + } + throw new ZipException(String.format("Zip file '%s' is malformed. It does not contain an end" + + " of central directory record.", file.getName())); + } + + /** + * Reads and parses ZIP file entries from the central directory. + * + * @param count the number of entries in the central directory + * @param fileOffset the file offset of the start of the central directory + * @return a map of all ZIP file entries read from the central directory and their names + * @throws ZipException if a ZIP format error has occurred + * @throws IOException if an I/O error has occurred + */ + private Map<String, ZipFileEntry> readCentralDirectoryFileHeaders(int count, long fileOffset) + throws IOException { + + InputStream centralDirectory = new BufferedInputStream( + Channels.newInputStream(in.getChannel().position(fileOffset))); + + Map<String, ZipFileEntry> entries = new LinkedHashMap<>(count); + for (int i = 0; i < count; i++) { + ZipFileEntry entry = CentralDirectoryFileHeader.read(centralDirectory, charset); + entries.put(entry.getName(), entry); + } + return entries; + } +} diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/zip/ZipUtil.java b/src/java_tools/singlejar/java/com/google/devtools/build/zip/ZipUtil.java new file mode 100644 index 0000000000..740f425ff4 --- /dev/null +++ b/src/java_tools/singlejar/java/com/google/devtools/build/zip/ZipUtil.java @@ -0,0 +1,418 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.zip; + +import com.google.devtools.build.zip.ZipFileEntry.Compression; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.Charset; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Collection; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.zip.ZipException; + +/** A utility class for reading and writing {@link ZipFileEntry}s from byte arrays. */ +public class ZipUtil { + + /** + * Midnight Jan 1st 1980. Uses the current time zone as the DOS format does not support time zones + * and will always assume the current zone. + */ + public static final long DOS_EPOCH = + new GregorianCalendar(1980, Calendar.JANUARY, 1, 0, 0, 0).getTimeInMillis(); + + /** 23:59:59 Dec 31st 2107. The maximum date representable in DOS format. */ + public static final long MAX_DOS_DATE = + new GregorianCalendar(2107, Calendar.DECEMBER, 31, 23, 59, 59).getTimeInMillis(); + + /** Converts a integral value to the corresponding little endian array. */ + private static byte[] integerToLittleEndian(byte[] buf, int offset, long value, int numBytes) { + for (int i = 0; i < numBytes; i++) { + buf[i + offset] = (byte) ((value & (0xffL << (i * 8))) >> (i * 8)); + } + return buf; + } + + /** Converts a short to the corresponding 2-byte little endian array. */ + static byte[] shortToLittleEndian(short value) { + return integerToLittleEndian(new byte[2], 0, value, 2); + } + + /** Writes a short to the buffer as a 2-byte little endian array starting at offset. */ + static byte[] shortToLittleEndian(byte[] buf, int offset, short value) { + return integerToLittleEndian(buf, offset, value, 2); + } + + /** Converts an int to the corresponding 4-byte little endian array. */ + static byte[] intToLittleEndian(int value) { + return integerToLittleEndian(new byte[4], 0, value, 4); + } + + /** Writes an int to the buffer as a 4-byte little endian array starting at offset. */ + static byte[] intToLittleEndian(byte[] buf, int offset, int value) { + return integerToLittleEndian(buf, offset, value, 4); + } + + /** Converts a long to the corresponding 8-byte little endian array. */ + static byte[] longToLittleEndian(long value) { + return integerToLittleEndian(new byte[8], 0, value, 8); + } + + /** Writes a long to the buffer as a 8-byte little endian array starting at offset. */ + static byte[] longToLittleEndian(byte[] buf, int offset, long value) { + return integerToLittleEndian(buf, offset, value, 8); + } + + /** Reads 16 bits in little-endian byte order from the buffer at the given offset. */ + static short get16(byte[] source, int offset) { + int a = source[offset + 0] & 0xff; + int b = source[offset + 1] & 0xff; + return (short) ((b << 8) | a); + } + + /** Reads 32 bits in little-endian byte order from the buffer at the given offset. */ + static int get32(byte[] source, int offset) { + int a = source[offset + 0] & 0xff; + int b = source[offset + 1] & 0xff; + int c = source[offset + 2] & 0xff; + int d = source[offset + 3] & 0xff; + return (d << 24) | (c << 16) | (b << 8) | a; + } + + /** + * Reads an unsigned short in little-endian byte order from the buffer at the given offset. + * Casts to an int to allow proper numerical comparison. + */ + static int getUnsignedShort(byte[] source, int offset) { + return get16(source, offset) & 0xffff; + } + + /** + * Reads an unsigned int in little-endian byte order from the buffer at the given offset. + * Casts to a long to allow proper numerical comparison. + */ + static long getUnsignedInt(byte[] source, int offset) { + return get32(source, offset) & 0xffffffffL; + } + + /** Checks if the timestamp is representable as a valid DOS timestamp. */ + private static boolean isValidInDos(long timestamp) { + Calendar time = Calendar.getInstance(); + time.setTimeInMillis(timestamp); + Calendar minTime = Calendar.getInstance(); + minTime.setTimeInMillis(DOS_EPOCH); + Calendar maxTime = Calendar.getInstance(); + maxTime.setTimeInMillis(MAX_DOS_DATE); + return (!time.before(minTime) && !time.after(maxTime)); + } + + /** Converts a unix timestamp into a 32-bit DOS timestamp. */ + static int unixToDosTime(long timestamp) { + Calendar time = Calendar.getInstance(); + time.setTimeInMillis(timestamp); + + if (!isValidInDos(timestamp)) { + DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + throw new IllegalArgumentException(String.format("%s is not representable in the DOS time" + + " format. It must be in the range %s to %s", df.format(time.getTime()), + df.format(new Date(DOS_EPOCH)), df.format(new Date(MAX_DOS_DATE)))); + } + + int dos = time.get(Calendar.SECOND) >> 1; + dos |= time.get(Calendar.MINUTE) << 5; + dos |= time.get(Calendar.HOUR_OF_DAY) << 11; + dos |= time.get(Calendar.DAY_OF_MONTH) << 16; + dos |= (time.get(Calendar.MONTH) + 1) << 21; + dos |= (time.get(Calendar.YEAR) - 1980) << 25; + return dos; + } + + /** Converts a 32-bit DOS timestamp into a unix timestamp. */ + static long dosToUnixTime(int timestamp) { + Calendar time = Calendar.getInstance(); + time.clear(); + time.set(Calendar.SECOND, (timestamp << 1) & 0x3e); + time.set(Calendar.MINUTE, (timestamp >> 5) & 0x3f); + time.set(Calendar.HOUR_OF_DAY, (timestamp >> 11) & 0x1f); + time.set(Calendar.DAY_OF_MONTH, (timestamp >> 16) & 0x1f); + time.set(Calendar.MONTH, ((timestamp >> 21) & 0x0f) - 1); + time.set(Calendar.YEAR, ((timestamp >> 25) & 0x7f) + 1980); + return time.getTimeInMillis(); + } + + /** Checks if array starts with target. */ + static boolean arrayStartsWith(byte[] array, byte[] target) { + if (array == null) { + return false; + } + if (target == null) { + return true; + } + if (target.length > array.length) { + return false; + } + for (int i = 0; i < target.length; i++) { + if (array[i] != target[i]) { + return false; + } + } + return true; + } + + static class LocalFileHeader { + static final int SIGNATURE = 0x04034b50; + static final int FIXED_DATA_SIZE = 30; + static final int SIGNATURE_OFFSET = 0; + static final int VERSION_OFFSET = 4; + static final int FLAGS_OFFSET = 6; + static final int METHOD_OFFSET = 8; + static final int MOD_TIME_OFFSET = 10; + static final int CRC_OFFSET = 14; + static final int COMPRESSED_SIZE_OFFSET = 18; + static final int UNCOMPRESSED_SIZE_OFFSET = 22; + static final int FILENAME_LENGTH_OFFSET = 26; + static final int EXTRA_FIELD_LENGTH_OFFSET = 28; + static final int VARIABLE_DATA_OFFSET = 30; + + /** + * Generates the raw byte data of the local file header for the ZipEntry. Uses the specified + * charset to encode the file name and comment. + */ + static byte[] create(ZipFileEntry entry, Charset charset) { + byte[] name = entry.getName().getBytes(charset); + byte[] extra = entry.getExtra() != null ? entry.getExtra() : new byte[]{}; + + byte[] buf = new byte[FIXED_DATA_SIZE + name.length + extra.length]; + + intToLittleEndian(buf, SIGNATURE_OFFSET, SIGNATURE); + shortToLittleEndian(buf, VERSION_OFFSET, entry.getVersionNeeded()); + shortToLittleEndian(buf, FLAGS_OFFSET, entry.getFlags()); + shortToLittleEndian(buf, METHOD_OFFSET, (short) (entry.getMethod().getValue() & 0xffff)); + intToLittleEndian(buf, MOD_TIME_OFFSET, unixToDosTime(entry.getTime())); + intToLittleEndian(buf, CRC_OFFSET, (int) (entry.getCrc() & 0xffffffff)); + intToLittleEndian(buf, COMPRESSED_SIZE_OFFSET, + (int) (entry.getCompressedSize() & 0xffffffff)); + intToLittleEndian(buf, UNCOMPRESSED_SIZE_OFFSET, (int) (entry.getSize() & 0xffffffff)); + shortToLittleEndian(buf, FILENAME_LENGTH_OFFSET, (short) name.length); + shortToLittleEndian(buf, EXTRA_FIELD_LENGTH_OFFSET, (short) extra.length); + System.arraycopy(name, 0, buf, FIXED_DATA_SIZE, name.length); + System.arraycopy(extra, 0, buf, FIXED_DATA_SIZE + name.length, extra.length); + + return buf; + } + } + + static class CentralDirectoryFileHeader { + static final int SIGNATURE = 0x02014b50; + static final int FIXED_DATA_SIZE = 46; + static final int SIGNATURE_OFFSET = 0; + static final int VERSION_OFFSET = 4; + static final int VERSION_NEEDED_OFFSET = 6; + static final int FLAGS_OFFSET = 8; + static final int METHOD_OFFSET = 10; + static final int MOD_TIME_OFFSET = 12; + static final int CRC_OFFSET = 16; + static final int COMPRESSED_SIZE_OFFSET = 20; + static final int UNCOMPRESSED_SIZE_OFFSET = 24; + static final int FILENAME_LENGTH_OFFSET = 28; + static final int EXTRA_FIELD_LENGTH_OFFSET = 30; + static final int COMMENT_LENGTH_OFFSET = 32; + static final int DISK_START_OFFSET = 34; + static final int INTERNAL_ATTRIBUTES_OFFSET = 36; + static final int EXTERNAL_ATTRIBUTES_OFFSET = 38; + static final int LOCAL_HEADER_OFFSET_OFFSET = 42; + + /** + * Reads a {@link ZipFileEntry} from the input stream, using the specified charset to decode the + * filename and comment. + */ + static ZipFileEntry read(InputStream in, Charset charset) + throws IOException { + byte[] fixedSizeData = new byte[FIXED_DATA_SIZE]; + + if (in.read(fixedSizeData) != FIXED_DATA_SIZE) { + throw new ZipException( + "Unexpected end of file while reading Central Directory File Header."); + } + if (!arrayStartsWith(fixedSizeData, intToLittleEndian(SIGNATURE))) { + throw new ZipException(String.format( + "Malformed Central Directory File Header; does not start with %08x", SIGNATURE)); + } + + byte[] name = new byte[getUnsignedShort(fixedSizeData, FILENAME_LENGTH_OFFSET)]; + byte[] extraField = new byte[getUnsignedShort(fixedSizeData, EXTRA_FIELD_LENGTH_OFFSET)]; + byte[] comment = new byte[getUnsignedShort(fixedSizeData, COMMENT_LENGTH_OFFSET)]; + + if (name.length > 0 && in.read(name) != name.length) { + throw new ZipException( + "Unexpected end of file while reading Central Directory File Header."); + } + if (extraField.length > 0 && in.read(extraField) != extraField.length) { + throw new ZipException( + "Unexpected end of file while reading Central Directory File Header."); + } + if (comment.length > 0 && in.read(comment) != comment.length) { + throw new ZipException( + "Unexpected end of file while reading Central Directory File Header."); + } + + ZipFileEntry entry = new ZipFileEntry(new String(name, charset)); + entry.setVersion(get16(fixedSizeData, VERSION_OFFSET)); + entry.setVersionNeeded(get16(fixedSizeData, VERSION_NEEDED_OFFSET)); + entry.setFlags(get16(fixedSizeData, FLAGS_OFFSET)); + entry.setMethod(Compression.fromValue(get16(fixedSizeData, METHOD_OFFSET))); + long time = dosToUnixTime(get32(fixedSizeData, MOD_TIME_OFFSET)); + entry.setTime(isValidInDos(time) ? time : DOS_EPOCH); + entry.setCrc(getUnsignedInt(fixedSizeData, CRC_OFFSET)); + entry.setCompressedSize(getUnsignedInt(fixedSizeData, COMPRESSED_SIZE_OFFSET)); + entry.setSize(getUnsignedInt(fixedSizeData, UNCOMPRESSED_SIZE_OFFSET)); + entry.setInternalAttributes(get16(fixedSizeData, INTERNAL_ATTRIBUTES_OFFSET)); + entry.setExternalAttributes(get32(fixedSizeData, EXTERNAL_ATTRIBUTES_OFFSET)); + entry.setLocalHeaderOffset(getUnsignedInt(fixedSizeData, LOCAL_HEADER_OFFSET_OFFSET)); + entry.setExtra(extraField); + entry.setComment(new String(comment, charset)); + + return entry; + } + + /** + * Generates the raw byte data of the central directory file header for the ZipEntry. Uses the + * specified charset to encode the file name and comment. + */ + static byte[] create(ZipFileEntry entry, Charset charset) { + byte[] name = entry.getName().getBytes(charset); + byte[] extra = entry.getExtra() != null ? entry.getExtra() : new byte[]{}; + byte[] comment = entry.getComment() != null + ? entry.getComment().getBytes(charset) : new byte[]{}; + + byte[] buf = new byte[FIXED_DATA_SIZE + name.length + extra.length + comment.length]; + + fillFixedSizeData(buf, entry, name.length, extra.length, comment.length); + System.arraycopy(name, 0, buf, FIXED_DATA_SIZE, name.length); + System.arraycopy(extra, 0, buf, FIXED_DATA_SIZE + name.length, extra.length); + System.arraycopy(comment, 0, buf, FIXED_DATA_SIZE + name.length + extra.length, + comment.length); + + return buf; + } + + /** + * Writes the central directory file header for the ZipEntry to an output stream. Uses the + * specified charset to encode the file name and comment. + */ + static int write(ZipFileEntry entry, Charset charset, byte[] buf, + OutputStream stream) throws IOException { + if (buf == null || buf.length < FIXED_DATA_SIZE) { + buf = new byte[FIXED_DATA_SIZE]; + } + + byte[] name = entry.getName().getBytes(charset); + byte[] extra = entry.getExtra() != null ? entry.getExtra() : new byte[]{}; + byte[] comment = entry.getComment() != null + ? entry.getComment().getBytes(charset) : new byte[]{}; + + fillFixedSizeData(buf, entry, name.length, extra.length, comment.length); + stream.write(buf, 0, FIXED_DATA_SIZE); + stream.write(name); + stream.write(extra); + stream.write(comment); + + return FIXED_DATA_SIZE + name.length + extra.length + comment.length; + } + + /** + * Write the fixed size data portion for the specified ZIP entry to the buffer. + */ + private static void fillFixedSizeData(byte[] buf, ZipFileEntry entry, int nameLength, + int extraLength, int commentLength) { + intToLittleEndian(buf, SIGNATURE_OFFSET, SIGNATURE); + shortToLittleEndian(buf, VERSION_OFFSET, entry.getVersion()); + shortToLittleEndian(buf, VERSION_NEEDED_OFFSET, entry.getVersionNeeded()); + shortToLittleEndian(buf, FLAGS_OFFSET, entry.getFlags()); + shortToLittleEndian(buf, METHOD_OFFSET, (short) (entry.getMethod().getValue() & 0xffff)); + intToLittleEndian(buf, MOD_TIME_OFFSET, unixToDosTime(entry.getTime())); + intToLittleEndian(buf, CRC_OFFSET, (int) (entry.getCrc() & 0xffffffff)); + intToLittleEndian(buf, COMPRESSED_SIZE_OFFSET, + (int) (entry.getCompressedSize() & 0xffffffff)); + intToLittleEndian(buf, UNCOMPRESSED_SIZE_OFFSET, (int) (entry.getSize() & 0xffffffff)); + shortToLittleEndian(buf, FILENAME_LENGTH_OFFSET, (short) (nameLength & 0xffff)); + shortToLittleEndian(buf, EXTRA_FIELD_LENGTH_OFFSET, (short) (extraLength & 0xffff)); + shortToLittleEndian(buf, COMMENT_LENGTH_OFFSET, (short) (commentLength & 0xffff)); + shortToLittleEndian(buf, DISK_START_OFFSET, (short) 0); + shortToLittleEndian(buf, INTERNAL_ATTRIBUTES_OFFSET, entry.getInternalAttributes()); + intToLittleEndian(buf, EXTERNAL_ATTRIBUTES_OFFSET, entry.getExternalAttributes()); + intToLittleEndian(buf, LOCAL_HEADER_OFFSET_OFFSET, + (int) (entry.getLocalHeaderOffset() & 0xffffffff)); + } + } + + static class EndOfCentralDirectoryRecord { + static final int SIGNATURE = 0x06054b50; + static final int FIXED_DATA_SIZE = 22; + static final int SIGNATURE_OFFSET = 0; + static final int DISK_NUMBER_OFFSET = 4; + static final int CD_DISK_OFFSET = 6; + static final int DISK_ENTRIES_OFFSET = 8; + static final int TOTAL_ENTRIES_OFFSET = 10; + static final int CD_SIZE_OFFSET = 12; + static final int CD_OFFSET_OFFSET = 16; + static final int COMMENT_LENGTH_OFFSET = 20; + + /** + * Generates the raw byte data of the end of central directory record, given the specifics of + * the ZIP file. + */ + static byte[] create(long fileOffset, long size, long numEntries, String fileComment, + Charset charset) { + byte[] comment = fileComment.getBytes(charset); + + byte[] buf = new byte[FIXED_DATA_SIZE + comment.length]; + + intToLittleEndian(buf, SIGNATURE_OFFSET, SIGNATURE); + shortToLittleEndian(buf, DISK_NUMBER_OFFSET, (short) 0); + shortToLittleEndian(buf, CD_DISK_OFFSET, (short) 0); + shortToLittleEndian(buf, DISK_ENTRIES_OFFSET, (short) (numEntries & 0xffff)); + shortToLittleEndian(buf, TOTAL_ENTRIES_OFFSET, (short) (numEntries & 0xffff)); + intToLittleEndian(buf, CD_SIZE_OFFSET, (int) (size & 0xffffffff)); + intToLittleEndian(buf, CD_OFFSET_OFFSET, (int) (fileOffset & 0xffffffff)); + shortToLittleEndian(buf, COMMENT_LENGTH_OFFSET, (short) (comment.length & 0xffff)); + System.arraycopy(comment, 0, buf, FIXED_DATA_SIZE, comment.length); + + return buf; + } + } + + static class CentralDirectory { + + /** + * Writes the central directory to an output stream, given the specifics of the ZIP file. + */ + static void write(Collection<ZipFileEntry> entries, String fileComment, long fileOffset, + Charset charset, OutputStream stream) throws IOException { + long directorySize = 0; + byte[] buf = new byte[CentralDirectoryFileHeader.FIXED_DATA_SIZE]; + for (ZipFileEntry entry : entries) { + directorySize += CentralDirectoryFileHeader.write(entry, charset, buf, stream); + } + stream.write(EndOfCentralDirectoryRecord.create(fileOffset, directorySize, entries.size(), + fileComment != null ? fileComment : "", charset)); + } + } +} diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/zip/ZipWriter.java b/src/java_tools/singlejar/java/com/google/devtools/build/zip/ZipWriter.java new file mode 100644 index 0000000000..ad8eafc5b8 --- /dev/null +++ b/src/java_tools/singlejar/java/com/google/devtools/build/zip/ZipWriter.java @@ -0,0 +1,231 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.zip; + +import com.google.devtools.build.zip.ZipFileEntry.Flag; +import com.google.devtools.build.zip.ZipUtil.CentralDirectory; +import com.google.devtools.build.zip.ZipUtil.LocalFileHeader; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.Charset; +import java.util.LinkedList; +import java.util.List; +import java.util.zip.ZipException; + +/** + * This class implements an output stream filter for writing files in the ZIP file format. It does + * not perform its own compression and so allows writing of already compressed file data. + */ +public class ZipWriter extends OutputStream { + private CountingOutputStream stream; + private Charset charset; + private String comment; + private boolean writingPrefix; + private ZipFileEntry entry; + private long bytesWritten; + private List<ZipFileEntry> entries; + private boolean finished; + + /** + * Creates a new raw ZIP output stream. + * + * @param out the actual output stream + * @param charset the {@link Charset} to be used to encode the entry names and comments + */ + public ZipWriter(OutputStream out, Charset charset) { + this.stream = new CountingOutputStream(out); + this.charset = charset; + this.entries = new LinkedList<>(); + this.finished = false; + } + + /** + * Sets the ZIP file comment. + * + * @param comment the ZIP file comment + */ + public void setComment(String comment) { + this.comment = comment; + } + + /** + * Configures the stream to write prefix file data. + * + * @throws ZipException if other contents have already been written to the output stream + */ + public void startPrefixFile() throws ZipException { + checkNotFinished(); + if (!entries.isEmpty() || entry != null) { + throw new ZipException("Cannot add a prefix file after the zip contents have been started."); + } + writingPrefix = true; + } + + /** Closes the prefix file and positions the output stream to write ZIP entries. */ + public void endPrefixFile() { + checkNotFinished(); + writingPrefix = false; + } + + /** + * Begins writing a new ZIP file entry and positions the stream to the start of the entry data. + * Closes the current entry if still active. + * + * <p><em>NOTE:</em> No defensive copying is performed on e. The local header offset and flags + * will be modified. + * + * @param e the ZIP entry to be written + * @throws IOException if an I/O error occurred + */ + public void putNextEntry(ZipFileEntry e) throws IOException { + checkNotFinished(); + writingPrefix = false; + if (entry != null) { + finishEntry(); + } + startEntry(e); + } + + /** + * Closes the current ZIP entry and positions the stream for writing the next entry. + * + * @throws ZipException if a ZIP format exception occurred + * @throws IOException if an I/O error occurred + */ + public void closeEntry() throws IOException { + checkNotFinished(); + if (entry != null) { + finishEntry(); + } + } + + @Override public void write(int b) throws IOException { + byte[] buf = new byte[1]; + buf[0] = (byte) (b & 0xff); + write(buf); + } + + @Override public void write(byte[] b) throws IOException { + write(b, 0, b.length); + } + + @Override public synchronized void write(byte[] b, int off, int len) throws IOException { + checkNotFinished(); + if (entry == null && !writingPrefix) { + throw new ZipException("Cannot write zip contents without first setting a ZipEntry or" + + " starting a prefix file."); + } + stream.write(b, off, len); + bytesWritten += len; + } + + /** + * Finishes writing the contents of the ZIP output stream without closing the underlying stream. + * Use this method when applying multiple filters in succession to the same output stream. + * + * @throws ZipException if a ZIP file error has occurred + * @throws IOException if an I/O exception has occurred + */ + public void finish() throws IOException { + checkNotFinished(); + if (entry != null) { + finishEntry(); + } + writeCentralDirectory(); + finished = true; + } + + @Override public void close() throws IOException { + if (!finished) { + finish(); + } + stream.close(); + } + + /** + * Writes the local file header for the ZIP entry and positions the stream to the start of the + * entry data. + * + * @param e the ZIP entry for which to write the local file header + * @throws ZipException if a ZIP file error has occurred + * @throws IOException if an I/O exception has occurred + */ + private void startEntry(ZipFileEntry e) throws IOException { + if (e.getName() == null) { + throw new IllegalArgumentException("Zip entry name must not be null"); + } + if (e.getTime() == -1) { + throw new IllegalArgumentException("Zip entry last modified time must be set"); + } + if (e.getCrc() == -1) { + throw new IllegalArgumentException("Zip entry CRC-32 must be set"); + } + if (e.getSize() == -1) { + throw new IllegalArgumentException("Zip entry uncompressed size must be set"); + } + if (e.getCompressedSize() == -1) { + throw new IllegalArgumentException("Zip entry compressed size must be set"); + } + if (e.getMethod() == null) { + throw new IllegalArgumentException("Zip entry compression method must not be null"); + } + if (e.getVersion() == -1) { + throw new IllegalArgumentException("Zip entry version made by must be set"); + } + if (e.getVersionNeeded() == -1) { + throw new IllegalArgumentException("Zip entry version needed must be set"); + } + bytesWritten = 0; + entry = e; + entry.setFlag(Flag.DATA_DESCRIPTOR, false); + entry.setLocalHeaderOffset(stream.getCount()); + stream.write(LocalFileHeader.create(entry, charset)); + } + + /** + * Closes the current ZIP entry and positions the stream for writing the next entry. Checks that + * the amount of data written matches the compressed size indicated by the ZipEntry. + * + * @throws ZipException if a ZIP file error has occurred + * @throws IOException if an I/O exception has occurred + */ + private void finishEntry() throws IOException { + if (entry.getCompressedSize() != bytesWritten) { + throw new ZipException(String.format("Number of bytes written for the entry %s (%d) does not" + + " match the reported compressed size (%d).", entry.getName(), bytesWritten, + entry.getCompressedSize())); + } + entries.add(entry); + entry = null; + } + + /** + * Writes the ZIP file's central directory. + * + * @throws ZipException if a ZIP file error has occurred + * @throws IOException if an I/O exception has occurred + */ + private void writeCentralDirectory() throws IOException { + CentralDirectory.write(entries, comment, stream.getCount(), charset, stream); + } + + /** Checks that the ZIP file has not been finished yet. */ + private void checkNotFinished() { + if (finished) { + throw new IllegalStateException(); + } + } +} diff --git a/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/MockSimpleFileSystem.java b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/MockSimpleFileSystem.java index 8fec585fe0..d6f801f23e 100644 --- a/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/MockSimpleFileSystem.java +++ b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/MockSimpleFileSystem.java @@ -21,10 +21,13 @@ import static org.junit.Assert.assertNull; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; import java.util.HashMap; import java.util.Map; @@ -74,6 +77,17 @@ public final class MockSimpleFileSystem implements SimpleFileSystem { } @Override + public File getFile(String filename) throws IOException { + byte[] data = files.get(filename); + if (data == null) { + throw new FileNotFoundException(); + } + File file = File.createTempFile(filename, null); + Files.copy(new ByteArrayInputStream(data), file.toPath(), StandardCopyOption.REPLACE_EXISTING); + return file; + } + + @Override public boolean delete(String filename) { assertEquals(outputFileName, filename); assertNotNull(out); diff --git a/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/SingleJarTest.java b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/SingleJarTest.java index 0c67b61e18..34c4cc0bf5 100644 --- a/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/SingleJarTest.java +++ b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/SingleJarTest.java @@ -14,8 +14,9 @@ package com.google.devtools.build.singlejar; +import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import com.google.common.base.Joiner; @@ -78,7 +79,7 @@ public class SingleJarTest { private final List<String> manifestLines; public ManifestValidator(List<String> manifestLines) { - this.manifestLines = new ArrayList<String>(manifestLines); + this.manifestLines = new ArrayList<>(manifestLines); Collections.sort(this.manifestLines); } @@ -146,7 +147,7 @@ public class SingleJarTest { private void assertStripFirstLine(String expected, String testCase) { byte[] result = SingleJar.stripFirstLine(testCase.getBytes(StandardCharsets.UTF_8)); - assertEquals(expected, new String(result)); + assertEquals(expected, new String(result, UTF_8)); } @Test @@ -428,7 +429,7 @@ public class SingleJarTest { MockSimpleFileSystem mockFs = new MockSimpleFileSystem("output.jar"); SingleJar singleJar = new SingleJar(mockFs); - List<String> args = new ArrayList<String>(); + List<String> args = new ArrayList<>(); args.add("--output"); args.add("output.jar"); args.addAll(infoPropertyArguments(buildInfo)); @@ -591,8 +592,8 @@ public class SingleJarTest { singleJar.run(ImmutableList.of("--output", "output.jar", "--exclude_build_data", "--resources", "a/b/c", "a/b/c")); fail(); - } catch (IllegalStateException e) { - assertTrue(e.getMessage().contains("already contains a file named a/b/c")); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage()).contains("already contains a file named 'a/b/c'."); } } @@ -616,19 +617,20 @@ public class SingleJarTest { public void testCanAddPreamble() throws IOException { MockSimpleFileSystem mockFs = new MockSimpleFileSystem("output.jar"); String preamble = "WeThePeople"; - mockFs.addFile(preamble, preamble.getBytes()); + mockFs.addFile(preamble, preamble.getBytes(UTF_8)); SingleJar singleJar = new SingleJar(mockFs); singleJar.run(ImmutableList.of("--output", "output.jar", "--java_launcher", preamble, "--main_class", "SomeClass")); - FakeZipFile expectedResult = new FakeZipFile() - .addPreamble(preamble.getBytes()) - .addEntry("META-INF/", EXTRA_FOR_META_INF) - .addEntry(JarFile.MANIFEST_NAME, new ManifestValidator( - "Manifest-Version: 1.0", - "Created-By: blaze-singlejar", - "Main-Class: SomeClass")) - .addEntry("build-data.properties", redactedBuildData("output.jar", "SomeClass")); + FakeZipFile expectedResult = + new FakeZipFile() + .addPreamble(preamble.getBytes(UTF_8)) + .addEntry("META-INF/", EXTRA_FOR_META_INF) + .addEntry( + JarFile.MANIFEST_NAME, + new ManifestValidator("Manifest-Version: 1.0", "Created-By: blaze-singlejar", + "Main-Class: SomeClass")) + .addEntry("build-data.properties", redactedBuildData("output.jar", "SomeClass")); expectedResult.assertSame(mockFs.toByteArray()); } } diff --git a/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/ZipCombinerTest.java b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/ZipCombinerTest.java index e5345cb1f8..aba561bfd3 100644 --- a/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/ZipCombinerTest.java +++ b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/ZipCombinerTest.java @@ -23,34 +23,44 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import com.google.common.base.Preconditions; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ListMultimap; import com.google.devtools.build.singlejar.ZipCombiner.OutputMode; import com.google.devtools.build.singlejar.ZipEntryFilter.CustomMergeStrategy; +import com.google.devtools.build.zip.ExtraData; +import com.google.devtools.build.zip.ZipFileEntry; +import com.google.devtools.build.zip.ZipReader; +import com.google.devtools.build.zip.ZipUtil; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.rules.TemporaryFolder; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.EOFException; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Arrays; -import java.util.Calendar; +import java.util.Collection; import java.util.Date; import java.util.GregorianCalendar; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.jar.JarOutputStream; import java.util.zip.ZipEntry; +import java.util.zip.ZipException; import java.util.zip.ZipInputStream; import java.util.zip.ZipOutputStream; @@ -59,39 +69,45 @@ import java.util.zip.ZipOutputStream; */ @RunWith(JUnit4.class) public class ZipCombinerTest { + @Rule public TemporaryFolder tmp = new TemporaryFolder(); + @Rule public ExpectedException thrown = ExpectedException.none(); - private static final Date DOS_EPOCH = ZipCombiner.DOS_EPOCH; - - private InputStream sampleZip() { + private File sampleZip() throws IOException { ZipFactory factory = new ZipFactory(); factory.addFile("hello.txt", "Hello World!"); - return factory.toInputStream(); + return writeInputStreamToFile(factory.toInputStream()); } - private InputStream sampleZip2() { + private File sampleZip2() throws IOException { ZipFactory factory = new ZipFactory(); factory.addFile("hello2.txt", "Hello World 2!"); - return factory.toInputStream(); + return writeInputStreamToFile(factory.toInputStream()); } - private InputStream sampleZipWithTwoEntries() { + private File sampleZipWithTwoEntries() throws IOException { ZipFactory factory = new ZipFactory(); factory.addFile("hello.txt", "Hello World!"); factory.addFile("hello2.txt", "Hello World 2!"); - return factory.toInputStream(); + return writeInputStreamToFile(factory.toInputStream()); } - private InputStream sampleZipWithOneUncompressedEntry() { + private File sampleZipWithOneUncompressedEntry() throws IOException { ZipFactory factory = new ZipFactory(); factory.addFile("hello.txt", "Hello World!", false); - return factory.toInputStream(); + return writeInputStreamToFile(factory.toInputStream()); } - private InputStream sampleZipWithTwoUncompressedEntries() { + private File sampleZipWithTwoUncompressedEntries() throws IOException { ZipFactory factory = new ZipFactory(); factory.addFile("hello.txt", "Hello World!", false); factory.addFile("hello2.txt", "Hello World 2!", false); - return factory.toInputStream(); + return writeInputStreamToFile(factory.toInputStream()); + } + + private File writeInputStreamToFile(InputStream in) throws IOException { + File out = tmp.newFile(); + Files.copy(in, out.toPath(), StandardCopyOption.REPLACE_EXISTING); + return out; } private void assertEntry(ZipInputStream zipInput, String filename, long time, byte[] content) @@ -111,7 +127,7 @@ public class ZipCombinerTest { private void assertEntry(ZipInputStream zipInput, String filename, byte[] content) throws IOException { - assertEntry(zipInput, filename, ZipCombiner.DOS_EPOCH.getTime(), content); + assertEntry(zipInput, filename, ZipUtil.DOS_EPOCH, content); } private void assertEntry(ZipInputStream zipInput, String filename, String content) @@ -124,210 +140,112 @@ public class ZipCombinerTest { assertEntry(zipInput, filename, date.getTime(), content.getBytes(ISO_8859_1)); } - @Test - public void testDateToDosTime() { - assertEquals(0x210000, ZipCombiner.dateToDosTime(ZipCombiner.DOS_EPOCH)); - Calendar calendar = new GregorianCalendar(); - for (int i = 1980; i <= 2107; i++) { - calendar.set(i, 0, 1, 0, 0, 0); - int result = ZipCombiner.dateToDosTime(calendar.getTime()); - assertEquals(i - 1980, result >>> 25); - assertEquals(1, (result >> 21) & 0xf); - assertEquals(1, (result >> 16) & 0x1f); - assertEquals(0, result & 0xffff); - } - } - - @Test - public void testDateToDosTimeFailsForBadValues() { - try { - Calendar calendar = new GregorianCalendar(); - calendar.set(1979, 0, 1, 0, 0, 0); - ZipCombiner.dateToDosTime(calendar.getTime()); - fail(); - } catch (IllegalArgumentException e) { - /* Expected exception. */ - } - try { - Calendar calendar = new GregorianCalendar(); - calendar.set(2108, 0, 1, 0, 0, 0); - ZipCombiner.dateToDosTime(calendar.getTime()); - fail(); - } catch (IllegalArgumentException e) { - /* Expected exception. */ - } - } - - @Test - public void testCompressedDontCare() throws IOException { + @Test public void testCompressedDontCare() throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner singleJar = new ZipCombiner(out); - singleJar.addZip(sampleZip()); - singleJar.close(); - FakeZipFile expectedResult = new FakeZipFile() - .addEntry("hello.txt", "Hello World!", true); + try (ZipCombiner zipCombiner = new ZipCombiner(out)) { + zipCombiner.addZip(sampleZip()); + } + FakeZipFile expectedResult = new FakeZipFile().addEntry("hello.txt", "Hello World!", true); expectedResult.assertSame(out.toByteArray()); } - @Test - public void testCompressedForceDeflate() throws IOException { + @Test public void testCompressedForceDeflate() throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner singleJar = new ZipCombiner(OutputMode.FORCE_DEFLATE, out); - singleJar.addZip(sampleZip()); - singleJar.close(); - FakeZipFile expectedResult = new FakeZipFile() - .addEntry("hello.txt", "Hello World!", true); + try (ZipCombiner zipCombiner = new ZipCombiner(OutputMode.FORCE_DEFLATE, out)) { + zipCombiner.addZip(sampleZip()); + } + FakeZipFile expectedResult = new FakeZipFile().addEntry("hello.txt", "Hello World!", true); expectedResult.assertSame(out.toByteArray()); } - @Test - public void testCompressedForceStored() throws IOException { + @Test public void testCompressedForceStored() throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner singleJar = new ZipCombiner(OutputMode.FORCE_STORED, out); - singleJar.addZip(sampleZip()); - singleJar.close(); - FakeZipFile expectedResult = new FakeZipFile() - .addEntry("hello.txt", "Hello World!", false); + try (ZipCombiner zipCombiner = new ZipCombiner(OutputMode.FORCE_STORED, out)) { + zipCombiner.addZip(sampleZip()); + } + FakeZipFile expectedResult = new FakeZipFile().addEntry("hello.txt", "Hello World!", false); expectedResult.assertSame(out.toByteArray()); } - @Test - public void testUncompressedDontCare() throws IOException { + @Test public void testUncompressedDontCare() throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner singleJar = new ZipCombiner(out); - singleJar.addZip(sampleZipWithOneUncompressedEntry()); - singleJar.close(); - FakeZipFile expectedResult = new FakeZipFile() - .addEntry("hello.txt", "Hello World!", false); + try (ZipCombiner zipCombiner = new ZipCombiner(out)) { + zipCombiner.addZip(sampleZipWithOneUncompressedEntry()); + } + FakeZipFile expectedResult = new FakeZipFile().addEntry("hello.txt", "Hello World!", false); expectedResult.assertSame(out.toByteArray()); } - @Test - public void testUncompressedForceDeflate() throws IOException { + @Test public void testUncompressedForceDeflate() throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner singleJar = new ZipCombiner(OutputMode.FORCE_DEFLATE, out); - singleJar.addZip(sampleZipWithOneUncompressedEntry()); - singleJar.close(); - FakeZipFile expectedResult = new FakeZipFile() - .addEntry("hello.txt", "Hello World!", true); + try (ZipCombiner zipCombiner = new ZipCombiner(OutputMode.FORCE_DEFLATE, out)) { + zipCombiner.addZip(sampleZipWithOneUncompressedEntry()); + } + FakeZipFile expectedResult = new FakeZipFile().addEntry("hello.txt", "Hello World!", true); expectedResult.assertSame(out.toByteArray()); } - @Test - public void testUncompressedForceStored() throws IOException { + @Test public void testUncompressedForceStored() throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner singleJar = new ZipCombiner(OutputMode.FORCE_STORED, out); - singleJar.addZip(sampleZipWithOneUncompressedEntry()); - singleJar.close(); - FakeZipFile expectedResult = new FakeZipFile() - .addEntry("hello.txt", "Hello World!", false); + try (ZipCombiner zipCombiner = new ZipCombiner(OutputMode.FORCE_STORED, out)) { + zipCombiner.addZip(sampleZipWithOneUncompressedEntry()); + } + FakeZipFile expectedResult = new FakeZipFile().addEntry("hello.txt", "Hello World!", false); expectedResult.assertSame(out.toByteArray()); } - @Test - public void testCopyTwoEntries() throws IOException { + @Test public void testCopyTwoEntries() throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner singleJar = new ZipCombiner(out); - singleJar.addZip(sampleZipWithTwoEntries()); - singleJar.close(); + try (ZipCombiner zipCombiner = new ZipCombiner(out)) { + zipCombiner.addZip(sampleZipWithTwoEntries()); + } ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray())); assertEntry(zipInput, "hello.txt", "Hello World!"); assertEntry(zipInput, "hello2.txt", "Hello World 2!"); assertNull(zipInput.getNextEntry()); } - @Test - public void testCopyTwoUncompressedEntries() throws IOException { + @Test public void testCopyTwoUncompressedEntries() throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner singleJar = new ZipCombiner(out); - singleJar.addZip(sampleZipWithTwoUncompressedEntries()); - singleJar.close(); + try (ZipCombiner zipCombiner = new ZipCombiner(out)) { + zipCombiner.addZip(sampleZipWithTwoUncompressedEntries()); + } ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray())); assertEntry(zipInput, "hello.txt", "Hello World!"); assertEntry(zipInput, "hello2.txt", "Hello World 2!"); assertNull(zipInput.getNextEntry()); } - @Test - public void testCombine() throws IOException { + @Test public void testCombine() throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner singleJar = new ZipCombiner(out); - singleJar.addZip(sampleZip()); - singleJar.addZip(sampleZip2()); - singleJar.close(); + try (ZipCombiner zipCombiner = new ZipCombiner(out)) { + zipCombiner.addZip(sampleZip()); + zipCombiner.addZip(sampleZip2()); + } ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray())); assertEntry(zipInput, "hello.txt", "Hello World!"); assertEntry(zipInput, "hello2.txt", "Hello World 2!"); assertNull(zipInput.getNextEntry()); } - @Test - public void testDuplicateEntry() throws IOException { + @Test public void testDuplicateEntry() throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner singleJar = new ZipCombiner(out); - singleJar.addZip(sampleZip()); - singleJar.addZip(sampleZip()); - singleJar.close(); - ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray())); - assertEntry(zipInput, "hello.txt", "Hello World!"); - assertNull(zipInput.getNextEntry()); - } - - // Returns an input stream that can only read one byte at a time. - private InputStream slowRead(final InputStream in) { - return new InputStream() { - @Override - public int read() throws IOException { - return in.read(); - } - @Override - public int read(byte b[], int off, int len) throws IOException { - Preconditions.checkArgument(b != null); - Preconditions.checkArgument((len >= 0) && (off >= 0)); - Preconditions.checkArgument(len <= b.length - off); - if (len == 0) { - return 0; - } - int value = read(); - if (value == -1) { - return -1; - } - b[off] = (byte) value; - return 1; - } - }; - } - - @Test - public void testDuplicateUncompressedEntryWithSlowRead() throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner singleJar = new ZipCombiner(out); - singleJar.addZip(slowRead(sampleZipWithOneUncompressedEntry())); - singleJar.addZip(slowRead(sampleZipWithOneUncompressedEntry())); - singleJar.close(); - ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray())); - assertEntry(zipInput, "hello.txt", "Hello World!"); - assertNull(zipInput.getNextEntry()); - } - - @Test - public void testDuplicateEntryWithSlowRead() throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner singleJar = new ZipCombiner(out); - singleJar.addZip(slowRead(sampleZip())); - singleJar.addZip(slowRead(sampleZip())); - singleJar.close(); + try (ZipCombiner zipCombiner = new ZipCombiner(out)) { + zipCombiner.addZip(sampleZip()); + zipCombiner.addZip(sampleZip()); + } ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray())); assertEntry(zipInput, "hello.txt", "Hello World!"); assertNull(zipInput.getNextEntry()); } - @Test - public void testBadZipFileNoEntry() throws IOException { + @Test public void testBadZipFileNoEntry() throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner singleJar = new ZipCombiner(out); - singleJar.addZip(new ByteArrayInputStream(new byte[] { 1, 2, 3, 4 })); - singleJar.close(); + try (ZipCombiner zipCombiner = new ZipCombiner(out)) { + thrown.expect(ZipException.class); + thrown.expectMessage("It does not contain an end of central directory record."); + zipCombiner.addZip(writeInputStreamToFile(new ByteArrayInputStream(new byte[] {1, 2, 3, 4}))); + } ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray())); assertNull(zipInput.getNextEntry()); } @@ -336,24 +254,22 @@ public class ZipCombinerTest { return new ByteArrayInputStream(content.getBytes(UTF_8)); } - @Test - public void testAddFile() throws IOException { + @Test public void testAddFile() throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner singleJar = new ZipCombiner(out); - singleJar.addFile("hello.txt", DOS_EPOCH, asStream("Hello World!")); - singleJar.close(); + try (ZipCombiner zipCombiner = new ZipCombiner(out)) { + zipCombiner.addFile("hello.txt", ZipCombiner.DOS_EPOCH, asStream("Hello World!")); + } ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray())); assertEntry(zipInput, "hello.txt", "Hello World!"); assertNull(zipInput.getNextEntry()); } - @Test - public void testAddFileAndDuplicateZipEntry() throws IOException { + @Test public void testAddFileAndDuplicateZipEntry() throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner singleJar = new ZipCombiner(out); - singleJar.addFile("hello.txt", DOS_EPOCH, asStream("Hello World!")); - singleJar.addZip(sampleZip()); - singleJar.close(); + try (ZipCombiner zipCombiner = new ZipCombiner(out)) { + zipCombiner.addFile("hello.txt", ZipCombiner.DOS_EPOCH, asStream("Hello World!")); + zipCombiner.addZip(sampleZip()); + } ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray())); assertEntry(zipInput, "hello.txt", "Hello World!"); assertNull(zipInput.getNextEntry()); @@ -381,7 +297,7 @@ public class ZipCombinerTest { */ class MockZipEntryFilter implements ZipEntryFilter { - private Date date = DOS_EPOCH; + private Date date = ZipCombiner.DOS_EPOCH; private final List<String> calls = new ArrayList<>(); // File name to merge strategy map. private final Map<String, CustomMergeStrategy> behavior = @@ -416,36 +332,33 @@ public class ZipCombinerTest { } } - @Test - public void testCopyCallsFilter() throws IOException { + @Test public void testCopyCallsFilter() throws IOException { MockZipEntryFilter mockFilter = new MockZipEntryFilter(); ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner singleJar = new ZipCombiner(mockFilter, out); - singleJar.addZip(sampleZip()); - singleJar.close(); + try (ZipCombiner zipCombiner = new ZipCombiner(mockFilter, out)) { + zipCombiner.addZip(sampleZip()); + } assertEquals(Arrays.asList("hello.txt"), mockFilter.calls); } - @Test - public void testDuplicateEntryCallsFilterOnce() throws IOException { + @Test public void testDuplicateEntryCallsFilterOnce() throws IOException { MockZipEntryFilter mockFilter = new MockZipEntryFilter(); ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner singleJar = new ZipCombiner(mockFilter, out); - singleJar.addZip(sampleZip()); - singleJar.addZip(sampleZip()); - singleJar.close(); + try (ZipCombiner zipCombiner = new ZipCombiner(mockFilter, out)) { + zipCombiner.addZip(sampleZip()); + zipCombiner.addZip(sampleZip()); + } assertEquals(Arrays.asList("hello.txt"), mockFilter.calls); } - @Test - public void testMergeStrategy() throws IOException { + @Test public void testMergeStrategy() throws IOException { MockZipEntryFilter mockFilter = new MockZipEntryFilter(); mockFilter.behavior.put("hello.txt", new ConcatenateStrategy()); ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner singleJar = new ZipCombiner(mockFilter, out); - singleJar.addZip(sampleZip()); - singleJar.addZip(sampleZipWithTwoEntries()); - singleJar.close(); + try (ZipCombiner zipCombiner = new ZipCombiner(mockFilter, out)) { + zipCombiner.addZip(sampleZip()); + zipCombiner.addZip(sampleZipWithTwoEntries()); + } assertEquals(Arrays.asList("hello.txt", "hello2.txt"), mockFilter.calls); ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray())); assertEntry(zipInput, "hello2.txt", "Hello World 2!"); @@ -453,47 +366,29 @@ public class ZipCombinerTest { assertNull(zipInput.getNextEntry()); } - @Test - public void testMergeStrategyWithUncompressedFiles() throws IOException { + @Test public void testMergeStrategyWithUncompressedFiles() throws IOException { MockZipEntryFilter mockFilter = new MockZipEntryFilter(); mockFilter.behavior.put("hello.txt", new ConcatenateStrategy()); mockFilter.behavior.put("hello2.txt", SKIP_PLACEHOLDER); ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner singleJar = new ZipCombiner(mockFilter, out); - singleJar.addZip(sampleZipWithTwoUncompressedEntries()); - singleJar.addZip(sampleZipWithTwoUncompressedEntries()); - singleJar.close(); - assertEquals(Arrays.asList("hello.txt", "hello2.txt"), mockFilter.calls); - ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray())); - assertEntry(zipInput, "hello.txt", "Hello World!\nHello World!"); - assertNull(zipInput.getNextEntry()); - } - - @Test - public void testMergeStrategyWithUncompressedEntriesAndSlowRead() throws IOException { - MockZipEntryFilter mockFilter = new MockZipEntryFilter(); - mockFilter.behavior.put("hello.txt", new ConcatenateStrategy()); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner singleJar = new ZipCombiner(mockFilter, out); - singleJar.addZip(slowRead(sampleZipWithOneUncompressedEntry())); - singleJar.addZip(slowRead(sampleZipWithTwoUncompressedEntries())); - singleJar.close(); + try (ZipCombiner zipCombiner = new ZipCombiner(mockFilter, out)) { + zipCombiner.addZip(sampleZipWithTwoUncompressedEntries()); + zipCombiner.addZip(sampleZipWithTwoUncompressedEntries()); + } assertEquals(Arrays.asList("hello.txt", "hello2.txt"), mockFilter.calls); ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray())); - assertEntry(zipInput, "hello2.txt", "Hello World 2!"); assertEntry(zipInput, "hello.txt", "Hello World!\nHello World!"); assertNull(zipInput.getNextEntry()); } - @Test - public void testMergeStrategyWithSlowCopy() throws IOException { + @Test public void testMergeStrategyWithSlowCopy() throws IOException { MockZipEntryFilter mockFilter = new MockZipEntryFilter(); mockFilter.behavior.put("hello.txt", new SlowConcatenateStrategy()); ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner singleJar = new ZipCombiner(mockFilter, out); - singleJar.addZip(sampleZip()); - singleJar.addZip(sampleZipWithTwoEntries()); - singleJar.close(); + try (ZipCombiner zipCombiner = new ZipCombiner(mockFilter, out)) { + zipCombiner.addZip(sampleZip()); + zipCombiner.addZip(sampleZipWithTwoEntries()); + } assertEquals(Arrays.asList("hello.txt", "hello2.txt"), mockFilter.calls); ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray())); assertEntry(zipInput, "hello2.txt", "Hello World 2!"); @@ -501,44 +396,41 @@ public class ZipCombinerTest { assertNull(zipInput.getNextEntry()); } - @Test - public void testMergeStrategyWithUncompressedFilesAndSlowCopy() throws IOException { + @Test public void testMergeStrategyWithUncompressedFilesAndSlowCopy() throws IOException { MockZipEntryFilter mockFilter = new MockZipEntryFilter(); mockFilter.behavior.put("hello.txt", new SlowConcatenateStrategy()); mockFilter.behavior.put("hello2.txt", SKIP_PLACEHOLDER); ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner singleJar = new ZipCombiner(mockFilter, out); - singleJar.addZip(sampleZipWithTwoUncompressedEntries()); - singleJar.addZip(sampleZipWithTwoUncompressedEntries()); - singleJar.close(); + try (ZipCombiner zipCombiner = new ZipCombiner(mockFilter, out)) { + zipCombiner.addZip(sampleZipWithTwoUncompressedEntries()); + zipCombiner.addZip(sampleZipWithTwoUncompressedEntries()); + } assertEquals(Arrays.asList("hello.txt", "hello2.txt"), mockFilter.calls); ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray())); assertEntry(zipInput, "hello.txt", "Hello World!Hello World!"); assertNull(zipInput.getNextEntry()); } - private InputStream specialZipWithMinusOne() { + private File specialZipWithMinusOne() throws IOException { ZipFactory factory = new ZipFactory(); factory.addFile("hello.txt", new byte[] {-1}); - return factory.toInputStream(); + return writeInputStreamToFile(factory.toInputStream()); } - @Test - public void testMergeStrategyWithSlowCopyAndNegativeBytes() throws IOException { + @Test public void testMergeStrategyWithSlowCopyAndNegativeBytes() throws IOException { MockZipEntryFilter mockFilter = new MockZipEntryFilter(); mockFilter.behavior.put("hello.txt", new SlowConcatenateStrategy()); ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner singleJar = new ZipCombiner(mockFilter, out); - singleJar.addZip(specialZipWithMinusOne()); - singleJar.close(); + try (ZipCombiner zipCombiner = new ZipCombiner(mockFilter, out)) { + zipCombiner.addZip(specialZipWithMinusOne()); + } assertEquals(Arrays.asList("hello.txt"), mockFilter.calls); ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray())); assertEntry(zipInput, "hello.txt", new byte[] { -1 }); assertNull(zipInput.getNextEntry()); } - @Test - public void testCopyDateHandling() throws IOException { + @Test public void testCopyDateHandling() throws IOException { final Date date = new GregorianCalendar(2009, 8, 2, 0, 0, 0).getTime(); ZipEntryFilter mockFilter = new ZipEntryFilter() { @Override @@ -548,33 +440,31 @@ public class ZipCombinerTest { } }; ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner singleJar = new ZipCombiner(mockFilter, out); - singleJar.addZip(sampleZip()); - singleJar.close(); + try (ZipCombiner zipCombiner = new ZipCombiner(mockFilter, out)) { + zipCombiner.addZip(sampleZip()); + } ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray())); assertEntry(zipInput, "hello.txt", date, "Hello World!"); assertNull(zipInput.getNextEntry()); } - @Test - public void testMergeDateHandling() throws IOException { + @Test public void testMergeDateHandling() throws IOException { MockZipEntryFilter mockFilter = new MockZipEntryFilter(); mockFilter.behavior.put("hello.txt", new ConcatenateStrategy()); mockFilter.date = new GregorianCalendar(2009, 8, 2, 0, 0, 0).getTime(); ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner singleJar = new ZipCombiner(mockFilter, out); - singleJar.addZip(sampleZip()); - singleJar.addZip(sampleZipWithTwoEntries()); - singleJar.close(); + try (ZipCombiner zipCombiner = new ZipCombiner(mockFilter, out)) { + zipCombiner.addZip(sampleZip()); + zipCombiner.addZip(sampleZipWithTwoEntries()); + } assertEquals(Arrays.asList("hello.txt", "hello2.txt"), mockFilter.calls); ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray())); - assertEntry(zipInput, "hello2.txt", DOS_EPOCH, "Hello World 2!"); + assertEntry(zipInput, "hello2.txt", ZipCombiner.DOS_EPOCH, "Hello World 2!"); assertEntry(zipInput, "hello.txt", mockFilter.date, "Hello World!\nHello World!"); assertNull(zipInput.getNextEntry()); } - @Test - public void testDuplicateCallThrowsException() throws IOException { + @Test public void testDuplicateCallThrowsException() throws IOException { ZipEntryFilter badFilter = new ZipEntryFilter() { @Override public void accept(String filename, StrategyCallback callback) throws IOException { @@ -584,16 +474,15 @@ public class ZipCombinerTest { } }; ByteArrayOutputStream out = new ByteArrayOutputStream(); - try (ZipCombiner singleJar = new ZipCombiner(badFilter, out)) { - singleJar.addZip(sampleZip()); + try (ZipCombiner zipCombiner = new ZipCombiner(badFilter, out)) { + zipCombiner.addZip(sampleZip()); fail(); } catch (IllegalStateException e) { // Expected exception. } } - @Test - public void testNoCallThrowsException() throws IOException { + @Test public void testNoCallThrowsException() throws IOException { ZipEntryFilter badFilter = new ZipEntryFilter() { @Override public void accept(String filename, StrategyCallback callback) { @@ -601,8 +490,8 @@ public class ZipCombinerTest { } }; ByteArrayOutputStream out = new ByteArrayOutputStream(); - try (ZipCombiner singleJar = new ZipCombiner(badFilter, out)) { - singleJar.addZip(sampleZip()); + try (ZipCombiner zipCombiner = new ZipCombiner(badFilter, out)) { + zipCombiner.addZip(sampleZip()); fail(); } catch (IllegalStateException e) { // Expected exception. @@ -612,20 +501,20 @@ public class ZipCombinerTest { // This test verifies that if an entry A is renamed as A (identy mapping), // then subsequent entries named A are still subject to filtering. // Note: this is different from a copy, where subsequent entries are skipped. - @Test - public void testRenameIdentityMapping() throws IOException { + @Test public void testRenameIdentityMapping() throws IOException { MockZipEntryFilter mockFilter = new MockZipEntryFilter(); mockFilter.behavior.put("hello.txt", COPY_PLACEHOLDER); mockFilter.behavior.put("hello2.txt", COPY_PLACEHOLDER); mockFilter.renameMap.put("hello.txt", "hello.txt"); // identity rename, not copy mockFilter.renameMap.put("hello2.txt", "hello2.txt"); // identity rename, not copy ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner singleJar = new ZipCombiner(mockFilter, out); - singleJar.addZip(sampleZipWithTwoEntries()); - singleJar.addZip(sampleZipWithTwoEntries()); - singleJar.close(); - assertThat(mockFilter.calls).containsExactly("hello.txt", "hello2.txt", - "hello.txt", "hello2.txt").inOrder(); + try (ZipCombiner zipCombiner = new ZipCombiner(mockFilter, out)) { + zipCombiner.addZip(sampleZipWithTwoEntries()); + zipCombiner.addZip(sampleZipWithTwoEntries()); + } + assertThat(mockFilter.calls) + .containsExactly("hello.txt", "hello2.txt", "hello.txt", "hello2.txt") + .inOrder(); ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray())); assertEntry(zipInput, "hello.txt", "Hello World!"); assertEntry(zipInput, "hello2.txt", "Hello World 2!"); @@ -634,20 +523,20 @@ public class ZipCombinerTest { // This test verifies that multiple entries with the same name can be // renamed to unique names. - @Test - public void testRenameNoConflictMapping() throws IOException { + @Test public void testRenameNoConflictMapping() throws IOException { MockZipEntryFilter mockFilter = new MockZipEntryFilter(); mockFilter.behavior.put("hello.txt", COPY_PLACEHOLDER); mockFilter.behavior.put("hello2.txt", COPY_PLACEHOLDER); mockFilter.renameMap.putAll("hello.txt", Arrays.asList("hello1.txt", "hello2.txt")); mockFilter.renameMap.putAll("hello2.txt", Arrays.asList("world1.txt", "world2.txt")); ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner singleJar = new ZipCombiner(mockFilter, out); - singleJar.addZip(sampleZipWithTwoEntries()); - singleJar.addZip(sampleZipWithTwoEntries()); - singleJar.close(); - assertThat(mockFilter.calls).containsExactly("hello.txt", "hello2.txt", - "hello.txt", "hello2.txt").inOrder(); + try (ZipCombiner zipCombiner = new ZipCombiner(mockFilter, out)) { + zipCombiner.addZip(sampleZipWithTwoEntries()); + zipCombiner.addZip(sampleZipWithTwoEntries()); + } + assertThat(mockFilter.calls) + .containsExactly("hello.txt", "hello2.txt", "hello.txt", "hello2.txt") + .inOrder(); ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray())); assertEntry(zipInput, "hello1.txt", "Hello World!"); assertEntry(zipInput, "world1.txt", "Hello World 2!"); @@ -659,8 +548,7 @@ public class ZipCombinerTest { // This tests verifies that an attempt to rename an entry to a // name already written, results in the entry being skipped, after // calling the filter. - @Test - public void testRenameSkipUsedName() throws IOException { + @Test public void testRenameSkipUsedName() throws IOException { MockZipEntryFilter mockFilter = new MockZipEntryFilter(); mockFilter.behavior.put("hello.txt", COPY_PLACEHOLDER); mockFilter.behavior.put("hello2.txt", COPY_PLACEHOLDER); @@ -668,13 +556,15 @@ public class ZipCombinerTest { Arrays.asList("hello1.txt", "hello2.txt", "hello3.txt")); mockFilter.renameMap.put("hello2.txt", "hello2.txt"); ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner singleJar = new ZipCombiner(mockFilter, out); - singleJar.addZip(sampleZipWithTwoEntries()); - singleJar.addZip(sampleZipWithTwoEntries()); - singleJar.addZip(sampleZipWithTwoEntries()); - singleJar.close(); - assertThat(mockFilter.calls).containsExactly("hello.txt", "hello2.txt", - "hello.txt", "hello2.txt", "hello.txt", "hello2.txt").inOrder(); + try (ZipCombiner zipCombiner = new ZipCombiner(mockFilter, out)) { + zipCombiner.addZip(sampleZipWithTwoEntries()); + zipCombiner.addZip(sampleZipWithTwoEntries()); + zipCombiner.addZip(sampleZipWithTwoEntries()); + } + assertThat(mockFilter.calls) + .containsExactly( + "hello.txt", "hello2.txt", "hello.txt", "hello2.txt", "hello.txt", "hello2.txt") + .inOrder(); ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray())); assertEntry(zipInput, "hello1.txt", "Hello World!"); assertEntry(zipInput, "hello2.txt", "Hello World 2!"); @@ -685,21 +575,21 @@ public class ZipCombinerTest { // This tests verifies that if an entry has been copied, then // further entries of the same name are skipped (filter not invoked), // and entries renamed to the same name are skipped (after calling filter). - @Test - public void testRenameAndCopy() throws IOException { + @Test public void testRenameAndCopy() throws IOException { MockZipEntryFilter mockFilter = new MockZipEntryFilter(); mockFilter.behavior.put("hello.txt", COPY_PLACEHOLDER); mockFilter.behavior.put("hello2.txt", COPY_PLACEHOLDER); mockFilter.renameMap.putAll("hello.txt", Arrays.asList("hello1.txt", "hello2.txt", "hello3.txt")); ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner singleJar = new ZipCombiner(mockFilter, out); - singleJar.addZip(sampleZipWithTwoEntries()); - singleJar.addZip(sampleZipWithTwoEntries()); - singleJar.addZip(sampleZipWithTwoEntries()); - singleJar.close(); - assertThat(mockFilter.calls).containsExactly("hello.txt", "hello2.txt", - "hello.txt", "hello.txt").inOrder(); + try (ZipCombiner zipCombiner = new ZipCombiner(mockFilter, out)) { + zipCombiner.addZip(sampleZipWithTwoEntries()); + zipCombiner.addZip(sampleZipWithTwoEntries()); + zipCombiner.addZip(sampleZipWithTwoEntries()); + } + assertThat(mockFilter.calls) + .containsExactly("hello.txt", "hello2.txt", "hello.txt", "hello.txt") + .inOrder(); ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray())); assertEntry(zipInput, "hello1.txt", "Hello World!"); assertEntry(zipInput, "hello2.txt", "Hello World 2!"); @@ -710,21 +600,21 @@ public class ZipCombinerTest { // This tests verifies that if an entry has been skipped, then // further entries of the same name are skipped (filter not invoked), // and entries renamed to the same name are skipped (after calling filter). - @Test - public void testRenameAndSkip() throws IOException { + @Test public void testRenameAndSkip() throws IOException { MockZipEntryFilter mockFilter = new MockZipEntryFilter(); mockFilter.behavior.put("hello.txt", COPY_PLACEHOLDER); mockFilter.behavior.put("hello2.txt", SKIP_PLACEHOLDER); mockFilter.renameMap.putAll("hello.txt", Arrays.asList("hello1.txt", "hello2.txt", "hello3.txt")); ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner singleJar = new ZipCombiner(mockFilter, out); - singleJar.addZip(sampleZipWithTwoEntries()); - singleJar.addZip(sampleZipWithTwoEntries()); - singleJar.addZip(sampleZipWithTwoEntries()); - singleJar.close(); - assertThat(mockFilter.calls).containsExactly("hello.txt", "hello2.txt", - "hello.txt", "hello.txt").inOrder(); + try (ZipCombiner zipCombiner = new ZipCombiner(mockFilter, out)) { + zipCombiner.addZip(sampleZipWithTwoEntries()); + zipCombiner.addZip(sampleZipWithTwoEntries()); + zipCombiner.addZip(sampleZipWithTwoEntries()); + } + assertThat(mockFilter.calls) + .containsExactly("hello.txt", "hello2.txt", "hello.txt", "hello.txt") + .inOrder(); ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray())); assertEntry(zipInput, "hello1.txt", "Hello World!"); assertEntry(zipInput, "hello3.txt", "Hello World!"); @@ -734,8 +624,7 @@ public class ZipCombinerTest { // This test verifies that renaming works when input and output // disagree on compression method. This is the simple case, where // content is read and rewritten, and no header repair is needed. - @Test - public void testRenameWithUncompressedFiles () throws IOException { + @Test public void testRenameWithUncompressedFiles () throws IOException { MockZipEntryFilter mockFilter = new MockZipEntryFilter(); mockFilter.behavior.put("hello.txt", COPY_PLACEHOLDER); mockFilter.behavior.put("hello2.txt", COPY_PLACEHOLDER); @@ -743,13 +632,15 @@ public class ZipCombinerTest { Arrays.asList("hello1.txt", "hello2.txt", "hello3.txt")); mockFilter.renameMap.put("hello2.txt", "hello2.txt"); ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner singleJar = new ZipCombiner(mockFilter, out); - singleJar.addZip(sampleZipWithTwoUncompressedEntries()); - singleJar.addZip(sampleZipWithTwoUncompressedEntries()); - singleJar.addZip(sampleZipWithTwoUncompressedEntries()); - singleJar.close(); - assertThat(mockFilter.calls).containsExactly("hello.txt", "hello2.txt", - "hello.txt", "hello2.txt", "hello.txt", "hello2.txt").inOrder(); + try (ZipCombiner zipCombiner = new ZipCombiner(mockFilter, out)) { + zipCombiner.addZip(sampleZipWithTwoUncompressedEntries()); + zipCombiner.addZip(sampleZipWithTwoUncompressedEntries()); + zipCombiner.addZip(sampleZipWithTwoUncompressedEntries()); + } + assertThat(mockFilter.calls) + .containsExactly( + "hello.txt", "hello2.txt", "hello.txt", "hello2.txt", "hello.txt", "hello2.txt") + .inOrder(); ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray())); assertEntry(zipInput, "hello1.txt", "Hello World!"); assertEntry(zipInput, "hello2.txt", "Hello World 2!"); @@ -762,66 +653,6 @@ public class ZipCombinerTest { // the data descriptor marker. It's unfortunately a bit tricky to create such // a ZIP. private static final int LOCAL_FILE_HEADER_MARKER = 0x04034b50; - private static final int DATA_DESCRIPTOR_MARKER = 0x08074b50; - private static final byte[] DATA_DESCRIPTOR_MARKER_AS_BYTES = new byte[] { - 0x50, 0x4b, 0x07, 0x08 - }; - - // Create a ZIP with an data descriptor marker in the DEFLATE content of a - // file. To do that, we build the ZIP byte by byte. - private InputStream zipWithUnexpectedDataDescriptorMarker() { - ByteBuffer out = ByteBuffer.wrap(new byte[200]).order(ByteOrder.LITTLE_ENDIAN); - out.clear(); - // file header - out.putInt(LOCAL_FILE_HEADER_MARKER); // file header signature - out.putShort((short) 6); // version to extract - out.putShort((short) 8); // general purpose bit flag - out.putShort((short) ZipOutputStream.DEFLATED); // compression method - out.putShort((short) 0); // mtime (00:00:00) - out.putShort((short) 0x21); // mdate (1.1.1980) - out.putInt(0); // crc32 - out.putInt(0); // compressed size - out.putInt(0); // uncompressed size - out.putShort((short) 1); // file name length - out.putShort((short) 0); // extra field length - out.put((byte) 'a'); // file name - - // file contents - out.put((byte) 0x01); // deflated content block is last block and uncompressed - out.putShort((short) 4); // uncompressed block length - out.putShort((short) ~4); // negated uncompressed block length - out.putInt(DATA_DESCRIPTOR_MARKER); // 4 bytes uncompressed data - - // data descriptor - out.putInt(DATA_DESCRIPTOR_MARKER); // data descriptor with marker - out.putInt((int) ZipFactory.calculateCrc32(DATA_DESCRIPTOR_MARKER_AS_BYTES)); - out.putInt(9); - out.putInt(4); - // We omit the central directory here. It's currently not used by - // ZipCombiner or by java.util.zip.ZipInputStream, so that shouldn't be a - // problem. - return new ByteArrayInputStream(out.array()); - } - - // Check that the created ZIP is correct. - @Test - public void testZipWithUnexpectedDataDescriptorMarkerIsCorrect() throws IOException { - ZipInputStream zipInput = new ZipInputStream(zipWithUnexpectedDataDescriptorMarker()); - assertEntry(zipInput, "a", DATA_DESCRIPTOR_MARKER_AS_BYTES); - assertNull(zipInput.getNextEntry()); - } - - // Check that ZipCombiner handles the ZIP correctly. - @Test - public void testZipWithUnexpectedDataDescriptorMarker() throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner singleJar = new ZipCombiner(out); - singleJar.addZip(zipWithUnexpectedDataDescriptorMarker()); - singleJar.close(); - ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray())); - assertEntry(zipInput, "a", DATA_DESCRIPTOR_MARKER_AS_BYTES); - assertNull(zipInput.getNextEntry()); - } // Create a ZIP with a partial entry. private InputStream zipWithPartialEntry() { @@ -848,89 +679,96 @@ public class ZipCombinerTest { return new ByteArrayInputStream(out.array()); } - @Test - public void testBadZipFilePartialEntry() throws IOException { + @Test public void testBadZipFilePartialEntry() throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); - try (ZipCombiner singleJar = new ZipCombiner(out)) { - singleJar.addZip(zipWithPartialEntry()); - fail(); - } catch (EOFException e) { - // Expected exception. + try (ZipCombiner zipCombiner = new ZipCombiner(out)) { + thrown.expect(ZipException.class); + thrown.expectMessage("It does not contain an end of central directory record."); + zipCombiner.addZip(writeInputStreamToFile(zipWithPartialEntry())); } } - @Test - public void testSimpleJarAgainstJavaUtil() throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - JarOutputStream jarOut = new JarOutputStream(out); - ZipEntry entry; - entry = new ZipEntry("META-INF/"); - entry.setTime(DOS_EPOCH.getTime()); - entry.setMethod(JarOutputStream.STORED); - entry.setSize(0); - entry.setCompressedSize(0); - entry.setCrc(0); - jarOut.putNextEntry(entry); - entry = new ZipEntry("META-INF/MANIFEST.MF"); - entry.setTime(DOS_EPOCH.getTime()); - entry.setMethod(JarOutputStream.DEFLATED); - jarOut.putNextEntry(entry); - jarOut.write(new byte[] { 1, 2, 3, 4 }); - jarOut.close(); - byte[] javaFile = out.toByteArray(); + @Test public void testZipCombinerAgainstJavaUtil() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (JarOutputStream jarOut = new JarOutputStream(out)) { + ZipEntry entry; + entry = new ZipEntry("META-INF/"); + entry.setTime(ZipCombiner.DOS_EPOCH.getTime()); + entry.setMethod(JarOutputStream.STORED); + entry.setSize(0); + entry.setCompressedSize(0); + entry.setCrc(0); + jarOut.putNextEntry(entry); + entry = new ZipEntry("META-INF/MANIFEST.MF"); + entry.setTime(ZipCombiner.DOS_EPOCH.getTime()); + entry.setMethod(JarOutputStream.DEFLATED); + jarOut.putNextEntry(entry); + jarOut.write(new byte[] {1, 2, 3, 4}); + } + File javaFile = writeInputStreamToFile(new ByteArrayInputStream(out.toByteArray())); out.reset(); - ZipCombiner singleJar = new ZipCombiner(out); - singleJar.addDirectory("META-INF/", DOS_EPOCH, - new ExtraData[] { new ExtraData((short) 0xCAFE, new byte[0]) }); - singleJar.addFile("META-INF/MANIFEST.MF", DOS_EPOCH, - new ByteArrayInputStream(new byte[] { 1, 2, 3, 4 })); - singleJar.close(); - byte[] singlejarFile = out.toByteArray(); - - new ZipTester(singlejarFile).validate(); - assertZipFilesEquivalent(singlejarFile, javaFile); - } - - void assertZipFilesEquivalent(byte[] x, byte[] y) { - assertEquals(x.length, y.length); - - for (int i = 0; i < x.length; i++) { - if (x[i] != y[i]) { - // Allow general purpose bit 11 (UTF-8 encoding) used in jdk7 to differ - assertEquals("at position " + i, 0x08, x[i] ^ y[i]); - // Check that x[i] is the second byte of a general purpose bit flag. - // Phil Katz, you will never be forgotten. - assertTrue( - // Local header - x[i-7] == 'P' && x[i-6] == 'K' && x[i-5] == 3 && x[i-4] == 4 || - // Central directory header - x[i-9] == 'P' && x[i-8] == 'K' && x[i-7] == 1 && x[i-6] == 2); - } + try (ZipCombiner zipcombiner = new ZipCombiner(out)) { + zipcombiner.addDirectory("META-INF/", ZipCombiner.DOS_EPOCH, + new ExtraData[] {new ExtraData((short) 0xCAFE, new byte[0])}); + zipcombiner.addFile("META-INF/MANIFEST.MF", ZipCombiner.DOS_EPOCH, + new ByteArrayInputStream(new byte[] {1, 2, 3, 4})); } + File zipCombinerFile = writeInputStreamToFile(new ByteArrayInputStream(out.toByteArray())); + byte[] zipCombinerRaw = out.toByteArray(); + + new ZipTester(zipCombinerRaw).validate(); + assertZipFilesEquivalent(new ZipReader(zipCombinerFile), new ZipReader(javaFile)); + } + + void assertZipFilesEquivalent(ZipReader x, ZipReader y) { + Collection<ZipFileEntry> xEntries = x.entries(); + Collection<ZipFileEntry> yEntries = y.entries(); + assertThat(xEntries).hasSize(yEntries.size()); + Iterator<ZipFileEntry> xIter = xEntries.iterator(); + Iterator<ZipFileEntry> yIter = yEntries.iterator(); + for (int i = 0; i < xEntries.size(); i++) { + assertZipEntryEquivalent(xIter.next(), yIter.next()); + } + } + + void assertZipEntryEquivalent(ZipFileEntry x, ZipFileEntry y) { + assertThat(x.getComment()).isEqualTo(y.getComment()); + assertThat(x.getCompressedSize()).isEqualTo(y.getCompressedSize()); + assertThat(x.getCrc()).isEqualTo(y.getCrc()); + assertThat(x.getExternalAttributes()).isEqualTo(y.getExternalAttributes()); + assertThat(x.getExtra()).isEqualTo(y.getExtra()); + assertThat(x.getInternalAttributes()).isEqualTo(y.getInternalAttributes()); + assertThat(x.getMethod()).isEqualTo(y.getMethod()); + assertThat(x.getName()).isEqualTo(y.getName()); + assertThat(x.getSize()).isEqualTo(y.getSize()); + assertThat(x.getTime()).isEqualTo(y.getTime()); + assertThat(x.getVersion()).isEqualTo(y.getVersion()); + assertThat(x.getVersionNeeded()).isEqualTo(y.getVersionNeeded()); + // Allow general purpose bit 3 (data descriptor) used in jdk7 to differ. + // Allow general purpose bit 11 (UTF-8 encoding) used in jdk7 to differ. + assertThat(x.getFlags() | (1 << 3) | (1 << 11)) + .isEqualTo(y.getFlags() | (1 << 3) | (1 << 11)); } /** * Ensures that the code that grows the central directory and the code that patches it is not * obviously broken. */ - @Test - public void testLotsOfFiles() throws IOException { + @Test public void testLotsOfFiles() throws IOException { int fileCount = 100; - for (int blockSize : new int[] { 1, 2, 3, 4, 10, 1000 }) { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - ZipCombiner zipCombiner = new ZipCombiner( - OutputMode.DONT_CARE, new CopyEntryFilter(), out, blockSize); - for (int i = 0; i < fileCount; i++) { - zipCombiner.addFile("hello" + i, DOS_EPOCH, asStream("Hello " + i + "!")); - } - zipCombiner.close(); - ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray())); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (ZipCombiner zipCombiner = + new ZipCombiner(OutputMode.DONT_CARE, new CopyEntryFilter(), out)) { for (int i = 0; i < fileCount; i++) { - assertEntry(zipInput, "hello" + i, "Hello " + i + "!"); + zipCombiner.addFile("hello" + i, ZipCombiner.DOS_EPOCH, asStream("Hello " + i + "!")); } - assertNull(zipInput.getNextEntry()); - new ZipTester(out.toByteArray()).validate(); } + ZipInputStream zipInput = new ZipInputStream(new ByteArrayInputStream(out.toByteArray())); + for (int i = 0; i < fileCount; i++) { + assertEntry(zipInput, "hello" + i, "Hello " + i + "!"); + } + assertNull(zipInput.getNextEntry()); + new ZipTester(out.toByteArray()).validate(); } } diff --git a/src/java_tools/singlejar/javatests/com/google/devtools/build/zip/ZipFileEntryTest.java b/src/java_tools/singlejar/javatests/com/google/devtools/build/zip/ZipFileEntryTest.java new file mode 100644 index 0000000000..de05c4a6e8 --- /dev/null +++ b/src/java_tools/singlejar/javatests/com/google/devtools/build/zip/ZipFileEntryTest.java @@ -0,0 +1,193 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.zip; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.testing.NullPointerTester; +import com.google.devtools.build.zip.ZipFileEntry.Compression; +import com.google.devtools.build.zip.ZipFileEntry.Flag; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class ZipFileEntryTest { + @Rule public ExpectedException thrown = ExpectedException.none(); + + @Test public void testNulls() { + NullPointerTester tester = new NullPointerTester(); + tester.testAllPublicConstructors(ZipFileEntry.class); + tester.testAllPublicInstanceMethods(new ZipFileEntry("foo")); + } + + @Test public void testCrc() { + ZipFileEntry foo = new ZipFileEntry("foo"); + foo.setCrc(32); + } + + @Test public void testCrc_Negative() { + ZipFileEntry foo = new ZipFileEntry("foo"); + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("invalid entry crc-32"); + foo.setCrc(-1); + } + + @Test public void testCrc_Large() { + ZipFileEntry foo = new ZipFileEntry("foo"); + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("invalid entry crc-32"); + foo.setCrc(0x100000000L); + } + + @Test public void testSize() { + ZipFileEntry foo = new ZipFileEntry("foo"); + foo.setSize(32); + } + + @Test public void testSize_Negative() { + ZipFileEntry foo = new ZipFileEntry("foo"); + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("invalid entry size"); + foo.setSize(-1); + } + + @Test public void testSize_Large() { + ZipFileEntry foo = new ZipFileEntry("foo"); + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("invalid entry size"); + foo.setSize(0x100000000L); + } + + @Test public void testCompressedSize() { + ZipFileEntry foo = new ZipFileEntry("foo"); + foo.setCompressedSize(32); + } + + @Test public void testCompressedSize_Negative() { + ZipFileEntry foo = new ZipFileEntry("foo"); + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("invalid entry size"); + foo.setCompressedSize(-1); + } + + @Test public void testCompressedSize_Large() { + ZipFileEntry foo = new ZipFileEntry("foo"); + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("invalid entry size"); + foo.setCompressedSize(0x100000000L); + } + + @Test public void testMinVersion_MethodUpdated() { + ZipFileEntry foo = new ZipFileEntry("foo"); + assertThat(foo.getVersion()).isEqualTo((short) -1); + foo.setMethod(Compression.STORED); + assertThat(foo.getVersion()).isEqualTo((short) 0x0a); + } + + @Test public void testMinVersion_Update() { + ZipFileEntry foo = new ZipFileEntry("foo"); + foo.setMethod(Compression.STORED); + foo.setVersion((short) 0x14); + assertThat(foo.getVersion()).isEqualTo((short) 0x14); + } + + @Test public void testMinVersion_BelowRequired() { + ZipFileEntry foo = new ZipFileEntry("foo"); + foo.setMethod(Compression.STORED); + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("The minimum allowable version for method STORED is 0x0a."); + foo.setVersion((short) 0); + } + + @Test public void testMinVersionNeeded_MethodUpdated() { + ZipFileEntry foo = new ZipFileEntry("foo"); + assertThat(foo.getVersionNeeded()).isEqualTo((short) -1); + foo.setMethod(Compression.DEFLATED); + assertThat(foo.getVersionNeeded()).isEqualTo((short) 0x14); + } + + @Test public void testMinVersionNeeded_Update() { + ZipFileEntry foo = new ZipFileEntry("foo"); + foo.setMethod(Compression.STORED); + foo.setVersionNeeded((short) 0x28); + assertThat(foo.getVersionNeeded()).isEqualTo((short) 0x28); + } + + @Test public void testMinVersionNeeded_BelowRequired() { + ZipFileEntry foo = new ZipFileEntry("foo"); + foo.setMethod(Compression.DEFLATED); + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("The minimum allowable version for method DEFLATED is 0x14."); + foo.setVersionNeeded((short) 0x0a); + } + + @Test public void testSetFlag() { + ZipFileEntry foo = new ZipFileEntry("foo"); + foo.setFlag(Flag.DATA_DESCRIPTOR, true); + assertThat(foo.getFlags()).isEqualTo((short) 0x08); + foo.setFlag(Flag.DATA_DESCRIPTOR, true); + assertThat(foo.getFlags()).isEqualTo((short) 0x08); + foo.setFlag(Flag.DATA_DESCRIPTOR, false); + assertThat(foo.getFlags()).isEqualTo((short) 0x00); + foo.setFlag(Flag.DATA_DESCRIPTOR, false); + assertThat(foo.getFlags()).isEqualTo((short) 0x00); + } + + @Test public void testLocalHeaderOffset() { + ZipFileEntry foo = new ZipFileEntry("foo"); + foo.setLocalHeaderOffset(32); + } + + @Test public void testLocalHeaderOffset_Negative() { + ZipFileEntry foo = new ZipFileEntry("foo"); + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("invalid local header offset"); + foo.setLocalHeaderOffset(-1); + } + + @Test public void testLocalHeaderOffset_Large() { + ZipFileEntry foo = new ZipFileEntry("foo"); + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("invalid local header offset"); + foo.setLocalHeaderOffset(0x100000000L); + } + + @Test public void testExtra() { + ZipFileEntry foo = new ZipFileEntry("foo"); + foo.setExtra(new byte[32]); + } + + @Test public void testExtra_Large() { + ZipFileEntry foo = new ZipFileEntry("foo"); + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("invalid extra field length"); + foo.setExtra(new byte[0x10000]); + } + + @Test public void testExtraData() { + ZipFileEntry foo = new ZipFileEntry("foo"); + ExtraData[] extra = new ExtraData[] { + new ExtraData((short) 0xCAFE, new byte[] { 0x01, 0x02 }), + new ExtraData((short) 0xADDE, new byte[] { (byte) 0xBE, (byte) 0xEF }) }; + foo.setExtra(extra); + assertThat(foo.getExtra()).isEqualTo(new byte[] { + (byte) 0xFE, (byte) 0xCA, 0x02, 0x00, 0x01, 0x02, + (byte) 0xDE, (byte) 0xAD, 0x02, 0x00, (byte) 0xBE, (byte) 0xEF }); + } +} diff --git a/src/java_tools/singlejar/javatests/com/google/devtools/build/zip/ZipReaderTest.java b/src/java_tools/singlejar/javatests/com/google/devtools/build/zip/ZipReaderTest.java new file mode 100644 index 0000000000..67346932d3 --- /dev/null +++ b/src/java_tools/singlejar/javatests/com/google/devtools/build/zip/ZipReaderTest.java @@ -0,0 +1,390 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.zip; + +import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.fail; + +import com.google.devtools.build.zip.ZipFileEntry.Compression; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Calendar; +import java.util.Date; +import java.util.zip.CRC32; +import java.util.zip.Deflater; +import java.util.zip.ZipEntry; +import java.util.zip.ZipException; +import java.util.zip.ZipOutputStream; + +@RunWith(JUnit4.class) +public class ZipReaderTest { + private void assertDateWithin(Date testDate, Date start, Date end) { + if (testDate.before(start) || testDate.after(end)) { + fail(); + } + } + + private void assertDateAboutNow(Date testDate) { + Date now = new Date(); + Calendar cal = Calendar.getInstance(); + cal.setTime(now); + cal.add(Calendar.MINUTE, -30); + Date start = cal.getTime(); + cal.add(Calendar.HOUR, 1); + Date end = cal.getTime(); + assertDateWithin(testDate, start, end); + } + + @Rule public TemporaryFolder tmp = new TemporaryFolder(); + @Rule public ExpectedException thrown = ExpectedException.none(); + + @Test public void testMalformed_Empty() throws IOException { + File test = tmp.newFile("test.zip"); + try (FileOutputStream out = new FileOutputStream(test)) { + } + thrown.expect(ZipException.class); + thrown.expectMessage("is malformed. It does not contain an end of central directory record."); + try (ZipReader reader = new ZipReader(test, UTF_8)) { + } + } + + @Test public void testMalformed_ShorterThanSignature() throws IOException { + File test = tmp.newFile("test.zip"); + try (FileOutputStream out = new FileOutputStream(test)) { + out.write(new byte[] { 1, 2, 3 }); + } + thrown.expect(ZipException.class); + thrown.expectMessage("is malformed. It does not contain an end of central directory record."); + try (ZipReader reader = new ZipReader(test, UTF_8)) { + } + } + + @Test public void testMalformed_SignatureLength() throws IOException { + File test = tmp.newFile("test.zip"); + try (FileOutputStream out = new FileOutputStream(test)) { + out.write(new byte[] { 1, 2, 3, 4 }); + } + thrown.expect(ZipException.class); + thrown.expectMessage("is malformed. It does not contain an end of central directory record."); + try (ZipReader reader = new ZipReader(test, UTF_8)) { + } + } + + @Test public void testEmpty() throws IOException { + File test = tmp.newFile("test.zip"); + try (ZipOutputStream zout = new ZipOutputStream(new FileOutputStream(test))) { + } + try (ZipReader reader = new ZipReader(test, UTF_8)) { + assertThat(reader.entries()).isEmpty(); + } + } + + @Test public void testFileComment() throws IOException { + File test = tmp.newFile("test.zip"); + try (ZipOutputStream zout = new ZipOutputStream(new FileOutputStream(test))) { + zout.setComment("test comment"); + } + try (ZipReader reader = new ZipReader(test, UTF_8)) { + assertThat(reader.entries()).isEmpty(); + assertThat(reader.getComment()).isEqualTo("test comment"); + } + } + + @Test public void testFileCommentWithSignature() throws IOException { + File test = tmp.newFile("test.zip"); + try (ZipOutputStream zout = new ZipOutputStream(new FileOutputStream(test))) { + zout.setComment("test comment\u0050\u004b\u0005\u0006abcdefghijklmnopqrstuvwxyz"); + } + try (ZipReader reader = new ZipReader(test, UTF_8)) { + assertThat(reader.entries()).isEmpty(); + assertThat(reader.getComment()) + .isEqualTo("test comment\u0050\u004b\u0005\u0006abcdefghijklmnopqrstuvwxyz"); + } + } + + @Test public void testSingleEntry() throws IOException { + File test = tmp.newFile("test.zip"); + try (ZipOutputStream zout = new ZipOutputStream(new FileOutputStream(test))) { + zout.putNextEntry(new ZipEntry("test")); + zout.write("foo".getBytes(UTF_8)); + zout.closeEntry(); + } + try (ZipReader reader = new ZipReader(test, UTF_8)) { + assertThat(reader.entries()).hasSize(1); + } + } + + @Test public void testMultipleEntries() throws IOException { + File test = tmp.newFile("test.zip"); + String[] names = new String[] { "test", "foo", "bar", "baz" }; + try (ZipOutputStream zout = new ZipOutputStream(new FileOutputStream(test))) { + for (String name : names) { + zout.putNextEntry(new ZipEntry(name)); + zout.write(name.getBytes(UTF_8)); + zout.closeEntry(); + } + } + + try (ZipReader reader = new ZipReader(test, UTF_8)) { + assertThat(reader.entries()).hasSize(names.length); + int i = 0; + for (ZipFileEntry entry : reader.entries()) { + assertThat(entry.getName()).isEqualTo(names[i++]); + } + assertThat(i).isEqualTo(names.length); + } + } + + @Test public void testZipEntryFields() throws IOException { + File test = tmp.newFile("test.zip"); + CRC32 crc = new CRC32(); + Deflater deflater = new Deflater(Deflater.DEFAULT_COMPRESSION, true); + long date = 791784306000L; // 2/3/1995 04:05:06 + byte[] extra = new byte[] { (byte) 0xaa, (byte) 0xbb, (byte) 0xcd }; + byte[] tmp = new byte[128]; + try (ZipOutputStream zout = new ZipOutputStream(new FileOutputStream(test))) { + + ZipEntry foo = new ZipEntry("foo"); + foo.setComment("foo comment."); + foo.setMethod(ZipEntry.DEFLATED); + foo.setTime(date); + foo.setExtra(extra); + zout.putNextEntry(foo); + zout.write("foo".getBytes(UTF_8)); + zout.closeEntry(); + + ZipEntry bar = new ZipEntry("bar"); + bar.setComment("bar comment."); + bar.setMethod(ZipEntry.STORED); + bar.setSize("bar".length()); + bar.setCompressedSize("bar".length()); + crc.reset(); + crc.update("bar".getBytes(UTF_8)); + bar.setCrc(crc.getValue()); + zout.putNextEntry(bar); + zout.write("bar".getBytes(UTF_8)); + zout.closeEntry(); + } + + try (ZipReader reader = new ZipReader(test, UTF_8)) { + ZipFileEntry fooEntry = reader.getEntry("foo"); + assertThat(fooEntry.getName()).isEqualTo("foo"); + assertThat(fooEntry.getComment()).isEqualTo("foo comment."); + assertThat(fooEntry.getMethod()).isEqualTo(Compression.DEFLATED); + assertThat(fooEntry.getVersion()).isEqualTo(Compression.DEFLATED.getMinVersion()); + assertThat(fooEntry.getTime()).isEqualTo(date); + assertThat(fooEntry.getSize()).isEqualTo("foo".length()); + deflater.reset(); + deflater.setInput("foo".getBytes(UTF_8)); + deflater.finish(); + assertThat(fooEntry.getCompressedSize()).isEqualTo(deflater.deflate(tmp)); + crc.reset(); + crc.update("foo".getBytes(UTF_8)); + assertThat(fooEntry.getCrc()).isEqualTo(crc.getValue()); + assertThat(fooEntry.getExtra()).isEqualTo(extra); + + ZipFileEntry barEntry = reader.getEntry("bar"); + assertThat(barEntry.getName()).isEqualTo("bar"); + assertThat(barEntry.getComment()).isEqualTo("bar comment."); + assertThat(barEntry.getMethod()).isEqualTo(Compression.STORED); + assertThat(barEntry.getVersion()).isEqualTo(Compression.STORED.getMinVersion()); + assertDateAboutNow(new Date(barEntry.getTime())); + assertThat(barEntry.getSize()).isEqualTo("bar".length()); + assertThat(barEntry.getCompressedSize()).isEqualTo("bar".length()); + crc.reset(); + crc.update("bar".getBytes(UTF_8)); + assertThat(barEntry.getCrc()).isEqualTo(crc.getValue()); + assertThat(barEntry.getExtra()).isEqualTo(new byte[] {}); + } + } + + @Test public void testZipEntryInvalidTime() throws IOException { + File test = tmp.newFile("test.zip"); + long date = 312796800000L; // 11/30/1979 00:00:00, which is also 0 in DOS format + byte[] extra = new byte[] { (byte) 0xaa, (byte) 0xbb, (byte) 0xcd }; + try (ZipOutputStream zout = new ZipOutputStream(new FileOutputStream(test))) { + ZipEntry foo = new ZipEntry("foo"); + foo.setComment("foo comment."); + foo.setMethod(ZipEntry.DEFLATED); + foo.setTime(date); + foo.setExtra(extra); + zout.putNextEntry(foo); + zout.write("foo".getBytes(UTF_8)); + zout.closeEntry(); + } + + try (ZipReader reader = new ZipReader(test, UTF_8)) { + ZipFileEntry fooEntry = reader.getEntry("foo"); + assertThat(fooEntry.getTime()).isEqualTo(ZipUtil.DOS_EPOCH); + } + } + + @Test public void testRawFileData() throws IOException { + File test = tmp.newFile("test.zip"); + CRC32 crc = new CRC32(); + Deflater deflator = new Deflater(Deflater.DEFAULT_COMPRESSION, true); + try (ZipOutputStream zout = new ZipOutputStream(new FileOutputStream(test))) { + ZipEntry foo = new ZipEntry("foo"); + foo.setComment("foo comment."); + foo.setMethod(ZipEntry.DEFLATED); + zout.putNextEntry(foo); + zout.write("foo".getBytes(UTF_8)); + zout.closeEntry(); + + ZipEntry bar = new ZipEntry("bar"); + bar.setComment("bar comment."); + bar.setMethod(ZipEntry.STORED); + bar.setSize("bar".length()); + bar.setCompressedSize("bar".length()); + crc.reset(); + crc.update("bar".getBytes(UTF_8)); + bar.setCrc(crc.getValue()); + zout.putNextEntry(bar); + zout.write("bar".getBytes(UTF_8)); + zout.closeEntry(); + } + + try (ZipReader reader = new ZipReader(test, UTF_8)) { + ZipFileEntry fooEntry = reader.getEntry("foo"); + InputStream fooIn = reader.getRawInputStream(fooEntry); + byte[] fooData = new byte[10]; + fooIn.read(fooData); + byte[] expectedFooData = new byte[10]; + deflator.reset(); + deflator.setInput("foo".getBytes(UTF_8)); + deflator.finish(); + deflator.deflate(expectedFooData); + assertThat(fooData).isEqualTo(expectedFooData); + + ZipFileEntry barEntry = reader.getEntry("bar"); + InputStream barIn = reader.getRawInputStream(barEntry); + byte[] barData = new byte[3]; + barIn.read(barData); + byte[] expectedBarData = "bar".getBytes(UTF_8); + assertThat(barData).isEqualTo(expectedBarData); + + assertThat(barIn.read()).isEqualTo(-1); + assertThat(barIn.read(barData)).isEqualTo(-1); + assertThat(barIn.read(barData, 0, 3)).isEqualTo(-1); + + thrown.expect(IOException.class); + thrown.expectMessage("Reset is not supported on this type of stream."); + barIn.reset(); + } + } + + @Test public void testFileData() throws IOException { + File test = tmp.newFile("test.zip"); + CRC32 crc = new CRC32(); + try (ZipOutputStream zout = new ZipOutputStream(new FileOutputStream(test))) { + ZipEntry foo = new ZipEntry("foo"); + foo.setComment("foo comment."); + foo.setMethod(ZipEntry.DEFLATED); + zout.putNextEntry(foo); + zout.write("foo".getBytes(UTF_8)); + zout.closeEntry(); + + ZipEntry bar = new ZipEntry("bar"); + bar.setComment("bar comment."); + bar.setMethod(ZipEntry.STORED); + bar.setSize("bar".length()); + bar.setCompressedSize("bar".length()); + crc.reset(); + crc.update("bar".getBytes(UTF_8)); + bar.setCrc(crc.getValue()); + zout.putNextEntry(bar); + zout.write("bar".getBytes(UTF_8)); + zout.closeEntry(); + } + + try (ZipReader reader = new ZipReader(test, UTF_8)) { + ZipFileEntry fooEntry = reader.getEntry("foo"); + InputStream fooIn = reader.getInputStream(fooEntry); + byte[] fooData = new byte[3]; + fooIn.read(fooData); + byte[] expectedFooData = "foo".getBytes(UTF_8); + assertThat(fooData).isEqualTo(expectedFooData); + + assertThat(fooIn.read()).isEqualTo(-1); + assertThat(fooIn.read(fooData)).isEqualTo(-1); + assertThat(fooIn.read(fooData, 0, 3)).isEqualTo(-1); + + ZipFileEntry barEntry = reader.getEntry("bar"); + InputStream barIn = reader.getInputStream(barEntry); + byte[] barData = new byte[3]; + barIn.read(barData); + byte[] expectedBarData = "bar".getBytes(UTF_8); + assertThat(barData).isEqualTo(expectedBarData); + + assertThat(barIn.read()).isEqualTo(-1); + assertThat(barIn.read(barData)).isEqualTo(-1); + assertThat(barIn.read(barData, 0, 3)).isEqualTo(-1); + + thrown.expect(IOException.class); + thrown.expectMessage("Reset is not supported on this type of stream."); + barIn.reset(); + } + } + + @Test public void testSimultaneousReads() throws IOException { + File test = tmp.newFile("test.zip"); + byte[] expectedFooData = "This if file foo. It contains a foo.".getBytes(UTF_8); + byte[] expectedBarData = "This is a different file bar. It contains only a bar." + .getBytes(UTF_8); + try (ZipOutputStream zout = new ZipOutputStream(new FileOutputStream(test))) { + ZipEntry foo = new ZipEntry("foo"); + foo.setComment("foo comment."); + foo.setMethod(ZipEntry.DEFLATED); + zout.putNextEntry(foo); + zout.write(expectedFooData); + zout.closeEntry(); + + ZipEntry bar = new ZipEntry("bar"); + bar.setComment("bar comment."); + bar.setMethod(ZipEntry.DEFLATED); + zout.putNextEntry(bar); + zout.write(expectedBarData); + zout.closeEntry(); + } + + try (ZipReader reader = new ZipReader(test, UTF_8)) { + ZipFileEntry fooEntry = reader.getEntry("foo"); + ZipFileEntry barEntry = reader.getEntry("bar"); + InputStream fooIn = reader.getInputStream(fooEntry); + InputStream barIn = reader.getInputStream(barEntry); + byte[] fooData = new byte[expectedFooData.length]; + byte[] barData = new byte[expectedBarData.length]; + fooIn.read(fooData, 0, 10); + barIn.read(barData, 0, 10); + fooIn.read(fooData, 10, 10); + barIn.read(barData, 10, 10); + fooIn.read(fooData, 20, fooData.length - 20); + barIn.read(barData, 20, barData.length - 20); + assertThat(fooData).isEqualTo(expectedFooData); + assertThat(barData).isEqualTo(expectedBarData); + } + } +} diff --git a/src/java_tools/singlejar/javatests/com/google/devtools/build/zip/ZipTests.java b/src/java_tools/singlejar/javatests/com/google/devtools/build/zip/ZipTests.java new file mode 100644 index 0000000000..2b05b5c2a5 --- /dev/null +++ b/src/java_tools/singlejar/javatests/com/google/devtools/build/zip/ZipTests.java @@ -0,0 +1,27 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.zip; + + +import com.google.devtools.build.lib.testutil.ClasspathSuite; + +import org.junit.runner.RunWith; + +/** + * A test-suite builder for this package. + */ +@RunWith(ClasspathSuite.class) +public class ZipTests { +} diff --git a/src/java_tools/singlejar/javatests/com/google/devtools/build/zip/ZipUtilTest.java b/src/java_tools/singlejar/javatests/com/google/devtools/build/zip/ZipUtilTest.java new file mode 100644 index 0000000000..60f2b732c7 --- /dev/null +++ b/src/java_tools/singlejar/javatests/com/google/devtools/build/zip/ZipUtilTest.java @@ -0,0 +1,172 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.zip; + +import static com.google.common.truth.Truth.assertThat; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.util.Calendar; +import java.util.GregorianCalendar; + +@RunWith(JUnit4.class) +public class ZipUtilTest { + @Rule public ExpectedException thrown = ExpectedException.none(); + + @Test public void testShortToLittleEndian() { + byte[] bytes = ZipUtil.shortToLittleEndian((short) 4660); + assertThat(bytes).isEqualTo(new byte[]{ 0x34, 0x12 }); + } + + @Test public void testShortToLittleEndian_Signed() { + byte[] bytes = ZipUtil.shortToLittleEndian((short) -3532); + assertThat(bytes).isEqualTo(new byte[]{ 0x34, (byte) 0xf2 }); + } + + @Test public void testIntToLittleEndian() { + byte[] bytes = ZipUtil.intToLittleEndian(305419896); + assertThat(bytes).isEqualTo(new byte[]{ 0x78, 0x56, 0x34, 0x12 }); + } + + @Test public void testIntToLittleEndian_Signed() { + byte[] bytes = ZipUtil.intToLittleEndian(-231451016); + assertThat(bytes).isEqualTo(new byte[]{ 0x78, 0x56, 0x34, (byte) 0xf2 }); + } + + @Test public void testLongToLittleEndian() { + byte[] bytes = ZipUtil.longToLittleEndian(305419896); + assertThat(bytes).isEqualTo(new byte[]{ 0x78, 0x56, 0x34, 0x12, 0x0, 0x0, 0x0, 0x0 }); + } + + @Test public void testLongToLittleEndian_Signed() { + byte[] bytes = ZipUtil.longToLittleEndian(-231451016); + assertThat(bytes).isEqualTo(new byte[]{ 0x78, 0x56, 0x34, (byte) 0xf2, + (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff }); + } + + @Test public void testGet16() { + short result = ZipUtil.get16(new byte[]{ 0x34, 0x12 }, 0); + assertThat(result).isEqualTo((short) 0x1234); + assertThat(result).isEqualTo((short) 4660); + } + + @Test public void testGet16_Signed() { + short result = ZipUtil.get16(new byte[]{ 0x34, (byte) 0xff }, 0); + assertThat(result).isEqualTo((short) 0xff34); + assertThat(result).isEqualTo((short) -204); + } + + @Test public void testGet32() { + int result = ZipUtil.get32(new byte[]{ 0x78, 0x56, 0x34, 0x12 }, 0); + assertThat(result).isEqualTo(0x12345678); + assertThat(result).isEqualTo(305419896); + } + + @Test public void testGet32_Short() { + int result = ZipUtil.get32(new byte[]{ 0x34, (byte) 0xff, 0x0, 0x0 }, 0); + assertThat(result).isEqualTo(0xff34); + assertThat(result).isEqualTo(65332); + } + + @Test public void testGet32_Signed() { + int result = ZipUtil.get32(new byte[]{ 0x34, (byte) 0xff, (byte) 0xff, (byte) 0xff }, 0); + assertThat(result).isEqualTo(0xffffff34); + assertThat(result).isEqualTo(-204); + } + + @Test public void testGetUnsignedShort() { + int result = ZipUtil.getUnsignedShort(new byte[]{ 0x34, 0x12 }, 0); + assertThat(result).isEqualTo(0x1234); + assertThat(result).isEqualTo(4660); + } + + @Test public void testGetUnsignedShort_Big() { + int result = ZipUtil.getUnsignedShort(new byte[]{ 0x34, (byte) 0xff }, 0); + assertThat(result).isEqualTo(0xff34); + assertThat(result).isEqualTo(65332); + } + + @Test public void testGetUnsignedInt() { + long result = ZipUtil.getUnsignedInt(new byte[]{ 0x34, 0x12, 0x0, 0x0 }, 0); + assertThat(result).isEqualTo(0x1234); + assertThat(result).isEqualTo(4660); + } + + @Test public void testGetUnsignedInt_Big() { + long result = ZipUtil.getUnsignedInt( + new byte[]{ 0x34, (byte) 0xff, (byte) 0xff, (byte) 0xff }, 0); + assertThat(result).isEqualTo(0xffffff34L); + assertThat(result).isEqualTo(4294967092L); + } + + @Test public void testTimeConversion_DosToUnix() { + int dos = (20 << 25) | (2 << 21) | (14 << 16) | (3 << 11) | (7 << 5) | (15 >> 1); + + Calendar time = new GregorianCalendar(2000, Calendar.FEBRUARY, 14, 3, 7, 14); + long expectedUnixTime = time.getTimeInMillis(); + assertThat(ZipUtil.dosToUnixTime(dos)).isEqualTo(expectedUnixTime); + } + + @Test public void testTimeConversion_UnixToDos() { + Calendar time = new GregorianCalendar(2000, Calendar.FEBRUARY, 14, 3, 7, 14); + long unix = time.getTimeInMillis(); + int expectedDosTime = (20 << 25) | (2 << 21) | (14 << 16) | (3 << 11) | (7 << 5) | (15 >> 1); + assertThat(ZipUtil.unixToDosTime(unix)).isEqualTo(expectedDosTime); + } + + @Test public void testTimeConversion_UnixToDos_LowBound() { + Calendar time = Calendar.getInstance(); + time.setTimeInMillis(ZipUtil.DOS_EPOCH); + time.add(Calendar.SECOND, -1); + thrown.expect(IllegalArgumentException.class); + ZipUtil.unixToDosTime(time.getTimeInMillis()); + } + + @Test public void testTimeConversion_UnixToDos_HighBound_Rounding() { + Calendar time = Calendar.getInstance(); + time.setTimeInMillis(ZipUtil.MAX_DOS_DATE); + ZipUtil.unixToDosTime(time.getTimeInMillis()); + } + + @Test public void testTimeConversion_UnixToDos_HighBound() { + Calendar time = Calendar.getInstance(); + time.setTimeInMillis(ZipUtil.MAX_DOS_DATE); + time.add(Calendar.SECOND, 1); + thrown.expect(IllegalArgumentException.class); + ZipUtil.unixToDosTime(time.getTimeInMillis()); + } + + @Test public void testTimeConversion_UnixToUnix() { + Calendar from = new GregorianCalendar(2000, Calendar.FEBRUARY, 14, 3, 7, 15); + Calendar to = new GregorianCalendar(2000, Calendar.FEBRUARY, 14, 3, 7, 14); + assertThat(ZipUtil.dosToUnixTime(ZipUtil.unixToDosTime(from.getTimeInMillis()))) + .isEqualTo(to.getTimeInMillis()); + } + + @Test public void testTimeConversion_DosToDos() { + int dos = (20 << 25) | (2 << 21) | (14 << 16) | (3 << 11) | (7 << 5) | (15 >> 1); + assertThat(ZipUtil.unixToDosTime(ZipUtil.dosToUnixTime(dos))).isEqualTo(dos); + } + + @Test public void testTimeConversion_DosToDos_Zero() { + int dos = 0; + thrown.expect(IllegalArgumentException.class); + assertThat(ZipUtil.unixToDosTime(ZipUtil.dosToUnixTime(dos))).isEqualTo(0); + } +} diff --git a/src/java_tools/singlejar/javatests/com/google/devtools/build/zip/ZipWriterTest.java b/src/java_tools/singlejar/javatests/com/google/devtools/build/zip/ZipWriterTest.java new file mode 100644 index 0000000000..e58a90c923 --- /dev/null +++ b/src/java_tools/singlejar/javatests/com/google/devtools/build/zip/ZipWriterTest.java @@ -0,0 +1,373 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.zip; + +import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.primitives.Bytes; +import com.google.devtools.build.zip.ZipFileEntry.Compression; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.Calendar; +import java.util.Random; +import java.util.zip.CRC32; +import java.util.zip.Deflater; +import java.util.zip.ZipEntry; +import java.util.zip.ZipException; +import java.util.zip.ZipFile; + +@RunWith(JUnit4.class) +public class ZipWriterTest { + @Rule public TemporaryFolder tmp = new TemporaryFolder(); + @Rule public ExpectedException thrown = ExpectedException.none(); + + private Random rand; + private Calendar cal; + private CRC32 crc; + private Deflater deflater; + private File test; + + @Before public void setup() throws IOException { + rand = new Random(); + cal = Calendar.getInstance(); + cal.clear(); + cal.set(Calendar.YEAR, rand.nextInt(128) + 1980); // Zip files have 7-bit year resolution. + cal.set(Calendar.MONTH, rand.nextInt(12)); + cal.set(Calendar.DAY_OF_MONTH, rand.nextInt(29)); + cal.set(Calendar.HOUR_OF_DAY, rand.nextInt(24)); + cal.set(Calendar.MINUTE, rand.nextInt(60)); + cal.set(Calendar.SECOND, rand.nextInt(30) * 2); // Zip files have 2 second resolution. + + crc = new CRC32(); + deflater = new Deflater(Deflater.DEFAULT_COMPRESSION, true); + test = tmp.newFile("test.zip"); + } + + @Test public void testEmpty() throws IOException { + try (ZipWriter writer = new ZipWriter(new FileOutputStream(test), UTF_8)) { + } + + try (ZipFile zipFile = new ZipFile(test)) { + assertThat(zipFile.entries().hasMoreElements()).isFalse(); + } + } + + @Test public void testComment() throws IOException { + try (ZipWriter writer = new ZipWriter(new FileOutputStream(test), UTF_8)) { + writer.setComment("test comment"); + } + + try (ZipFile zipFile = new ZipFile(test)) { + assertThat(zipFile.entries().hasMoreElements()).isFalse(); + assertThat(zipFile.getComment()).isEqualTo("test comment"); + } + } + + @Test public void testFileDataBeforeEntry() throws IOException { + try (ZipWriter writer = new ZipWriter(new FileOutputStream(test), UTF_8)) { + thrown.expect(ZipException.class); + thrown.expectMessage("Cannot write zip contents without first setting a ZipEntry or starting" + + " a prefix file."); + writer.write(new byte[] { 0xf, 0xa, 0xb }); + } + + try (ZipFile zipFile = new ZipFile(test)) { + assertThat(zipFile.entries().hasMoreElements()).isFalse(); + } + } + + @Test public void testSingleEntry() throws IOException { + ZipWriter writer = new ZipWriter(new FileOutputStream(test), UTF_8); + byte[] content = "content".getBytes(UTF_8); + crc.update(content); + ZipFileEntry entry = new ZipFileEntry("foo"); + entry.setMethod(Compression.STORED); + entry.setSize(content.length); + entry.setCompressedSize(content.length); + entry.setCrc(crc.getValue()); + entry.setTime(cal.getTimeInMillis()); + + writer.putNextEntry(entry); + writer.write(content); + writer.closeEntry(); + writer.close(); + + byte[] buf = new byte[128]; + try (ZipFile zipFile = new ZipFile(test)) { + ZipEntry foo = zipFile.getEntry("foo"); + assertThat(foo.getMethod()).isEqualTo(ZipEntry.STORED); + assertThat(foo.getSize()).isEqualTo(content.length); + assertThat(foo.getCompressedSize()).isEqualTo(content.length); + assertThat(foo.getCrc()).isEqualTo(crc.getValue()); + assertThat(foo.getTime()).isEqualTo(cal.getTimeInMillis()); + zipFile.getInputStream(foo).read(buf); + assertThat(Bytes.indexOf(buf, content)).isEqualTo(0); + } + } + + @Test public void testMultipleEntry() throws IOException { + ZipWriter writer = new ZipWriter(new FileOutputStream(test), UTF_8); + writer.setComment("file comment"); + + byte[] fooContent = "content".getBytes(UTF_8); + crc.update(fooContent); + long fooCrc = crc.getValue(); + ZipFileEntry rawFoo = new ZipFileEntry("foo"); + rawFoo.setMethod(Compression.STORED); + rawFoo.setSize(fooContent.length); + rawFoo.setCompressedSize(fooContent.length); + rawFoo.setCrc(crc.getValue()); + rawFoo.setTime(cal.getTimeInMillis()); + rawFoo.setComment("foo comment"); + + writer.putNextEntry(rawFoo); + writer.write(fooContent); + writer.closeEntry(); + + byte[] barContent = "stuff".getBytes(UTF_8); + byte[] deflatedBarContent = new byte[128]; + crc.reset(); + crc.update(barContent); + long barCrc = crc.getValue(); + deflater.setInput(barContent); + deflater.finish(); + int deflatedSize = deflater.deflate(deflatedBarContent); + ZipFileEntry rawBar = new ZipFileEntry("bar"); + rawBar.setMethod(Compression.DEFLATED); + rawBar.setSize(barContent.length); + rawBar.setCompressedSize(deflatedSize); + rawBar.setCrc(barCrc); + rawBar.setTime(cal.getTimeInMillis()); + + writer.putNextEntry(rawBar); + writer.write(deflatedBarContent, 0, deflatedSize); + writer.closeEntry(); + + writer.close(); + + byte[] buf = new byte[128]; + try (ZipFile zipFile = new ZipFile(test)) { + assertThat(zipFile.getComment()).isEqualTo("file comment"); + + ZipEntry foo = zipFile.getEntry("foo"); + assertThat(foo.getMethod()).isEqualTo(ZipEntry.STORED); + assertThat(foo.getSize()).isEqualTo(fooContent.length); + assertThat(foo.getCompressedSize()).isEqualTo(fooContent.length); + assertThat(foo.getCrc()).isEqualTo(fooCrc); + assertThat(foo.getTime()).isEqualTo(cal.getTimeInMillis()); + assertThat(foo.getComment()).isEqualTo("foo comment"); + zipFile.getInputStream(foo).read(buf); + assertThat(Bytes.indexOf(buf, fooContent)).isEqualTo(0); + + ZipEntry bar = zipFile.getEntry("bar"); + assertThat(bar.getMethod()).isEqualTo(ZipEntry.DEFLATED); + assertThat(bar.getSize()).isEqualTo(barContent.length); + assertThat(bar.getCompressedSize()).isEqualTo(deflatedSize); + assertThat(bar.getCrc()).isEqualTo(barCrc); + assertThat(bar.getTime()).isEqualTo(cal.getTimeInMillis()); + zipFile.getInputStream(bar).read(buf); + assertThat(Bytes.indexOf(buf, barContent)).isEqualTo(0); + } + } + + @Test public void testWrongSizeContent() throws IOException { + try (ZipWriter writer = new ZipWriter(new FileOutputStream(test), UTF_8)) { + byte[] content = "content".getBytes(UTF_8); + crc.update(content); + ZipFileEntry entry = new ZipFileEntry("foo"); + entry.setMethod(Compression.STORED); + entry.setSize(content.length); + entry.setCompressedSize(content.length); + entry.setCrc(crc.getValue()); + entry.setTime(cal.getTimeInMillis()); + + writer.putNextEntry(entry); + writer.write("some other content".getBytes(UTF_8)); + thrown.expect(ZipException.class); + thrown.expectMessage("Number of bytes written for the entry"); + writer.closeEntry(); + } + } + + @Test public void testRawZipEntry() throws IOException { + ZipWriter writer = new ZipWriter(new FileOutputStream(test), UTF_8); + byte[] content = "content".getBytes(UTF_8); + crc.update(content); + ZipFileEntry entry = new ZipFileEntry("foo"); + entry.setVersion((short) 1); + entry.setVersionNeeded((short) 2); + entry.setMethod(Compression.STORED); + entry.setSize(content.length); + entry.setCompressedSize(content.length); + entry.setCrc(crc.getValue()); + entry.setTime(cal.getTimeInMillis()); + entry.setFlags(ZipUtil.get16(new byte[]{ 0x08, 0x00 }, 0)); + entry.setInternalAttributes(ZipUtil.get16(new byte[]{ 0x34, 0x12 }, 0)); + entry.setExternalAttributes(ZipUtil.get32(new byte[]{ 0x0a, 0x09, 0x78, 0x56 }, 0)); + entry.setLocalHeaderOffset(rand.nextInt(Integer.MAX_VALUE)); + + writer.putNextEntry(entry); + writer.write(content); + writer.closeEntry(); + writer.close(); + + byte[] buf = new byte[128]; + try (ZipFile zipFile = new ZipFile(test)) { + ZipEntry foo = zipFile.getEntry("foo"); + assertThat(foo.getMethod()).isEqualTo(ZipEntry.STORED); + assertThat(foo.getSize()).isEqualTo(content.length); + assertThat(foo.getCompressedSize()).isEqualTo(content.length); + assertThat(foo.getCrc()).isEqualTo(crc.getValue()); + assertThat(foo.getTime()).isEqualTo(cal.getTimeInMillis()); + zipFile.getInputStream(foo).read(buf); + assertThat(Bytes.indexOf(buf, content)).isEqualTo(0); + } + + try (ZipReader zipFile = new ZipReader(test)) { + ZipFileEntry foo = zipFile.getEntry("foo"); + // Versions should be increased to minimum required for STORED compression. + assertThat(foo.getVersion()).isEqualTo((short) 0xa); + assertThat(foo.getVersionNeeded()).isEqualTo((short) 0xa); + assertThat(foo.getFlags()).isEqualTo((short) 0); // Data descriptor bit should be cleared. + assertThat(foo.getInternalAttributes()).isEqualTo((short) 4660); + assertThat(foo.getExternalAttributes()).isEqualTo(1450707210); + } + } + + @Test public void testPrefixFile() throws IOException, InterruptedException { + ZipWriter writer = new ZipWriter(new FileOutputStream(test), UTF_8); + + writer.startPrefixFile(); + writer.write("#!/bin/bash\necho 'hello world'\n".getBytes(UTF_8)); + writer.endPrefixFile(); + + byte[] content = "content".getBytes(UTF_8); + crc.update(content); + ZipFileEntry entry = new ZipFileEntry("foo"); + entry.setMethod(Compression.STORED); + entry.setSize(content.length); + entry.setCompressedSize(content.length); + entry.setCrc(crc.getValue()); + entry.setTime(cal.getTimeInMillis()); + + writer.putNextEntry(entry); + writer.write(content); + writer.closeEntry(); + writer.close(); + + byte[] buf = new byte[128]; + try (ZipFile zipFile = new ZipFile(test)) { + ZipEntry foo = zipFile.getEntry("foo"); + assertThat(foo.getMethod()).isEqualTo(ZipEntry.STORED); + assertThat(foo.getSize()).isEqualTo(content.length); + assertThat(foo.getCompressedSize()).isEqualTo(content.length); + assertThat(foo.getCrc()).isEqualTo(crc.getValue()); + assertThat(foo.getTime()).isEqualTo(cal.getTimeInMillis()); + zipFile.getInputStream(foo).read(buf); + assertThat(Bytes.indexOf(buf, content)).isEqualTo(0); + } + + Process pr = new ProcessBuilder("chmod", "750", test.getAbsolutePath()).start(); + pr.waitFor(); + pr = new ProcessBuilder(test.getAbsolutePath()).start(); + pr.getInputStream().read(buf); + pr.waitFor(); + assertThat(Bytes.indexOf(buf, "hello world".getBytes(UTF_8))).isEqualTo(0); + } + + @Test public void testPrefixFileAfterZip() throws IOException { + try (ZipWriter writer = new ZipWriter(new FileOutputStream(test), UTF_8)) { + byte[] content = "content".getBytes(UTF_8); + crc.update(content); + ZipFileEntry entry = new ZipFileEntry("foo"); + entry.setMethod(Compression.STORED); + entry.setSize(content.length); + entry.setCompressedSize(content.length); + entry.setCrc(crc.getValue()); + entry.setTime(cal.getTimeInMillis()); + + writer.putNextEntry(entry); + thrown.expect(ZipException.class); + thrown.expectMessage("Cannot add a prefix file after the zip contents have been started."); + writer.startPrefixFile(); + writer.write("#!/bin/bash\necho 'hello world'\n".getBytes(UTF_8)); + writer.endPrefixFile(); + } + } + + @Test public void testPrefixAfterFinish() throws IOException { + try (ZipWriter writer = new ZipWriter(new FileOutputStream(test), UTF_8)) { + writer.finish(); + thrown.expect(IllegalStateException.class); + writer.startPrefixFile(); + writer.write("#!/bin/bash\necho 'hello world'\n".getBytes(UTF_8)); + writer.endPrefixFile(); + } + } + + @Test public void testPutEntryAfterFinish() throws IOException { + try (ZipWriter writer = new ZipWriter(new FileOutputStream(test), UTF_8)) { + writer.finish(); + thrown.expect(IllegalStateException.class); + writer.putNextEntry(new ZipFileEntry("foo")); + } + } + + @Test public void testCloseEntryAfterFinish() throws IOException { + try (ZipWriter writer = new ZipWriter(new FileOutputStream(test), UTF_8)) { + byte[] content = "content".getBytes(UTF_8); + crc.update(content); + ZipFileEntry entry = new ZipFileEntry("foo"); + entry.setMethod(Compression.STORED); + entry.setSize(content.length); + entry.setCompressedSize(content.length); + entry.setCrc(crc.getValue()); + entry.setTime(cal.getTimeInMillis()); + + writer.putNextEntry(entry); + writer.write(content); + writer.finish(); + thrown.expect(IllegalStateException.class); + writer.closeEntry(); + } + } + + @Test public void testFinishAfterFinish() throws IOException { + try (ZipWriter writer = new ZipWriter(new FileOutputStream(test), UTF_8)) { + writer.finish(); + thrown.expect(IllegalStateException.class); + writer.finish(); + } + } + + @Test public void testWriteAfterFinish() throws IOException { + try (ZipWriter writer = new ZipWriter(new FileOutputStream(test), UTF_8)) { + writer.finish(); + thrown.expect(IllegalStateException.class); + writer.write("content".getBytes(UTF_8)); + } + } +} |