diff options
Diffstat (limited to 'src/java_tools/singlejar/java/com/google')
15 files changed, 3042 insertions, 1390 deletions
diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ExtraData.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ExtraData.java deleted file mode 100644 index 2e8cb75e02..0000000000 --- a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ExtraData.java +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2014 Google Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.devtools.build.singlejar; - -/** - * A holder class for extra data in a ZIP entry. - * - * <p>Note: This class performs no defensive copying of the byte array, so the - * byte array passed into this class or returned from this class may not be - * modified. - */ -final class ExtraData { - - private final short id; - private final byte[] data; - - public ExtraData(short id, byte[] data) { - this.id = id; - this.data = data; - } - - public short getId() { - return id; - } - - public byte[] getData() { - return data; - } -} diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/JarUtils.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/JarUtils.java index a9c8ee3f65..a21f58307d 100644 --- a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/JarUtils.java +++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/JarUtils.java @@ -14,6 +14,8 @@ package com.google.devtools.build.singlejar; +import com.google.devtools.build.zip.ExtraData; + import java.io.IOException; import java.util.Date; diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/JavaIoFileSystem.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/JavaIoFileSystem.java index 0da6e33040..f66f23dba2 100644 --- a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/JavaIoFileSystem.java +++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/JavaIoFileSystem.java @@ -37,6 +37,11 @@ public final class JavaIoFileSystem implements SimpleFileSystem { } @Override + public File getFile(String filename) throws IOException { + return new File(filename); + } + + @Override public boolean delete(String filename) { return new File(filename).delete(); } diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/SimpleFileSystem.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/SimpleFileSystem.java index 844f12b714..890ab093e7 100644 --- a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/SimpleFileSystem.java +++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/SimpleFileSystem.java @@ -16,6 +16,7 @@ package com.google.devtools.build.singlejar; import com.google.devtools.build.singlejar.OptionFileExpander.OptionFileProvider; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -36,6 +37,11 @@ public interface SimpleFileSystem extends OptionFileProvider { OutputStream getOutputStream(String filename) throws IOException; /** + * Returns the File object for this filename. + */ + File getFile(String filename) throws IOException; + + /** * Delete the file with the given name and return whether deleting it was * successfull. */ diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/SingleJar.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/SingleJar.java index 4551fd1813..8fd677e1b7 100644 --- a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/SingleJar.java +++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/SingleJar.java @@ -19,6 +19,7 @@ import com.google.devtools.build.singlejar.ZipCombiner.OutputMode; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; @@ -80,10 +81,10 @@ public class SingleJar { protected boolean includeBuildData = true; /** List of build information properties files */ - protected List<String> buildInformationFiles = new ArrayList<String>(); + protected List<String> buildInformationFiles = new ArrayList<>(); /** Extraneous build informations (key=value) */ - protected List<String> buildInformations = new ArrayList<String>(); + protected List<String> buildInformations = new ArrayList<>(); /** The (optional) native executable that will be prepended to this JAR. */ private String launcherBin = null; @@ -223,21 +224,8 @@ public class SingleJar { // Copy the jars into the jar file. for (String inputJar : inputJars) { - InputStream in = fileSystem.getInputStream(inputJar); - try { - combiner.addZip(inputJar, in); - InputStream inToClose = in; - in = null; - inToClose.close(); - } finally { - if (in != null) { - try { - in.close(); - } catch (IOException e) { - // Preserve original exception. - } - } - } + File jar = fileSystem.getFile(inputJar); + combiner.addZip(jar); } // Close the output file. If something goes wrong here, delete the file. diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ZipCombiner.java b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ZipCombiner.java index d38c6d4493..64e9695e07 100644 --- a/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ZipCombiner.java +++ b/src/java_tools/singlejar/java/com/google/devtools/build/singlejar/ZipCombiner.java @@ -14,44 +14,46 @@ package com.google.devtools.build.singlejar; -import static java.nio.charset.StandardCharsets.ISO_8859_1; import static java.nio.charset.StandardCharsets.UTF_8; import com.google.devtools.build.singlejar.ZipEntryFilter.CustomMergeStrategy; import com.google.devtools.build.singlejar.ZipEntryFilter.StrategyCallback; +import com.google.devtools.build.zip.ExtraData; +import com.google.devtools.build.zip.ExtraDataList; +import com.google.devtools.build.zip.ZipFileEntry; +import com.google.devtools.build.zip.ZipFileEntry.Compression; +import com.google.devtools.build.zip.ZipReader; +import com.google.devtools.build.zip.ZipUtil; +import com.google.devtools.build.zip.ZipWriter; import java.io.BufferedOutputStream; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.EOFException; -import java.io.FilterOutputStream; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Calendar; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; import java.util.Date; -import java.util.GregorianCalendar; import java.util.HashMap; -import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.concurrent.atomic.AtomicBoolean; import java.util.zip.CRC32; -import java.util.zip.DataFormatException; import java.util.zip.Deflater; +import java.util.zip.DeflaterInputStream; import java.util.zip.Inflater; +import java.util.zip.InflaterInputStream; import javax.annotation.Nullable; -import javax.annotation.concurrent.NotThreadSafe; /** * An object that combines multiple ZIP files into a single file. It only * supports a subset of the ZIP format, specifically: * <ul> * <li>It only supports STORE and DEFLATE storage methods.</li> - * <li>There may be no data before the first file or between files.</li> - * <li>It ignores any data after the last file.</li> + * <li>It only supports 32-bit ZIP files.</li> * </ul> * * <p>These restrictions are also present in the JDK implementations @@ -68,63 +70,8 @@ import javax.annotation.concurrent.NotThreadSafe; * <p>Also see: * <a href="http://www.pkware.com/documents/casestudies/APPNOTE.TXT">ZIP format</a> */ -@NotThreadSafe -public final class ZipCombiner implements AutoCloseable { - - /** - * A Date set to the 1/1/1980, 00:00:00, the minimum value that can be stored - * in a ZIP file. - */ - public static final Date DOS_EPOCH = new GregorianCalendar(1980, 0, 1, 0, 0, 0).getTime(); - - private static final int DEFAULT_CENTRAL_DIRECTORY_BLOCK_SIZE = 1048576; // 1 MB for each block - - // The following constants are ZIP-specific. - private static final int LOCAL_FILE_HEADER_MARKER = 0x04034b50; - private static final int DATA_DESCRIPTOR_MARKER = 0x08074b50; - private static final int CENTRAL_DIRECTORY_MARKER = 0x02014b50; - private static final int END_OF_CENTRAL_DIRECTORY_MARKER = 0x06054b50; - - private static final int FILE_HEADER_BUFFER_SIZE = 30; - - private static final int VERSION_TO_EXTRACT_OFFSET = 4; - private static final int GENERAL_PURPOSE_FLAGS_OFFSET = 6; - private static final int COMPRESSION_METHOD_OFFSET = 8; - private static final int MTIME_OFFSET = 10; - private static final int MDATE_OFFSET = 12; - private static final int CRC32_OFFSET = 14; - private static final int COMPRESSED_SIZE_OFFSET = 18; - private static final int UNCOMPRESSED_SIZE_OFFSET = 22; - private static final int FILENAME_LENGTH_OFFSET = 26; - private static final int EXTRA_LENGTH_OFFSET = 28; - - private static final int DIRECTORY_ENTRY_BUFFER_SIZE = 46; - - // Set if the size, compressed size and CRC are set to zero, and present in - // the data descriptor after the data. - private static final int SIZE_MASKED_FLAG = 1 << 3; - - private static final int STORED_METHOD = 0; - private static final int DEFLATE_METHOD = 8; - - private static final int VERSION_STORED = 10; // Version 1.0 - private static final int VERSION_DEFLATE = 20; // Version 2.0 - - private static final long MAXIMUM_DATA_SIZE = 0xffffffffL; - - // This class relies on the buffer to have sufficient space for a complete - // file name. 2^16 is the maximum number of bytes in a file name. - private static final int BUFFER_SIZE = 65536; - - /** An empty entry used to skip files that have already been copied (or skipped). */ - private static final FileEntry COPIED_FILE_ENTRY = new FileEntry(null, null, 0); - - /** An empty entry used to mark files that have already been renamed. */ - private static final FileEntry RENAMED_FILE_ENTRY = new FileEntry(null, null, 0); - - /** A zero length array of ExtraData. */ - public static final ExtraData[] NO_EXTRA_ENTRIES = new ExtraData[0]; - +public class ZipCombiner implements AutoCloseable { + public static final Date DOS_EPOCH = new Date(ZipUtil.DOS_EPOCH); /** * Whether to compress or decompress entries. */ @@ -136,47 +83,41 @@ public final class ZipCombiner implements AutoCloseable { DONT_CARE, /** - * Output all entries using DEFLATE method, except directory entries. It is - * always more efficient to store directory entries uncompressed. + * Output all entries using DEFLATE method, except directory entries. It is always more + * efficient to store directory entries uncompressed. */ FORCE_DEFLATE, /** * Output all entries using STORED method. */ - FORCE_STORED; + FORCE_STORED, } - // A two-element enum for copyOrSkip type methods. - private static enum SkipMode { + /** + * The type of action to take for a ZIP file entry. + */ + private enum ActionType { + + /** + * Skip the entry. + */ + SKIP, /** - * Copy the read data to the output stream. + * Copy the entry. */ COPY, /** - * Do not write anything to the output stream. + * Rename the entry. */ - SKIP; - } + RENAME, - /** - * Stores internal information about merges or skips. - */ - private static final class FileEntry { - - /** If null, the file should be skipped. Otherwise, it should be merged. */ - private final CustomMergeStrategy mergeStrategy; - private final ByteArrayOutputStream outputBuffer; - private final int dosTime; - - private FileEntry(CustomMergeStrategy mergeStrategy, ByteArrayOutputStream outputBuffer, - int dosTime) { - this.mergeStrategy = mergeStrategy; - this.outputBuffer = outputBuffer; - this.dosTime = dosTime; - } + /** + * Merge the entry. + */ + MERGE; } /** @@ -188,6 +129,7 @@ public final class ZipCombiner implements AutoCloseable { * whose meaning depends on the value of {@code madeByVersion}, but is usually a reasonable * default. */ + @Deprecated public static final DirectoryEntryInfo DEFAULT_DIRECTORY_ENTRY_INFO = new DirectoryEntryInfo((short) -1, 0); @@ -196,6 +138,7 @@ public final class ZipCombiner implements AutoCloseable { * This does not contain all the information stored in the central directory record, only the * information that can be customized and is not automatically calculated or detected. */ + @Deprecated public static final class DirectoryEntryInfo { private final short madeByVersion; private final int externalFileAttribute; @@ -225,1394 +168,545 @@ public final class ZipCombiner implements AutoCloseable { } /** - * The central directory, which is grown as required; instead of using a single large buffer, we - * store a sequence of smaller buffers. With a single large buffer, whenever we grow the buffer by - * 2x, we end up requiring 3x the memory temporarily, which can lead to OOM problems even if there - * would still be enough memory. - * - * <p>The invariants for the fields are as follows: + * Encapsulates the action to take for a ZIP file entry along with optional details specific to + * the action type. The minimum requirements per type are: * <ul> - * <li>All blocks must have the same size. - * <li>The list of blocks must contain all blocks, including the current block (even if empty). - * <li>The current block offset must apply to the last block in the list, which is - * simultaneously the current block. - * <li>The current block may only be {@code null} if the list is empty. + * <li>SKIP: none.</li> + * <li>COPY: none.</li> + * <li>RENAME: newName.</li> + * <li>MERGE: strategy, mergeBuffer.</li> * </ul> + * + * <p>An action can be easily changed from one type to another by using + * {@link EntryAction#EntryAction(ActionType, EntryAction)}. */ - private static final class CentralDirectory { - private final int blockSize; // We allow this to be overridden for testing. - private List<byte[]> blockList = new ArrayList<>(); - private byte[] currentBlock; - private int currentBlockOffset = 0; - private int size = 0; - - CentralDirectory(int centralDirectoryBlockSize) { - this.blockSize = centralDirectoryBlockSize; - } + private static final class EntryAction { + private final ActionType type; + @Nullable private final Date date; + @Nullable private final String newName; + @Nullable private final CustomMergeStrategy strategy; + @Nullable private final ByteArrayOutputStream mergeBuffer; /** - * Appends the given data to the central directory and returns the start - * offset within the central directory to allow back-patching. + * Create an action of the specified type with no extra details. */ - int writeToCentralDirectory(byte[] b, int off, int len) { - checkArgument(len >= 0); - int offsetStarted = size; - while (len > 0) { - if (currentBlock == null - || currentBlockOffset >= currentBlock.length) { - currentBlock = new byte[blockSize]; - currentBlockOffset = 0; - blockList.add(currentBlock); - } - int maxCopy = Math.min(blockSize - currentBlockOffset, len); - System.arraycopy(b, off, currentBlock, currentBlockOffset, maxCopy); - off += maxCopy; - len -= maxCopy; - size += maxCopy; - currentBlockOffset += maxCopy; - } - return offsetStarted; - } - - /** Calls through to {@link #writeToCentralDirectory(byte[], int, int)}. */ - int writeToCentralDirectory(byte[] b) { - return writeToCentralDirectory(b, 0, b.length); + public EntryAction(ActionType type) { + this(type, null, null, null, null); } /** - * Writes an unsigned int in little-endian byte order to the central directory at the - * given offset. Does not perform range checking. + * Create a duplicate action with a different {@link ActionType}. */ - void setUnsignedInt(int offset, int value) { - blockList.get(cdIndex(offset + 0))[cdOffset(offset + 0)] = (byte) (value & 0xff); - blockList.get(cdIndex(offset + 1))[cdOffset(offset + 1)] = (byte) ((value >> 8) & 0xff); - blockList.get(cdIndex(offset + 2))[cdOffset(offset + 2)] = (byte) ((value >> 16) & 0xff); - blockList.get(cdIndex(offset + 3))[cdOffset(offset + 3)] = (byte) ((value >> 24) & 0xff); - } - - private int cdIndex(int offset) { - return offset / blockSize; - } - - private int cdOffset(int offset) { - return offset % blockSize; + public EntryAction(ActionType type, EntryAction action) { + this(type, action.getDate(), action.getNewName(), action.getStrategy(), + action.getMergeBuffer()); } /** - * Writes the central directory to the given output stream and returns the size, i.e., the - * number of bytes written. + * Create an action of the specified type and details. + * + * @param type the type of action + * @param date the custom date to set on the entry + * @param newName the custom name to create the entry as + * @param strategy the {@link CustomMergeStrategy} to use for merging this entry + * @param mergeBuffer the output stream to use for merge results */ - int writeTo(OutputStream out) throws IOException { - for (int i = 0; i < blockList.size() - 1; i++) { - out.write(blockList.get(i)); - } - if (currentBlock != null) { - out.write(currentBlock, 0, currentBlockOffset); - } - return size; + public EntryAction(ActionType type, Date date, String newName, CustomMergeStrategy strategy, + ByteArrayOutputStream mergeBuffer) { + checkArgument(type != ActionType.RENAME || newName != null, + "NewName must not be null if the ActionType is RENAME."); + checkArgument(type != ActionType.MERGE || strategy != null, + "Strategy must not be null if the ActionType is MERGE."); + checkArgument(type != ActionType.MERGE || mergeBuffer != null, + "MergeBuffer must not be null if the ActionType is MERGE."); + this.type = type; + this.date = date; + this.newName = newName; + this.strategy = strategy; + this.mergeBuffer = mergeBuffer; } - } - - /** - * An output stream that counts how many bytes were written. - */ - private static final class ByteCountingOutputStream extends FilterOutputStream { - private long bytesWritten = 0L; - ByteCountingOutputStream(OutputStream out) { - super(out); + /** Returns the type. */ + public ActionType getType() { + return type; } - @Override - public void write(byte[] b, int off, int len) throws IOException { - out.write(b, off, len); - bytesWritten += len; + /** Returns the date. */ + public Date getDate() { + return date; } - @Override - public void write(int b) throws IOException { - out.write(b); - bytesWritten++; + /** Returns the new name. */ + public String getNewName() { + return newName; } - } - - private final OutputMode mode; - private final ZipEntryFilter entryFilter; - - private final ByteCountingOutputStream out; - - // An input buffer to allow reading blocks of data. Keeping it here avoids - // another copy operation that would be required by the BufferedInputStream. - // The valid data is between bufferOffset and bufferOffset+bufferLength (exclusive). - private final byte[] buffer = new byte[BUFFER_SIZE]; - private int bufferOffset = 0; - private int bufferLength = 0; - - private String currentInputFile; - - // An intermediate buffer for the file header data. Keeping it here avoids - // creating a new buffer for every entry. - private final byte[] headerBuffer = new byte[FILE_HEADER_BUFFER_SIZE]; - - // An intermediate buffer for a central directory entry. Keeping it here - // avoids creating a new buffer for every entry. - private final byte[] directoryEntryBuffer = new byte[DIRECTORY_ENTRY_BUFFER_SIZE]; - - // The Inflater is a class member to avoid creating a new instance for every - // entry in the ZIP file. - private final Inflater inflater = new Inflater(true); - - // The contents of this buffer are never read. The Inflater is only used to - // determine the length of the compressed data, and the buffer is a throw- - // away buffer for the decompressed data. - private final byte[] inflaterBuffer = new byte[BUFFER_SIZE]; - - private final Map<String, FileEntry> fileNames = new HashMap<>(); - - private final CentralDirectory centralDirectory; - private int fileCount = 0; - - private boolean finished = false; - - // Package private for testing. - ZipCombiner(OutputMode mode, ZipEntryFilter entryFilter, OutputStream out, - int centralDirectoryBlockSize) { - this.mode = mode; - this.entryFilter = entryFilter; - this.out = new ByteCountingOutputStream(new BufferedOutputStream(out)); - this.centralDirectory = new CentralDirectory(centralDirectoryBlockSize); - } - - /** - * Creates a new instance with the given parameters. The {@code entryFilter} - * is called for every entry in the ZIP files and the combined ZIP file is - * written to {@code out}. The output mode determines whether entries must be - * written in compressed or decompressed form. Note that the result is - * invalid if an exception is thrown from any of the methods in this class, - * and before a call to {@link #close} or {@link #finish}. - */ - public ZipCombiner(OutputMode mode, ZipEntryFilter entryFilter, OutputStream out) { - this(mode, entryFilter, out, DEFAULT_CENTRAL_DIRECTORY_BLOCK_SIZE); - } - - /** - * Creates a new instance with the given parameters and the DONT_CARE mode. - */ - public ZipCombiner(ZipEntryFilter entryFilter, OutputStream out) { - this(OutputMode.DONT_CARE, entryFilter, out); - } - - /** - * Creates a new instance with the {@link CopyEntryFilter} as the filter and - * the given mode and output stream. - */ - public ZipCombiner(OutputMode mode, OutputStream out) { - this(mode, new CopyEntryFilter(), out); - } - - /** - * Creates a new instance with the {@link CopyEntryFilter} as the filter, the - * DONT_CARE mode and the given output stream. - */ - public ZipCombiner(OutputStream out) { - this(OutputMode.DONT_CARE, new CopyEntryFilter(), out); - } - - /** - * Returns whether the output zip already contains a file or directory with - * the given name. - */ - public boolean containsFile(String filename) { - return fileNames.containsKey(filename); - } - - /** - * Makes a write call to the output stream, and updates the current offset. - */ - private void write(byte[] b, int off, int len) throws IOException { - out.write(b, off, len); - } - - /** Calls through to {@link #write(byte[], int, int)}. */ - private void write(byte[] b) throws IOException { - write(b, 0, b.length); - } - /** - * Reads at least one more byte into the internal buffer. This method must - * only be called when more data is necessary to correctly decode the ZIP - * format. - * - * <p>This method automatically compacts the existing data in the buffer by - * moving it to the beginning of the buffer. - * - * @throws EOFException if no more data is available from the input stream - * @throws IOException if the underlying stream throws one - */ - private void readMoreData(InputStream in) throws IOException { - if ((bufferLength > 0) && (bufferOffset > 0)) { - System.arraycopy(buffer, bufferOffset, buffer, 0, bufferLength); + /** Returns the strategy. */ + public CustomMergeStrategy getStrategy() { + return strategy; } - if (bufferLength >= buffer.length) { - // The buffer size is specifically chosen to avoid this situation. - throw new AssertionError("Internal error: buffer overrun."); - } - bufferOffset = 0; - int bytesRead = in.read(buffer, bufferLength, buffer.length - bufferLength); - if (bytesRead <= 0) { - throw new EOFException(); - } - bufferLength += bytesRead; - } - /** - * Reads data until the buffer is filled with at least {@code length} bytes. - * - * @throws IllegalArgumentException if not 0 <= length <= buffer.length - * @throws IOException if the underlying input stream throws one or the end - * of the input stream is reached before the required - * number of bytes is read - */ - private void readFully(InputStream in, int length) throws IOException { - checkArgument(length >= 0, "length too small: %s", length); - checkArgument(length <= buffer.length, "length too large: %s", length); - while (bufferLength < length) { - readMoreData(in); + /** Returns the mergeBuffer. */ + public ByteArrayOutputStream getMergeBuffer() { + return mergeBuffer; } } - /** - * Reads an unsigned short in little-endian byte order from the buffer at the - * given offset. Does not perform range checking. - */ - private int getUnsignedShort(byte[] source, int offset) { - int a = source[offset + 0] & 0xff; - int b = source[offset + 1] & 0xff; - return (b << 8) | a; - } - - /** - * Reads an unsigned int in little-endian byte order from the buffer at the - * given offset. Does not perform range checking. - */ - private long getUnsignedInt(byte[] source, int offset) { - int a = source[offset + 0] & 0xff; - int b = source[offset + 1] & 0xff; - int c = source[offset + 2] & 0xff; - int d = source[offset + 3] & 0xff; - return ((d << 24) | (c << 16) | (b << 8) | a) & 0xffffffffL; - } - - /** - * Writes an unsigned short in little-endian byte order to the buffer at the - * given offset. Does not perform range checking. - */ - private void setUnsignedShort(byte[] target, int offset, short value) { - target[offset + 0] = (byte) (value & 0xff); - target[offset + 1] = (byte) ((value >> 8) & 0xff); - } - - /** - * Writes an unsigned int in little-endian byte order to the buffer at the - * given offset. Does not perform range checking. - */ - private void setUnsignedInt(byte[] target, int offset, int value) { - target[offset + 0] = (byte) (value & 0xff); - target[offset + 1] = (byte) ((value >> 8) & 0xff); - target[offset + 2] = (byte) ((value >> 16) & 0xff); - target[offset + 3] = (byte) ((value >> 24) & 0xff); - } + private final class FilterCallback implements StrategyCallback { + private String filename; + private final AtomicBoolean called = new AtomicBoolean(); - /** - * Copies or skips {@code length} amount of bytes from the input stream to the - * output stream. If the internal buffer is not empty, those bytes are copied - * first. When the method returns, there may be more bytes remaining in the - * buffer. - * - * @throws IOException if the underlying stream throws one - */ - private void copyOrSkipData(InputStream in, long length, SkipMode skip) throws IOException { - checkArgument(length >= 0); - while (length > 0) { - if (bufferLength == 0) { - readMoreData(in); - } - int bytesToWrite = (length < bufferLength) ? (int) length : bufferLength; - if (skip == SkipMode.COPY) { - write(buffer, bufferOffset, bytesToWrite); - } - bufferOffset += bytesToWrite; - bufferLength -= bytesToWrite; - length -= bytesToWrite; + public void resetForFile(String filename) { + this.filename = filename; + this.called.set(false); } - } - /** - * Copies or skips {@code length} amount of bytes from the input stream to the - * output stream. If the internal buffer is not empty, those bytes are copied - * first. When the method returns, there may be more bytes remaining in the - * buffer. In addition to writing to the output stream, it also writes to the - * central directory. - * - * @throws IOException if the underlying stream throws one - */ - private void forkOrSkipData(InputStream in, long length, SkipMode skip) throws IOException { - checkArgument(length >= 0); - while (length > 0) { - if (bufferLength == 0) { - readMoreData(in); - } - int bytesToWrite = (length < bufferLength) ? (int) length : bufferLength; - if (skip == SkipMode.COPY) { - write(buffer, bufferOffset, bytesToWrite); - centralDirectory.writeToCentralDirectory(buffer, bufferOffset, bytesToWrite); - } - bufferOffset += bytesToWrite; - bufferLength -= bytesToWrite; - length -= bytesToWrite; + @Override public void skip() throws IOException { + checkCall(); + actions.put(filename, new EntryAction(ActionType.SKIP)); } - } - - /** - * A mutable integer reference value to allow returning two values from a - * method. - */ - private static class MutableInt { - private int value; - - MutableInt(int initialValue) { - this.value = initialValue; + @Override public void copy(Date date) throws IOException { + checkCall(); + actions.put(filename, new EntryAction(ActionType.COPY, date, null, null, null)); } - public void setValue(int value) { - this.value = value; + @Override public void rename(String newName, Date date) throws IOException { + checkCall(); + actions.put(filename, new EntryAction(ActionType.RENAME, date, newName, null, null)); } - public int getValue() { - return value; + @Override public void customMerge(Date date, CustomMergeStrategy strategy) throws IOException { + checkCall(); + actions.put(filename, new EntryAction(ActionType.MERGE, date, null, strategy, + new ByteArrayOutputStream())); } - } - /** - * Uses the inflater to decompress some data into the given buffer. This - * method performs no error checking on the input parameters and also does - * not update the buffer parameters of the input buffer (such as bufferOffset - * and bufferLength). It's only here to avoid code duplication. - * - * <p>The Inflater may not be in the finished state when this method is - * called. - * - * <p>This method returns 0 if it read data and reached the end of the - * DEFLATE stream without producing output. In that case, {@link - * Inflater#finished} is guaranteed to return true. - * - * @throws IOException if the underlying stream throws an IOException or if - * illegal data is encountered - */ - private int inflateData(InputStream in, byte[] dest, int off, int len, MutableInt consumed) - throws IOException { - // Defend against Inflater.finished() returning true. - consumed.setValue(0); - int bytesProduced = 0; - int bytesConsumed = 0; - while ((bytesProduced == 0) && !inflater.finished()) { - inflater.setInput(buffer, bufferOffset + bytesConsumed, bufferLength - bytesConsumed); - int remainingBefore = inflater.getRemaining(); - try { - bytesProduced = inflater.inflate(dest, off, len); - } catch (DataFormatException e) { - throw new IOException("Invalid deflate stream in ZIP file.", e); - } - bytesConsumed += remainingBefore - inflater.getRemaining(); - consumed.setValue(bytesConsumed); - if (bytesProduced == 0) { - if (inflater.needsDictionary()) { - // The DEFLATE algorithm as used in the ZIP file format does not - // require an additional dictionary. - throw new AssertionError("Inflater unexpectedly requires a dictionary."); - } else if (inflater.needsInput()) { - readMoreData(in); - } else if (inflater.finished()) { - return 0; - } else { - // According to the Inflater specification, this cannot happen. - throw new AssertionError("Inflater unexpectedly produced no output."); - } - } + private void checkCall() { + checkState(called.compareAndSet(false, true), "The callback was already called once."); } - return bytesProduced; } - /** - * Copies or skips data from the input stream to the output stream. To - * determine the length of the data, the data is decompressed with the - * DEFLATE algorithm, which stores the length implicitly as part of the - * compressed data, using a combination of end markers and length indicators. - * - * @see <a href="http://www.ietf.org/rfc/rfc1951.txt">RFC 1951</a> - * - * @throws IOException if the underlying stream throws an IOException - */ - private long copyOrSkipDeflateData(InputStream in, SkipMode skip) throws IOException { - long bytesCopied = 0; - inflater.reset(); - MutableInt consumedBytes = new MutableInt(0); - while (!inflater.finished()) { - // Neither the uncompressed data nor the length of it is used. The - // decompression is only required to determine the correct length of the - // compressed data to copy. - inflateData(in, inflaterBuffer, 0, inflaterBuffer.length, consumedBytes); - int bytesRead = consumedBytes.getValue(); - if (skip == SkipMode.COPY) { - write(buffer, bufferOffset, bytesRead); - } - bufferOffset += bytesRead; - bufferLength -= bytesRead; - bytesCopied += bytesRead; - } - return bytesCopied; + /** Returns a {@link Deflater} for performing ZIP compression. */ + private static Deflater getDeflater() { + return new Deflater(Deflater.DEFAULT_COMPRESSION, true); } - /** - * Returns a 32-bit integer containing a ZIP-compatible encoding of the given - * date. Only dates between 1980 and 2107 (inclusive) are supported. - * - * <p>The upper 16 bits contain the year, month, and day. The lower 16 bits - * contain the hour, minute, and second. The resolution of the second field - * is only 4 bits, which means that the only even second values can be - * stored - this method rounds down to the nearest even value. - * - * @throws IllegalArgumentException if the given date is outside the - * supported range - */ - // Only visible for testing. - static int dateToDosTime(Date date) { - Calendar calendar = new GregorianCalendar(); - calendar.setTime(date); - int year = calendar.get(Calendar.YEAR); - if (year < 1980) { - throw new IllegalArgumentException("date must be in or after 1980"); - } - // The ZIP format only provides 7 bits for the year. - if (year > 2107) { - throw new IllegalArgumentException("date must before 2107"); - } - int month = calendar.get(Calendar.MONTH) + 1; // Months from Calendar are zero-based. - int day = calendar.get(Calendar.DAY_OF_MONTH); - int hour = calendar.get(Calendar.HOUR_OF_DAY); - int minute = calendar.get(Calendar.MINUTE); - int second = calendar.get(Calendar.SECOND); - return ((year - 1980) << 25) | (month << 21) | (day << 16) - | (hour << 11) | (minute << 5) | (second >> 1); + /** Returns a {@link Inflater} for performing ZIP decompression. */ + private static Inflater getInflater() { + return new Inflater(true); } - /** - * Fills the directory entry, using the information from the header buffer, - * and writes it to the central directory. It returns the offset into the - * central directory that can be used for patching the entry. Requires that - * the entire entry header is present in {@link #headerBuffer}. It also uses - * the {@link ByteCountingOutputStream#bytesWritten}, so it must be called - * just before the header is written to the output stream. - * - * @throws IOException if the current offset is too large for the ZIP format - */ - private int fillDirectoryEntryBuffer( - DirectoryEntryInfo directoryEntryInfo) throws IOException { - // central file header signature - setUnsignedInt(directoryEntryBuffer, 0, CENTRAL_DIRECTORY_MARKER); - short version = (short) getUnsignedShort(headerBuffer, VERSION_TO_EXTRACT_OFFSET); - short curMadeMyVersion = (directoryEntryInfo.madeByVersion == -1) - ? version : directoryEntryInfo.madeByVersion; - setUnsignedShort(directoryEntryBuffer, 4, curMadeMyVersion); // version made by - // version needed to extract - setUnsignedShort(directoryEntryBuffer, 6, version); - // general purpose bit flag - setUnsignedShort(directoryEntryBuffer, 8, - (short) getUnsignedShort(headerBuffer, GENERAL_PURPOSE_FLAGS_OFFSET)); - // compression method - setUnsignedShort(directoryEntryBuffer, 10, - (short) getUnsignedShort(headerBuffer, COMPRESSION_METHOD_OFFSET)); - // last mod file time, last mod file date - setUnsignedShort(directoryEntryBuffer, 12, - (short) getUnsignedShort(headerBuffer, MTIME_OFFSET)); - setUnsignedShort(directoryEntryBuffer, 14, - (short) getUnsignedShort(headerBuffer, MDATE_OFFSET)); - // crc-32 - setUnsignedInt(directoryEntryBuffer, 16, (int) getUnsignedInt(headerBuffer, CRC32_OFFSET)); - // compressed size - setUnsignedInt(directoryEntryBuffer, 20, - (int) getUnsignedInt(headerBuffer, COMPRESSED_SIZE_OFFSET)); - // uncompressed size - setUnsignedInt(directoryEntryBuffer, 24, - (int) getUnsignedInt(headerBuffer, UNCOMPRESSED_SIZE_OFFSET)); - // file name length - setUnsignedShort(directoryEntryBuffer, 28, - (short) getUnsignedShort(headerBuffer, FILENAME_LENGTH_OFFSET)); - // extra field length - setUnsignedShort(directoryEntryBuffer, 30, - (short) getUnsignedShort(headerBuffer, EXTRA_LENGTH_OFFSET)); - setUnsignedShort(directoryEntryBuffer, 32, (short) 0); // file comment length - setUnsignedShort(directoryEntryBuffer, 34, (short) 0); // disk number start - setUnsignedShort(directoryEntryBuffer, 36, (short) 0); // internal file attributes - setUnsignedInt(directoryEntryBuffer, 38, directoryEntryInfo.externalFileAttribute); - if (out.bytesWritten >= MAXIMUM_DATA_SIZE) { - throw new IOException("Unable to handle files bigger than 2^32 bytes."); + /** Copies all data from the input stream to the output stream. */ + private static long copyStream(InputStream from, OutputStream to) throws IOException { + byte[] buf = new byte[0x1000]; + long total = 0; + int r; + while ((r = from.read(buf)) != -1) { + to.write(buf, 0, r); + total += r; } - // relative offset of local header - setUnsignedInt(directoryEntryBuffer, 42, (int) out.bytesWritten); - fileCount++; - return centralDirectory.writeToCentralDirectory(directoryEntryBuffer); + return total; } - /** - * Fix the directory entry with the correct crc32, compressed size, and - * uncompressed size. - */ - private void fixDirectoryEntry(int offset, long crc32, long compressedSize, - long uncompressedSize) { - // The constants from the top don't apply here, because this is the central directory entry. - centralDirectory.setUnsignedInt(offset + 16, (int) crc32); // crc-32 - centralDirectory.setUnsignedInt(offset + 20, (int) compressedSize); // compressed size - centralDirectory.setUnsignedInt(offset + 24, (int) uncompressedSize); // uncompressed size - } + private final OutputMode mode; + private final ZipEntryFilter entryFilter; + private final FilterCallback callback; + private final ZipWriter out; - /** - * (Un)Compresses and copies the current ZIP file entry. Requires that the - * entire entry header is present in {@link #headerBuffer}. It currently - * drops the extra data in the process. - * - * @throws IOException if the underlying stream throws an IOException - */ - private void modifyAndCopyEntry(String filename, InputStream in, int dosTime) - throws IOException { - final int method = getUnsignedShort(headerBuffer, COMPRESSION_METHOD_OFFSET); - final int flags = getUnsignedShort(headerBuffer, GENERAL_PURPOSE_FLAGS_OFFSET); - final int fileNameLength = getUnsignedShort(headerBuffer, FILENAME_LENGTH_OFFSET); - final int extraFieldLength = getUnsignedShort(headerBuffer, EXTRA_LENGTH_OFFSET); - // TODO(bazel-team): Read and copy the extra data if present. - - forkOrSkipData(in, fileNameLength, SkipMode.SKIP); - forkOrSkipData(in, extraFieldLength, SkipMode.SKIP); - if (method == STORED_METHOD) { - long compressedSize = getUnsignedInt(headerBuffer, COMPRESSED_SIZE_OFFSET); - copyStreamToEntry(filename, new FixedLengthInputStream(in, compressedSize), dosTime, - NO_EXTRA_ENTRIES, true, DEFAULT_DIRECTORY_ENTRY_INFO); - } else if (method == DEFLATE_METHOD) { - inflater.reset(); - copyStreamToEntry(filename, new DeflateInputStream(in), dosTime, NO_EXTRA_ENTRIES, false, - DEFAULT_DIRECTORY_ENTRY_INFO); - if ((flags & SIZE_MASKED_FLAG) != 0) { - copyOrSkipData(in, 16, SkipMode.SKIP); - } - } else { - throw new AssertionError("This should have been checked in validateHeader()."); - } - } + private final Map<String, ZipFileEntry> entries; + private final Map<String, EntryAction> actions; /** - * Copies or skips the current ZIP file entry. Requires that the entire entry - * header is present in {@link #headerBuffer}. It uses the current mode to - * decide whether to compress or decompress the entry. + * Creates a {@link ZipCombiner} for combining ZIP files using the specified {@link OutputMode}, + * {@link ZipEntryFilter}, and destination {@link OutputStream}. * - * @throws IOException if the underlying stream throws an IOException + * @param mode the compression preference for the output ZIP file + * @param entryFilter the filter to use when adding ZIP files to the combined output + * @param out the {@link OutputStream} for writing the combined ZIP file */ - private void copyOrSkipEntry(String filename, InputStream in, SkipMode skip, Date date, - DirectoryEntryInfo directoryEntryInfo) throws IOException { - copyOrSkipEntry(filename, in, skip, date, directoryEntryInfo, false); + public ZipCombiner(OutputMode mode, ZipEntryFilter entryFilter, OutputStream out) { + this.mode = mode; + this.entryFilter = entryFilter; + this.callback = new FilterCallback(); + this.out = new ZipWriter(new BufferedOutputStream(out), UTF_8); + this.entries = new HashMap<>(); + this.actions = new HashMap<>(); } /** - * Renames and otherwise copies the current ZIP file entry. Requires that the entire - * entry header is present in {@link #headerBuffer}. It uses the current mode to - * decide whether to compress or decompress the entry. + * Creates a {@link ZipCombiner} for combining ZIP files using the specified + * {@link ZipEntryFilter}, and destination {@link OutputStream}. Uses the DONT_CARE + * {@link OutputMode}. * - * @throws IOException if the underlying stream throws an IOException + * @param entryFilter the filter to use when adding ZIP files to the combined output + * @param out the {@link OutputStream} for writing the combined ZIP file */ - private void renameEntry(String filename, InputStream in, Date date, - DirectoryEntryInfo directoryEntryInfo) throws IOException { - copyOrSkipEntry(filename, in, SkipMode.COPY, date, directoryEntryInfo, true); + public ZipCombiner(ZipEntryFilter entryFilter, OutputStream out) { + this(OutputMode.DONT_CARE, entryFilter, out); } /** - * Copies or skips the current ZIP file entry. Requires that the entire entry - * header is present in {@link #headerBuffer}. It uses the current mode to - * decide whether to compress or decompress the entry. + * Creates a {@link ZipCombiner} for combining ZIP files using the specified {@link OutputMode}, + * and destination {@link OutputStream}. Uses a {@link CopyEntryFilter} as the + * {@link ZipEntryFilter}. * - * @throws IOException if the underlying stream throws an IOException + * @param mode the compression preference for the output ZIP file + * @param out the {@link OutputStream} for writing the combined ZIP file */ - private void copyOrSkipEntry(String filename, InputStream in, SkipMode skip, Date date, - DirectoryEntryInfo directoryEntryInfo, boolean rename) throws IOException { - final int method = getUnsignedShort(headerBuffer, COMPRESSION_METHOD_OFFSET); - - // We can cast here, because the result is only treated as a bitmask. - int dosTime = date == null ? (int) getUnsignedInt(headerBuffer, MTIME_OFFSET) - : dateToDosTime(date); - if (skip == SkipMode.COPY) { - if ((mode == OutputMode.FORCE_DEFLATE) && (method == STORED_METHOD) - && !filename.endsWith("/")) { - modifyAndCopyEntry(filename, in, dosTime); - return; - } else if ((mode == OutputMode.FORCE_STORED) && (method == DEFLATE_METHOD)) { - modifyAndCopyEntry(filename, in, dosTime); - return; - } - } - - int directoryOffset = copyOrSkipEntryHeader(filename, in, date, directoryEntryInfo, - skip, rename); - - copyOrSkipEntryData(filename, in, skip, directoryOffset); + public ZipCombiner(OutputMode mode, OutputStream out) { + this(mode, new CopyEntryFilter(), out); } /** - * Copies or skips the header of an entry, including filename and extra data. - * Requires that the entire entry header is present in {@link #headerBuffer}. + * Creates a {@link ZipCombiner} for combining ZIP files using the specified destination + * {@link OutputStream}. Uses the DONT_CARE {@link OutputMode} and a {@link CopyEntryFilter} as + * the {@link ZipEntryFilter}. * - * @returns the enrty offset in the central directory - * @throws IOException if the underlying stream throws an IOException + * @param out the {@link OutputStream} for writing the combined ZIP file */ - private int copyOrSkipEntryHeader(String filename, InputStream in, Date date, - DirectoryEntryInfo directoryEntryInfo, SkipMode skip, boolean rename) - throws IOException { - final int fileNameLength = getUnsignedShort(headerBuffer, FILENAME_LENGTH_OFFSET); - final int extraFieldLength = getUnsignedShort(headerBuffer, EXTRA_LENGTH_OFFSET); - - byte[] fileNameAsBytes = null; - if (rename) { - // If the entry is renamed, we patch the filename length in the buffer - // before it's copied, and before writing to the central directory. - fileNameAsBytes = filename.getBytes(UTF_8); - checkArgument(fileNameAsBytes.length <= 65535, - "File name too long: %s bytes (max. 65535)", fileNameAsBytes.length); - setUnsignedShort(headerBuffer, FILENAME_LENGTH_OFFSET, (short) fileNameAsBytes.length); - } - - int directoryOffset = 0; - if (skip == SkipMode.COPY) { - if (date != null) { - int dosTime = dateToDosTime(date); - setUnsignedShort(headerBuffer, MTIME_OFFSET, (short) dosTime); // lower 16 bits - setUnsignedShort(headerBuffer, MDATE_OFFSET, (short) (dosTime >> 16)); // upper 16 bits - } - // Call this before writing the data out, so that we get the correct offset. - directoryOffset = fillDirectoryEntryBuffer(directoryEntryInfo); - write(headerBuffer, 0, FILE_HEADER_BUFFER_SIZE); - } - if (!rename) { - forkOrSkipData(in, fileNameLength, skip); - } else { - forkOrSkipData(in, fileNameLength, SkipMode.SKIP); - write(fileNameAsBytes); - centralDirectory.writeToCentralDirectory(fileNameAsBytes); - } - forkOrSkipData(in, extraFieldLength, skip); - return directoryOffset; + public ZipCombiner(OutputStream out) { + this(OutputMode.DONT_CARE, new CopyEntryFilter(), out); } /** - * Copy or skip the data of an entry. Requires that the - * entire entry header is present in {@link #headerBuffer}. + * Write all contents from the {@link InputStream} as a prefix file for the combined ZIP file. * - * @throws IOException if the underlying stream throws an IOException + * @param in the {@link InputStream} containing the prefix file data + * @throws IOException if there is an error writing the prefix file */ - private void copyOrSkipEntryData(String filename, InputStream in, SkipMode skip, - int directoryOffset) throws IOException { - final int flags = getUnsignedShort(headerBuffer, GENERAL_PURPOSE_FLAGS_OFFSET); - final int method = getUnsignedShort(headerBuffer, COMPRESSION_METHOD_OFFSET); - if ((flags & SIZE_MASKED_FLAG) != 0) { - // The compressed data size is unknown. - if (method != DEFLATE_METHOD) { - throw new AssertionError("This should have been checked in validateHeader()."); - } - copyOrSkipDeflateData(in, skip); - // The flags indicate that a data descriptor must follow the data. - readFully(in, 16); - if (getUnsignedInt(buffer, bufferOffset) != DATA_DESCRIPTOR_MARKER) { - throw new IOException("Missing data descriptor for " + filename + " in " + currentInputFile - + "."); - } - long crc32 = getUnsignedInt(buffer, bufferOffset + 4); - long compressedSize = getUnsignedInt(buffer, bufferOffset + 8); - long uncompressedSize = getUnsignedInt(buffer, bufferOffset + 12); - if (skip == SkipMode.COPY) { - fixDirectoryEntry(directoryOffset, crc32, compressedSize, uncompressedSize); - } - copyOrSkipData(in, 16, skip); - } else { - // The size value is present in the header, so just copy that amount. - long compressedSize = getUnsignedInt(headerBuffer, COMPRESSED_SIZE_OFFSET); - copyOrSkipData(in, compressedSize, skip); - } + public void prependExecutable(InputStream in) throws IOException { + out.startPrefixFile(); + copyStream(in, out); + out.endPrefixFile(); } /** - * An input stream that reads a fixed number of bytes from the given input - * stream before it returns end-of-input. It uses the local buffer, so it - * can't be static. + * Adds a directory entry to the combined ZIP file using the specified filename and date. + * + * @param filename the name of the directory to create + * @param date the modified time to assign to the directory + * @throws IOException if there is an error writing the directory entry */ - private class FixedLengthInputStream extends InputStream { - - private final InputStream in; - private long remainingBytes; - private final byte[] singleByteBuffer = new byte[1]; - - FixedLengthInputStream(InputStream in, long remainingBytes) { - this.in = in; - this.remainingBytes = remainingBytes; - } - - @Override - public int read() throws IOException { - int bytesRead = read(singleByteBuffer, 0, 1); - return (bytesRead == -1) ? -1 : singleByteBuffer[0]; - } - - @Override - public int read(byte b[], int off, int len) throws IOException { - checkArgument(len >= 0); - checkArgument(off >= 0); - checkArgument(off + len <= b.length); - if (remainingBytes == 0) { - return -1; - } - if (bufferLength == 0) { - readMoreData(in); - } - int bytesToCopy = len; - if (remainingBytes < bytesToCopy) { - bytesToCopy = (int) remainingBytes; - } - if (bufferLength < bytesToCopy) { - bytesToCopy = bufferLength; - } - System.arraycopy(buffer, bufferOffset, b, off, bytesToCopy); - bufferOffset += bytesToCopy; - bufferLength -= bytesToCopy; - remainingBytes -= bytesToCopy; - return bytesToCopy; - } + public void addDirectory(String filename, Date date) throws IOException { + addDirectory(filename, date, new ExtraData[0]); } /** - * An input stream that reads from a given input stream, decoding that data - * according to the DEFLATE algorithm. The DEFLATE data stream implicitly - * contains its own end-of-input marker. It uses the local buffer, so it - * can't be static. + * Adds a directory entry to the combined ZIP file using the specified filename, date, and extra + * data. + * + * @param filename the name of the directory to create + * @param date the modified time to assign to the directory + * @param extra the extra field data to add to the directory entry + * @throws IOException if there is an error writing the directory entry */ - private class DeflateInputStream extends InputStream { - - private final InputStream in; - private final byte[] singleByteBuffer = new byte[1]; - private final MutableInt consumedBytes = new MutableInt(0); + public void addDirectory(String filename, Date date, ExtraData[] extra) throws IOException { + checkArgument(filename.endsWith("/"), "Directory names must end with a /"); + checkState(!entries.containsKey(filename), + "Zip already contains a directory named %s", filename); - DeflateInputStream(InputStream in) { - this.in = in; - } - - @Override - public int read() throws IOException { - int bytesRead = read(singleByteBuffer, 0, 1); - // Do an unsigned cast on the byte from the buffer if it exists. - return (bytesRead == -1) ? -1 : (singleByteBuffer[0] & 0xff); - } - - @Override - public int read(byte b[], int off, int len) throws IOException { - if (inflater.finished()) { - return -1; - } - int length = inflateData(in, b, off, len, consumedBytes); - int bytesRead = consumedBytes.getValue(); - bufferOffset += bytesRead; - bufferLength -= bytesRead; - return length == 0 ? -1 : length; - } + ZipFileEntry entry = new ZipFileEntry(filename); + entry.setMethod(Compression.STORED); + entry.setCrc(0); + entry.setSize(0); + entry.setCompressedSize(0); + entry.setTime(date != null ? date.getTime() : new Date().getTime()); + entry.setExtra(new ExtraDataList(extra)); + out.putNextEntry(entry); + out.closeEntry(); + entries.put(filename, entry); } /** - * Handles a custom merge operation with the given strategy. This method - * creates an appropriate input stream and hands it to the strategy for - * processing. Requires that the entire entry header is present in {@link - * #headerBuffer}. + * Adds a file with the specified name to the combined ZIP file. * - * @throws IOException if one of the underlying stream throws an IOException, - * if the ZIP entry data is inconsistent, or if the - * implementation cannot handle the compression method - * given in the ZIP entry + * @param filename the name of the file to create + * @param in the {@link InputStream} containing the file data + * @throws IOException if there is an error writing the file entry + * @throws IllegalArgumentException if the combined ZIP file already contains a file of the same + * name. */ - private void handleCustomMerge(final InputStream in, CustomMergeStrategy mergeStrategy, - ByteArrayOutputStream outputBuffer) throws IOException { - final int flags = getUnsignedShort(headerBuffer, GENERAL_PURPOSE_FLAGS_OFFSET); - final int method = getUnsignedShort(headerBuffer, COMPRESSION_METHOD_OFFSET); - final long compressedSize = getUnsignedInt(headerBuffer, COMPRESSED_SIZE_OFFSET); - - final int fileNameLength = getUnsignedShort(headerBuffer, FILENAME_LENGTH_OFFSET); - final int extraFieldLength = getUnsignedShort(headerBuffer, EXTRA_LENGTH_OFFSET); - - copyOrSkipData(in, fileNameLength, SkipMode.SKIP); - copyOrSkipData(in, extraFieldLength, SkipMode.SKIP); - if (method == STORED_METHOD) { - mergeStrategy.merge(new FixedLengthInputStream(in, compressedSize), outputBuffer); - } else if (method == DEFLATE_METHOD) { - inflater.reset(); - // TODO(bazel-team): Defend against the mergeStrategy not reading the complete input. - mergeStrategy.merge(new DeflateInputStream(in), outputBuffer); - if ((flags & SIZE_MASKED_FLAG) != 0) { - copyOrSkipData(in, 16, SkipMode.SKIP); - } - } else { - throw new AssertionError("This should have been checked in validateHeader()."); - } + public void addFile(String filename, InputStream in) throws IOException { + addFile(filename, null, in); } /** - * Implementation of the strategy callback. + * Adds a file with the specified name and date to the combined ZIP file. + * + * @param filename the name of the file to create + * @param date the modified time to assign to the file + * @param in the {@link InputStream} containing the file data + * @throws IOException if there is an error writing the file entry + * @throws IllegalArgumentException if the combined ZIP file already contains a file of the same + * name. */ - private class TheStrategyCallback implements StrategyCallback { - - private String filename; - private final InputStream in; - - // Use an atomic boolean to make sure that only a single call goes - // through, even if there are multiple concurrent calls. Paranoid - // defensive programming. - private final AtomicBoolean callDone = new AtomicBoolean(); - - TheStrategyCallback(String filename, InputStream in) { - this.filename = filename; - this.in = in; - } - - // Verify that this is the first call and throw an exception if not. - private void checkCall() { - checkState(callDone.compareAndSet(false, true), "The callback was already called once."); - } - - @Override - public void copy(Date date) throws IOException { - checkCall(); - if (!containsFile(filename)) { - fileNames.put(filename, COPIED_FILE_ENTRY); - copyOrSkipEntry(filename, in, SkipMode.COPY, date, DEFAULT_DIRECTORY_ENTRY_INFO); - } else { // can't copy, name already used for renamed entry - copyOrSkipEntry(filename, in, SkipMode.SKIP, null, DEFAULT_DIRECTORY_ENTRY_INFO); - } - } - - @Override - public void rename(String newName, Date date) throws IOException { - checkCall(); - if (!containsFile(newName)) { - fileNames.put(newName, RENAMED_FILE_ENTRY); - renameEntry(newName, in, date, DEFAULT_DIRECTORY_ENTRY_INFO); - } else { - copyOrSkipEntry(filename, in, SkipMode.SKIP, null, DEFAULT_DIRECTORY_ENTRY_INFO); - } - filename = newName; - } - - @Override - public void skip() throws IOException { - checkCall(); - if (!containsFile(filename)) {// don't overwrite possible RENAMED_FILE_ENTRY value - fileNames.put(filename, COPIED_FILE_ENTRY); - } - copyOrSkipEntry(filename, in, SkipMode.SKIP, null, DEFAULT_DIRECTORY_ENTRY_INFO); - } - - @Override - public void customMerge(Date date, CustomMergeStrategy strategy) throws IOException { - checkCall(); - ByteArrayOutputStream outputBuffer = new ByteArrayOutputStream(); - fileNames.put(filename, new FileEntry(strategy, outputBuffer, dateToDosTime(date))); - handleCustomMerge(in, strategy, outputBuffer); - } + public void addFile(String filename, Date date, InputStream in) throws IOException { + ZipFileEntry entry = new ZipFileEntry(filename); + entry.setTime(date != null ? date.getTime() : new Date().getTime()); + addFile(entry, in); } /** - * Validates that the current entry obeys all the restrictions of this implementation. + * Adds a file with attributes specified by the {@link ZipFileEntry} to the combined ZIP file. * - * @throws IOException if the current entry doesn't obey the restrictions + * @param entry the {@link ZipFileEntry} containing the entry meta-data + * @param in the {@link InputStream} containing the file data + * @throws IOException if there is an error writing the file entry + * @throws IllegalArgumentException if the combined ZIP file already contains a file of the same + * name. */ - private void validateHeader() throws IOException { - // We only handle DEFLATE and STORED, like java.util.zip. - final int method = getUnsignedShort(headerBuffer, COMPRESSION_METHOD_OFFSET); - if ((method != DEFLATE_METHOD) && (method != STORED_METHOD)) { - throw new IOException("Unable to handle compression methods other than DEFLATE!"); - } + public void addFile(ZipFileEntry entry, InputStream in) throws IOException { + checkNotNull(entry, "Zip entry must not be null."); + checkNotNull(in, "Input stream must not be null."); + checkArgument(!entries.containsKey(entry.getName()), "Zip already contains a file named '%s'.", + entry.getName()); - // If the method is STORED, then the size must be available in the header. - final int flags = getUnsignedShort(headerBuffer, GENERAL_PURPOSE_FLAGS_OFFSET); - if ((method == STORED_METHOD) && ((flags & SIZE_MASKED_FLAG) != 0)) { - throw new IOException("If the method is STORED, then the size must be available in the" - + " header!"); - } - - // If the method is STORED, the compressed and uncompressed sizes must be equal. - final long compressedSize = getUnsignedInt(headerBuffer, COMPRESSED_SIZE_OFFSET); - final long uncompressedSize = getUnsignedInt(headerBuffer, UNCOMPRESSED_SIZE_OFFSET); - if ((method == STORED_METHOD) && (compressedSize != uncompressedSize)) { - throw new IOException("Compressed and uncompressed sizes for STORED entry differ!"); - } + ByteArrayOutputStream uncompressed = new ByteArrayOutputStream(); + copyStream(in, uncompressed); - // The compressed or uncompressed size being set to 0xffffffff is a strong indicator that the - // ZIP file is in ZIP64 mode, which supports files larger than 2^32. - // TODO(bazel-team): Support the ZIP64 extension. - if ((compressedSize == MAXIMUM_DATA_SIZE) || (uncompressedSize == MAXIMUM_DATA_SIZE)) { - throw new IOException("Unable to handle ZIP64 compressed files."); - } + writeEntryFromBuffer(new ZipFileEntry(entry), uncompressed.toByteArray()); } /** - * Reads a file entry from the input stream, calls the entryFilter to - * determine what to do with the entry, and performs the requested operation. - * Returns true if the input stream contained another entry. + * Adds a new entry into the output, by reading the input stream until it returns end of stream. + * This method does not call {@link ZipEntryFilter#accept}. * - * @throws IOException if one of the underlying stream throws an IOException, - * if the ZIP contains unsupported, inconsistent or - * incomplete data or if the filter throws an IOException + * @throws IOException if one of the underlying streams throws an IOException + * or if the input stream returns more data than + * supported by the ZIP format + * @throws IllegalStateException if an entry with the given name already + * exists + * @throws IllegalArgumentException if the given file name is longer than + * supported by the ZIP format */ - private boolean handleNextEntry(final InputStream in) throws IOException { - // Just try to read the complete header and fail if it didn't work. - try { - readFully(in, FILE_HEADER_BUFFER_SIZE); - } catch (EOFException e) { - return false; - } - - System.arraycopy(buffer, bufferOffset, headerBuffer, 0, FILE_HEADER_BUFFER_SIZE); - bufferOffset += FILE_HEADER_BUFFER_SIZE; - bufferLength -= FILE_HEADER_BUFFER_SIZE; - if (getUnsignedInt(headerBuffer, 0) != LOCAL_FILE_HEADER_MARKER) { - return false; - } - validateHeader(); - - final int fileNameLength = getUnsignedShort(headerBuffer, FILENAME_LENGTH_OFFSET); - readFully(in, fileNameLength); - // TODO(bazel-team): If I read the spec correctly, this should be UTF-8 rather than ISO-8859-1. - final String filename = new String(buffer, bufferOffset, fileNameLength, ISO_8859_1); - - FileEntry handler = fileNames.get(filename); - // The handler is null if this is the first time we see an entry with this filename, - // or if all previous entries with this name were renamed by the filter (and we can - // pretend we didn't encounter the name yet). - // If the handler is RENAMED_FILE_ENTRY, a previous entry was renamed as filename, - // in which case the filter should now be invoked for this name for the first time, - // giving the filter a chance to choose an unique name. - if (handler == null || handler == RENAMED_FILE_ENTRY) { - TheStrategyCallback callback = new TheStrategyCallback(filename, in); - entryFilter.accept(filename, callback); - if (fileNames.get(callback.filename) == null && fileNames.get(filename) == null) { - throw new IllegalStateException(); - } - } else if (handler.mergeStrategy == null) { - copyOrSkipEntry(filename, in, SkipMode.SKIP, null, DEFAULT_DIRECTORY_ENTRY_INFO); - } else { - handleCustomMerge(in, handler.mergeStrategy, handler.outputBuffer); - } - return true; + @Deprecated + public void addFile(String filename, Date date, InputStream in, + DirectoryEntryInfo directoryEntryInfo) throws IOException { + ZipFileEntry entry = new ZipFileEntry(filename); + entry.setTime(date != null ? date.getTime() : new Date().getTime()); + entry.setVersion(directoryEntryInfo.madeByVersion); + entry.setExternalAttributes(directoryEntryInfo.externalFileAttribute); + addFile(entry, in); } /** - * Clears the internal buffer. + * Adds the contents of a ZIP file to the combined ZIP file using the specified + * {@link ZipEntryFilter} to determine the appropriate action for each file. + * + * @param in the InputStream of the ZIP file to add to the combined ZIP file + * @throws IOException if there is an error reading the ZIP file or writing entries to the + * combined ZIP file */ - private void clearBuffer() { - bufferOffset = 0; - bufferLength = 0; + @Deprecated + public void addZip(InputStream in) throws IOException { + addZip(null, in); } /** - * Copies another ZIP file into the output. If multiple entries with the same - * name are present, the first such entry is copied, but the others are - * ignored. This is also true for multiple invocations of this method. The - * {@code inputName} parameter is used to provide better error messages in the - * case of a failure to decode the ZIP file. + * Adds the contents of a ZIP file to the combined ZIP file using the specified + * {@link ZipEntryFilter} to determine the appropriate action for each file. * - * @throws IOException if one of the underlying stream throws an IOException, - * if the ZIP contains unsupported, inconsistent or - * incomplete data or if the filter throws an IOException + * @param inputName the name of the ZIP file to add for providing better error messages + * @param in the InputStream of the ZIP file to add to the combined ZIP file + * @throws IOException if there is an error reading the ZIP file or writing entries to the + * combined ZIP file */ + @Deprecated public void addZip(String inputName, InputStream in) throws IOException { - if (finished) { - throw new IllegalStateException(); - } - if (in == null) { - throw new NullPointerException(); - } - clearBuffer(); - currentInputFile = inputName; - while (handleNextEntry(in)) {/*handleNextEntry has side-effect.*/} - } - - public void addZip(InputStream in) throws IOException { - addZip(null, in); - } - - private void copyStreamToEntry(String filename, InputStream in, int dosTime, - ExtraData[] extraDataEntries, boolean compress, DirectoryEntryInfo directoryEntryInfo) - throws IOException { - fileNames.put(filename, COPIED_FILE_ENTRY); - - byte[] fileNameAsBytes = filename.getBytes(UTF_8); - checkArgument(fileNameAsBytes.length <= 65535, - "File name too long: %s bytes (max. 65535)", fileNameAsBytes.length); - - // Note: This method can be called with an input stream that uses the buffer field of this - // class. We use a local buffer here to avoid conflicts. - byte[] localBuffer = new byte[4096]; - - byte[] uncompressedData = null; - if (!compress) { - ByteArrayOutputStream temp = new ByteArrayOutputStream(); - int bytesRead; - while ((bytesRead = in.read(localBuffer)) != -1) { - temp.write(localBuffer, 0, bytesRead); - } - uncompressedData = temp.toByteArray(); - } - byte[] extraData = null; - if (extraDataEntries.length != 0) { - int totalLength = 0; - for (ExtraData extra : extraDataEntries) { - int length = extra.getData().length; - if (totalLength > 0xffff - 4 - length) { - throw new IOException("Total length of extra data too big."); + File file = Files.createTempFile(inputName, null).toFile(); + Files.copy(in, file.toPath(), StandardCopyOption.REPLACE_EXISTING); + addZip(file); + file.deleteOnExit(); + } + + /** + * Adds the contents of a ZIP file to the combined ZIP file using the specified + * {@link ZipEntryFilter} to determine the appropriate action for each file. + * + * @param zipFile the ZIP file to add to the combined ZIP file + * @throws IOException if there is an error reading the ZIP file or writing entries to the + * combined ZIP file + */ + public void addZip(File zipFile) throws IOException { + try (ZipReader zip = new ZipReader(zipFile)) { + for (ZipFileEntry entry : zip.entries()) { + String filename = entry.getName(); + EntryAction action = getAction(filename); + switch (action.getType()) { + case SKIP: + break; + case COPY: + case RENAME: + writeEntry(zip, entry, action); + break; + case MERGE: + entries.put(filename, null); + InputStream in = zip.getRawInputStream(entry); + if (entry.getMethod() == Compression.DEFLATED) { + in = new InflaterInputStream(in, getInflater()); + } + action.getStrategy().merge(in, action.getMergeBuffer()); + break; } - totalLength += length + 4; - } - extraData = new byte[totalLength]; - int position = 0; - for (ExtraData extra : extraDataEntries) { - byte[] data = extra.getData(); - setUnsignedShort(extraData, position + 0, extra.getId()); - setUnsignedShort(extraData, position + 2, (short) data.length); - System.arraycopy(data, 0, extraData, position + 4, data.length); - position += data.length + 4; } } + } - // write header - Arrays.fill(headerBuffer, (byte) 0); - setUnsignedInt(headerBuffer, 0, LOCAL_FILE_HEADER_MARKER); // file header signature - if (compress) { - setUnsignedShort(headerBuffer, 4, (short) VERSION_DEFLATE); // version to extract - setUnsignedShort(headerBuffer, 6, (short) SIZE_MASKED_FLAG); // general purpose bit flag - setUnsignedShort(headerBuffer, 8, (short) DEFLATE_METHOD); // compression method - } else { - setUnsignedShort(headerBuffer, 4, (short) VERSION_STORED); // version to extract - setUnsignedShort(headerBuffer, 6, (short) 0); // general purpose bit flag - setUnsignedShort(headerBuffer, 8, (short) STORED_METHOD); // compression method - } - setUnsignedShort(headerBuffer, 10, (short) dosTime); // mtime - setUnsignedShort(headerBuffer, 12, (short) (dosTime >> 16)); // mdate - if (uncompressedData != null) { - CRC32 crc = new CRC32(); - crc.update(uncompressedData); - setUnsignedInt(headerBuffer, 14, (int) crc.getValue()); // crc32 - setUnsignedInt(headerBuffer, 18, uncompressedData.length); // compressed size - setUnsignedInt(headerBuffer, 22, uncompressedData.length); // uncompressed size - } else { - setUnsignedInt(headerBuffer, 14, 0); // crc32 - setUnsignedInt(headerBuffer, 18, 0); // compressed size - setUnsignedInt(headerBuffer, 22, 0); // uncompressed size - } - setUnsignedShort(headerBuffer, 26, (short) fileNameAsBytes.length); // file name length - if (extraData != null) { - setUnsignedShort(headerBuffer, 28, (short) extraData.length); // extra field length - } else { - setUnsignedShort(headerBuffer, 28, (short) 0); // extra field length + /** Returns the action to take for a file of the given filename. */ + private EntryAction getAction(String filename) throws IOException { + // If this filename has not been encountered before (no entry for filename) or this filename + // has been renamed (RENAME entry for filename), the desired action should be recomputed. + if (!actions.containsKey(filename) || actions.get(filename).getType() == ActionType.RENAME) { + callback.resetForFile(filename); + entryFilter.accept(filename, callback); } + checkState(actions.containsKey(filename), + "Action for file '%s' should have been set by ZipEntryFilter.", filename); - // This call works for both compressed or uncompressed entries. - int directoryOffset = fillDirectoryEntryBuffer(directoryEntryInfo); - write(headerBuffer); - write(fileNameAsBytes); - centralDirectory.writeToCentralDirectory(fileNameAsBytes); - if (extraData != null) { - write(extraData); - centralDirectory.writeToCentralDirectory(extraData); + EntryAction action = actions.get(filename); + // Only copy if this is the first instance of filename. + if (action.getType() == ActionType.COPY && entries.containsKey(filename)) { + action = new EntryAction(ActionType.SKIP, action); + actions.put(filename, action); } - - // write data - if (uncompressedData != null) { - write(uncompressedData); - } else { - try (DeflaterOutputStream deflaterStream = new DeflaterOutputStream()) { - int bytesRead; - while ((bytesRead = in.read(localBuffer)) != -1) { - deflaterStream.write(localBuffer, 0, bytesRead); - } - deflaterStream.finish(); - - // write data descriptor - Arrays.fill(headerBuffer, (byte) 0); - setUnsignedInt(headerBuffer, 0, DATA_DESCRIPTOR_MARKER); - setUnsignedInt(headerBuffer, 4, deflaterStream.getCRC()); // crc32 - setUnsignedInt(headerBuffer, 8, deflaterStream.getCompressedSize()); // compressed size - setUnsignedInt(headerBuffer, 12, deflaterStream.getUncompressedSize()); // uncompressed size - write(headerBuffer, 0, 16); - fixDirectoryEntry(directoryOffset, deflaterStream.getCRC(), - deflaterStream.getCompressedSize(), deflaterStream.getUncompressedSize()); + // Only rename if there is not already an entry with filename or filename's action is SKIP. + if (action.getType() == ActionType.RENAME) { + if (actions.containsKey(action.getNewName()) + && actions.get(action.getNewName()).getType() == ActionType.SKIP) { + action = new EntryAction(ActionType.SKIP, action); + } + if (entries.containsKey(action.getNewName())) { + action = new EntryAction(ActionType.SKIP, action); } } + return action; } - /** - * Adds a new entry into the output, by reading the input stream until it - * returns end of stream. Equivalent to - * {@link #addFile(String, Date, InputStream, DirectoryEntryInfo)}, but uses - * {@link #DEFAULT_DIRECTORY_ENTRY_INFO} for the file's directory entry. - */ - public void addFile(String filename, Date date, InputStream in) throws IOException { - addFile(filename, date, in, DEFAULT_DIRECTORY_ENTRY_INFO); - } + /** Writes an entry with the given name, date and external file attributes from the buffer. */ + private void writeEntryFromBuffer(ZipFileEntry entry, byte[] uncompressed) throws IOException { + CRC32 crc = new CRC32(); + crc.update(uncompressed); - /** - * Adds a new entry into the output, by reading the input stream until it - * returns end of stream. This method does not call {@link - * ZipEntryFilter#accept}. - * - * @throws IOException if one of the underlying streams throws an IOException - * or if the input stream returns more data than - * supported by the ZIP format - * @throws IllegalStateException if an entry with the given name already - * exists - * @throws IllegalArgumentException if the given file name is longer than - * supported by the ZIP format - */ - public void addFile(String filename, Date date, InputStream in, - DirectoryEntryInfo directoryEntryInfo) throws IOException { - checkNotFinished(); - if (in == null) { - throw new NullPointerException(); - } - if (filename == null) { - throw new NullPointerException(); + entry.setCrc(crc.getValue()); + entry.setSize(uncompressed.length); + if (mode == OutputMode.FORCE_STORED) { + entry.setMethod(Compression.STORED); + entry.setCompressedSize(uncompressed.length); + writeEntry(entry, new ByteArrayInputStream(uncompressed)); + } else { + ByteArrayOutputStream compressed = new ByteArrayOutputStream(); + copyStream(new DeflaterInputStream(new ByteArrayInputStream(uncompressed), getDeflater()), + compressed); + entry.setMethod(Compression.DEFLATED); + entry.setCompressedSize(compressed.size()); + writeEntry(entry, new ByteArrayInputStream(compressed.toByteArray())); } - checkState(!fileNames.containsKey(filename), - "jar already contains a file named %s", filename); - int dosTime = dateToDosTime(date != null ? date : new Date()); - copyStreamToEntry(filename, in, dosTime, NO_EXTRA_ENTRIES, - mode != OutputMode.FORCE_STORED, // Always compress if we're allowed to. - directoryEntryInfo); } /** - * Adds a new directory entry into the output. This method does not call - * {@link ZipEntryFilter#accept}. Uses {@link #DEFAULT_DIRECTORY_ENTRY_INFO} for the added - * directory entry. - * - * @throws IOException if one of the underlying streams throws an IOException - * @throws IllegalStateException if an entry with the given name already - * exists - * @throws IllegalArgumentException if the given file name is longer than - * supported by the ZIP format + * Writes an entry from the specified source {@link ZipReader} and {@link ZipFileEntry} using the + * specified {@link EntryAction}. + * + * <p>Writes the output entry from the input entry performing inflation or deflation as needed + * and applies any values from the {@link EntryAction} as needed. */ - public void addDirectory(String filename, Date date, ExtraData[] extraDataEntries) + private void writeEntry(ZipReader zip, ZipFileEntry entry, EntryAction action) throws IOException { - checkNotFinished(); - checkArgument(filename.endsWith("/")); // Can also throw NPE. - checkState(!fileNames.containsKey(filename), - "jar already contains a directory named %s", filename); - int dosTime = dateToDosTime(date != null ? date : new Date()); - copyStreamToEntry(filename, new ByteArrayInputStream(new byte[0]), dosTime, extraDataEntries, - false, // Never compress directory entries. - DEFAULT_DIRECTORY_ENTRY_INFO); + checkArgument(action.getType() != ActionType.SKIP, + "Cannot write a zip entry whose action is of type SKIP."); + + ZipFileEntry outEntry = new ZipFileEntry(entry); + if (action.getType() == ActionType.RENAME) { + checkNotNull(action.getNewName(), + "ZipEntryFilter actions of type RENAME must not have a null filename."); + outEntry.setName(action.getNewName()); + } + + if (action.getDate() != null) { + outEntry.setTime(action.getDate().getTime()); + } + + InputStream data; + if (mode == OutputMode.FORCE_DEFLATE && entry.getMethod() != Compression.DEFLATED) { + // The output mode is deflate, but the entry compression is not. Create a deflater stream + // from the raw file data and deflate to a temporary byte array to determine the deflated + // size. Then use this byte array as the input stream for writing the entry. + ByteArrayOutputStream tmp = new ByteArrayOutputStream(); + copyStream(new DeflaterInputStream(zip.getRawInputStream(entry), getDeflater()), tmp); + data = new ByteArrayInputStream(tmp.toByteArray()); + outEntry.setMethod(Compression.DEFLATED); + outEntry.setCompressedSize(tmp.size()); + } else if (mode == OutputMode.FORCE_STORED && entry.getMethod() != Compression.STORED) { + // The output mode is stored, but the entry compression is not; create an inflater stream + // from the raw file data. + data = new InflaterInputStream(zip.getRawInputStream(entry), getInflater()); + outEntry.setMethod(Compression.STORED); + outEntry.setCompressedSize(entry.getSize()); + } else { + // Entry compression agrees with output mode; use the raw file data as is. + data = zip.getRawInputStream(entry); + } + writeEntry(outEntry, data); } /** - * Adds a new directory entry into the output. This method does not call - * {@link ZipEntryFilter#accept}. - * - * @throws IOException if one of the underlying streams throws an IOException - * @throws IllegalStateException if an entry with the given name already - * exists - * @throws IllegalArgumentException if the given file name is longer than - * supported by the ZIP format + * Writes the specified {@link ZipFileEntry} using the data from the given {@link InputStream}. */ - public void addDirectory(String filename, Date date) - throws IOException { - addDirectory(filename, date, NO_EXTRA_ENTRIES); + private void writeEntry(ZipFileEntry entry, InputStream data) throws IOException { + out.putNextEntry(entry); + copyStream(data, out); + out.closeEntry(); + entries.put(entry.getName(), entry); } /** - * A deflater output stream that also counts uncompressed and compressed - * numbers of bytes and computes the CRC so that the data descriptor marker - * is written correctly. + * Returns true if the combined ZIP file already contains a file of the specified file name. * - * <p>Not static, so it can access the write() methods. + * @param filename the filename of the file whose presence in the combined ZIP file is to be + * tested + * @return true if the combined ZIP file contains the specified file */ - private class DeflaterOutputStream extends OutputStream { - - private final Deflater deflater = new Deflater(Deflater.DEFAULT_COMPRESSION, true); - private final CRC32 crc = new CRC32(); - private final byte[] outputBuffer = new byte[4096]; - private long uncompressedBytes = 0; - private long compressedBytes = 0; - - @Override - public void write(int b) throws IOException { - byte[] buf = new byte[] { (byte) (b & 0xff) }; - write(buf, 0, buf.length); - } - - @Override - public void write(byte b[], int off, int len) throws IOException { - checkNotFinished(); - uncompressedBytes += len; - crc.update(b, off, len); - deflater.setInput(b, off, len); - while (!deflater.needsInput()) { - deflate(); - } - } - - @Override - public void close() throws IOException { - super.close(); - deflater.end(); - } - - /** - * Writes out the remaining buffered data without closing the output - * stream. - */ - public void finish() throws IOException { - checkNotFinished(); - deflater.finish(); - while (!deflater.finished()) { - deflate(); - } - if ((compressedBytes >= MAXIMUM_DATA_SIZE) || (uncompressedBytes >= MAXIMUM_DATA_SIZE)) { - throw new IOException("Too much data for ZIP entry."); - } - } - - private void deflate() throws IOException { - int length = deflater.deflate(outputBuffer); - ZipCombiner.this.write(outputBuffer, 0, length); - compressedBytes += length; - } - - public int getCRC() { - return (int) crc.getValue(); - } - - public int getCompressedSize() { - return (int) compressedBytes; - } - - public int getUncompressedSize() { - return (int) uncompressedBytes; - } - - private void checkNotFinished() { - if (deflater.finished()) { - throw new IllegalStateException(); - } - } + public boolean containsFile(String filename) { + // TODO(apell): may be slightly different behavior because v1 returns true on skipped names. + return entries.containsKey(filename); } /** - * Writes any remaining output data to the output stream and also creates the - * merged entries by calling the {@link CustomMergeStrategy} implementations - * given back from the ZIP entry filter. + * Writes any remaining output data to the output stream and also creates the merged entries by + * calling the {@link CustomMergeStrategy} implementations given back from the + * {@link ZipEntryFilter}. * - * @throws IOException if the output stream or the filter throws an - * IOException + * @throws IOException if the output stream or the filter throws an IOException * @throws IllegalStateException if this method was already called earlier */ public void finish() throws IOException { - checkNotFinished(); - finished = true; - for (Map.Entry<String, FileEntry> entry : fileNames.entrySet()) { + for (Entry<String, EntryAction> entry : actions.entrySet()) { String filename = entry.getKey(); - CustomMergeStrategy mergeStrategy = entry.getValue().mergeStrategy; - ByteArrayOutputStream outputBuffer = entry.getValue().outputBuffer; - int dosTime = entry.getValue().dosTime; - if (mergeStrategy == null) { - // Do nothing. - } else { - mergeStrategy.finish(outputBuffer); - copyStreamToEntry(filename, new ByteArrayInputStream(outputBuffer.toByteArray()), dosTime, - NO_EXTRA_ENTRIES, true, DEFAULT_DIRECTORY_ENTRY_INFO); - } - } - - // Write central directory. - if (out.bytesWritten >= MAXIMUM_DATA_SIZE) { - throw new IOException("Unable to handle files bigger than 2^32 bytes."); - } - int startOfCentralDirectory = (int) out.bytesWritten; - int centralDirectorySize = centralDirectory.writeTo(out); - - // end of central directory signature - setUnsignedInt(directoryEntryBuffer, 0, END_OF_CENTRAL_DIRECTORY_MARKER); - // number of this disk - setUnsignedShort(directoryEntryBuffer, 4, (short) 0); - // number of the disk with the start of the central directory - setUnsignedShort(directoryEntryBuffer, 6, (short) 0); - // total number of entries in the central directory on this disk - setUnsignedShort(directoryEntryBuffer, 8, (short) fileCount); - // total number of entries in the central directory - setUnsignedShort(directoryEntryBuffer, 10, (short) fileCount); - // size of the central directory - setUnsignedInt(directoryEntryBuffer, 12, centralDirectorySize); - // offset of start of central directory with respect to the starting disk number - setUnsignedInt(directoryEntryBuffer, 16, startOfCentralDirectory); - // .ZIP file comment length - setUnsignedShort(directoryEntryBuffer, 20, (short) 0); - write(directoryEntryBuffer, 0, 22); - - out.flush(); - } + EntryAction action = entry.getValue(); + if (action.getType() == ActionType.MERGE) { + ByteArrayOutputStream uncompressed = action.getMergeBuffer(); + action.getStrategy().finish(uncompressed); - private void checkNotFinished() { - if (finished) { - throw new IllegalStateException(); + ZipFileEntry e = new ZipFileEntry(filename); + e.setTime(action.getDate() != null ? action.getDate().getTime() : new Date().getTime()); + writeEntryFromBuffer(e, uncompressed.toByteArray()); + } } + out.finish(); } /** * Writes any remaining output data to the output stream and closes it. * - * @throws IOException if the output stream or the filter throws an - * IOException + * @throws IOException if the output stream or the filter throws an IOException */ - @Override - public void close() throws IOException { - if (!finished) { - finish(); - } + @Override public void close() throws IOException { + finish(); out.close(); } - /** - * Turns this JAR file into an executable JAR by prepending an executable. - * JAR files are placed at the end of a file, and executables are placed - * at the beginning, so a file can be both, if desired. - * - * @param launcherIn The InputStream, from which the launcher is read. - * @throws NullPointerException if launcherIn is null - * @throws IOException if reading from launcherIn or writing to the output - * stream throws an IOException. - */ - public void prependExecutable(InputStream launcherIn) throws IOException { - if (launcherIn == null) { - throw new NullPointerException("No launcher specified"); - } - byte[] buf = new byte[BUFFER_SIZE]; - int bytesRead; - while ((bytesRead = launcherIn.read(buf)) > 0) { - out.write(buf, 0, bytesRead); - } - } - - /** - * Ensures the truth of an expression involving one or more parameters to the calling method. - */ + /** Ensures the truth of an expression involving one or more parameters to the calling method. */ private static void checkArgument(boolean expression, @Nullable String errorMessageTemplate, @Nullable Object... errorMessageArgs) { @@ -1621,18 +715,18 @@ public final class ZipCombiner implements AutoCloseable { } } - /** - * Ensures the truth of an expression involving one or more parameters to the calling method. - */ - private static void checkArgument(boolean expression) { - if (!expression) { - throw new IllegalArgumentException(); + /** Ensures that an object reference passed as a parameter to the calling method is not null. */ + public static <T> T checkNotNull(T reference, + @Nullable String errorMessageTemplate, + @Nullable Object... errorMessageArgs) { + if (reference == null) { + // If either of these parameters is null, the right thing happens anyway + throw new NullPointerException(String.format(errorMessageTemplate, errorMessageArgs)); } + return reference; } - /** - * Ensures the truth of an expression involving state. - */ + /** Ensures the truth of an expression involving state. */ private static void checkState(boolean expression, @Nullable String errorMessageTemplate, @Nullable Object... errorMessageArgs) { diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/zip/CountingInputStream.java b/src/java_tools/singlejar/java/com/google/devtools/build/zip/CountingInputStream.java new file mode 100644 index 0000000000..bac26e2d95 --- /dev/null +++ b/src/java_tools/singlejar/java/com/google/devtools/build/zip/CountingInputStream.java @@ -0,0 +1,85 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.zip; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; + +/** + * An {@link InputStream} that counts the number of bytes read. + */ +public final class CountingInputStream extends FilterInputStream { + + private static <T> T checkNotNull(T reference) { + if (reference == null) { + throw new NullPointerException(); + } + return reference; + } + + private long count; + private long mark = -1; + + /** + * Wraps another input stream, counting the number of bytes read. + * + * @param in the input stream to be wrapped + */ + public CountingInputStream(InputStream in) { + super(checkNotNull(in)); + } + + /** Returns the number of bytes read. */ + public long getCount() { + return count; + } + + @Override public int read() throws IOException { + int result = in.read(); + count += result == -1 ? 0 : 1; + return result; + } + + @Override public int read(byte[] b, int off, int len) throws IOException { + int result = in.read(b, off, len); + count += result == -1 ? 0 : result; + return result; + } + + @Override public long skip(long n) throws IOException { + long result = in.skip(n); + count += result; + return result; + } + + @Override public synchronized void mark(int readlimit) { + in.mark(readlimit); + mark = count; + // it's okay to mark even if mark isn't supported, as reset won't work + } + + @Override public synchronized void reset() throws IOException { + if (!in.markSupported()) { + throw new IOException("Mark not supported"); + } + if (mark == -1) { + throw new IOException("Mark not set"); + } + + in.reset(); + count = mark; + } +}
\ No newline at end of file diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/zip/CountingOutputStream.java b/src/java_tools/singlejar/java/com/google/devtools/build/zip/CountingOutputStream.java new file mode 100644 index 0000000000..a08159ff14 --- /dev/null +++ b/src/java_tools/singlejar/java/com/google/devtools/build/zip/CountingOutputStream.java @@ -0,0 +1,54 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.zip; + +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +/** An OutputStream that counts the number of bytes written. */ +final class CountingOutputStream extends FilterOutputStream { + + private long count; + + /** + * Wraps another output stream, counting the number of bytes written. + * + * @param out the output stream to be wrapped + */ + public CountingOutputStream(OutputStream out) { + super(out); + } + + /** Returns the number of bytes written. */ + public long getCount() { + return count; + } + + @Override public void write(int b) throws IOException { + out.write(b); + count++; + } + + @Override public void write(byte[] b) throws IOException { + out.write(b); + count += b.length; + } + + @Override public void write(byte[] b, int off, int len) throws IOException { + out.write(b, off, len); + count += len; + } +}
\ No newline at end of file diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/zip/ExtraData.java b/src/java_tools/singlejar/java/com/google/devtools/build/zip/ExtraData.java new file mode 100644 index 0000000000..197f0f4246 --- /dev/null +++ b/src/java_tools/singlejar/java/com/google/devtools/build/zip/ExtraData.java @@ -0,0 +1,103 @@ +// Copyright 2014 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.zip; + +import java.util.Arrays; + +/** + * A holder class for extra data in a ZIP entry. + */ +public final class ExtraData { + static final int ID_OFFSET = 0; + static final int LENGTH_OFFSET = 2; + static final int FIXED_DATA_SIZE = 4; + + private final int index; + private final byte[] buffer; + + /** + * Creates a new {@link ExtraData} record with the specified id and data. + * + * @param id the ID tag for this extra data record + * @param data the data payload for this extra data record + */ + public ExtraData(short id, byte[] data) { + if (data.length > 0xffff) { + throw new IllegalArgumentException(String.format("Data is too long. Is %d; max %d", + data.length, 0xffff)); + } + index = 0; + buffer = new byte[FIXED_DATA_SIZE + data.length]; + ZipUtil.shortToLittleEndian(buffer, ID_OFFSET, id); + ZipUtil.shortToLittleEndian(buffer, LENGTH_OFFSET, (short) data.length); + System.arraycopy(data, 0, buffer, FIXED_DATA_SIZE, data.length); + } + + /** + * Creates a new {@link ExtraData} record using the buffer as the backing data store. + * + * <p><em>NOTE:</em> does not perform any defensive copying. Any modification to the buffer will + * alter the extra data record and can make it invalid. + * + * @param buffer the array containing the extra data record + * @param index the index where the extra data record is located + * @throws IllegalArgumentException if buffer does not contain a well formed extra data record + * at index + */ + ExtraData(byte[] buffer, int index) { + if (index >= buffer.length) { + throw new IllegalArgumentException("index past end of buffer"); + } + if (buffer.length - index < FIXED_DATA_SIZE) { + throw new IllegalArgumentException("incomplete extra data entry in buffer"); + } + int length = ZipUtil.getUnsignedShort(buffer, index + LENGTH_OFFSET); + if (buffer.length - index - FIXED_DATA_SIZE < length) { + throw new IllegalArgumentException("incomplete extra data entry in buffer"); + } + this.buffer = buffer; + this.index = index; + } + + /** Returns the Id of the extra data record. */ + public short getId() { + return ZipUtil.get16(buffer, index + ID_OFFSET); + } + + /** Returns the total length of the extra data record in bytes. */ + public int getLength() { + return getDataLength() + FIXED_DATA_SIZE; + } + + /** Returns the length of the data payload of the extra data record in bytes. */ + public int getDataLength() { + return ZipUtil.getUnsignedShort(buffer, index + LENGTH_OFFSET); + } + + /** Returns a byte array copy of the data payload. */ + public byte[] getData() { + return Arrays.copyOfRange(buffer, index + FIXED_DATA_SIZE, index + getLength()); + } + + /** Returns a byte array copy of the entire record. */ + public byte[] getBytes() { + return Arrays.copyOfRange(buffer, index, index + getLength()); + } + + /** Returns the byte at index from the entire record. */ + byte getByte(int index) { + return buffer[this.index + index]; + } +} diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/zip/ExtraDataList.java b/src/java_tools/singlejar/java/com/google/devtools/build/zip/ExtraDataList.java new file mode 100644 index 0000000000..b3cd252e2e --- /dev/null +++ b/src/java_tools/singlejar/java/com/google/devtools/build/zip/ExtraDataList.java @@ -0,0 +1,161 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.zip; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Iterator; +import java.util.LinkedHashMap; + +/** + * A list of {@link ExtraData} records to be associated with a {@link ZipFileEntry}. Supports + * creating the list directly from a byte array and modifying the list without reallocating the + * underlying buffer. + */ +public class ExtraDataList { + private final LinkedHashMap<Short, ExtraData> entries; + + /** + * Create a new empty extra data list. + * + * <p><em>NOTE:</em> entries in a list created this way will be backed by their own storage. + */ + public ExtraDataList() { + entries = new LinkedHashMap<>(); + } + + /** + * Creates an extra data list from the given extra data records. + * + * <p><em>NOTE:</em> entries in a list created this way will be backed by their own storage. + * + * @param extra the extra data records + */ + public ExtraDataList(ExtraData... extra) { + this(); + for (ExtraData e : extra) { + add(e); + } + } + + /** + * Creates an extra data list from the entries contained in the given array. + * + * <p><em>NOTE:</em> entries in a list created this way will be backed by the buffer. No defensive + * copying is performed. + * + * @param buffer the array containing sequential extra data entries + */ + public ExtraDataList(byte[] buffer) { + if (buffer.length > 0xffff) { + throw new IllegalArgumentException("invalid extra field length"); + } + entries = new LinkedHashMap<>(); + int index = 0; + while (index < buffer.length) { + ExtraData extra = new ExtraData(buffer, index); + entries.put(extra.getId(), extra); + index += extra.getLength(); + } + } + + /** + * Returns the extra data record with the specified id, or null if it does not exist. + */ + public ExtraData get(short id) { + return entries.get(id); + } + + /** + * Removes and returns the extra data record with the specified id if it exists. + * + * <p><em>NOTE:</em> does not modify the underlying storage, only marks the record as removed. + */ + public ExtraData remove(short id) { + return entries.remove(id); + } + + /** + * Returns if the list contains an extra data record with the specified id. + */ + public boolean contains(short id) { + return entries.containsKey(id); + } + + /** + * Adds a new entry to the end of the list. + * + * @throws IllegalArgumentException if adding the entry will make the list too long for the ZIP + * format + */ + public void add(ExtraData entry) { + if (getLength() + entry.getLength() > 0xffff) { + throw new IllegalArgumentException("adding entry will make the extra field be too long"); + } + entries.put(entry.getId(), entry); + } + + /** + * Returns the overall length of the list in bytes. + */ + public int getLength() { + int length = 0; + for (ExtraData e : entries.values()) { + length += e.getLength(); + } + return length; + } + + /** + * Creates and returns a byte array of the extra data list. + */ + public byte[] getBytes() { + byte[] extra = new byte[getLength()]; + try { + getByteStream().read(extra); + } catch (IOException impossible) { + throw new AssertionError(impossible); + } + return extra; + } + + /** + * Returns an input stream for reading the extra data list entries. + */ + public InputStream getByteStream() { + return new InputStream() { + private final Iterator<ExtraData> itr = entries.values().iterator(); + private ExtraData entry; + private int index; + + @Override + public int read() { + if (entry == null) { + if (itr.hasNext()) { + entry = itr.next(); + index = 0; + } else { + return -1; + } + } + byte val = entry.getByte(index++); + if (index >= entry.getLength()) { + entry = null; + } + return val & 0xff; + } + }; + } +} diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/zip/ZipFileData.java b/src/java_tools/singlejar/java/com/google/devtools/build/zip/ZipFileData.java new file mode 100644 index 0000000000..aaea5b90ad --- /dev/null +++ b/src/java_tools/singlejar/java/com/google/devtools/build/zip/ZipFileData.java @@ -0,0 +1,288 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.zip; + +import com.google.devtools.build.zip.ZipFileEntry.Feature; + +import java.nio.charset.Charset; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.zip.ZipException; + +import javax.annotation.Nullable; + +/** + * A representation of a ZIP file. Contains the file comment, encoding, and entries. Also contains + * internal information about the structure and location of ZIP file parts. + */ +class ZipFileData { + private final Charset charset; + private String comment; + + private long centralDirectorySize; + private long centralDirectoryOffset; + private long expectedEntries; + private long numEntries; + private final Map<String, ZipFileEntry> entries; + + private boolean maybeZip64; + private boolean isZip64; + private long zip64EndOfCentralDirectoryOffset; + + /** + * Creates a new ZIP file with the specified charset encoding. + */ + public ZipFileData(Charset charset) { + if (charset == null) { + throw new NullPointerException(); + } + this.charset = charset; + comment = ""; + entries = new LinkedHashMap<>(); + } + + /** + * Returns the encoding of the file. + */ + public Charset getCharset() { + return charset; + } + + /** + * Returns the file comment. + */ + public String getComment() { + return comment; + } + + /** + * Sets the file comment from the raw byte data in the file. Converts the bytes to a string using + * the file's charset encoding. + * + * @throws ZipException if the comment is longer than allowed by the ZIP format + */ + public void setComment(byte[] comment) throws ZipException { + if (comment == null) { + throw new NullPointerException(); + } + if (comment.length > 0xffff) { + throw new ZipException(String.format("File comment too long. Is %d; max %d.", + comment.length, 0xffff)); + } + this.comment = fromBytes(comment); + } + + /** + * Sets the file comment. + * + * @throws ZipException if the comment will be longer than allowed by the ZIP format when encoded + * using the file's charset encoding + */ + public void setComment(String comment) throws ZipException { + setComment(getBytes(comment)); + } + + /** + * Returns the size of the central directory in bytes. + */ + public long getCentralDirectorySize() { + return centralDirectorySize; + } + + /** + * Sets the size of the central directory in bytes. If the size is larger than 0xffffffff, the + * file is set to Zip64 mode. + * + * <p>See <a href="http://www.pkware.com/documents/casestudies/APPNOTE.TXT">ZIP Format</a> + * section 4.4.23 + */ + public void setCentralDirectorySize(long centralDirectorySize) { + this.centralDirectorySize = centralDirectorySize; + if (centralDirectorySize > 0xffffffffL) { + setZip64(true); + } + } + + /** + * Returns the file offset of the start of the central directory. + */ + public long getCentralDirectoryOffset() { + return centralDirectoryOffset; + } + + /** + * Sets the file offset of the start of the central directory. If the offset is larger than + * 0xffffffff, the file is set to Zip64 mode. + * + * <p>See <a href="http://www.pkware.com/documents/casestudies/APPNOTE.TXT">ZIP Format</a> + * section 4.4.24 + */ + public void setCentralDirectoryOffset(long offset) { + this.centralDirectoryOffset = offset; + if (centralDirectoryOffset > 0xffffffffL) { + setZip64(true); + } + } + + /** + * Returns the number of entries expected to be in the ZIP file. This value is determined from the + * end of central directory record. + */ + public long getExpectedEntries() { + return expectedEntries; + } + + /** + * Sets the number of entries expected to be in the ZIP file. This value should be set by reading + * the end of central directory record. + * + * <p>See <a href="http://www.pkware.com/documents/casestudies/APPNOTE.TXT">ZIP Format</a> + * section 4.4.22 + */ + public void setExpectedEntries(long count) { + this.expectedEntries = count; + if (expectedEntries > 0xffff) { + setZip64(true); + } + } + + /** + * Returns the number of entries actually in the ZIP file. This value is derived from the number + * of times {@link #addEntry(ZipFileEntry)} was called. + * + * <p><em>NOTE:</em> This value should be used rather than getting the size from the + * {@link Collection} returned from {@link #getEntries()}, because the value may be too large to + * be properly represented by an int. + */ + public long getNumEntries() { + return numEntries; + } + + /** + * Sets the number of entries actually in the ZIP file. If the value is larger than 0xffff, the + * file is set to Zip64 mode. + */ + private void setNumEntries(long numEntries) { + this.numEntries = numEntries; + if (numEntries > 0xffff) { + setZip64(true); + } + } + + /** + * Returns a collection of all entries in the ZIP file. + */ + public Collection<ZipFileEntry> getEntries() { + return entries.values(); + } + + /** + * Returns the entry with the given name, or null if it does not exist. + */ + public ZipFileEntry getEntry(@Nullable String name) { + return entries.get(name); + } + + /** + * Adds an entry to the ZIP file. If this causes the actual number of entries to exceed + * 0xffffffff, or if the file requires Zip64 features, the file is set to Zip64 mode. + */ + public void addEntry(ZipFileEntry entry) { + entries.put(entry.getName(), entry); + setNumEntries(numEntries + 1); + if (entry.getFeatureSet().contains(Feature.ZIP64_SIZE) + || entry.getFeatureSet().contains(Feature.ZIP64_CSIZE) + || entry.getFeatureSet().contains(Feature.ZIP64_OFFSET)) { + setZip64(true); + } + } + + /** + * Returns if the file may be in Zip64 mode. This is true if any of the values in the end of + * central directory record are -1. + * + * <p>See <a href="http://www.pkware.com/documents/casestudies/APPNOTE.TXT">ZIP Format</a> + * section 4.4.19 - 4.4.24 + */ + public boolean isMaybeZip64() { + return maybeZip64; + } + + /** + * Set if the file may be in Zip64 mode. This is true if any of the values in the end of + * central directory record are -1. + * + * <p>See <a href="http://www.pkware.com/documents/casestudies/APPNOTE.TXT">ZIP Format</a> + * section 4.4.19 - 4.4.24 + */ + public void setMaybeZip64(boolean maybeZip64) { + this.maybeZip64 = maybeZip64; + } + + /** + * Returns if the file is in Zip64 mode. This is true if any of a number of fields exceed the + * maximum value. + * + * <p>See <a href="http://www.pkware.com/documents/casestudies/APPNOTE.TXT">ZIP Format</a> for + * details + */ + public boolean isZip64() { + return isZip64; + } + + /** + * Set if the file is in Zip64 mode. This is true if any of a number of fields exceed the maximum + * value. + * + * <p>See <a href="http://www.pkware.com/documents/casestudies/APPNOTE.TXT">ZIP Format</a> for + * details + */ + public void setZip64(boolean isZip64) { + this.isZip64 = isZip64; + setMaybeZip64(true); + } + + /** + * Returns the file offset of the Zip64 end of central directory record. The record is only + * present if {@link #isZip64()} returns true. + */ + public long getZip64EndOfCentralDirectoryOffset() { + return zip64EndOfCentralDirectoryOffset; + } + + /** + * Sets the file offset of the Zip64 end of central directory record and sets the file to Zip64 + * mode. + */ + public void setZip64EndOfCentralDirectoryOffset(long offset) { + this.zip64EndOfCentralDirectoryOffset = offset; + setZip64(true); + } + + /** + * Returns the byte representation of the specified string using the file's charset encoding. + */ + public byte[] getBytes(String string) { + return string.getBytes(charset); + } + + /** + * Returns the string represented by the specified byte array using the file's charset encoding. + */ + public String fromBytes(byte[] bytes) { + return new String(bytes, charset); + } +} diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/zip/ZipFileEntry.java b/src/java_tools/singlejar/java/com/google/devtools/build/zip/ZipFileEntry.java new file mode 100644 index 0000000000..e8687f1c74 --- /dev/null +++ b/src/java_tools/singlejar/java/com/google/devtools/build/zip/ZipFileEntry.java @@ -0,0 +1,440 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.zip; + +import java.util.EnumSet; + +import javax.annotation.Nullable; + +/** + * A full representation of a ZIP file entry. + * + * <p>See <a href="http://www.pkware.com/documents/casestudies/APPNOTE.TXT">ZIP Format</a> for + * a description of the entry fields. (Section 4.3.7 and 4.4) + */ +public final class ZipFileEntry { + + /** Compression method for ZIP entries. */ + public enum Compression { + STORED((short) 0, Feature.STORED), + DEFLATED((short) 8, Feature.DEFLATED); + + public static Compression fromValue(int value) { + for (Compression c : Compression.values()) { + if (c.getValue() == value) { + return c; + } + } + return null; + } + + private short value; + private Feature feature; + + private Compression(short value, Feature feature) { + this.value = value; + this.feature = feature; + } + + public short getValue() { + return value; + } + + public short getMinVersion() { + return feature.getMinVersion(); + } + + Feature getFeature() { + return feature; + } + } + + /** General purpose bit flag for ZIP entries. */ + public enum Flag { + DATA_DESCRIPTOR(3); + + private int bit; + + private Flag(int bit) { + this.bit = bit; + } + + public int getBit() { + return bit; + } + } + + /** Zip file features that entries may use. */ + enum Feature { + DEFAULT((short) 0x0a), + STORED((short) 0x0a), + DEFLATED((short) 0x14), + ZIP64_SIZE((short) 0x2d), + ZIP64_CSIZE((short) 0x2d), + ZIP64_OFFSET((short) 0x2d); + + private short minVersion; + + private Feature(short minVersion) { + this.minVersion = minVersion; + } + + public short getMinVersion() { + return minVersion; + } + + static short getMinRequiredVersion(EnumSet<Feature> featureSet) { + short minVersion = Feature.DEFAULT.getMinVersion(); + for (Feature feature : featureSet) { + minVersion = (short) Math.max(minVersion, feature.getMinVersion()); + } + return minVersion; + } + } + + private String name; + private long time = -1; + private long crc = -1; + private long size = -1; + private long csize = -1; + private Compression method; + private short version = -1; + private short versionNeeded = -1; + private short flags; + private short internalAttributes; + private int externalAttributes; + private long localHeaderOffset = -1; + private ExtraDataList extra; + @Nullable private String comment; + + private EnumSet<Feature> featureSet; + + /** + * Creates a new ZIP entry with the specified name. + * + * @throws NullPointerException if the entry name is null + */ + public ZipFileEntry(String name) { + this.featureSet = EnumSet.of(Feature.DEFAULT); + setName(name); + setMethod(Compression.STORED); + setExtra(new ExtraDataList()); + } + + /** + * Creates a new ZIP entry with fields taken from the specified ZIP entry. + */ + public ZipFileEntry(ZipFileEntry e) { + this.name = e.getName(); + this.time = e.getTime(); + this.crc = e.getCrc(); + this.size = e.getSize(); + this.csize = e.getCompressedSize(); + this.method = e.getMethod(); + this.version = e.getVersion(); + this.versionNeeded = e.getVersionNeeded(); + this.flags = e.getFlags(); + this.internalAttributes = e.getInternalAttributes(); + this.externalAttributes = e.getExternalAttributes(); + this.localHeaderOffset = e.getLocalHeaderOffset(); + this.extra = e.getExtra(); + this.comment = e.getComment(); + this.featureSet = EnumSet.copyOf(e.getFeatureSet()); + } + + /** + * Sets the name of the entry. + */ + public void setName(String name) { + if (name == null) { + throw new NullPointerException(); + } + this.name = name; + } + + /** + * Returns the name of the entry. + */ + public String getName() { + return name; + } + + /** + * Sets the modification time of the entry. + * + * @param time the entry modification time in number of milliseconds since the epoch + */ + public void setTime(long time) { + this.time = time; + } + + /** + * Returns the modification time of the entry, or -1 if not specified. + */ + public long getTime() { + return time; + } + + /** + * Sets the CRC-32 checksum of the uncompressed entry data. + * + * @throws IllegalArgumentException if the specified CRC-32 value is less than 0 or greater than + * 0xFFFFFFFF + */ + public void setCrc(long crc) { + if (crc < 0 || crc > 0xffffffffL) { + throw new IllegalArgumentException("invalid entry crc-32"); + } + this.crc = crc; + } + + /** + * Returns the CRC-32 checksum of the uncompressed entry data, or -1 if not known. + */ + public long getCrc() { + return crc; + } + + /** + * Sets the uncompressed size of the entry data in bytes. + * + * @throws IllegalArgumentException if the specified size is less than 0 + */ + public void setSize(long size) { + if (size < 0) { + throw new IllegalArgumentException("invalid entry size"); + } + if (size > 0xffffffffL) { + featureSet.add(Feature.ZIP64_SIZE); + } else { + featureSet.remove(Feature.ZIP64_SIZE); + } + this.size = size; + } + + /** + * Returns the uncompressed size of the entry data, or -1 if not known. + */ + public long getSize() { + return size; + } + + /** + * Sets the size of the compressed entry data in bytes. + * + * @throws IllegalArgumentException if the specified size is less than 0 + */ + public void setCompressedSize(long csize) { + if (csize < 0) { + throw new IllegalArgumentException("invalid entry size"); + } + if (csize > 0xffffffffL) { + featureSet.add(Feature.ZIP64_CSIZE); + } else { + featureSet.remove(Feature.ZIP64_CSIZE); + } + this.csize = csize; + } + + /** + * Returns the size of the compressed entry data, or -1 if not known. In the case of a stored + * entry, the compressed size will be the same as the uncompressed size of the entry. + */ + public long getCompressedSize() { + return csize; + } + + /** + * Sets the compression method for the entry. + */ + public void setMethod(Compression method) { + if (method == null) { + throw new NullPointerException(); + } + if (this.method != null) { + featureSet.remove(this.method.getFeature()); + } + this.method = method; + featureSet.add(this.method.getFeature()); + } + + /** + * Returns the compression method of the entry. + */ + public Compression getMethod() { + return method; + } + + /** + * Sets the made by version for the entry. + */ + public void setVersion(short version) { + this.version = version; + } + + /** + * Returns the made by version of the entry, accounting for assigned version and feature set. + */ + public short getVersion() { + return (short) Math.max(version, Feature.getMinRequiredVersion(featureSet)); + } + + /** + * Sets the version needed to extract the entry. + */ + public void setVersionNeeded(short versionNeeded) { + this.versionNeeded = versionNeeded; + } + + /** + * Returns the version needed to extract the entry, accounting for assigned version and feature + * set. + */ + public short getVersionNeeded() { + return (short) Math.max(versionNeeded, Feature.getMinRequiredVersion(featureSet)); + } + + /** + * Sets the general purpose bit flags for the entry. + */ + public void setFlags(short flags) { + this.flags = flags; + } + + /** + * Sets or clears the specified bit of the general purpose bit flags. + * + * @param flag the flag to set or clear + * @param set whether the flag is to be set or cleared + */ + public void setFlag(Flag flag, boolean set) { + short mask = 0x0000; + mask |= 1 << flag.getBit(); + if (set) { + flags |= mask; + } else { + flags &= ~mask; + } + } + + /** + * Returns the general purpose bit flags of the entry. + * + * <p>See <a href="http://www.pkware.com/documents/casestudies/APPNOTE.TXT">ZIP Format</a> + * section 4.4.4. + */ + public short getFlags() { + return flags; + } + + /** + * Sets the internal file attributes of the entry. + */ + public void setInternalAttributes(short internalAttributes) { + this.internalAttributes = internalAttributes; + } + + /** + * Returns the internal file attributes of the entry. + */ + public short getInternalAttributes() { + return internalAttributes; + } + + /** + * Sets the external file attributes of the entry. + */ + public void setExternalAttributes(int externalAttributes) { + this.externalAttributes = externalAttributes; + } + + /** + * Returns the external file attributes of the entry. + */ + public int getExternalAttributes() { + return externalAttributes; + } + + /** + * Sets the file offset, in bytes, of the location of the local file header for the entry. + * + * <p>See <a href="http://www.pkware.com/documents/casestudies/APPNOTE.TXT">ZIP Format</a> + * section 4.4.16 + * + * @throws IllegalArgumentException if the specified local header offset is less than 0 + */ + void setLocalHeaderOffset(long localHeaderOffset) { + if (localHeaderOffset < 0) { + throw new IllegalArgumentException("invalid local header offset"); + } + if (localHeaderOffset > 0xffffffffL) { + featureSet.add(Feature.ZIP64_OFFSET); + } else { + featureSet.remove(Feature.ZIP64_OFFSET); + } + this.localHeaderOffset = localHeaderOffset; + } + + /** + * Returns the file offset of the local header of the entry. + */ + public long getLocalHeaderOffset() { + return localHeaderOffset; + } + + /** + * Sets the optional extra field data for the entry. + * + * @throws IllegalArgumentException if the length of the specified extra field data is greater + * than 0xFFFF bytes + */ + public void setExtra(ExtraDataList extra) { + if (extra == null) { + throw new NullPointerException(); + } + if (extra.getLength() > 0xffff) { + throw new IllegalArgumentException("invalid extra field length"); + } + this.extra = extra; + } + + /** + * Returns the extra field data for the entry. + */ + public ExtraDataList getExtra() { + return extra; + } + + /** + * Sets the optional comment string for the entry. + */ + public void setComment(@Nullable String comment) { + this.comment = comment; + } + + /** + * Returns the comment string for the entry, or null if none. + */ + public String getComment() { + return comment; + } + + /** + * Returns the feature set that this entry uses. + */ + EnumSet<Feature> getFeatureSet() { + return featureSet; + } +} diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/zip/ZipReader.java b/src/java_tools/singlejar/java/com/google/devtools/build/zip/ZipReader.java new file mode 100644 index 0000000000..b4ccd5ceb2 --- /dev/null +++ b/src/java_tools/singlejar/java/com/google/devtools/build/zip/ZipReader.java @@ -0,0 +1,510 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.zip; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.devtools.build.zip.ZipFileEntry.Compression; +import com.google.devtools.build.zip.ZipUtil.CentralDirectoryFileHeader; +import com.google.devtools.build.zip.ZipUtil.EndOfCentralDirectoryRecord; +import com.google.devtools.build.zip.ZipUtil.LocalFileHeader; +import com.google.devtools.build.zip.ZipUtil.Zip64EndOfCentralDirectory; +import com.google.devtools.build.zip.ZipUtil.Zip64EndOfCentralDirectoryLocator; + +import java.io.BufferedInputStream; +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.RandomAccessFile; +import java.nio.channels.Channels; +import java.nio.charset.Charset; +import java.util.Collection; +import java.util.zip.Inflater; +import java.util.zip.InflaterInputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipException; +import java.util.zip.ZipFile; + +/** + * A ZIP file reader. + * + * <p>This class provides entry data in the form of {@link ZipFileEntry}, which provides more detail + * about the entry than the JDK equivalent {@link ZipEntry}. In addition to providing + * {@link InputStream}s for entries, similar to JDK {@link ZipFile#getInputStream(ZipEntry)}, it + * also provides access to the raw byte entry data via {@link #getRawInputStream(ZipFileEntry)}. + * + * <p>Using the raw access capabilities allows for more efficient ZIP file processing, such as + * merging, by not requiring each entry's data to be decompressed when read. + * + * <p><em>NOTE:</em> The entries are read from the central directory. If the entry is not listed + * there, it will not be returned from {@link #entries()} or {@link #getEntry(String)}. + */ +public class ZipReader implements Closeable, AutoCloseable { + + /** An input stream for reading the file data of a ZIP file entry. */ + private class ZipEntryInputStream extends InputStream { + private InputStream stream; + private long rem; + + /** + * Opens an input stream for reading at the beginning of the ZIP file entry's content. + * + * @param zipEntry the ZIP file entry to open the input stream for + * @throws ZipException if a ZIP format error has occurred + * @throws IOException if an I/O error has occurred + */ + private ZipEntryInputStream(ZipFileEntry zipEntry) throws IOException { + stream = new BufferedInputStream(Channels.newInputStream( + in.getChannel().position(zipEntry.getLocalHeaderOffset()))); + + byte[] fileHeader = new byte[LocalFileHeader.FIXED_DATA_SIZE]; + stream.read(fileHeader); + + if (!ZipUtil.arrayStartsWith(fileHeader, + ZipUtil.intToLittleEndian(LocalFileHeader.SIGNATURE))) { + throw new ZipException(String.format("The file '%s' is not a correctly formatted zip file: " + + "Expected a File Header at file offset %d, but was not present.", + file.getName(), zipEntry.getLocalHeaderOffset())); + } + + int nameLength = ZipUtil.getUnsignedShort(fileHeader, + LocalFileHeader.FILENAME_LENGTH_OFFSET); + int extraFieldLength = ZipUtil.getUnsignedShort(fileHeader, + LocalFileHeader.EXTRA_FIELD_LENGTH_OFFSET); + stream.skip(nameLength + extraFieldLength); + rem = zipEntry.getSize(); + if (zipEntry.getMethod() == Compression.DEFLATED) { + stream = new InflaterInputStream(stream, new Inflater(true)); + } + } + + @Override public int available() throws IOException { + return rem > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) rem; + } + + @Override public void close() throws IOException { + } + + @Override public void mark(int readlimit) { + } + + @Override public boolean markSupported() { + return false; + } + + @Override public int read() throws IOException { + byte[] b = new byte[1]; + if (read(b, 0, 1) == 1) { + return b[0] & 0xff; + } else { + return -1; + } + } + + @Override public int read(byte[] b) throws IOException { + return read(b, 0, b.length); + } + + @Override public int read(byte[] b, int off, int len) throws IOException { + if (rem == 0) { + return -1; + } + if (len > rem) { + len = available(); + } + len = stream.read(b, off, len); + rem -= len; + return len; + } + + @Override public long skip(long n) throws IOException { + if (n > rem) { + n = rem; + } + n = stream.skip(n); + rem -= n; + return n; + } + + @Override public void reset() throws IOException { + throw new IOException("Reset is not supported on this type of stream."); + } + } + + /** An input stream for reading the raw file data of a ZIP file entry. */ + private class RawZipEntryInputStream extends InputStream { + private InputStream stream; + private long rem; + + /** + * Opens an input stream for reading at the beginning of the ZIP file entry's content. + * + * @param zipEntry the ZIP file entry to open the input stream for + * @throws ZipException if a ZIP format error has occurred + * @throws IOException if an I/O error has occurred + */ + private RawZipEntryInputStream(ZipFileEntry zipEntry) throws IOException { + stream = new BufferedInputStream(Channels.newInputStream( + in.getChannel().position(zipEntry.getLocalHeaderOffset()))); + + byte[] fileHeader = new byte[LocalFileHeader.FIXED_DATA_SIZE]; + stream.read(fileHeader); + + if (!ZipUtil.arrayStartsWith(fileHeader, + ZipUtil.intToLittleEndian(LocalFileHeader.SIGNATURE))) { + throw new ZipException(String.format("The file '%s' is not a correctly formatted zip file: " + + "Expected a File Header at file offset %d, but was not present.", + file.getName(), zipEntry.getLocalHeaderOffset())); + } + + int nameLength = ZipUtil.getUnsignedShort(fileHeader, + LocalFileHeader.FILENAME_LENGTH_OFFSET); + int extraFieldLength = ZipUtil.getUnsignedShort(fileHeader, + LocalFileHeader.EXTRA_FIELD_LENGTH_OFFSET); + stream.skip(nameLength + extraFieldLength); + rem = zipEntry.getCompressedSize(); + } + + @Override public int available() throws IOException { + return rem > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) rem; + } + + @Override public void close() throws IOException { + } + + @Override public void mark(int readlimit) { + } + + @Override public boolean markSupported() { + return false; + } + + @Override public int read() throws IOException { + byte[] b = new byte[1]; + if (read(b, 0, 1) == 1) { + return b[0] & 0xff; + } else { + return -1; + } + } + + @Override public int read(byte[] b) throws IOException { + return read(b, 0, b.length); + } + + @Override public int read(byte[] b, int off, int len) throws IOException { + if (rem == 0) { + return -1; + } + if (len > rem) { + len = available(); + } + len = stream.read(b, off, len); + rem -= len; + return len; + } + + @Override public long skip(long n) throws IOException { + if (n > rem) { + n = rem; + } + n = stream.skip(n); + rem -= n; + return n; + } + + @Override public void reset() throws IOException { + throw new IOException("Reset is not supported on this type of stream."); + } + } + + private final File file; + private final RandomAccessFile in; + private final ZipFileData zipData; + + /** + * Opens a zip file for raw acceess. + * + * <p>The UTF-8 charset is used to decode the entry names and comments. + * + * @param file the zip file + * @throws ZipException if a ZIP format error has occurred + * @throws IOException if an I/O error has occurred + */ + public ZipReader(File file) throws IOException { + this(file, UTF_8); + } + + /** + * Opens a zip file for raw acceess. + * + * @param file the zip file + * @param charset the charset to use to decode the entry names and comments + * @throws ZipException if a ZIP format error has occurred + * @throws IOException if an I/O error has occurred + */ + public ZipReader(File file, Charset charset) throws IOException { + this(file, charset, false); + } + + /** + * Opens a zip file for raw acceess. + * + * @param file the zip file + * @param charset the charset to use to decode the entry names and comments + * @param strictEntries force parsing to use the number of entries recorded in the end of + * central directory as the correct value, not as an estimate + * @throws ZipException if a ZIP format error has occurred + * @throws IOException if an I/O error has occurred + */ + public ZipReader(File file, Charset charset, boolean strictEntries) throws IOException { + if (file == null || charset == null) { + throw new NullPointerException(); + } + this.file = file; + this.in = new RandomAccessFile(file, "r"); + this.zipData = new ZipFileData(charset); + readCentralDirectory(strictEntries); + } + + /** + * Returns the ZIP file comment. + */ + public String getComment() { + return zipData.getComment(); + } + + /** + * Returns a collection of the ZIP file entries. + */ + public Collection<ZipFileEntry> entries() { + return zipData.getEntries(); + } + + /** + * Returns the ZIP file entry for the specified name, or null if not found. + */ + public ZipFileEntry getEntry(String name) { + return zipData.getEntry(name); + } + + /** + * Returns the number of entries in the ZIP file. + */ + public long size() { + return zipData.getNumEntries(); + } + + /** + * Returns an input stream for reading the contents of the specified ZIP file entry. + * + * <p>Closing this ZIP file will, in turn, close all input streams that have been returned by + * invocations of this method. + * + * @param entry the ZIP file entry + * @return the input stream for reading the contents of the specified zip file entry + * @throws ZipException if a ZIP format error has occurred + * @throws IOException if an I/O error has occurred + */ + public InputStream getInputStream(ZipFileEntry entry) throws IOException { + if (!zipData.getEntry(entry.getName()).equals(entry)) { + throw new ZipException(String.format( + "Zip file '%s' does not contain the requested entry '%s'.", file.getName(), + entry.getName())); + } + return new ZipEntryInputStream(entry); + } + + /** + * Returns an input stream for reading the raw contents of the specified ZIP file entry. + * + * <p><em>NOTE:</em> No inflating will take place; The data read from the input stream will be + * the exact byte content of the ZIP file entry on disk. + * + * <p>Closing this ZIP file will, in turn, close all input streams that have been returned by + * invocations of this method. + * + * @param entry the ZIP file entry + * @return the input stream for reading the contents of the specified zip file entry + * @throws ZipException if a ZIP format error has occurred + * @throws IOException if an I/O error has occurred + */ + public InputStream getRawInputStream(ZipFileEntry entry) throws IOException { + if (!zipData.getEntry(entry.getName()).equals(entry)) { + throw new ZipException(String.format( + "Zip file '%s' does not contain the requested entry '%s'.", file.getName(), + entry.getName())); + } + return new RawZipEntryInputStream(entry); + } + + /** + * Closes the ZIP file. + * + * <p>Closing this ZIP file will close all of the input streams previously returned by invocations + * of the {@link #getRawInputStream(ZipFileEntry)} method. + */ + @Override public void close() throws IOException { + in.close(); + } + + /** + * Finds, reads and parses ZIP file entries from the central directory. + * + * @param strictEntries force parsing to use the number of entries recorded in the end of + * central directory as the correct value, not as an estimate + * @throws ZipException if a ZIP format error has occurred + * @throws IOException if an I/O error has occurred + */ + private void readCentralDirectory(boolean strictEntries) throws IOException { + long eocdLocation = findEndOfCentralDirectoryRecord(); + InputStream stream = new BufferedInputStream(Channels.newInputStream( + in.getChannel().position(eocdLocation))); + EndOfCentralDirectoryRecord.read(stream, zipData); + + if (zipData.isMaybeZip64()) { + try { + stream = new BufferedInputStream(Channels.newInputStream(in.getChannel() + .position(eocdLocation - Zip64EndOfCentralDirectoryLocator.FIXED_DATA_SIZE))); + Zip64EndOfCentralDirectoryLocator.read(stream, zipData); + + stream = new BufferedInputStream(Channels.newInputStream(in.getChannel() + .position(zipData.getZip64EndOfCentralDirectoryOffset()))); + Zip64EndOfCentralDirectory.read(stream, zipData); + } catch (ZipException e) { + // expected if not in Zip64 format + } + } + + if (zipData.isZip64() || strictEntries) { + // If in Zip64 format or using strict entry numbers, use the parsed information as is to read + // the central directory file headers. + readCentralDirectoryFileHeaders(zipData.getExpectedEntries(), + zipData.getCentralDirectoryOffset()); + } else { + // If not in Zip64 format, compute central directory offset by end of central directory record + // offset and central directory size to allow reading large non-compliant Zip32 directories. + long centralDirectoryOffset = eocdLocation - zipData.getCentralDirectorySize(); + // If the lower 4 bytes match, the above calculation is correct; otherwise fallback to + // reported offset. + if ((int) centralDirectoryOffset == (int) zipData.getCentralDirectoryOffset()) { + readCentralDirectoryFileHeaders(centralDirectoryOffset); + } else { + readCentralDirectoryFileHeaders(zipData.getExpectedEntries(), + zipData.getCentralDirectoryOffset()); + } + } + } + + /** + * Looks for the target sub array in the buffer scanning backwards starting at offset. Returns the + * index where the target is found or -1 if not found. + * + * @param target the sub array to find + * @param buffer the array to scan + * @param offset the index of where to begin scanning + * @return the index of target within buffer or -1 if not found + */ + private int scanBackwards(byte[] target, byte[] buffer, int offset) { + int start = Math.min(offset, buffer.length - target.length); + for (int i = start; i >= 0; i--) { + for (int j = 0; j < target.length; j++) { + if (buffer[i + j] != target[j]) { + break; + } else if (j == target.length - 1) { + return i; + } + } + } + return -1; + } + + /** + * Finds the file offset of the end of central directory record. + * + * @return the file offset of the end of central directory record + * @throws ZipException if a ZIP format error has occurred + * @throws IOException if an I/O error has occurred + */ + private long findEndOfCentralDirectoryRecord() throws IOException { + byte[] signature = ZipUtil.intToLittleEndian(EndOfCentralDirectoryRecord.SIGNATURE); + byte[] buffer = new byte[(int) Math.min(64, in.length())]; + int readLength = buffer.length; + if (readLength < EndOfCentralDirectoryRecord.FIXED_DATA_SIZE) { + throw new ZipException(String.format("Zip file '%s' is malformed. It does not contain an end" + + " of central directory record.", file.getName())); + } + + long offset = in.length() - buffer.length; + while (offset >= 0) { + in.seek(offset); + in.readFully(buffer, 0, readLength); + int signatureLocation = scanBackwards(signature, buffer, buffer.length); + while (signatureLocation != -1) { + long eocdSize = in.length() - offset - signatureLocation; + if (eocdSize >= EndOfCentralDirectoryRecord.FIXED_DATA_SIZE) { + int commentLength = ZipUtil.getUnsignedShort(buffer, signatureLocation + + EndOfCentralDirectoryRecord.COMMENT_LENGTH_OFFSET); + long readCommentLength = eocdSize - EndOfCentralDirectoryRecord.FIXED_DATA_SIZE; + if (commentLength == readCommentLength) { + return offset + signatureLocation; + } + } + signatureLocation = scanBackwards(signature, buffer, signatureLocation - 1); + } + readLength = buffer.length - 3; + buffer[buffer.length - 3] = buffer[0]; + buffer[buffer.length - 2] = buffer[1]; + buffer[buffer.length - 1] = buffer[2]; + offset -= readLength; + } + throw new ZipException(String.format("Zip file '%s' is malformed. It does not contain an end" + + " of central directory record.", file.getName())); + } + + /** + * Reads and parses ZIP file entries from the central directory. + * + * @param count the number of entries in the central directory + * @param fileOffset the file offset of the start of the central directory + * @throws ZipException if a ZIP format error has occurred + * @throws IOException if an I/O error has occurred + */ + private void readCentralDirectoryFileHeaders(long count, long fileOffset) throws IOException { + InputStream centralDirectory = new BufferedInputStream( + Channels.newInputStream(in.getChannel().position(fileOffset))); + for (long i = 0; i < count; i++) { + ZipFileEntry entry = CentralDirectoryFileHeader.read(centralDirectory, zipData.getCharset()); + zipData.addEntry(entry); + } + } + + /** + * Reads and parses ZIP file entries from the central directory. + * + * @param fileOffset the file offset of the start of the central directory + * @throws ZipException if a ZIP format error has occurred + * @throws IOException if an I/O error has occurred + */ + private void readCentralDirectoryFileHeaders(long fileOffset) throws IOException { + CountingInputStream centralDirectory = new CountingInputStream(new BufferedInputStream( + Channels.newInputStream(in.getChannel().position(fileOffset)))); + while (centralDirectory.getCount() < zipData.getCentralDirectorySize()) { + ZipFileEntry entry = CentralDirectoryFileHeader.read(centralDirectory, zipData.getCharset()); + zipData.addEntry(entry); + } + } +} diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/zip/ZipUtil.java b/src/java_tools/singlejar/java/com/google/devtools/build/zip/ZipUtil.java new file mode 100644 index 0000000000..2ba4caf4f2 --- /dev/null +++ b/src/java_tools/singlejar/java/com/google/devtools/build/zip/ZipUtil.java @@ -0,0 +1,728 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.zip; + +import com.google.devtools.build.zip.ZipFileEntry.Compression; +import com.google.devtools.build.zip.ZipFileEntry.Feature; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.Charset; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.EnumSet; +import java.util.GregorianCalendar; +import java.util.zip.ZipException; + +/** A utility class for reading and writing {@link ZipFileEntry}s from byte arrays. */ +public class ZipUtil { + + /** + * Midnight Jan 1st 1980. Uses the current time zone as the DOS format does not support time zones + * and will always assume the current zone. + */ + public static final long DOS_EPOCH = + new GregorianCalendar(1980, Calendar.JANUARY, 1, 0, 0, 0).getTimeInMillis(); + + /** 23:59:59 Dec 31st 2107. The maximum date representable in DOS format. */ + public static final long MAX_DOS_DATE = + new GregorianCalendar(2107, Calendar.DECEMBER, 31, 23, 59, 59).getTimeInMillis(); + + /** Converts a integral value to the corresponding little endian array. */ + private static byte[] integerToLittleEndian(byte[] buf, int offset, long value, int numBytes) { + for (int i = 0; i < numBytes; i++) { + buf[i + offset] = (byte) ((value & (0xffL << (i * 8))) >> (i * 8)); + } + return buf; + } + + /** Converts a short to the corresponding 2-byte little endian array. */ + static byte[] shortToLittleEndian(short value) { + return integerToLittleEndian(new byte[2], 0, value, 2); + } + + /** Writes a short to the buffer as a 2-byte little endian array starting at offset. */ + static byte[] shortToLittleEndian(byte[] buf, int offset, short value) { + return integerToLittleEndian(buf, offset, value, 2); + } + + /** Converts an int to the corresponding 4-byte little endian array. */ + static byte[] intToLittleEndian(int value) { + return integerToLittleEndian(new byte[4], 0, value, 4); + } + + /** Writes an int to the buffer as a 4-byte little endian array starting at offset. */ + static byte[] intToLittleEndian(byte[] buf, int offset, int value) { + return integerToLittleEndian(buf, offset, value, 4); + } + + /** Converts a long to the corresponding 8-byte little endian array. */ + static byte[] longToLittleEndian(long value) { + return integerToLittleEndian(new byte[8], 0, value, 8); + } + + /** Writes a long to the buffer as a 8-byte little endian array starting at offset. */ + static byte[] longToLittleEndian(byte[] buf, int offset, long value) { + return integerToLittleEndian(buf, offset, value, 8); + } + + /** Reads 16 bits in little-endian byte order from the buffer at the given offset. */ + static short get16(byte[] source, int offset) { + int a = source[offset + 0] & 0xff; + int b = source[offset + 1] & 0xff; + return (short) ((b << 8) | a); + } + + /** Reads 32 bits in little-endian byte order from the buffer at the given offset. */ + static int get32(byte[] source, int offset) { + int a = source[offset + 0] & 0xff; + int b = source[offset + 1] & 0xff; + int c = source[offset + 2] & 0xff; + int d = source[offset + 3] & 0xff; + return (d << 24) | (c << 16) | (b << 8) | a; + } + + /** Reads 64 bits in little-endian byte order from the buffer at the given offset. */ + static long get64(byte[] source, int offset) { + long a = source[offset + 0] & 0xffL; + long b = source[offset + 1] & 0xffL; + long c = source[offset + 2] & 0xffL; + long d = source[offset + 3] & 0xffL; + long e = source[offset + 4] & 0xffL; + long f = source[offset + 5] & 0xffL; + long g = source[offset + 6] & 0xffL; + long h = source[offset + 7] & 0xffL; + return (h << 56) | (g << 48) | (f << 40) | (e << 32) | (d << 24) | (c << 16) | (b << 8) | a; + } + + /** + * Reads an unsigned short in little-endian byte order from the buffer at the given offset. + * Casts to an int to allow proper numerical comparison. + */ + static int getUnsignedShort(byte[] source, int offset) { + return get16(source, offset) & 0xffff; + } + + /** + * Reads an unsigned int in little-endian byte order from the buffer at the given offset. + * Casts to a long to allow proper numerical comparison. + */ + static long getUnsignedInt(byte[] source, int offset) { + return get32(source, offset) & 0xffffffffL; + } + + /** + * Reads an unsigned long in little-endian byte order from the buffer at the given offset. + * Performs bounds checking to see if the unsigned long will be properly represented in Java's + * signed value. + */ + static long getUnsignedLong(byte[] source, int offset) throws ZipException { + long result = get64(source, offset); + if (result < 0) { + throw new ZipException("The requested unsigned long value is too large for Java's signed" + + "values. This Zip file is unsupported"); + } + return result; + } + + /** Checks if the timestamp is representable as a valid DOS timestamp. */ + private static boolean isValidInDos(long timestamp) { + Calendar time = Calendar.getInstance(); + time.setTimeInMillis(timestamp); + Calendar minTime = Calendar.getInstance(); + minTime.setTimeInMillis(DOS_EPOCH); + Calendar maxTime = Calendar.getInstance(); + maxTime.setTimeInMillis(MAX_DOS_DATE); + return (!time.before(minTime) && !time.after(maxTime)); + } + + /** Converts a unix timestamp into a 32-bit DOS timestamp. */ + static int unixToDosTime(long timestamp) { + Calendar time = Calendar.getInstance(); + time.setTimeInMillis(timestamp); + + if (!isValidInDos(timestamp)) { + DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + throw new IllegalArgumentException(String.format("%s is not representable in the DOS time" + + " format. It must be in the range %s to %s", df.format(time.getTime()), + df.format(new Date(DOS_EPOCH)), df.format(new Date(MAX_DOS_DATE)))); + } + + int dos = time.get(Calendar.SECOND) >> 1; + dos |= time.get(Calendar.MINUTE) << 5; + dos |= time.get(Calendar.HOUR_OF_DAY) << 11; + dos |= time.get(Calendar.DAY_OF_MONTH) << 16; + dos |= (time.get(Calendar.MONTH) + 1) << 21; + dos |= (time.get(Calendar.YEAR) - 1980) << 25; + return dos; + } + + /** Converts a 32-bit DOS timestamp into a unix timestamp. */ + static long dosToUnixTime(int timestamp) { + Calendar time = Calendar.getInstance(); + time.clear(); + time.set(Calendar.SECOND, (timestamp << 1) & 0x3e); + time.set(Calendar.MINUTE, (timestamp >> 5) & 0x3f); + time.set(Calendar.HOUR_OF_DAY, (timestamp >> 11) & 0x1f); + time.set(Calendar.DAY_OF_MONTH, (timestamp >> 16) & 0x1f); + time.set(Calendar.MONTH, ((timestamp >> 21) & 0x0f) - 1); + time.set(Calendar.YEAR, ((timestamp >> 25) & 0x7f) + 1980); + return time.getTimeInMillis(); + } + + /** Checks if array starts with target. */ + static boolean arrayStartsWith(byte[] array, byte[] target) { + if (array == null) { + return false; + } + if (target == null) { + return true; + } + if (target.length > array.length) { + return false; + } + for (int i = 0; i < target.length; i++) { + if (array[i] != target[i]) { + return false; + } + } + return true; + } + + static class LocalFileHeader { + static final int SIGNATURE = 0x04034b50; + static final int FIXED_DATA_SIZE = 30; + static final int SIGNATURE_OFFSET = 0; + static final int VERSION_OFFSET = 4; + static final int FLAGS_OFFSET = 6; + static final int METHOD_OFFSET = 8; + static final int MOD_TIME_OFFSET = 10; + static final int CRC_OFFSET = 14; + static final int COMPRESSED_SIZE_OFFSET = 18; + static final int UNCOMPRESSED_SIZE_OFFSET = 22; + static final int FILENAME_LENGTH_OFFSET = 26; + static final int EXTRA_FIELD_LENGTH_OFFSET = 28; + static final int VARIABLE_DATA_OFFSET = 30; + + /** + * Generates the raw byte data of the local file header for the {@link ZipFileEntry}. Uses the + * specified {@link ZipFileData} to encode the file name and comment. + * @throws IOException + */ + static byte[] create(ZipFileEntry entry, ZipFileData file, boolean allowZip64) + throws IOException { + byte[] name = entry.getName().getBytes(file.getCharset()); + ExtraDataList extra = entry.getExtra(); + + EnumSet<Feature> features = entry.getFeatureSet(); + int size = (int) entry.getSize(); + int csize = (int) entry.getCompressedSize(); + + if (features.contains(Feature.ZIP64_SIZE) || features.contains(Feature.ZIP64_CSIZE)) { + if (!allowZip64) { + throw new ZipException(String.format("Writing an entry of size %d(%d) without Zip64" + + " extensions is not supported.", entry.getSize(), entry.getCompressedSize())); + } + extra.remove((short) 0x0001); + int extraSize = 0; + if (features.contains(Feature.ZIP64_SIZE)) { + size = -1; + extraSize += 8; + } + if (features.contains(Feature.ZIP64_CSIZE)) { + csize = -1; + extraSize += 8; + } + byte[] zip64Extra = new byte[ExtraData.FIXED_DATA_SIZE + extraSize]; + shortToLittleEndian(zip64Extra, ExtraData.ID_OFFSET, (short) 0x0001); + shortToLittleEndian(zip64Extra, ExtraData.LENGTH_OFFSET, (short) extraSize); + int offset = ExtraData.FIXED_DATA_SIZE; + if (features.contains(Feature.ZIP64_SIZE)) { + longToLittleEndian(zip64Extra, offset, entry.getSize()); + offset += 8; + } + if (features.contains(Feature.ZIP64_CSIZE)) { + longToLittleEndian(zip64Extra, offset, entry.getCompressedSize()); + offset += 8; + } + extra.add(new ExtraData(zip64Extra, 0)); + } else { + extra.remove((short) 0x0001); + } + + byte[] buf = new byte[FIXED_DATA_SIZE + name.length + extra.getLength()]; + intToLittleEndian(buf, SIGNATURE_OFFSET, SIGNATURE); + shortToLittleEndian(buf, VERSION_OFFSET, entry.getVersionNeeded()); + shortToLittleEndian(buf, FLAGS_OFFSET, entry.getFlags()); + shortToLittleEndian(buf, METHOD_OFFSET, entry.getMethod().getValue()); + intToLittleEndian(buf, MOD_TIME_OFFSET, unixToDosTime(entry.getTime())); + intToLittleEndian(buf, CRC_OFFSET, (int) (entry.getCrc() & 0xffffffff)); + intToLittleEndian(buf, COMPRESSED_SIZE_OFFSET, csize); + intToLittleEndian(buf, UNCOMPRESSED_SIZE_OFFSET, size); + shortToLittleEndian(buf, FILENAME_LENGTH_OFFSET, (short) name.length); + shortToLittleEndian(buf, EXTRA_FIELD_LENGTH_OFFSET, (short) extra.getLength()); + System.arraycopy(name, 0, buf, FIXED_DATA_SIZE, name.length); + extra.getByteStream().read(buf, FIXED_DATA_SIZE + name.length, extra.getLength()); + + return buf; + } + } + + static class CentralDirectoryFileHeader { + static final int SIGNATURE = 0x02014b50; + static final int FIXED_DATA_SIZE = 46; + static final int SIGNATURE_OFFSET = 0; + static final int VERSION_OFFSET = 4; + static final int VERSION_NEEDED_OFFSET = 6; + static final int FLAGS_OFFSET = 8; + static final int METHOD_OFFSET = 10; + static final int MOD_TIME_OFFSET = 12; + static final int CRC_OFFSET = 16; + static final int COMPRESSED_SIZE_OFFSET = 20; + static final int UNCOMPRESSED_SIZE_OFFSET = 24; + static final int FILENAME_LENGTH_OFFSET = 28; + static final int EXTRA_FIELD_LENGTH_OFFSET = 30; + static final int COMMENT_LENGTH_OFFSET = 32; + static final int DISK_START_OFFSET = 34; + static final int INTERNAL_ATTRIBUTES_OFFSET = 36; + static final int EXTERNAL_ATTRIBUTES_OFFSET = 38; + static final int LOCAL_HEADER_OFFSET_OFFSET = 42; + + /** + * Reads a {@link ZipFileEntry} from the input stream, using the specified {@link Charset} to + * decode the filename and comment. + */ + static ZipFileEntry read(InputStream in, Charset charset) + throws IOException { + byte[] fixedSizeData = new byte[FIXED_DATA_SIZE]; + + if (in.read(fixedSizeData) != FIXED_DATA_SIZE) { + throw new ZipException( + "Unexpected end of file while reading Central Directory File Header."); + } + if (!arrayStartsWith(fixedSizeData, intToLittleEndian(SIGNATURE))) { + throw new ZipException(String.format( + "Malformed Central Directory File Header; does not start with %08x", SIGNATURE)); + } + + byte[] name = new byte[getUnsignedShort(fixedSizeData, FILENAME_LENGTH_OFFSET)]; + byte[] extraField = new byte[getUnsignedShort(fixedSizeData, EXTRA_FIELD_LENGTH_OFFSET)]; + byte[] comment = new byte[getUnsignedShort(fixedSizeData, COMMENT_LENGTH_OFFSET)]; + + if (name.length > 0 && in.read(name) != name.length) { + throw new ZipException( + "Unexpected end of file while reading Central Directory File Header."); + } + if (extraField.length > 0 && in.read(extraField) != extraField.length) { + throw new ZipException( + "Unexpected end of file while reading Central Directory File Header."); + } + if (comment.length > 0 && in.read(comment) != comment.length) { + throw new ZipException( + "Unexpected end of file while reading Central Directory File Header."); + } + + ExtraDataList extra = new ExtraDataList(extraField); + + long csize = getUnsignedInt(fixedSizeData, COMPRESSED_SIZE_OFFSET); + long size = getUnsignedInt(fixedSizeData, UNCOMPRESSED_SIZE_OFFSET); + long offset = getUnsignedInt(fixedSizeData, LOCAL_HEADER_OFFSET_OFFSET); + if (csize == 0xffffffffL || size == 0xffffffffL || offset == 0xffffffffL) { + ExtraData zip64Extra = extra.get((short) 0x0001); + if (zip64Extra != null) { + int index = 0; + if (size == 0xffffffffL) { + size = ZipUtil.getUnsignedLong(zip64Extra.getData(), index); + index += 8; + } + if (csize == 0xffffffffL) { + csize = ZipUtil.getUnsignedLong(zip64Extra.getData(), index); + index += 8; + } + if (offset == 0xffffffffL) { + offset = ZipUtil.getUnsignedLong(zip64Extra.getData(), index); + index += 8; + } + } + } + + ZipFileEntry entry = new ZipFileEntry(new String(name, charset)); + entry.setVersion(get16(fixedSizeData, VERSION_OFFSET)); + entry.setVersionNeeded(get16(fixedSizeData, VERSION_NEEDED_OFFSET)); + entry.setFlags(get16(fixedSizeData, FLAGS_OFFSET)); + entry.setMethod(Compression.fromValue(get16(fixedSizeData, METHOD_OFFSET))); + long time = dosToUnixTime(get32(fixedSizeData, MOD_TIME_OFFSET)); + entry.setTime(isValidInDos(time) ? time : DOS_EPOCH); + entry.setCrc(getUnsignedInt(fixedSizeData, CRC_OFFSET)); + entry.setCompressedSize(csize); + entry.setSize(size); + entry.setInternalAttributes(get16(fixedSizeData, INTERNAL_ATTRIBUTES_OFFSET)); + entry.setExternalAttributes(get32(fixedSizeData, EXTERNAL_ATTRIBUTES_OFFSET)); + entry.setLocalHeaderOffset(offset); + entry.setExtra(extra); + entry.setComment(new String(comment, charset)); + + return entry; + } + + /** + * Generates the raw byte data of the central directory file header for the ZipEntry. Uses the + * specified {@link ZipFileData} to encode the file name and comment. + * @throws ZipException + */ + static byte[] create(ZipFileEntry entry, ZipFileData file, boolean allowZip64) + throws ZipException { + if (allowZip64) { + addZip64Extra(entry); + } else { + entry.getExtra().remove((short) 0x0001); + } + byte[] name = file.getBytes(entry.getName()); + byte[] extra = entry.getExtra().getBytes(); + byte[] comment = entry.getComment() != null + ? file.getBytes(entry.getComment()) : new byte[]{}; + + byte[] buf = new byte[FIXED_DATA_SIZE + name.length + extra.length + comment.length]; + + fillFixedSizeData(buf, entry, name.length, extra.length, comment.length, allowZip64); + System.arraycopy(name, 0, buf, FIXED_DATA_SIZE, name.length); + System.arraycopy(extra, 0, buf, FIXED_DATA_SIZE + name.length, extra.length); + System.arraycopy(comment, 0, buf, FIXED_DATA_SIZE + name.length + extra.length, + comment.length); + + return buf; + } + + /** + * Writes the central directory file header for the ZipEntry to an output stream. Uses the + * specified {@link ZipFileData} to encode the file name and comment. + */ + static int write(ZipFileEntry entry, ZipFileData file, boolean allowZip64, byte[] buf, + OutputStream stream) throws IOException { + if (buf == null || buf.length < FIXED_DATA_SIZE) { + buf = new byte[FIXED_DATA_SIZE]; + } + + if (allowZip64) { + addZip64Extra(entry); + } else { + entry.getExtra().remove((short) 0x0001); + } + byte[] name = entry.getName().getBytes(file.getCharset()); + byte[] extra = entry.getExtra().getBytes(); + byte[] comment = entry.getComment() != null + ? entry.getComment().getBytes(file.getCharset()) : new byte[]{}; + + fillFixedSizeData(buf, entry, name.length, extra.length, comment.length, allowZip64); + stream.write(buf, 0, FIXED_DATA_SIZE); + stream.write(name); + stream.write(extra); + stream.write(comment); + + return FIXED_DATA_SIZE + name.length + extra.length + comment.length; + } + + /** + * Write the fixed size data portion for the specified ZIP entry to the buffer. + * @throws ZipException + */ + private static void fillFixedSizeData(byte[] buf, ZipFileEntry entry, int nameLength, + int extraLength, int commentLength, boolean allowZip64) throws ZipException { + if (!allowZip64 && entry.getFeatureSet().contains(Feature.ZIP64_CSIZE)) { + throw new ZipException(String.format("Writing an entry with compressed size %d without" + + " Zip64 extensions is not supported.", entry.getCompressedSize())); + } + if (!allowZip64 && entry.getFeatureSet().contains(Feature.ZIP64_SIZE)) { + throw new ZipException(String.format("Writing an entry of size %d without" + + " Zip64 extensions is not supported.", entry.getSize())); + } + if (!allowZip64 && entry.getFeatureSet().contains(Feature.ZIP64_OFFSET)) { + throw new ZipException(String.format("Writing an entry with local header offset %d without" + + " Zip64 extensions is not supported.", entry.getLocalHeaderOffset())); + } + int csize = (int) (entry.getFeatureSet().contains(Feature.ZIP64_CSIZE) + ? -1 : entry.getCompressedSize()); + int size = (int) (entry.getFeatureSet().contains(Feature.ZIP64_SIZE) + ? -1 : entry.getSize()); + int offset = (int) (entry.getFeatureSet().contains(Feature.ZIP64_OFFSET) + ? -1 : entry.getLocalHeaderOffset()); + intToLittleEndian(buf, SIGNATURE_OFFSET, SIGNATURE); + shortToLittleEndian(buf, VERSION_OFFSET, entry.getVersion()); + shortToLittleEndian(buf, VERSION_NEEDED_OFFSET, entry.getVersionNeeded()); + shortToLittleEndian(buf, FLAGS_OFFSET, entry.getFlags()); + shortToLittleEndian(buf, METHOD_OFFSET, entry.getMethod().getValue()); + intToLittleEndian(buf, MOD_TIME_OFFSET, unixToDosTime(entry.getTime())); + intToLittleEndian(buf, CRC_OFFSET, (int) (entry.getCrc() & 0xffffffff)); + intToLittleEndian(buf, COMPRESSED_SIZE_OFFSET, csize); + intToLittleEndian(buf, UNCOMPRESSED_SIZE_OFFSET, size); + shortToLittleEndian(buf, FILENAME_LENGTH_OFFSET, (short) (nameLength & 0xffff)); + shortToLittleEndian(buf, EXTRA_FIELD_LENGTH_OFFSET, (short) (extraLength & 0xffff)); + shortToLittleEndian(buf, COMMENT_LENGTH_OFFSET, (short) (commentLength & 0xffff)); + shortToLittleEndian(buf, DISK_START_OFFSET, (short) 0); + shortToLittleEndian(buf, INTERNAL_ATTRIBUTES_OFFSET, entry.getInternalAttributes()); + intToLittleEndian(buf, EXTERNAL_ATTRIBUTES_OFFSET, entry.getExternalAttributes()); + intToLittleEndian(buf, LOCAL_HEADER_OFFSET_OFFSET, offset); + } + + /** + * Update the extra data fields to contain a Zip64 extended information field if required + */ + private static void addZip64Extra(ZipFileEntry entry) { + EnumSet<Feature> features = entry.getFeatureSet(); + ExtraDataList extra = entry.getExtra(); + int extraSize = 0; + if (features.contains(Feature.ZIP64_SIZE)) { + extraSize += 8; + } + if (features.contains(Feature.ZIP64_CSIZE)) { + extraSize += 8; + } + if (features.contains(Feature.ZIP64_OFFSET)) { + extraSize += 8; + } + if (extraSize > 0) { + extra.remove((short) 0x0001); + byte[] zip64Extra = new byte[ExtraData.FIXED_DATA_SIZE + extraSize]; + shortToLittleEndian(zip64Extra, ExtraData.ID_OFFSET, (short) 0x0001); + shortToLittleEndian(zip64Extra, ExtraData.LENGTH_OFFSET, (short) extraSize); + int offset = ExtraData.FIXED_DATA_SIZE; + if (features.contains(Feature.ZIP64_SIZE)) { + longToLittleEndian(zip64Extra, offset, entry.getSize()); + offset += 8; + } + if (features.contains(Feature.ZIP64_CSIZE)) { + longToLittleEndian(zip64Extra, offset, entry.getCompressedSize()); + offset += 8; + } + if (features.contains(Feature.ZIP64_OFFSET)) { + longToLittleEndian(zip64Extra, offset, entry.getLocalHeaderOffset()); + } + extra.add(new ExtraData(zip64Extra, 0)); + } + } + } + + static class Zip64EndOfCentralDirectory { + static final int SIGNATURE = 0x06064b50; + static final int FIXED_DATA_SIZE = 56; + static final int SIGNATURE_OFFSET = 0; + static final int SIZE_OFFSET = 4; + static final int VERSION_OFFSET = 12; + static final int VERSION_NEEDED_OFFSET = 14; + static final int DISK_NUMBER_OFFSET = 16; + static final int CD_DISK_OFFSET = 20; + static final int DISK_ENTRIES_OFFSET = 24; + static final int TOTAL_ENTRIES_OFFSET = 32; + static final int CD_SIZE_OFFSET = 40; + static final int CD_OFFSET_OFFSET = 48; + + /** + * Read the Zip64 end of central directory record from the input stream and parse additional + * {@link ZipFileData} from it. + */ + static ZipFileData read(InputStream in, ZipFileData file) throws IOException { + if (file == null) { + throw new NullPointerException(); + } + + byte[] fixedSizeData = new byte[FIXED_DATA_SIZE]; + if (in.read(fixedSizeData) != FIXED_DATA_SIZE) { + throw new ZipException( + "Unexpected end of file while reading Zip64 End of Central Directory Record."); + } + if (!arrayStartsWith(fixedSizeData, intToLittleEndian(SIGNATURE))) { + throw new ZipException(String.format( + "Malformed Zip64 End of Central Directory; does not start with %08x", SIGNATURE)); + } + file.setZip64(true); + file.setCentralDirectoryOffset(getUnsignedLong(fixedSizeData, CD_OFFSET_OFFSET)); + file.setExpectedEntries(getUnsignedLong(fixedSizeData, TOTAL_ENTRIES_OFFSET)); + return file; + } + + /** + * Generates the raw byte data of the Zip64 end of central directory record for the file. + */ + static byte[] create(ZipFileData file) { + byte[] buf = new byte[FIXED_DATA_SIZE]; + intToLittleEndian(buf, SIGNATURE_OFFSET, SIGNATURE); + longToLittleEndian(buf, SIZE_OFFSET, FIXED_DATA_SIZE - 12); + shortToLittleEndian(buf, VERSION_OFFSET, (short) 0x2d); + shortToLittleEndian(buf, VERSION_NEEDED_OFFSET, (short) 0x2d); + intToLittleEndian(buf, DISK_NUMBER_OFFSET, 0); + intToLittleEndian(buf, CD_DISK_OFFSET, 0); + longToLittleEndian(buf, DISK_ENTRIES_OFFSET, file.getNumEntries()); + longToLittleEndian(buf, TOTAL_ENTRIES_OFFSET, file.getNumEntries()); + longToLittleEndian(buf, CD_SIZE_OFFSET, file.getCentralDirectorySize()); + longToLittleEndian(buf, CD_OFFSET_OFFSET, file.getCentralDirectoryOffset()); + return buf; + } + } + + static class Zip64EndOfCentralDirectoryLocator { + static final int SIGNATURE = 0x07064b50; + static final int FIXED_DATA_SIZE = 20; + static final int SIGNATURE_OFFSET = 0; + static final int ZIP64_EOCD_DISK_OFFSET = 4; + static final int ZIP64_EOCD_OFFSET_OFFSET = 8; + static final int DISK_NUMBER_OFFSET = 16; + + /** + * Read the Zip64 end of central directory locator from the input stream and parse additional + * {@link ZipFileData} from it. + */ + static ZipFileData read(InputStream in, ZipFileData file) throws IOException { + if (file == null) { + throw new NullPointerException(); + } + + byte[] fixedSizeData = new byte[FIXED_DATA_SIZE]; + if (in.read(fixedSizeData) != FIXED_DATA_SIZE) { + throw new ZipException( + "Unexpected end of file while reading Zip64 End of Central Directory Locator."); + } + if (!arrayStartsWith(fixedSizeData, intToLittleEndian(SIGNATURE))) { + throw new ZipException(String.format( + "Malformed Zip64 Central Directory Locator; does not start with %08x", SIGNATURE)); + } + file.setZip64(true); + file.setZip64EndOfCentralDirectoryOffset( + getUnsignedLong(fixedSizeData, ZIP64_EOCD_OFFSET_OFFSET)); + return file; + } + + /** + * Generates the raw byte data of the Zip64 end of central directory locator for the file. + */ + static byte[] create(ZipFileData file) { + byte[] buf = new byte[FIXED_DATA_SIZE]; + intToLittleEndian(buf, SIGNATURE_OFFSET, SIGNATURE); + intToLittleEndian(buf, ZIP64_EOCD_DISK_OFFSET, 0); + longToLittleEndian(buf, ZIP64_EOCD_OFFSET_OFFSET, file.getZip64EndOfCentralDirectoryOffset()); + intToLittleEndian(buf, DISK_NUMBER_OFFSET, 1); + return buf; + } + } + + static class EndOfCentralDirectoryRecord { + static final int SIGNATURE = 0x06054b50; + static final int FIXED_DATA_SIZE = 22; + static final int SIGNATURE_OFFSET = 0; + static final int DISK_NUMBER_OFFSET = 4; + static final int CD_DISK_OFFSET = 6; + static final int DISK_ENTRIES_OFFSET = 8; + static final int TOTAL_ENTRIES_OFFSET = 10; + static final int CD_SIZE_OFFSET = 12; + static final int CD_OFFSET_OFFSET = 16; + static final int COMMENT_LENGTH_OFFSET = 20; + + /** + * Read the end of central directory record from the input stream and parse {@link ZipFileData} + * from it. + */ + static ZipFileData read(InputStream in, ZipFileData file) throws IOException { + if (file == null) { + throw new NullPointerException(); + } + + byte[] fixedSizeData = new byte[FIXED_DATA_SIZE]; + if (in.read(fixedSizeData) != FIXED_DATA_SIZE) { + throw new ZipException( + "Unexpected end of file while reading End of Central Directory Record."); + } + if (!arrayStartsWith(fixedSizeData, intToLittleEndian(SIGNATURE))) { + throw new ZipException(String.format( + "Malformed End of Central Directory Record; does not start with %08x", SIGNATURE)); + } + + byte[] comment = new byte[getUnsignedShort(fixedSizeData, COMMENT_LENGTH_OFFSET)]; + if (comment.length > 0 && in.read(comment) != comment.length) { + throw new ZipException( + "Unexpected end of file while reading End of Central Directory Record."); + } + short diskNumber = get16(fixedSizeData, DISK_NUMBER_OFFSET); + short centralDirectoryDisk = get16(fixedSizeData, CD_DISK_OFFSET); + short entriesOnDisk = get16(fixedSizeData, DISK_ENTRIES_OFFSET); + short totalEntries = get16(fixedSizeData, TOTAL_ENTRIES_OFFSET); + int centralDirectorySize = get32(fixedSizeData, CD_SIZE_OFFSET); + int centralDirectoryOffset = get32(fixedSizeData, CD_OFFSET_OFFSET); + if (diskNumber == -1 || centralDirectoryDisk == -1 || entriesOnDisk == -1 + || totalEntries == -1 || centralDirectorySize == -1 || centralDirectoryOffset == -1) { + file.setMaybeZip64(true); + } + file.setComment(comment); + file.setCentralDirectorySize(getUnsignedInt(fixedSizeData, CD_SIZE_OFFSET)); + file.setCentralDirectoryOffset(getUnsignedInt(fixedSizeData, CD_OFFSET_OFFSET)); + file.setExpectedEntries(getUnsignedShort(fixedSizeData, TOTAL_ENTRIES_OFFSET)); + return file; + } + + /** + * Generates the raw byte data of the end of central directory record for the specified + * {@link ZipFileData}. + * @throws ZipException if the file comment is too long + */ + static byte[] create(ZipFileData file, boolean allowZip64) throws ZipException { + byte[] comment = file.getBytes(file.getComment()); + + byte[] buf = new byte[FIXED_DATA_SIZE + comment.length]; + + // Allow writing of Zip file without Zip64 extensions for large archives as a special case + // since many reading implementations can handle this. + short numEntries = (short) (file.getNumEntries() > 0xffff && allowZip64 + ? -1 : file.getNumEntries()); + int cdSize = (int) (file.getCentralDirectorySize() > 0xffffffffL && allowZip64 + ? -1 : file.getCentralDirectorySize()); + int cdOffset = (int) (file.getCentralDirectoryOffset() > 0xffffffffL && allowZip64 + ? -1 : file.getCentralDirectoryOffset()); + intToLittleEndian(buf, SIGNATURE_OFFSET, SIGNATURE); + shortToLittleEndian(buf, DISK_NUMBER_OFFSET, (short) 0); + shortToLittleEndian(buf, CD_DISK_OFFSET, (short) 0); + shortToLittleEndian(buf, DISK_ENTRIES_OFFSET, numEntries); + shortToLittleEndian(buf, TOTAL_ENTRIES_OFFSET, numEntries); + intToLittleEndian(buf, CD_SIZE_OFFSET, cdSize); + intToLittleEndian(buf, CD_OFFSET_OFFSET, cdOffset); + shortToLittleEndian(buf, COMMENT_LENGTH_OFFSET, (short) comment.length); + System.arraycopy(comment, 0, buf, FIXED_DATA_SIZE, comment.length); + + return buf; + } + } + + static class CentralDirectory { + /** + * Writes the central directory to an output stream for the specified {@link ZipFileData}. + */ + static void write(ZipFileData file, boolean allowZip64, OutputStream stream) + throws IOException { + long directorySize = 0; + byte[] buf = new byte[CentralDirectoryFileHeader.FIXED_DATA_SIZE]; + for (ZipFileEntry entry : file.getEntries()) { + directorySize += CentralDirectoryFileHeader.write(entry, file, allowZip64, buf, stream); + } + file.setCentralDirectorySize(directorySize); + if (file.isZip64() && allowZip64) { + file.setZip64EndOfCentralDirectoryOffset(file.getCentralDirectoryOffset() + + file.getCentralDirectorySize()); + stream.write(Zip64EndOfCentralDirectory.create(file)); + stream.write(Zip64EndOfCentralDirectoryLocator.create(file)); + } + stream.write(EndOfCentralDirectoryRecord.create(file, allowZip64)); + } + } +} diff --git a/src/java_tools/singlejar/java/com/google/devtools/build/zip/ZipWriter.java b/src/java_tools/singlejar/java/com/google/devtools/build/zip/ZipWriter.java new file mode 100644 index 0000000000..f2d0cdee55 --- /dev/null +++ b/src/java_tools/singlejar/java/com/google/devtools/build/zip/ZipWriter.java @@ -0,0 +1,229 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.zip; + +import com.google.devtools.build.zip.ZipFileEntry.Flag; +import com.google.devtools.build.zip.ZipUtil.CentralDirectory; +import com.google.devtools.build.zip.ZipUtil.LocalFileHeader; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.Charset; +import java.util.zip.ZipException; + +/** + * This class implements an output stream filter for writing files in the ZIP file format. It does + * not perform its own compression and so allows writing of already compressed file data. + */ +public class ZipWriter extends OutputStream { + private final CountingOutputStream stream; + private final ZipFileData zipData; + private final boolean allowZip64; + private boolean writingPrefix; + private ZipFileEntry entry; + private long bytesWritten; + private boolean finished; + + /** + * Creates a new raw ZIP output stream. + * + * @param out the actual output stream + * @param charset the {@link Charset} to be used to encode the entry names and comments + */ + public ZipWriter(OutputStream out, Charset charset) { + this(out, charset, false); + } + + /** + * Creates a new raw ZIP output stream. + * + * @param out the actual output stream + * @param charset the {@link Charset} to be used to encode the entry names and comments + * @param allowZip64 whether the output Zip should be allowed to use Zip64 extensions + */ + public ZipWriter(OutputStream out, Charset charset, boolean allowZip64) { + this.stream = new CountingOutputStream(out); + this.zipData = new ZipFileData(charset); + this.allowZip64 = allowZip64; + this.finished = false; + } + + /** + * Sets the ZIP file comment. + * + * @param comment the ZIP file comment + * @throws ZipException if the comment is longer than 0xffff bytes + */ + public void setComment(String comment) throws ZipException { + zipData.setComment(comment); + } + + /** + * Configures the stream to write prefix file data. + * + * @throws ZipException if other contents have already been written to the output stream + */ + public void startPrefixFile() throws ZipException { + checkNotFinished(); + if (!zipData.getEntries().isEmpty() || entry != null) { + throw new ZipException("Cannot add a prefix file after the zip contents have been started."); + } + writingPrefix = true; + } + + /** Closes the prefix file and positions the output stream to write ZIP entries. */ + public void endPrefixFile() { + checkNotFinished(); + writingPrefix = false; + } + + /** + * Begins writing a new ZIP file entry and positions the stream to the start of the entry data. + * Closes the current entry if still active. + * + * <p><em>NOTE:</em> No defensive copying is performed on e. The local header offset and flags + * will be modified. + * + * @param e the ZIP entry to be written + * @throws IOException if an I/O error occurred + */ + public void putNextEntry(ZipFileEntry e) throws IOException { + checkNotFinished(); + writingPrefix = false; + if (entry != null) { + finishEntry(); + } + startEntry(e); + } + + /** + * Closes the current ZIP entry and positions the stream for writing the next entry. + * + * @throws ZipException if a ZIP format exception occurred + * @throws IOException if an I/O error occurred + */ + public void closeEntry() throws IOException { + checkNotFinished(); + if (entry != null) { + finishEntry(); + } + } + + @Override public void write(int b) throws IOException { + byte[] buf = new byte[1]; + buf[0] = (byte) (b & 0xff); + write(buf); + } + + @Override public void write(byte[] b) throws IOException { + write(b, 0, b.length); + } + + @Override public synchronized void write(byte[] b, int off, int len) throws IOException { + checkNotFinished(); + if (entry == null && !writingPrefix) { + throw new ZipException("Cannot write zip contents without first setting a ZipEntry or" + + " starting a prefix file."); + } + stream.write(b, off, len); + bytesWritten += len; + } + + /** + * Finishes writing the contents of the ZIP output stream without closing the underlying stream. + * Use this method when applying multiple filters in succession to the same output stream. + * + * @throws ZipException if a ZIP file error has occurred + * @throws IOException if an I/O exception has occurred + */ + public void finish() throws IOException { + checkNotFinished(); + if (entry != null) { + finishEntry(); + } + writeCentralDirectory(); + finished = true; + } + + @Override public void close() throws IOException { + if (!finished) { + finish(); + } + stream.close(); + } + + /** + * Writes the local file header for the ZIP entry and positions the stream to the start of the + * entry data. + * + * @param e the ZIP entry for which to write the local file header + * @throws ZipException if a ZIP file error has occurred + * @throws IOException if an I/O exception has occurred + */ + private void startEntry(ZipFileEntry e) throws IOException { + if (e.getTime() == -1) { + throw new IllegalArgumentException("Zip entry last modified time must be set"); + } + if (e.getCrc() == -1) { + throw new IllegalArgumentException("Zip entry CRC-32 must be set"); + } + if (e.getSize() == -1) { + throw new IllegalArgumentException("Zip entry uncompressed size must be set"); + } + if (e.getCompressedSize() == -1) { + throw new IllegalArgumentException("Zip entry compressed size must be set"); + } + bytesWritten = 0; + entry = e; + entry.setFlag(Flag.DATA_DESCRIPTOR, false); + entry.setLocalHeaderOffset(stream.getCount()); + stream.write(LocalFileHeader.create(entry, zipData, allowZip64)); + } + + /** + * Closes the current ZIP entry and positions the stream for writing the next entry. Checks that + * the amount of data written matches the compressed size indicated by the ZipEntry. + * + * @throws ZipException if a ZIP file error has occurred + * @throws IOException if an I/O exception has occurred + */ + private void finishEntry() throws IOException { + if (entry.getCompressedSize() != bytesWritten) { + throw new ZipException(String.format("Number of bytes written for the entry %s (%d) does not" + + " match the reported compressed size (%d).", entry.getName(), bytesWritten, + entry.getCompressedSize())); + } + zipData.addEntry(entry); + entry = null; + } + + /** + * Writes the ZIP file's central directory. + * + * @throws ZipException if a ZIP file error has occurred + * @throws IOException if an I/O exception has occurred + */ + private void writeCentralDirectory() throws IOException { + zipData.setCentralDirectoryOffset(stream.getCount()); + CentralDirectory.write(zipData, allowZip64, stream); + } + + /** Checks that the ZIP file has not been finished yet. */ + private void checkNotFinished() { + if (finished) { + throw new IllegalStateException(); + } + } +} |