diff options
Diffstat (limited to 'src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/ZipTester.java')
| -rw-r--r-- | src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/ZipTester.java | 412 |
1 files changed, 412 insertions, 0 deletions
diff --git a/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/ZipTester.java b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/ZipTester.java new file mode 100644 index 0000000000..c1293d6f92 --- /dev/null +++ b/src/java_tools/singlejar/javatests/com/google/devtools/build/singlejar/ZipTester.java @@ -0,0 +1,412 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.singlejar; + + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.zip.CRC32; +import java.util.zip.DataFormatException; +import java.util.zip.Inflater; + +/** + * A helper class to validate zip files and provide reasonable diagnostics (better than what zip + * does). We might want to make this into a fully-fledged binary some day. + */ +final class ZipTester { + + // The following constants are ZIP-specific. + private static final int LOCAL_FILE_HEADER_MARKER = 0x04034b50; + private static final int DATA_DESCRIPTOR_MARKER = 0x08074b50; + private static final int CENTRAL_DIRECTORY_MARKER = 0x02014b50; + private static final int END_OF_CENTRAL_DIRECTORY_MARKER = 0x06054b50; + + private static final int FILE_HEADER_BUFFER_SIZE = 26; // without marker + private static final int DATA_DESCRIPTOR_BUFFER_SIZE = 12; // without marker + + private static final int DIRECTORY_ENTRY_BUFFER_SIZE = 42; // without marker + private static final int END_OF_CENTRAL_DIRECTORY_BUFFER_SIZE = 18; // without marker + + // Set if the size, compressed size and CRC are set to zero, and present in + // the data descriptor after the data. + private static final int SIZE_MASKED_FLAG = 1 << 3; + + private static final int STORED_METHOD = 0; + private static final int DEFLATE_METHOD = 8; + + private static final int VERSION_STORED = 10; // Version 1.0 + private static final int VERSION_DEFLATE = 20; // Version 2.0 + + private static class Entry { + private final long pos; + private final String name; + private final int flags; + private final int method; + private final int dosTime; + Entry(long pos, String name, int flags, int method, int dosTime) { + this.pos = pos; + this.name = name; + this.flags = flags; + this.method = method; + this.dosTime = dosTime; + } + } + + private final InputStream in; + private final byte[] buffer = new byte[1024]; + private int bufferLength; + private int bufferOffset; + private long pos; + + private List<Entry> entries = new ArrayList<Entry>(); + + public ZipTester(InputStream in) { + this.in = in; + } + + public ZipTester(byte[] data) { + this(new ByteArrayInputStream(data)); + } + + private void warn(String msg) { + System.err.println("WARNING: " + msg); + } + + private void readMoreData(String action) throws IOException { + if ((bufferLength > 0) && (bufferOffset > 0)) { + System.arraycopy(buffer, bufferOffset, buffer, 0, bufferLength); + } + if (bufferLength >= buffer.length) { + // The buffer size is specifically chosen to avoid this situation. + throw new AssertionError("Internal error: buffer overrun."); + } + bufferOffset = 0; + int bytesRead = in.read(buffer, bufferLength, buffer.length - bufferLength); + if (bytesRead <= 0) { + throw new IOException("Unexpected end of file, while " + action); + } + bufferLength += bytesRead; + } + + private int readByte(String action) throws IOException { + if (bufferLength == 0) { + readMoreData(action); + } + byte result = buffer[bufferOffset]; + bufferOffset++; bufferLength--; + pos++; + return result & 0xff; + } + + private long getUnsignedInt(String action) throws IOException { + int a = readByte(action); + int b = readByte(action); + int c = readByte(action); + int d = readByte(action); + return ((d << 24) | (c << 16) | (b << 8) | a) & 0xffffffffL; + } + + private void readFully(byte[] buffer, String action) throws IOException { + for (int i = 0; i < buffer.length; i++) { + buffer[i] = (byte) readByte(action); + } + } + + private void skip(long length, String action) throws IOException { + for (long i = 0; i < length; i++) { + readByte(action); + } + } + + private int getUnsignedShort(byte[] source, int offset) { + int a = source[offset + 0] & 0xff; + int b = source[offset + 1] & 0xff; + return (b << 8) | a; + } + + private long getUnsignedInt(byte[] source, int offset) { + int a = source[offset + 0] & 0xff; + int b = source[offset + 1] & 0xff; + int c = source[offset + 2] & 0xff; + int d = source[offset + 3] & 0xff; + return ((d << 24) | (c << 16) | (b << 8) | a) & 0xffffffffL; + } + + private class DeflateInputStream extends InputStream { + + private final byte[] singleByteBuffer = new byte[1]; + private int consumedBytes; + private final Inflater inflater = new Inflater(true); + private long totalBytesRead; + + private int inflateData(byte[] dest, int off, int len) + throws IOException { + consumedBytes = 0; + int bytesProduced = 0; + int bytesConsumed = 0; + while ((bytesProduced == 0) && !inflater.finished()) { + inflater.setInput(buffer, bufferOffset + bytesConsumed, bufferLength - bytesConsumed); + int remainingBefore = inflater.getRemaining(); + try { + bytesProduced = inflater.inflate(dest, off, len); + } catch (DataFormatException e) { + throw new IOException("Invalid deflate stream in ZIP file.", e); + } + bytesConsumed += remainingBefore - inflater.getRemaining(); + consumedBytes = bytesConsumed; + if (bytesProduced == 0) { + if (inflater.needsDictionary()) { + // The DEFLATE algorithm as used in the ZIP file format does not + // require an additional dictionary. + throw new AssertionError("Inflater unexpectedly requires a dictionary."); + } else if (inflater.needsInput()) { + readMoreData("need more data for deflate"); + } else if (inflater.finished()) { + return 0; + } else { + // According to the Inflater specification, this cannot happen. + throw new AssertionError("Inflater unexpectedly produced no output."); + } + } + } + return bytesProduced; + } + + @Override + public int read(byte b[], int off, int len) throws IOException { + if (inflater.finished()) { + return -1; + } + int length = inflateData(b, off, len); + totalBytesRead += consumedBytes; + bufferLength -= consumedBytes; + bufferOffset += consumedBytes; + pos += consumedBytes; + return length == 0 ? -1 : length; + } + + @Override + public int read() throws IOException { + int bytesRead = read(singleByteBuffer, 0, 1); + return (bytesRead == -1) ? -1 : (singleByteBuffer[0] & 0xff); + } + } + + private void readEntry() throws IOException { + long entrypos = pos - 4; + String entryDesc = "file entry at " + Long.toHexString(entrypos); + byte[] entryBuffer = new byte[FILE_HEADER_BUFFER_SIZE]; + readFully(entryBuffer, "reading file header"); + int versionToExtract = getUnsignedShort(entryBuffer, 0); + int flags = getUnsignedShort(entryBuffer, 2); + int method = getUnsignedShort(entryBuffer, 4); + int dosTime = (int) getUnsignedInt(entryBuffer, 6); + int crc32 = (int) getUnsignedInt(entryBuffer, 10); + long compressedSize = getUnsignedInt(entryBuffer, 14); + long uncompressedSize = getUnsignedInt(entryBuffer, 18); + int filenameLength = getUnsignedShort(entryBuffer, 22); + int extraLength = getUnsignedShort(entryBuffer, 24); + + byte[] filename = new byte[filenameLength]; + readFully(filename, "reading file name"); + skip(extraLength, "skipping extra data"); + + String name = new String(filename, "UTF-8"); + for (int i = 0; i < filename.length; i++) { + if ((filename[i] < ' ') || (filename[i] > 127)) { + warn(entryDesc + ": file name has unexpected non-ascii characters"); + } + } + entryDesc = "file entry '" + name + "' at " + Long.toHexString(entrypos); + + if ((method != STORED_METHOD) && (method != DEFLATE_METHOD)) { + throw new IOException(entryDesc + ": unknown method " + method); + } + if ((flags != 0) && (flags != SIZE_MASKED_FLAG)) { + throw new IOException(entryDesc + ": unknown flags " + flags); + } + if ((method == STORED_METHOD) && (versionToExtract != VERSION_STORED)) { + warn(entryDesc + ": unexpected version to extract for stored entry " + versionToExtract); + } + if ((method == DEFLATE_METHOD) && (versionToExtract != VERSION_DEFLATE)) { +// warn(entryDesc + ": unexpected version to extract for deflated entry " + versionToExtract); + } + + if (method == STORED_METHOD) { + if (compressedSize != uncompressedSize) { + throw new IOException(entryDesc + ": stored entries should have identical compressed and " + + "uncompressed sizes"); + } + skip(compressedSize, entryDesc + "skipping data"); + } else { + // No OS resources are actually allocated. + @SuppressWarnings("resource") DeflateInputStream deflater = new DeflateInputStream(); + long generatedBytes = 0; + byte[] deflated = new byte[1024]; + int readBytes; + CRC32 crc = new CRC32(); + while ((readBytes = deflater.read(deflated)) > 0) { + crc.update(deflated, 0, readBytes); + generatedBytes += readBytes; + } + int actualCrc32 = (int) crc.getValue(); + long consumedBytes = deflater.totalBytesRead; + if (flags == SIZE_MASKED_FLAG) { + long id = getUnsignedInt("reading footer marker"); + if (id != DATA_DESCRIPTOR_MARKER) { + throw new IOException(entryDesc + ": expected footer at " + Long.toHexString(pos - 4) + + ", but found " + Long.toHexString(id)); + } + byte[] footer = new byte[DATA_DESCRIPTOR_BUFFER_SIZE]; + readFully(footer, "reading footer"); + crc32 = (int) getUnsignedInt(footer, 0); + compressedSize = getUnsignedInt(footer, 4); + uncompressedSize = getUnsignedInt(footer, 8); + } + + if (consumedBytes != compressedSize) { + throw new IOException(entryDesc + ": amount of compressed data does not match value " + + "specified in the zip (specified: " + compressedSize + ", actual: " + consumedBytes + + ")"); + } + if (generatedBytes != uncompressedSize) { + throw new IOException(entryDesc + ": amount of uncompressed data does not match value " + + "specified in the zip (specified: " + uncompressedSize + ", actual: " + + generatedBytes + ")"); + } + if (crc32 != actualCrc32) { + throw new IOException(entryDesc + ": specified crc checksum does not match actual check " + + "sum"); + } + } + entries.add(new Entry(entrypos, name, flags, method, dosTime)); + } + + @SuppressWarnings("unused") // A couple of unused local variables. + private void validateCentralDirectoryEntry(Entry entry) throws IOException { + long entrypos = pos - 4; + String entryDesc = "file directory entry '" + entry.name + "' at " + Long.toHexString(entrypos); + + byte[] entryBuffer = new byte[DIRECTORY_ENTRY_BUFFER_SIZE]; + readFully(entryBuffer, "reading central directory entry"); + int versionMadeBy = getUnsignedShort(entryBuffer, 0); + int versionToExtract = getUnsignedShort(entryBuffer, 2); + int flags = getUnsignedShort(entryBuffer, 4); + int method = getUnsignedShort(entryBuffer, 6); + int dosTime = (int) getUnsignedInt(entryBuffer, 8); + int crc32 = (int) getUnsignedInt(entryBuffer, 12); + long compressedSize = getUnsignedInt(entryBuffer, 16); + long uncompressedSize = getUnsignedInt(entryBuffer, 20); + int filenameLength = getUnsignedShort(entryBuffer, 24); + int extraLength = getUnsignedShort(entryBuffer, 26); + int commentLength = getUnsignedShort(entryBuffer, 28); + int diskNumberStart = getUnsignedShort(entryBuffer, 30); + int internalAttributes = getUnsignedShort(entryBuffer, 32); + int externalAttributes = (int) getUnsignedInt(entryBuffer, 34); + long offset = getUnsignedInt(entryBuffer, 38); + + byte[] filename = new byte[filenameLength]; + readFully(filename, "reading file name"); + skip(extraLength, "skipping extra data"); + String name = new String(filename, "UTF-8"); + + if (!name.equals(entry.name)) { + throw new IOException(entryDesc + ": file name in central directory does not match original " + + "name"); + } + if (offset != entry.pos) { + throw new IOException(entryDesc); + } + if (flags != entry.flags) { + throw new IOException(entryDesc); + } + if (method != entry.method) { + throw new IOException(entryDesc); + } + if (dosTime != entry.dosTime) { + throw new IOException(entryDesc); + } + } + + private void validateCentralDirectory() throws IOException { + boolean first = true; + for (Entry entry : entries) { + if (first) { + first = false; + } else { + long id = getUnsignedInt("reading marker"); + if (id != CENTRAL_DIRECTORY_MARKER) { + throw new IOException(); + } + } + validateCentralDirectoryEntry(entry); + } + } + + @SuppressWarnings("unused") // A couple of unused local variables. + private void validateEndOfCentralDirectory() throws IOException { + long id = getUnsignedInt("expecting end of central directory"); + byte[] entryBuffer = new byte[END_OF_CENTRAL_DIRECTORY_BUFFER_SIZE]; + readFully(entryBuffer, "reading end of central directory"); + int diskNumber = getUnsignedShort(entryBuffer, 0); + int startDiskNumber = getUnsignedShort(entryBuffer, 2); + int numEntries = getUnsignedShort(entryBuffer, 4); + int numTotalEntries = getUnsignedShort(entryBuffer, 6); + long centralDirectorySize = getUnsignedInt(entryBuffer, 8); + long centralDirectoryOffset = getUnsignedInt(entryBuffer, 12); + int commentLength = getUnsignedShort(entryBuffer, 16); + if (diskNumber != 0) { + throw new IOException(String.format("diskNumber=%d", diskNumber)); + } + if (startDiskNumber != 0) { + throw new IOException(String.format("startDiskNumber=%d", diskNumber)); + } + if (numEntries != numTotalEntries) { + throw new IOException(String.format("numEntries=%d numTotalEntries=%d", + numEntries, numTotalEntries)); + } + if (numEntries != (entries.size() % 0x10000)) { + throw new IOException("bad number of entries in central directory footer"); + } + if (numTotalEntries != (entries.size() % 0x10000)) { + throw new IOException("bad number of entries in central directory footer"); + } + if (commentLength != 0) { + throw new IOException("Zip file comment is unexpected"); + } + if (id != END_OF_CENTRAL_DIRECTORY_MARKER) { + throw new IOException("Expected end of central directory marker"); + } + } + + public void validate() throws IOException { + while (true) { + long id = getUnsignedInt("reading marker"); + if (id == LOCAL_FILE_HEADER_MARKER) { + readEntry(); + } else if (id == CENTRAL_DIRECTORY_MARKER) { + validateCentralDirectory(); + validateEndOfCentralDirectory(); + return; + } else { + throw new IOException("unexpected result for marker: " + + Long.toHexString(id) + " at position " + Long.toHexString(pos - 4)); + } + } + } +} |