diff options
author | Liam Miller-Cushon <cushon@google.com> | 2015-12-03 18:23:09 +0000 |
---|---|---|
committer | Kristina Chodorow <kchodorow@google.com> | 2015-12-03 18:39:00 +0000 |
commit | 220be7a7362bb27f3a23b2328b36d1a55f6f9197 (patch) | |
tree | 559e6c9a458305c84b28c191265413cf2c5c2662 /third_party | |
parent | 188d8eb7838ecc8aeaa9ba146908e2ad2db7de03 (diff) |
Add zip64 support to ijar
This allows ijar to process jars with >65535 entries.
--
MOS_MIGRATED_REVID=109321374
Diffstat (limited to 'third_party')
-rw-r--r-- | third_party/ijar/common.h | 12 | ||||
-rw-r--r-- | third_party/ijar/test/BUILD | 34 | ||||
-rw-r--r-- | third_party/ijar/test/GenZipWithEntries.java | 64 | ||||
-rw-r--r-- | third_party/ijar/test/ZipCount.java | 36 | ||||
-rwxr-xr-x | third_party/ijar/test/ijar_test.sh | 21 | ||||
-rw-r--r-- | third_party/ijar/zip.cc | 258 |
6 files changed, 384 insertions, 41 deletions
diff --git a/third_party/ijar/common.h b/third_party/ijar/common.h index f7df14bb75..513e0019a9 100644 --- a/third_party/ijar/common.h +++ b/third_party/ijar/common.h @@ -59,6 +59,13 @@ inline u4 get_u4le(const u1 *&p) { return x; } +inline u8 get_u8le(const u1 *&p) { + u4 lo = get_u4le(p); + u4 hi = get_u4le(p); + u8 x = ((u8)hi << 32) | lo; + return x; +} + inline void put_u1(u1 *&p, u1 x) { *p++ = x; } @@ -87,6 +94,11 @@ inline void put_u4le(u1 *&p, u4 x) { *p++ = x >> 24; } +inline void put_u8le(u1 *&p, u8 x) { + put_u4le(p, x & 0xffffffff); + put_u4le(p, (x >> 32) & 0xffffffff); +} + // Copy n bytes from src to p, and advance p. inline void put_n(u1 *&p, const u1 *src, size_t n) { memcpy(p, src, n); diff --git a/third_party/ijar/test/BUILD b/third_party/ijar/test/BUILD index 05bca7c100..15ebdb37ea 100644 --- a/third_party/ijar/test/BUILD +++ b/third_party/ijar/test/BUILD @@ -18,6 +18,7 @@ sh_test( # We assume unzip and zip to be on the path "unzip", "zip", + "$(location :zip_count)", ], data = [ "testenv.sh", @@ -31,6 +32,10 @@ sh_test( ":libtypeannotations2.jar", ":libmethodparameters.jar", ":source_debug_extension.jar", + ":largest_regular.jar", + ":smallest_zip64.jar", + ":definitely_zip64.jar", + ":zip_count", "TypeAnnotationTest2.java", # invokedynamic/ClassWithLambda.java, compiled with javac8 ":libinvokedynamic.jar", @@ -151,3 +156,32 @@ test_suite( testonly = 1, visibility = ["//visibility:public"], ) + +java_binary( + name = "gen_zip_with_entries", + srcs = ["GenZipWithEntries.java"], + jvm_flags = ["-Djdk.util.zip.inhibitZip64=false"], + main_class = "test.GenZipWithEntries", + deps = ["//third_party:asm"], +) + +[ + genrule( + name = name, + outs = [name + ".jar"], + cmd = "$(location :gen_zip_with_entries) %s $@" % entries, + tools = [":gen_zip_with_entries"], + ) + for name, entries in [ + ("largest_regular", 65535), + ("smallest_zip64", 65536), + ("definitely_zip64", 70000), + ] +] + +java_binary( + name = "zip_count", + srcs = ["ZipCount.java"], + jvm_flags = ["-Djdk.util.zip.inhibitZip64=false"], + main_class = "test.ZipCount", +) diff --git a/third_party/ijar/test/GenZipWithEntries.java b/third_party/ijar/test/GenZipWithEntries.java new file mode 100644 index 0000000000..1654e7abc3 --- /dev/null +++ b/third_party/ijar/test/GenZipWithEntries.java @@ -0,0 +1,64 @@ +// Copyright 2015 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test; + +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.jar.JarOutputStream; +import java.util.zip.ZipEntry; + +/** + * Generates a jar file with the specified number of .class entries. + * + * usage: GenZipWithEntries <number of zip entries> <output file> + */ +public class GenZipWithEntries { + + public static void main(String[] args) throws IOException { + int entries = Integer.parseInt(args[0]); + Path out = Paths.get(args[1]); + try (OutputStream os = Files.newOutputStream(out); + JarOutputStream jos = new JarOutputStream(os)) { + for (int i = 1; i <= entries; i++) { + String name = String.format("Test%d", i); + jos.putNextEntry(new ZipEntry(String.format("%s.class", name))); + jos.write(dump(name)); + } + } + } + + public static byte[] dump(String name) { + ClassWriter cw = new ClassWriter(0); + cw.visit(52, Opcodes.ACC_PUBLIC + Opcodes.ACC_SUPER, name, null, "java/lang/Object", null); + { + MethodVisitor mv = cw.visitMethod(Opcodes.ACC_PUBLIC, "<init>", "()V", null, null); + mv.visitCode(); + mv.visitVarInsn(Opcodes.ALOAD, 0); + mv.visitMethodInsn(Opcodes.INVOKESPECIAL, "java/lang/Object", "<init>", "()V", false); + mv.visitInsn(Opcodes.RETURN); + mv.visitMaxs(1, 1); + mv.visitEnd(); + } + cw.visitEnd(); + return cw.toByteArray(); + } +} diff --git a/third_party/ijar/test/ZipCount.java b/third_party/ijar/test/ZipCount.java new file mode 100644 index 0000000000..c72e8498f6 --- /dev/null +++ b/third_party/ijar/test/ZipCount.java @@ -0,0 +1,36 @@ +// Copyright 2015 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test; + +import java.io.IOException; +import java.util.zip.ZipFile; + +/** + * Checks the number of entries in a zip file. + * + * <p>usage: ZipCount <zip file> <expected entries> + */ +public class ZipCount { + public static void main(String[] args) throws IOException { + int expect = Integer.parseInt(args[1]); + int count; + try (ZipFile zf = new ZipFile(args[0])) { + count = zf.size(); + } + if (count != expect) { + throw new AssertionError(String.format("expected %d entries, saw %d", expect, count)); + } + } +} diff --git a/third_party/ijar/test/ijar_test.sh b/third_party/ijar/test/ijar_test.sh index 6f1a241df9..93f858690c 100755 --- a/third_party/ijar/test/ijar_test.sh +++ b/third_party/ijar/test/ijar_test.sh @@ -35,6 +35,8 @@ UNZIP=$1 shift ZIP=$1 shift +ZIP_COUNT=$1 +shift ## Test framework source ${DIR}/testenv.sh || { echo "testenv.sh not found!" >&2; exit 1; } @@ -48,6 +50,7 @@ source ${DIR}/testenv.sh || { echo "testenv.sh not found!" >&2; exit 1; } [[ "$UNZIP" =~ ^(/|[^/]+$) ]] || UNZIP="$PWD/$UNZIP" [[ "$ZIP" =~ ^(/|[^/]+$) ]] || ZIP="$PWD/$ZIP" [[ "$JAVAP" =~ ^(/|[^/]+$) ]] || JAVAP="$PWD/$JAVAP" +[[ "$ZIP_COUNT" =~ ^(/|[^/]+$) ]] || ZIP_COUNT="$PWD/$ZIP_COUNT" IJAR_SRCDIR=$(dirname ${IJAR}) A_JAR=$TEST_TMPDIR/A.jar @@ -70,6 +73,9 @@ METHODPARAM_JAR=$IJAR_SRCDIR/test/libmethodparameters.jar METHODPARAM_IJAR=$TEST_TMPDIR/methodparameters_interface.jar SOURCEDEBUGEXT_JAR=$IJAR_SRCDIR/test/source_debug_extension.jar SOURCEDEBUGEXT_IJAR=$TEST_TMPDIR/source_debug_extension.jar +CENTRAL_DIR_LARGEST_REGULAR=$IJAR_SRCDIR/test/largest_regular.jar +CENTRAL_DIR_SMALLEST_ZIP64=$IJAR_SRCDIR/test/smallest_zip64.jar +CENTRAL_DIR_ZIP64=$IJAR_SRCDIR/test/definitely_zip64.jar #### Setup @@ -515,6 +521,19 @@ function test_source_debug_extension_attribute() { expect_not_log "SourceDebugExtension" "SourceDebugExtension preserved!" } -SOURCEDEBUGEXT_JAR=$IJAR_SRCDIR/test/source_debug_extension.jar +function test_central_dir_largest_regular() { + $IJAR $CENTRAL_DIR_LARGEST_REGULAR $TEST_TMPDIR/ijar.jar || fail "ijar failed" + $ZIP_COUNT $TEST_TMPDIR/ijar.jar 65535 || fail +} + +function test_central_dir_smallest_zip64() { + $IJAR $CENTRAL_DIR_SMALLEST_ZIP64 $TEST_TMPDIR/ijar.jar || fail "ijar failed" + $ZIP_COUNT $TEST_TMPDIR/ijar.jar 65536 || fail +} + +function test_central_dir_zip64() { + $IJAR $CENTRAL_DIR_ZIP64 $TEST_TMPDIR/ijar.jar || fail "ijar failed" + $ZIP_COUNT $TEST_TMPDIR/ijar.jar 70000 || fail +} run_suite "ijar tests" diff --git a/third_party/ijar/zip.cc b/third_party/ijar/zip.cc index b06e7cf38e..3c83b92b05 100644 --- a/third_party/ijar/zip.cc +++ b/third_party/ijar/zip.cc @@ -39,10 +39,20 @@ #include "third_party/ijar/zip.h" #include <zlib.h> -#define LOCAL_FILE_HEADER_SIGNATURE 0x04034b50 -#define CENTRAL_FILE_HEADER_SIGNATURE 0x02014b50 -#define END_OF_CENTRAL_DIR_SIGNATURE 0x06054b50 -#define DATA_DESCRIPTOR_SIGNATURE 0x08074b50 +#define LOCAL_FILE_HEADER_SIGNATURE 0x04034b50 +#define CENTRAL_FILE_HEADER_SIGNATURE 0x02014b50 +#define DIGITAL_SIGNATURE 0x05054b50 +#define ZIP64_EOCD_SIGNATURE 0x06064b50 +#define ZIP64_EOCD_LOCATOR_SIGNATURE 0x07064b50 +#define EOCD_SIGNATURE 0x06054b50 +#define DATA_DESCRIPTOR_SIGNATURE 0x08074b50 + +#define U2_MAX 0xffff +#define U4_MAX 0xffffffffUL + +#define ZIP64_EOCD_LOCATOR_SIZE 20 +// zip64 eocd is fixed size in the absence of a zip64 extensible data sector +#define ZIP64_EOCD_FIXED_SIZE 56 // version to extract: 1.0 - default value from APPNOTE.TXT. // Output JAR files contain no extra ZIP features, so this is enough. @@ -63,14 +73,6 @@ namespace devtools_ijar { // http://www.info-zip.org/FAQ.html#limits static const u8 kMaximumOutputSize = std::numeric_limits<uint32_t>::max(); -static bool ProcessCentralDirEntry(const u1 *&p, - size_t *compressed_size, - size_t *uncompressed_size, - char *filename, - size_t filename_size, - u4 *attr, - u4 *offset); - // // A class representing a ZipFile for reading. Its public API is exposed // using the ZipExtractor abstract class. @@ -96,6 +98,11 @@ class InputZipFile : public ZipExtractor { virtual u8 CalculateOutputLength(); + virtual bool ProcessCentralDirEntry(const u1 *&p, size_t *compressed_size, + size_t *uncompressed_size, char *filename, + size_t filename_size, u4 *attr, + u4 *offset); + private: ZipExtractorProcessor *processor; @@ -551,11 +558,17 @@ int InputZipFile::ProcessFile(const bool compressed) { // Of course, in the latter case, the size output variables are not changed. // Note that the central directory is always followed by another data structure // that has a signature, so parsing it this way is safe. -static bool ProcessCentralDirEntry( - const u1 *&p, size_t *compressed_size, size_t *uncompressed_size, - char *filename, size_t filename_size, u4 *attr, u4 *offset) { +bool InputZipFile::ProcessCentralDirEntry(const u1 *&p, size_t *compressed_size, + size_t *uncompressed_size, + char *filename, size_t filename_size, + u4 *attr, u4 *offset) { u4 signature = get_u4le(p); + if (signature != CENTRAL_FILE_HEADER_SIGNATURE) { + if (signature != DIGITAL_SIGNATURE && signature != EOCD_SIGNATURE && + signature != ZIP64_EOCD_SIGNATURE) { + error("invalid central file header signature: 0x%x\n", signature); + } return false; } @@ -617,10 +630,128 @@ u8 InputZipFile::CalculateOutputLength() { + (uncompressed_size - compressed_size); } +// An end of central directory record, sized for optional zip64 contents. +struct EndOfCentralDirectoryRecord { + u4 number_of_this_disk; + u4 disk_with_central_dir; + u8 central_dir_entries_on_this_disk; + u8 central_dir_entries; + u8 central_dir_size; + u8 central_dir_offset; +}; + +// Checks for a zip64 end of central directory record. If a valid zip64 EOCD is +// found, updates the original EOCD record and returns true. +bool MaybeReadZip64CentralDirectory(const u1 *bytes, size_t in_length, + const u1 *current, + const u1 **end_of_central_dir, + EndOfCentralDirectoryRecord *cd) { + if (current < bytes) { + return false; + } + const u1 *candidate = current; + u4 zip64_directory_signature = get_u4le(current); + if (zip64_directory_signature != ZIP64_EOCD_SIGNATURE) { + return false; + } + + // size of zip64 end of central directory record + // (fixed size unless there's a zip64 extensible data sector, which + // we don't need to read) + get_u8le(current); + get_u2be(current); // version made by + get_u2be(current); // version needed to extract + + u4 number_of_this_disk = get_u4be(current); + u4 disk_with_central_dir = get_u4le(current); + u8 central_dir_entries_on_this_disk = get_u8le(current); + u8 central_dir_entries = get_u8le(current); + u8 central_dir_size = get_u8le(current); + u8 central_dir_offset = get_u8le(current); + + // check for a zip64 EOCD that matches the regular EOCD + if (number_of_this_disk != cd->number_of_this_disk && + cd->number_of_this_disk != U2_MAX) { + return false; + } + if (disk_with_central_dir != cd->disk_with_central_dir && + cd->disk_with_central_dir != U2_MAX) { + return false; + } + if (central_dir_entries_on_this_disk != + cd->central_dir_entries_on_this_disk && + cd->central_dir_entries_on_this_disk != U2_MAX) { + return false; + } + if (central_dir_entries != cd->central_dir_entries && + cd->central_dir_entries != U2_MAX) { + return false; + } + if (central_dir_size != cd->central_dir_size && + cd->central_dir_size != U4_MAX) { + return false; + } + if (central_dir_offset != cd->central_dir_offset && + cd->central_dir_offset != U4_MAX) { + return false; + } + + *end_of_central_dir = candidate; + cd->number_of_this_disk = number_of_this_disk; + cd->disk_with_central_dir = disk_with_central_dir; + cd->central_dir_entries_on_this_disk = central_dir_entries_on_this_disk; + cd->central_dir_entries = central_dir_entries; + cd->central_dir_size = central_dir_size; + cd->central_dir_offset = central_dir_offset; + return true; +} + +// Starting from the end of central directory record, attempts to locate a zip64 +// end of central directory record. If found, updates the given record and +// offset with the zip64 data. Returns false on error. +bool FindZip64CentralDirectory(const u1 *bytes, size_t in_length, + const u1 **end_of_central_dir, + EndOfCentralDirectoryRecord *cd) { + // In the absence of a zip64 extensible data sector, the zip64 EOCD is at a + // fixed offset from the regular central directory. + if (MaybeReadZip64CentralDirectory( + bytes, in_length, + *end_of_central_dir - ZIP64_EOCD_LOCATOR_SIZE - ZIP64_EOCD_FIXED_SIZE, + end_of_central_dir, cd)) { + return true; + } + + // If we couldn't find a zip64 EOCD at a fixed offset, either it doesn't exist + // or there was a zip64 extensible data sector, so try going through the + // locator. This approach doesn't work if data was prepended to the archive + // without updating the offset in the locator. + const u1 *zip64_locator = *end_of_central_dir - ZIP64_EOCD_LOCATOR_SIZE; + if (zip64_locator - ZIP64_EOCD_FIXED_SIZE < bytes) { + return true; + } + u4 zip64_locator_signature = get_u4le(zip64_locator); + if (zip64_locator_signature != ZIP64_EOCD_LOCATOR_SIGNATURE) { + return true; + } + u4 disk_with_zip64_central_directory = get_u4le(zip64_locator); + u8 zip64_end_of_central_dir_offset = get_u8le(zip64_locator); + u4 zip64_total_disks = get_u4le(zip64_locator); + if (MaybeReadZip64CentralDirectory(bytes, in_length, + bytes + zip64_end_of_central_dir_offset, + end_of_central_dir, cd)) { + if (disk_with_zip64_central_directory != 0 || zip64_total_disks != 1) { + fprintf(stderr, "multi-disk JAR files are not supported\n"); + return false; + } + return true; + } + return true; +} + // Given the data in the zip file, returns the offset of the central directory // and the number of files contained in it. -bool FindZipCentralDirectory(const u1* bytes, size_t in_length, - u4* offset, const u1** central_dir) { +bool FindZipCentralDirectory(const u1 *bytes, size_t in_length, u4 *offset, + const u1 **central_dir) { static const int MAX_COMMENT_LENGTH = 0xffff; static const int CENTRAL_DIR_LOCATOR_SIZE = 22; // Maximum distance of start of central dir locator from end of file @@ -635,7 +766,7 @@ bool FindZipCentralDirectory(const u1* bytes, size_t in_length, current >= last_pos_to_check; current-- ) { const u1* p = current; - if (get_u4le(p) != END_OF_CENTRAL_DIR_SIGNATURE) { + if (get_u4le(p) != EOCD_SIGNATURE) { continue; } @@ -659,30 +790,34 @@ bool FindZipCentralDirectory(const u1* bytes, size_t in_length, return false; } + EndOfCentralDirectoryRecord cd; const u1* end_of_central_dir = current; get_u4le(current); // central directory locator signature, already checked - u2 number_of_this_disk = get_u2le(current); - u2 disk_with_central_dir = get_u2le(current); - u2 central_dir_entries_on_this_disk = get_u2le(current); - u2 central_dir_entries = get_u2le(current); - u4 central_dir_size = get_u4le(current); - u4 central_dir_offset = get_u4le(current); + cd.number_of_this_disk = get_u2le(current); + cd.disk_with_central_dir = get_u2le(current); + cd.central_dir_entries_on_this_disk = get_u2le(current); + cd.central_dir_entries = get_u2le(current); + cd.central_dir_size = get_u4le(current); + cd.central_dir_offset = get_u4le(current); u2 file_comment_length = get_u2le(current); current += file_comment_length; // set current to the end of the central dir - if (number_of_this_disk != 0 - || disk_with_central_dir != 0 - || central_dir_entries_on_this_disk != central_dir_entries) { + if (!FindZip64CentralDirectory(bytes, in_length, &end_of_central_dir, &cd)) { + return false; + } + + if (cd.number_of_this_disk != 0 || cd.disk_with_central_dir != 0 || + cd.central_dir_entries_on_this_disk != cd.central_dir_entries) { fprintf(stderr, "multi-disk JAR files are not supported\n"); return false; } // Do not change output values before determining that they are OK. - *offset = central_dir_offset; + *offset = cd.central_dir_offset; // Central directory start can then be used to determine the actual // starts of the zip file (which can be different in case of a non-zip // header like for auto-extractable binaries). - *central_dir = end_of_central_dir - central_dir_size; + *central_dir = end_of_central_dir - cd.central_dir_size; return true; } @@ -821,17 +956,60 @@ void OutputZipFile::WriteCentralDirectory() { put_n(q, entry->file_name, entry->file_name_length); put_n(q, entry->extra_field, entry->extra_field_length); } - u4 central_directory_size = q - central_directory_start; - - put_u4le(q, END_OF_CENTRAL_DIR_SIGNATURE); - put_u2le(q, 0); // number of this disk - put_u2le(q, 0); // number of the disk with the start of the central directory - put_u2le(q, entries_.size()); // # central dir entries on this disk - put_u2le(q, entries_.size()); // total # entries in the central directory - put_u4le(q, central_directory_size); // size of the central directory - put_u4le(q, Offset(central_directory_start)); // offset of start of central - // directory wrt starting disk - put_u2le(q, 0); // .ZIP file comment length + u8 central_directory_size = q - central_directory_start; + + if (entries_.size() > U2_MAX || central_directory_size > U4_MAX || + Offset(central_directory_start) > U4_MAX) { + u1 *zip64_end_of_central_directory_start = q; + + put_u4le(q, ZIP64_EOCD_SIGNATURE); + // signature and size field doesn't count towards size + put_u8le(q, ZIP64_EOCD_FIXED_SIZE - 12); + put_u2le(q, 0); // version made by + put_u2le(q, 0); // version needed to extract + put_u4le(q, 0); // number of this disk + put_u4le(q, 0); // # of the disk with the start of the central directory + put_u8le(q, entries_.size()); // # central dir entries on this disk + put_u8le(q, entries_.size()); // total # entries in the central directory + put_u8le(q, central_directory_size); // size of the central directory + // offset of start of central directory wrt starting disk + put_u8le(q, Offset(central_directory_start)); + + put_u4le(q, ZIP64_EOCD_LOCATOR_SIGNATURE); + // number of the disk with the start of the zip64 end of central directory + put_u4le(q, 0); + // relative offset of the zip64 end of central directory record + put_u8le(q, Offset(zip64_end_of_central_directory_start)); + // total number of disks + put_u4le(q, 1); + + put_u4le(q, EOCD_SIGNATURE); + put_u2le(q, 0); // number of this disk + put_u2le(q, 0); // # of disk with the start of the central directory + // # central dir entries on this disk + put_u2le(q, entries_.size() > 0xffff ? 0xffff : entries_.size()); + // total # entries in the central directory + put_u2le(q, entries_.size() > 0xffff ? 0xffff : entries_.size()); + // size of the central directory + put_u4le(q, + central_directory_size > U4_MAX ? U4_MAX : central_directory_size); + // offset of start of central + put_u4le(q, Offset(central_directory_start) > U4_MAX + ? U4_MAX + : Offset(central_directory_start)); + put_u2le(q, 0); // .ZIP file comment length + + } else { + put_u4le(q, EOCD_SIGNATURE); + put_u2le(q, 0); // number of this disk + put_u2le(q, 0); // # of the disk with the start of the central directory + put_u2le(q, entries_.size()); // # central dir entries on this disk + put_u2le(q, entries_.size()); // total # entries in the central directory + put_u4le(q, central_directory_size); // size of the central directory + // offset of start of central directory wrt starting disk + put_u4le(q, Offset(central_directory_start)); + put_u2le(q, 0); // .ZIP file comment length + } } u1* OutputZipFile::WriteLocalFileHeader(const char* filename, const u4 attr) { |