From c731c28ae539cb0ff1eac2f32c538ff4f99c00f4 Mon Sep 17 00:00:00 2001 From: sherman Date: Thu, 2 Apr 2009 15:35:46 -0700 Subject: [PATCH] 4681995: Add support for large (> 4GB) zip/jar files Summary: The ZIP64 format support is added for > 4GB jar/zip files Reviewed-by: alanb, martin --- .../classes/java/util/zip/ZipConstants64.java | 77 +++++++ src/share/classes/java/util/zip/ZipEntry.java | 8 +- .../classes/java/util/zip/ZipInputStream.java | 76 +++++-- .../java/util/zip/ZipOutputStream.java | 174 +++++++++++++--- src/share/classes/java/util/zip/package.html | 8 +- src/share/native/java/util/zip/zip_util.c | 104 +++++++++- src/share/native/java/util/zip/zip_util.h | 38 +++- .../native/java/util/zip/zlib-1.1.3/zlib.h | 4 +- test/java/util/zip/LargeZip.java | 193 ++++++++++++++++++ test/java/util/zip/ZipFile/LargeZipFile.java | 1 - 10 files changed, 629 insertions(+), 54 deletions(-) create mode 100644 src/share/classes/java/util/zip/ZipConstants64.java create mode 100644 test/java/util/zip/LargeZip.java diff --git a/src/share/classes/java/util/zip/ZipConstants64.java b/src/share/classes/java/util/zip/ZipConstants64.java new file mode 100644 index 000000000..1bf3b1841 --- /dev/null +++ b/src/share/classes/java/util/zip/ZipConstants64.java @@ -0,0 +1,77 @@ +/* + * Copyright 1995-1996 Sun Microsystems, Inc. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Sun designates this + * particular file as subject to the "Classpath" exception as provided + * by Sun in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, + * CA 95054 USA or visit www.sun.com if you need additional information or + * have any questions. + */ + +package java.util.zip; + +/* + * This class defines the constants that are used by the classes + * which manipulate Zip64 files. + */ + +class ZipConstants64 { + + /* + * ZIP64 constants + */ + static final long ZIP64_ENDSIG = 0x06064b50L; // "PK\006\006" + static final long ZIP64_LOCSIG = 0x07064b50L; // "PK\006\007" + static final int ZIP64_ENDHDR = 56; // ZIP64 end header size + static final int ZIP64_LOCHDR = 20; // ZIP64 end loc header size + static final int ZIP64_EXTHDR = 24; // EXT header size + static final int ZIP64_EXTID = 0x0001; // Extra field Zip64 header ID + + static final int ZIP64_MAGICCOUNT = 0xFFFF; + static final long ZIP64_MAGICVAL = 0xFFFFFFFFL; + + /* + * Zip64 End of central directory (END) header field offsets + */ + static final int ZIP64_ENDLEN = 4; // size of zip64 end of central dir + static final int ZIP64_ENDVEM = 12; // version made by + static final int ZIP64_ENDVER = 14; // version needed to extract + static final int ZIP64_ENDNMD = 16; // number of this disk + static final int ZIP64_ENDDSK = 20; // disk number of start + static final int ZIP64_ENDTOD = 24; // total number of entries on this disk + static final int ZIP64_ENDTOT = 32; // total number of entries + static final int ZIP64_ENDSIZ = 40; // central directory size in bytes + static final int ZIP64_ENDOFF = 48; // offset of first CEN header + static final int ZIP64_ENDEXT = 56; // zip64 extensible data sector + + /* + * Zip64 End of central directory locator field offsets + */ + static final int ZIP64_LOCDSK = 4; // disk number start + static final int ZIP64_LOCOFF = 8; // offset of zip64 end + static final int ZIP64_LOCTOT = 16; // total number of disks + + /* + * Zip64 Extra local (EXT) header field offsets + */ + static final int ZIP64_EXTCRC = 4; // uncompressed file crc-32 value + static final int ZIP64_EXTSIZ = 8; // compressed size, 8-byte + static final int ZIP64_EXTLEN = 16; // uncompressed size, 8-byte + + private ZipConstants64() {} +} diff --git a/src/share/classes/java/util/zip/ZipEntry.java b/src/share/classes/java/util/zip/ZipEntry.java index bcc27e63a..6c16a9adb 100644 --- a/src/share/classes/java/util/zip/ZipEntry.java +++ b/src/share/classes/java/util/zip/ZipEntry.java @@ -1,5 +1,5 @@ /* - * Copyright 1995-2005 Sun Microsystems, Inc. All Rights Reserved. + * Copyright 1995-2009 Sun Microsystems, Inc. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -144,11 +144,13 @@ class ZipEntry implements ZipConstants, Cloneable { * Sets the uncompressed size of the entry data. * @param size the uncompressed size in bytes * @exception IllegalArgumentException if the specified size is less - * than 0 or greater than 0xFFFFFFFF bytes + * than 0, is greater than 0xFFFFFFFF when + * ZIP64 format is not supported, + * or is less than 0 when ZIP64 is supported * @see #getSize() */ public void setSize(long size) { - if (size < 0 || size > 0xFFFFFFFFL) { + if (size < 0) { throw new IllegalArgumentException("invalid entry size"); } this.size = size; diff --git a/src/share/classes/java/util/zip/ZipInputStream.java b/src/share/classes/java/util/zip/ZipInputStream.java index e8f0ebdc5..1b9b93415 100644 --- a/src/share/classes/java/util/zip/ZipInputStream.java +++ b/src/share/classes/java/util/zip/ZipInputStream.java @@ -1,5 +1,5 @@ /* - * Copyright 1996-2006 Sun Microsystems, Inc. All Rights Reserved. + * Copyright 1996-2009 Sun Microsystems, Inc. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -29,6 +29,7 @@ import java.io.InputStream; import java.io.IOException; import java.io.EOFException; import java.io.PushbackInputStream; +import static java.util.zip.ZipConstants64.*; /** * This class implements an input stream filter for reading files in the @@ -285,6 +286,29 @@ class ZipInputStream extends InflaterInputStream implements ZipConstants { byte[] bb = new byte[len]; readFully(bb, 0, len); e.setExtra(bb); + // extra fields are in "HeaderID(2)DataSize(2)Data... format + if (e.csize == ZIP64_MAGICVAL || e.size == ZIP64_MAGICVAL) { + int off = 0; + while (off + 4 < len) { + int sz = get16(bb, off + 2); + if (get16(bb, off) == ZIP64_EXTID) { + off += 4; + // LOC extra zip64 entry MUST include BOTH original and + // compressed file size fields + if (sz < 16 || (off + sz) > len ) { + // Invalid zip64 extra fields, simply skip. Even it's + // rare, it's possible the entry size happens to be + // the magic value and it "accidnetly" has some bytes + // in extra match the id. + return e; + } + e.size = get64(bb, off); + e.csize = get64(bb, off + 8); + break; + } + off += (sz + 4); + } + } } return e; } @@ -375,18 +399,36 @@ class ZipInputStream extends InflaterInputStream implements ZipConstants { } if ((flag & 8) == 8) { /* "Data Descriptor" present */ - readFully(tmpbuf, 0, EXTHDR); - long sig = get32(tmpbuf, 0); - if (sig != EXTSIG) { // no EXTSIG present - e.crc = sig; - e.csize = get32(tmpbuf, EXTSIZ - EXTCRC); - e.size = get32(tmpbuf, EXTLEN - EXTCRC); - ((PushbackInputStream)in).unread( - tmpbuf, EXTHDR - EXTCRC - 1, EXTCRC); + if (inf.getBytesWritten() > ZIP64_MAGICVAL || + inf.getBytesRead() > ZIP64_MAGICVAL) { + // ZIP64 format + readFully(tmpbuf, 0, ZIP64_EXTHDR); + long sig = get32(tmpbuf, 0); + if (sig != EXTSIG) { // no EXTSIG present + e.crc = sig; + e.csize = get64(tmpbuf, ZIP64_EXTSIZ - ZIP64_EXTCRC); + e.size = get64(tmpbuf, ZIP64_EXTLEN - ZIP64_EXTCRC); + ((PushbackInputStream)in).unread( + tmpbuf, ZIP64_EXTHDR - ZIP64_EXTCRC - 1, ZIP64_EXTCRC); + } else { + e.crc = get32(tmpbuf, ZIP64_EXTCRC); + e.csize = get64(tmpbuf, ZIP64_EXTSIZ); + e.size = get64(tmpbuf, ZIP64_EXTLEN); + } } else { - e.crc = get32(tmpbuf, EXTCRC); - e.csize = get32(tmpbuf, EXTSIZ); - e.size = get32(tmpbuf, EXTLEN); + readFully(tmpbuf, 0, EXTHDR); + long sig = get32(tmpbuf, 0); + if (sig != EXTSIG) { // no EXTSIG present + e.crc = sig; + e.csize = get32(tmpbuf, EXTSIZ - EXTCRC); + e.size = get32(tmpbuf, EXTLEN - EXTCRC); + ((PushbackInputStream)in).unread( + tmpbuf, EXTHDR - EXTCRC - 1, EXTCRC); + } else { + e.crc = get32(tmpbuf, EXTCRC); + e.csize = get32(tmpbuf, EXTSIZ); + e.size = get32(tmpbuf, EXTLEN); + } } } if (e.size != inf.getBytesWritten()) { @@ -433,6 +475,14 @@ class ZipInputStream extends InflaterInputStream implements ZipConstants { * The bytes are assumed to be in Intel (little-endian) byte order. */ private static final long get32(byte b[], int off) { - return get16(b, off) | ((long)get16(b, off+2) << 16); + return (get16(b, off) | ((long)get16(b, off+2) << 16)) & 0xffffffffL; + } + + /* + * Fetches signed 64-bit value from byte array at specified offset. + * The bytes are assumed to be in Intel (little-endian) byte order. + */ + private static final long get64(byte b[], int off) { + return get32(b, off) | (get32(b, off+4) << 32); } } diff --git a/src/share/classes/java/util/zip/ZipOutputStream.java b/src/share/classes/java/util/zip/ZipOutputStream.java index 797a37392..bd44d3213 100644 --- a/src/share/classes/java/util/zip/ZipOutputStream.java +++ b/src/share/classes/java/util/zip/ZipOutputStream.java @@ -1,5 +1,5 @@ /* - * Copyright 1996-2008 Sun Microsystems, Inc. All Rights Reserved. + * Copyright 1996-2009 Sun Microsystems, Inc. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -29,6 +29,7 @@ import java.io.OutputStream; import java.io.IOException; import java.util.Vector; import java.util.HashSet; +import static java.util.zip.ZipConstants64.*; /** * This class implements an output stream filter for writing files in the @@ -343,26 +344,52 @@ class ZipOutputStream extends DeflaterOutputStream implements ZipConstants { private void writeLOC(XEntry xentry) throws IOException { ZipEntry e = xentry.entry; int flag = xentry.flag; + int elen = (e.extra != null) ? e.extra.length : 0; + boolean hasZip64 = false; + writeInt(LOCSIG); // LOC header signature - writeShort(version(e)); // version needed to extract - writeShort(flag); // general purpose bit flag - writeShort(e.method); // compression method - writeInt(e.time); // last modification time + if ((flag & 8) == 8) { + writeShort(version(e)); // version needed to extract + writeShort(flag); // general purpose bit flag + writeShort(e.method); // compression method + writeInt(e.time); // last modification time + // store size, uncompressed size, and crc-32 in data descriptor // immediately following compressed entry data writeInt(0); writeInt(0); writeInt(0); } else { - writeInt(e.crc); // crc-32 - writeInt(e.csize); // compressed size - writeInt(e.size); // uncompressed size + if (e.csize >= ZIP64_MAGICVAL || e.size >= ZIP64_MAGICVAL) { + hasZip64 = true; + writeShort(45); // ver 4.5 for zip64 + } else { + writeShort(version(e)); // version needed to extract + } + writeShort(flag); // general purpose bit flag + writeShort(e.method); // compression method + writeInt(e.time); // last modification time + writeInt(e.crc); // crc-32 + if (hasZip64) { + writeInt(ZIP64_MAGICVAL); + writeInt(ZIP64_MAGICVAL); + elen += 20; //headid(2) + size(2) + size(8) + csize(8) + } else { + writeInt(e.csize); // compressed size + writeInt(e.size); // uncompressed size + } } byte[] nameBytes = getUTF8Bytes(e.name); writeShort(nameBytes.length); - writeShort(e.extra != null ? e.extra.length : 0); + writeShort(elen); writeBytes(nameBytes, 0, nameBytes.length); + if (hasZip64) { + writeShort(ZIP64_EXTID); + writeShort(16); + writeLong(e.size); + writeLong(e.csize); + } if (e.extra != null) { writeBytes(e.extra, 0, e.extra.length); } @@ -375,8 +402,13 @@ class ZipOutputStream extends DeflaterOutputStream implements ZipConstants { private void writeEXT(ZipEntry e) throws IOException { writeInt(EXTSIG); // EXT header signature writeInt(e.crc); // crc-32 - writeInt(e.csize); // compressed size - writeInt(e.size); // uncompressed size + if (e.csize >= ZIP64_MAGICVAL || e.size >= ZIP64_MAGICVAL) { + writeLong(e.csize); + writeLong(e.size); + } else { + writeInt(e.csize); // compressed size + writeInt(e.size); // uncompressed size + } } /* @@ -387,18 +419,49 @@ class ZipOutputStream extends DeflaterOutputStream implements ZipConstants { ZipEntry e = xentry.entry; int flag = xentry.flag; int version = version(e); + + long csize = e.csize; + long size = e.size; + long offset = xentry.offset; + int e64len = 0; + boolean hasZip64 = false; + if (e.csize >= ZIP64_MAGICVAL) { + csize = ZIP64_MAGICVAL; + e64len += 8; // csize(8) + hasZip64 = true; + } + if (e.size >= ZIP64_MAGICVAL) { + size = ZIP64_MAGICVAL; // size(8) + e64len += 8; + hasZip64 = true; + } + if (xentry.offset >= ZIP64_MAGICVAL) { + offset = ZIP64_MAGICVAL; + e64len += 8; // offset(8) + hasZip64 = true; + } writeInt(CENSIG); // CEN header signature - writeShort(version); // version made by - writeShort(version); // version needed to extract + if (hasZip64) { + writeShort(45); // ver 4.5 for zip64 + writeShort(45); + } else { + writeShort(version); // version made by + writeShort(version); // version needed to extract + } writeShort(flag); // general purpose bit flag writeShort(e.method); // compression method writeInt(e.time); // last modification time writeInt(e.crc); // crc-32 - writeInt(e.csize); // compressed size - writeInt(e.size); // uncompressed size + writeInt(csize); // compressed size + writeInt(size); // uncompressed size byte[] nameBytes = getUTF8Bytes(e.name); writeShort(nameBytes.length); - writeShort(e.extra != null ? e.extra.length : 0); + if (hasZip64) { + // + headid(2) + datasize(2) + writeShort(e64len + 4 + (e.extra != null ? e.extra.length : 0)); + } else { + writeShort(e.extra != null ? e.extra.length : 0); + } byte[] commentBytes; if (e.comment != null) { commentBytes = getUTF8Bytes(e.comment); @@ -410,8 +473,18 @@ class ZipOutputStream extends DeflaterOutputStream implements ZipConstants { writeShort(0); // starting disk number writeShort(0); // internal file attributes (unused) writeInt(0); // external file attributes (unused) - writeInt(xentry.offset); // relative offset of local header + writeInt(offset); // relative offset of local header writeBytes(nameBytes, 0, nameBytes.length); + if (hasZip64) { + writeShort(ZIP64_EXTID);// Zip64 extra + writeShort(e64len); + if (size == ZIP64_MAGICVAL) + writeLong(e.size); + if (csize == ZIP64_MAGICVAL) + writeLong(e.csize); + if (offset == ZIP64_MAGICVAL) + writeLong(xentry.offset); + } if (e.extra != null) { writeBytes(e.extra, 0, e.extra.length); } @@ -424,15 +497,50 @@ class ZipOutputStream extends DeflaterOutputStream implements ZipConstants { * Writes end of central directory (END) header. */ private void writeEND(long off, long len) throws IOException { + boolean hasZip64 = false; + long xlen = len; + long xoff = off; + if (xlen >= ZIP64_MAGICVAL) { + xlen = ZIP64_MAGICVAL; + hasZip64 = true; + } + if (xoff >= ZIP64_MAGICVAL) { + xoff = ZIP64_MAGICVAL; + hasZip64 = true; + } int count = xentries.size(); - writeInt(ENDSIG); // END record signature - writeShort(0); // number of this disk - writeShort(0); // central directory start disk - writeShort(count); // number of directory entries on disk - writeShort(count); // total number of directory entries - writeInt(len); // length of central directory - writeInt(off); // offset of central directory - if (comment != null) { // zip file comment + if (count >= ZIP64_MAGICCOUNT) { + count = ZIP64_MAGICCOUNT; + hasZip64 = true; + } + if (hasZip64) { + long off64 = written; + //zip64 end of central directory record + writeInt(ZIP64_ENDSIG); // zip64 END record signature + writeLong(ZIP64_ENDHDR - 12); // size of zip64 end + writeShort(45); // version made by + writeShort(45); // version needed to extract + writeInt(0); // number of this disk + writeInt(0); // central directory start disk + writeLong(xentries.size()); // number of directory entires on disk + writeLong(xentries.size()); // number of directory entires + writeLong(len); // length of central directory + writeLong(off); // offset of central directory + + //zip64 end of central directory locator + writeInt(ZIP64_LOCSIG); // zip64 END locator signature + writeInt(0); // zip64 END start disk + writeLong(off64); // offset of zip64 END + writeInt(1); // total number of disks (?) + } + writeInt(ENDSIG); // END record signature + writeShort(0); // number of this disk + writeShort(0); // central directory start disk + writeShort(count); // number of directory entries on disk + writeShort(count); // total number of directory entries + writeInt(xlen); // length of central directory + writeInt(xoff); // offset of central directory + if (comment != null) { // zip file comment byte[] b = getUTF8Bytes(comment); writeShort(b.length); writeBytes(b, 0, b.length); @@ -463,6 +571,22 @@ class ZipOutputStream extends DeflaterOutputStream implements ZipConstants { written += 4; } + /* + * Writes a 64-bit int to the output stream in little-endian byte order. + */ + private void writeLong(long v) throws IOException { + OutputStream out = this.out; + out.write((int)((v >>> 0) & 0xff)); + out.write((int)((v >>> 8) & 0xff)); + out.write((int)((v >>> 16) & 0xff)); + out.write((int)((v >>> 24) & 0xff)); + out.write((int)((v >>> 32) & 0xff)); + out.write((int)((v >>> 40) & 0xff)); + out.write((int)((v >>> 48) & 0xff)); + out.write((int)((v >>> 56) & 0xff)); + written += 8; + } + /* * Writes an array of bytes to the output stream. */ diff --git a/src/share/classes/java/util/zip/package.html b/src/share/classes/java/util/zip/package.html index d32e0fa0f..d4b592636 100644 --- a/src/share/classes/java/util/zip/package.html +++ b/src/share/classes/java/util/zip/package.html @@ -45,6 +45,13 @@ input streams. Info-ZIP Application Note 970311 - a detailed description of the Info-ZIP format upon which the java.util.zip classes are based. +

+ +

  • An implementation may optionally support the ZIP64(tm) format extensions + defined by the + + PKWARE ZIP File Format Specification. The ZIP64(tm) format extensions + are used to overcome the size limitations of the original ZIP format.

  • ZLIB Compressed Data Format Specification version 3.3 @@ -70,7 +77,6 @@ input streams.
  • CRC-32 checksum is described in RFC 1952 (above)

  • Adler-32 checksum is described in RFC 1950 (above) - diff --git a/src/share/native/java/util/zip/zip_util.c b/src/share/native/java/util/zip/zip_util.c index f38162971..89f52ae3b 100644 --- a/src/share/native/java/util/zip/zip_util.c +++ b/src/share/native/java/util/zip/zip_util.c @@ -312,6 +312,38 @@ findEND(jzfile *zip, void *endbuf) return -1; /* END header not found */ } +/* + * Searches for the ZIP64 end of central directory (END) header. The + * contents of the ZIP64 END header will be read and placed in end64buf. + * Returns the file position of the ZIP64 END header, otherwise returns + * -1 if the END header was not found or an error occurred. + * + * The ZIP format specifies the "position" of each related record as + * ... + * [central directory] + * [zip64 end of central directory record] + * [zip64 end of central directory locator] + * [end of central directory record] + * + * The offset of zip64 end locator can be calculated from endpos as + * "endpos - ZIP64_LOCHDR". + * The "offset" of zip64 end record is stored in zip64 end locator. + */ +static jlong +findEND64(jzfile *zip, void *end64buf, jlong endpos) +{ + char loc64[ZIP64_LOCHDR]; + jlong end64pos; + if (readFullyAt(zip->zfd, loc64, ZIP64_LOCHDR, endpos - ZIP64_LOCHDR) == -1) { + return -1; // end64 locator not found + } + end64pos = ZIP64_LOCOFF(loc64); + if (readFullyAt(zip->zfd, end64buf, ZIP64_ENDHDR, end64pos) == -1) { + return -1; // end64 record not found + } + return end64pos; +} + /* * Returns a hash code value for a C-style NUL-terminated string. */ @@ -463,7 +495,7 @@ static jlong readCEN(jzfile *zip, jint knownTotal) { /* Following are unsigned 32-bit */ - jlong endpos, cenpos, cenlen; + jlong endpos, end64pos, cenpos, cenlen, cenoff; /* Following are unsigned 16-bit */ jint total, tablelen, i, j; unsigned char *cenbuf = NULL; @@ -474,6 +506,7 @@ readCEN(jzfile *zip, jint knownTotal) jlong offset; #endif unsigned char endbuf[ENDHDR]; + jint endhdrlen = ENDHDR; jzcell *entries; jint *table; @@ -490,13 +523,27 @@ readCEN(jzfile *zip, jint knownTotal) /* Get position and length of central directory */ cenlen = ENDSIZ(endbuf); + cenoff = ENDOFF(endbuf); + total = ENDTOT(endbuf); + if (cenlen == ZIP64_MAGICVAL || cenoff == ZIP64_MAGICVAL || + total == ZIP64_MAGICCOUNT) { + unsigned char end64buf[ZIP64_ENDHDR]; + if ((end64pos = findEND64(zip, end64buf, endpos)) != -1) { + cenlen = ZIP64_ENDSIZ(end64buf); + cenoff = ZIP64_ENDOFF(end64buf); + total = (jint)ZIP64_ENDTOT(end64buf); + endpos = end64pos; + endhdrlen = ZIP64_ENDHDR; + } + } + if (cenlen > endpos) ZIP_FORMAT_ERROR("invalid END header (bad central directory size)"); cenpos = endpos - cenlen; /* Get position of first local file (LOC) header, taking into * account that there may be a stub prefixed to the zip file. */ - zip->locpos = cenpos - ENDOFF(endbuf); + zip->locpos = cenpos - cenoff; if (zip->locpos < 0) ZIP_FORMAT_ERROR("invalid END header (bad central directory offset)"); @@ -527,7 +574,7 @@ readCEN(jzfile *zip, jint knownTotal) out the page size in order to make offset to be multiples of page size. */ - zip->mlen = cenpos - offset + cenlen + ENDHDR; + zip->mlen = cenpos - offset + cenlen + endhdrlen; zip->offset = offset; mappedAddr = mmap64(0, zip->mlen, PROT_READ, MAP_SHARED, zip->zfd, (off64_t) offset); zip->maddr = (mappedAddr == (void*) MAP_FAILED) ? NULL : @@ -551,8 +598,13 @@ readCEN(jzfile *zip, jint knownTotal) * is a 2-byte field, but we (and other zip implementations) * support approx. 2**31 entries, we do not trust ENDTOT, but * treat it only as a strong hint. When we call ourselves - * recursively, knownTotal will have the "true" value. */ - total = (knownTotal != -1) ? knownTotal : ENDTOT(endbuf); + * recursively, knownTotal will have the "true" value. + * + * Keep this path alive even with the Zip64 END support added, just + * for zip files that have more than 0xffff entries but don't have + * the Zip64 enabled. + */ + total = (knownTotal != -1) ? knownTotal : total; entries = zip->entries = calloc(total, sizeof(entries[0])); tablelen = zip->tablelen = ((total/2) | 1); // Odd -> fewer collisions table = zip->table = malloc(tablelen * sizeof(table[0])); @@ -854,6 +906,7 @@ typedef enum { ACCESS_RANDOM, ACCESS_SEQUENTIAL } AccessHint; static jzentry * newEntry(jzfile *zip, jzcell *zc, AccessHint accessHint) { + jlong locoff; jint nlen, elen, clen; jzentry *ze; char *cen; @@ -880,18 +933,55 @@ newEntry(jzfile *zip, jzcell *zc, AccessHint accessHint) ze->size = CENLEN(cen); ze->csize = (CENHOW(cen) == STORED) ? 0 : CENSIZ(cen); ze->crc = CENCRC(cen); - ze->pos = -(zip->locpos + CENOFF(cen)); + locoff = CENOFF(cen); + ze->pos = -(zip->locpos + locoff); if ((ze->name = malloc(nlen + 1)) == NULL) goto Catch; memcpy(ze->name, cen + CENHDR, nlen); ze->name[nlen] = '\0'; if (elen > 0) { + char *extra = cen + CENHDR + nlen; + /* This entry has "extra" data */ if ((ze->extra = malloc(elen + 2)) == NULL) goto Catch; ze->extra[0] = (unsigned char) elen; ze->extra[1] = (unsigned char) (elen >> 8); - memcpy(ze->extra+2, cen + CENHDR + nlen, elen); + memcpy(ze->extra+2, extra, elen); + if (ze->csize == ZIP64_MAGICVAL || ze->size == ZIP64_MAGICVAL || + locoff == ZIP64_MAGICVAL) { + jint off = 0; + while ((off + 4) < elen) { // spec: HeaderID+DataSize+Data + jint sz = SH(extra, off + 2); + if (SH(extra, off) == ZIP64_EXTID) { + off += 4; + if (ze->size == ZIP64_MAGICVAL) { + // if invalid zip64 extra fields, just skip + if (sz < 8 || (off + 8) > elen) + break; + ze->size = LL(extra, off); + sz -= 8; + off += 8; + } + if (ze->csize == ZIP64_MAGICVAL) { + if (sz < 8 || (off + 8) > elen) + break; + ze->csize = LL(extra, off); + sz -= 8; + off += 8; + } + if (locoff == ZIP64_MAGICVAL) { + if (sz < 8 || (off + 8) > elen) + break; + ze->pos = -(zip->locpos + LL(extra, off)); + sz -= 8; + off += 8; + } + break; + } + off += (sz + 4); + } + } } if (clen > 0) { diff --git a/src/share/native/java/util/zip/zip_util.h b/src/share/native/java/util/zip/zip_util.h index 3814ad02d..7ad11715e 100644 --- a/src/share/native/java/util/zip/zip_util.h +++ b/src/share/native/java/util/zip/zip_util.h @@ -38,9 +38,13 @@ #define CENSIG 0x02014b50L /* "PK\001\002" */ #define ENDSIG 0x06054b50L /* "PK\005\006" */ +#define ZIP64_ENDSIG 0x06064b50L /* "PK\006\006" */ +#define ZIP64_LOCSIG 0x07064b50L /* "PK\006\007" */ + /* * Header sizes including signatures */ + #ifdef USE_MMAP #define SIGSIZ 4 #endif @@ -49,12 +53,22 @@ #define CENHDR 46 #define ENDHDR 22 +#define ZIP64_ENDHDR 56 // ZIP64 end header size +#define ZIP64_LOCHDR 20 // ZIP64 end loc header size +#define ZIP64_EXTHDR 24 // EXT header size +#define ZIP64_EXTID 1 // Extra field Zip64 header ID + +#define ZIP64_MAGICVAL 0xffffffffLL +#define ZIP64_MAGICCOUNT 0xffff + + /* * Header field access macros */ #define CH(b, n) (((unsigned char *)(b))[n]) #define SH(b, n) (CH(b, n) | (CH(b, n+1) << 8)) -#define LG(b, n) (SH(b, n) | (SH(b, n+2) << 16)) +#define LG(b, n) ((SH(b, n) | (SH(b, n+2) << 16)) &0xffffffffUL) +#define LL(b, n) (((jlong)LG(b, n)) | (((jlong)LG(b, n+4)) << 32)) #define GETSIG(b) LG(b, 0) /* @@ -105,6 +119,26 @@ #define ENDOFF(b) LG(b, 16) /* central directory offset */ #define ENDCOM(b) SH(b, 20) /* size of zip file comment */ +/* + * Macros for getting Zip64 end of central directory header fields + */ +#define ZIP64_ENDLEN(b) LL(b, 4) /* size of zip64 end of central dir */ +#define ZIP64_ENDVEM(b) SH(b, 12) /* version made by */ +#define ZIP64_ENDVER(b) SH(b, 14) /* version needed to extract */ +#define ZIP64_ENDNMD(b) LG(b, 16) /* number of this disk */ +#define ZIP64_ENDDSK(b) LG(b, 20) /* disk number of start */ +#define ZIP64_ENDTOD(b) LL(b, 24) /* total number of entries on this disk */ +#define ZIP64_ENDTOT(b) LL(b, 32) /* total number of entries */ +#define ZIP64_ENDSIZ(b) LL(b, 40) /* central directory size in bytes */ +#define ZIP64_ENDOFF(b) LL(b, 48) /* offset of first CEN header */ + +/* + * Macros for getting Zip64 end of central directory locator fields + */ +#define ZIP64_LOCDSK(b) LG(b, 4) /* disk number start */ +#define ZIP64_LOCOFF(b) LL(b, 8) /* offset of zip64 end */ +#define ZIP64_LOCTOT(b) LG(b, 16) /* total number of disks */ + /* * Supported compression methods */ @@ -145,7 +179,7 @@ typedef struct jzentry { /* Zip file entry */ */ typedef struct jzcell { unsigned int hash; /* 32 bit hashcode on name */ - unsigned int cenpos; /* Offset of central directory file header */ + jlong cenpos; /* Offset of central directory file header */ unsigned int next; /* hash chain: index into jzfile->entries */ } jzcell; diff --git a/src/share/native/java/util/zip/zlib-1.1.3/zlib.h b/src/share/native/java/util/zip/zlib-1.1.3/zlib.h index 9576715b2..00fdf0600 100644 --- a/src/share/native/java/util/zip/zlib-1.1.3/zlib.h +++ b/src/share/native/java/util/zip/zlib-1.1.3/zlib.h @@ -106,11 +106,11 @@ struct internal_state; typedef struct z_stream_s { Bytef *next_in; /* next input byte */ uInt avail_in; /* number of bytes available at next_in */ - uLong total_in; /* total nb of input bytes read so far */ + long long total_in; /* total nb of input bytes read so far */ Bytef *next_out; /* next output byte should be put there */ uInt avail_out; /* remaining free space at next_out */ - uLong total_out; /* total nb of bytes output so far */ + long long total_out; /* total nb of bytes output so far */ char *msg; /* last error message, NULL if no error */ struct internal_state FAR *state; /* not visible by applications */ diff --git a/test/java/util/zip/LargeZip.java b/test/java/util/zip/LargeZip.java new file mode 100644 index 000000000..e49950261 --- /dev/null +++ b/test/java/util/zip/LargeZip.java @@ -0,0 +1,193 @@ +/* + * Copyright 2009 Sun Microsystems, Inc. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Sun designates this + * particular file as subject to the "Classpath" exception as provided + * by Sun in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, + * CA 95054 USA or visit www.sun.com if you need additional information or + * have any questions. + */ + +import java.io.*; +import java.nio.*; +import java.util.*; +import java.util.zip.*; + +public class LargeZip { + // If true, don't delete large ZIP file created for test. + static final boolean debug = System.getProperty("debug") != null; + + //static final int DATA_LEN = 1024 * 1024; + static final int DATA_LEN = 80 * 1024; + static final int DATA_SIZE = 8; + + static long fileSize = 6L * 1024L * 1024L * 1024L; // 6GB + + static boolean userFile = false; + + static byte[] data; + static File largeFile; + static String lastEntryName; + + /* args can be empty, in which case check a 3 GB file which is created for + * this test (and then deleted). Or it can be a number, in which case + * that designates the size of the file that's created for this test (and + * then deleted). Or it can be the name of a file to use for the test, in + * which case it is *not* deleted. Note that in this last case, the data + * comparison might fail. + */ + static void realMain (String[] args) throws Throwable { + if (args.length > 0) { + try { + fileSize = Long.parseLong(args[0]); + System.out.println("Testing with file of size " + fileSize); + } catch (NumberFormatException ex) { + largeFile = new File(args[0]); + if (!largeFile.exists()) { + throw new Exception("Specified file " + args[0] + " does not exist"); + } + userFile = true; + System.out.println("Testing with user-provided file " + largeFile); + } + } + File testDir = null; + if (largeFile == null) { + testDir = new File(System.getProperty("test.scratch", "."), + "LargeZip"); + if (testDir.exists()) { + if (!testDir.delete()) { + throw new Exception("Cannot delete already-existing test directory"); + } + } + check(!testDir.exists() && testDir.mkdirs()); + largeFile = new File(testDir, "largezip.zip"); + createLargeZip(); + } + + readLargeZip1(); + readLargeZip2(); + + if (!userFile && !debug) { + check(largeFile.delete()); + check(testDir.delete()); + } + } + + static void createLargeZip() throws Throwable { + int iterations = DATA_LEN / DATA_SIZE; + ByteBuffer bb = ByteBuffer.allocate(DATA_SIZE); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + for (int i = 0; i < iterations; i++) { + bb.putDouble(0, Math.random()); + baos.write(bb.array(), 0, DATA_SIZE); + } + data = baos.toByteArray(); + + ZipOutputStream zos = new ZipOutputStream( + new BufferedOutputStream(new FileOutputStream(largeFile))); + long length = 0; + while (length < fileSize) { + ZipEntry ze = new ZipEntry("entry-" + length); + lastEntryName = ze.getName(); + zos.putNextEntry(ze); + zos.write(data, 0, data.length); + zos.closeEntry(); + length = largeFile.length(); + } + System.out.println("Last entry written is " + lastEntryName); + zos.close(); + } + + static void readLargeZip1() throws Throwable { + ZipFile zipFile = new ZipFile(largeFile); + ZipEntry entry = null; + String entryName = null; + int count = 0; + Enumeration entries = zipFile.entries(); + while (entries.hasMoreElements()) { + entry = entries.nextElement(); + entryName = entry.getName(); + count++; + } + System.out.println("Number of entries read: " + count); + System.out.println("Last entry read is " + entryName); + check(!entry.isDirectory()); + if (check(entryName.equals(lastEntryName))) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + InputStream is = zipFile.getInputStream(entry); + byte buf[] = new byte[4096]; + int len; + while ((len = is.read(buf)) >= 0) { + baos.write(buf, 0, len); + } + baos.close(); + is.close(); + check(Arrays.equals(data, baos.toByteArray())); + } + } + + + static void readLargeZip2() throws Throwable { + ZipInputStream zis = new ZipInputStream( + new BufferedInputStream(new FileInputStream(largeFile))); + ZipEntry entry = null; + String entryName = null; + int count = 0; + while ((entry = zis.getNextEntry()) != null) { + entryName = entry.getName(); + if (entryName.equals(lastEntryName)) { + break; + } + count++; + } + System.out.println("Number of entries read: " + count); + System.out.println("Last entry read is " + entryName); + check(!entry.isDirectory()); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + + byte buf[] = new byte[4096]; + int len; + while ((len = zis.read(buf)) >= 0) { + baos.write(buf, 0, len); + } + baos.close(); + check(Arrays.equals(data, baos.toByteArray())); + check(zis.getNextEntry() == null); + zis.close(); + } + + + //--------------------- Infrastructure --------------------------- + static volatile int passed = 0, failed = 0; + static void pass() {passed++;} + static void pass(String msg) {System.out.println(msg); passed++;} + static void fail() {failed++; Thread.dumpStack();} + static void fail(String msg) {System.out.println(msg); fail();} + static void unexpected(Throwable t) {failed++; t.printStackTrace();} + static void unexpected(Throwable t, String msg) { + System.out.println(msg); failed++; t.printStackTrace();} + static boolean check(boolean cond) {if (cond) pass(); else fail(); return cond;} + static void equal(Object x, Object y) { + if (x == null ? y == null : x.equals(y)) pass(); + else fail(x + " not equal to " + y);} + public static void main(String[] args) throws Throwable { + try {realMain(args);} catch (Throwable t) {unexpected(t);} + System.out.println("\nPassed = " + passed + " failed = " + failed); + if (failed > 0) throw new AssertionError("Some tests failed");} +} diff --git a/test/java/util/zip/ZipFile/LargeZipFile.java b/test/java/util/zip/ZipFile/LargeZipFile.java index 479bac48b..d228a5f93 100644 --- a/test/java/util/zip/ZipFile/LargeZipFile.java +++ b/test/java/util/zip/ZipFile/LargeZipFile.java @@ -158,4 +158,3 @@ public class LargeZipFile { System.out.println("\nPassed = " + passed + " failed = " + failed); if (failed > 0) throw new AssertionError("Some tests failed");} } - -- GitLab