8029646: [pack200] should support the new zip64 format.

Reviewed-by: ksrini, sherman

src/share/native/com/sun/java/util/jar/pack/zip.cpp

 /*
- * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -73,8 +73,9 @@ inline uint jar::get_crc32(uint c, uchar *ptr, uint len) { return crc32(c, ptr,
     SWAP_BYTES(a & 0xFFFF)
 
 #define GET_INT_HI(a) \
-    SWAP_BYTES((a >> 16) & 0xFFFF);
+    SWAP_BYTES((a >> 16) & 0xFFFF)
 
+static const ushort jarmagic[2] = { SWAP_BYTES(0xCAFE), 0 };
 
 void jar::init(unpacker* u_) {
   BYTES_OF(*this).clear();
@@ -105,13 +106,14 @@ void jar::add_to_jar_directory(const char* fname, bool store, int modtime,
 
   header[0] = (ushort)SWAP_BYTES(0x4B50);
   header[1] = (ushort)SWAP_BYTES(0x0201);
-  header[2] = (ushort)SWAP_BYTES(0xA);
+  header[2] = (ushort)SWAP_BYTES(( store ) ? 0x0A : 0x14);
 
   // required version
-  header[3] = (ushort)SWAP_BYTES(0xA);
+  header[3] = (ushort)SWAP_BYTES(( store ) ? 0x0A : 0x14);
 
-  // flags 02 = maximum  sub-compression flag
-  header[4] = ( store ) ? 0x0 : SWAP_BYTES(0x2);
+  // Flags - UTF-8 compression and separating crc and sizes
+  // into separate headers for deflated file
+  header[4] = ( store ) ? SWAP_BYTES(0x0800) : 0x0808;
 
   // Compression method 8=deflate.
   header[5] = ( store ) ? 0x0 : SWAP_BYTES(0x08);
@@ -135,7 +137,8 @@ void jar::add_to_jar_directory(const char* fname, bool store, int modtime,
   // Filename length
   header[14] = (ushort)SWAP_BYTES(fname_length);
   // So called "extra field" length.
-  header[15] = 0;
+  // If it's the first record we must add JAR magic sequence
+  header[15] = ( central_directory_count ) ? 0 : (ushort)SWAP_BYTES(4);
   // So called "comment" length.
   header[16] = 0;
   // Disk number start
@@ -155,6 +158,11 @@ void jar::add_to_jar_directory(const char* fname, bool store, int modtime,
   // Copy the fname to the header.
   central_directory.append(fname, fname_length);
 
+  // Add jar magic for the first record
+  if (central_directory_count == 0) {
+    central_directory.append((void *)jarmagic, sizeof(jarmagic));
+  }
+
   central_directory_count++;
 }
 
@@ -170,10 +178,10 @@ void jar::write_jar_header(const char* fname, bool store, int modtime,
   header[1] = (ushort)SWAP_BYTES(0x0403);
 
   // Version
-  header[2] = (ushort)SWAP_BYTES(0xA);
+  header[2] = (ushort)SWAP_BYTES(( store ) ? 0x0A : 0x14);
 
-  // flags 02 = maximum  sub-compression flag
-  header[3] = ( store ) ? 0x0 : SWAP_BYTES(0x2);
+  // General purpose flags - same as in the Central Directory
+  header[3] = ( store ) ? SWAP_BYTES(0x0800) : 0x0808;
 
   // Compression method = deflate
   header[4] = ( store ) ? 0x0 : SWAP_BYTES(0x08);
@@ -182,28 +190,51 @@ void jar::write_jar_header(const char* fname, bool store, int modtime,
   header[5] = (ushort)GET_INT_LO(dostime);
   header[6] = (ushort)GET_INT_HI(dostime);
 
-  // CRC
-  header[7] = (ushort)GET_INT_LO(crc);
-  header[8] = (ushort)GET_INT_HI(crc);
+  // CRC, 0 if deflated, will come separately in extra header
+  header[7] = ( store ) ? (ushort)GET_INT_LO(crc) : 0;
+  header[8] = ( store ) ? (ushort)GET_INT_HI(crc) : 0;
 
-  // Compressed length:
-  header[9] = (ushort)GET_INT_LO(clen);
-  header[10] = (ushort)GET_INT_HI(clen);
+  // Compressed length, 0 if deflated
+  header[9] = ( store ) ? (ushort)GET_INT_LO(clen) : 0;
+  header[10] = ( store ) ? (ushort)GET_INT_HI(clen) : 0;
 
-  // Uncompressed length.
-  header[11] = (ushort)GET_INT_LO(len);
-  header[12] = (ushort)GET_INT_HI(len);
+  // Uncompressed length, 0 if deflated
+  header[11] = ( store ) ? (ushort)GET_INT_LO(len) : 0;
+  header[12] = ( store ) ? (ushort)GET_INT_HI(len) : 0;
 
   // Filename length
   header[13] = (ushort)SWAP_BYTES(fname_length);
   // So called "extra field" length.
-  header[14] = 0;
+  header[14] = ( central_directory_count - 1 ) ? 0 : (ushort)SWAP_BYTES(4);
 
   // Write the LOC header to the output file.
   write_data(header, (int)sizeof(header));
 
   // Copy the fname to the header.
   write_data((char*)fname, (int)fname_length);
+
+  if (central_directory_count == 1) {
+    // Write JAR magic sequence
+    write_data((void *)jarmagic, (int)sizeof(jarmagic));
+  }
+}
+
+void jar::write_jar_extra(int len, int clen, uint crc) {
+  ushort header[8];
+  // Extra field signature
+  header[0] = (ushort)SWAP_BYTES(0x4B50);
+  header[1] = (ushort)SWAP_BYTES(0x0807);
+  // CRC
+  header[2] = (ushort)GET_INT_LO(crc);
+  header[3] = (ushort)GET_INT_HI(crc);
+  // Compressed length
+  header[4] = (ushort)GET_INT_LO(clen);
+  header[5] = (ushort)GET_INT_HI(clen);
+  // Uncompressed length
+  header[6] = (ushort)GET_INT_LO(len);
+  header[7] = (ushort)GET_INT_HI(len);
+
+  write_data(header, sizeof(header));
 }
 
 static const char marker_comment[] = ZIP_ARCHIVE_MARKER_COMMENT;
@@ -212,6 +243,7 @@ void jar::write_central_directory() {
   bytes mc; mc.set(marker_comment);
 
   ushort header[11];
+  ushort header64[38];
 
   // Create the End of Central Directory structure.
   header[0] = (ushort)SWAP_BYTES(0x4B50);
@@ -220,8 +252,8 @@ void jar::write_central_directory() {
   header[2] = 0;
   header[3] = 0;
   // Number of entries in central directory.
-  header[4] = (ushort)SWAP_BYTES(central_directory_count);
-  header[5] = (ushort)SWAP_BYTES(central_directory_count);
+  header[4] = ( central_directory_count >= 0xffff ) ? 0xffff : (ushort)SWAP_BYTES(central_directory_count);
+  header[5] = ( central_directory_count >= 0xffff ) ? 0xffff : (ushort)SWAP_BYTES(central_directory_count);
   // Size of the central directory}
   header[6] = (ushort)GET_INT_LO((int)central_directory.size());
   header[7] = (ushort)GET_INT_HI((int)central_directory.size());
@@ -229,12 +261,71 @@ void jar::write_central_directory() {
   header[8] = (ushort)GET_INT_LO(output_file_offset);
   header[9] = (ushort)GET_INT_HI(output_file_offset);
   // zipfile comment length;
-  header [10] = (ushort)SWAP_BYTES((int)mc.len);
+  header[10] = (ushort)SWAP_BYTES((int)mc.len);
 
   // Write the central directory.
   PRINTCR((2, "Central directory at %d\n", output_file_offset));
   write_data(central_directory.b);
 
+  // If number of records exceeds the 0xFFFF we need to prepend extended
+  // Zip64 End of Central Directory record and its locator to the old
+  // style ECD record
+  if (central_directory_count > 0xFFFF) {
+    // Zip64 END signature
+    header64[0] = (ushort)SWAP_BYTES(0x4B50);
+    header64[1] = (ushort)0x0606;
+    // Size of header (long)
+    header64[2] = (ushort)SWAP_BYTES(44);;
+    header64[3] = 0;
+    header64[4] = 0;
+    header64[5] = 0;
+    // Version produced and required (short)
+    header64[6] = (ushort)SWAP_BYTES(45);
+    header64[7] = (ushort)SWAP_BYTES(45);
+    // Current disk number (int)
+    header64[8] = 0;
+    header64[9] = 0;
+    // Central directory start disk (int)
+    header64[10] = 0;
+    header64[11] = 0;
+    // Count of records on disk (long)
+    header64[12] = (ushort)GET_INT_LO(central_directory_count);
+    header64[13] = (ushort)GET_INT_HI(central_directory_count);
+    header64[14] = 0;
+    header64[15] = 0;
+    // Count of records totally (long)
+    header64[16] = (ushort)GET_INT_LO(central_directory_count);
+    header64[17] = (ushort)GET_INT_HI(central_directory_count);
+    header64[18] = 0;
+    header64[19] = 0;
+    // Length of the central directory (long)
+    header64[20] = header[6];
+    header64[21] = header[7];
+    header64[22] = 0;
+    header64[23] = 0;
+    // Offset of central directory (long)
+    header64[24] = header[8];
+    header64[25] = header[9];
+    header64[26] = 0;
+    header64[27] = 0;
+    // Zip64 end of central directory locator
+    // Locator signature
+    header64[28] = (ushort)SWAP_BYTES(0x4B50);
+    header64[29] = (ushort)SWAP_BYTES(0x0706);
+    // Start disk number (int)
+    header64[30] = 0;
+    header64[31] = 0;
+    // Offset of zip64 END record (long)
+    header64[32] = (ushort)GET_INT_LO(output_file_offset);
+    header64[33] = (ushort)GET_INT_HI(output_file_offset);
+    header64[34] = 0;
+    header64[35] = 0;
+    // Total number of disks (int)
+    header64[36] = (ushort)SWAP_BYTES(1);
+    header64[37] = 0;
+    write_data(header64, (int)sizeof(header64));
+  }
+
   // Write the End of Central Directory structure.
   PRINTCR((2, "end-of-directory at %d\n", output_file_offset));
   write_data(header, (int)sizeof(header));
@@ -286,6 +377,8 @@ void jar::addJarEntry(const char* fname,
 
   if (deflate) {
     write_data(deflated.b);
+    // Write deflated information in extra header
+    write_jar_extra(len, clen, crc);
   } else {
     write_data(head);
     write_data(tail);
@@ -368,7 +461,7 @@ bool jar::deflate_bytes(bytes& head, bytes& tail) {
   // NOTE: the window size should always be -MAX_WBITS normally -15.
   // unzip/zipup.c and java/Deflater.c
 
-  int error = deflateInit2(&zs, Z_BEST_COMPRESSION, Z_DEFLATED,
+  int error = deflateInit2(&zs, Z_DEFAULT_COMPRESSION, Z_DEFLATED,
                            -MAX_WBITS, 8, Z_DEFAULT_STRATEGY);
   if (error != Z_OK) {
     switch (error) {
@@ -414,7 +507,8 @@ bool jar::deflate_bytes(bytes& head, bytes& tail) {
     error = deflate(&zs, Z_FINISH);
   }
   if (error == Z_STREAM_END) {
-    if (len > (int)zs.total_out ) {
+    if ((int)zs.total_out > 0) {
+      // Even if compressed size is bigger than uncompressed, write it
       PRINTCR((2, "deflate compressed data %d -> %d\n", len, zs.total_out));
       deflated.b.len = zs.total_out;
       deflateEnd(&zs);

src/share/native/com/sun/java/util/jar/pack/zip.h

 /*
- * Copyright (c) 2001, 2008, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -40,7 +40,7 @@ struct jar {
 
   // Private members
   fillbytes   central_directory;
-  ushort      central_directory_count;
+  uint        central_directory_count;
   uint        output_file_offset;
   fillbytes   deflated;  // temporary buffer
 
@@ -74,6 +74,7 @@ struct jar {
                             int len, int clen, uLong crc);
   void write_jar_header(const char* fname, bool store, int modtime,
                         int len, int clen, unsigned int crc);
+  void write_jar_extra(int len, int clen, unsigned int crc);
   void write_central_directory();
   uLong dostime(int y, int n, int d, int h, int m, int s);
   uLong get_dostime(int modtime);

test/tools/pack200/PackTestZip64.java

+/*
+ * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+import java.io.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Enumeration;
+import java.util.List;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+import java.util.jar.JarInputStream;
+import java.util.jar.JarOutputStream;
+import java.util.zip.ZipEntry;
+/*
+ * @test
+ * @bug 8029646
+ * @summary tests that native unpacker produces the same result as Java one
+ * @compile -XDignore.symbol.file Utils.java PackTestZip64.java
+ * @run main PackTestZip64
+ * @author kizune
+ */
+
+public class PackTestZip64 {
+    public static void main(String... args) throws Exception {
+        testPacking();
+        Utils.cleanup();
+    }
+
+    // 1KB buffer is enough to copy jar content
+    private static final byte[] BUFFER = new byte[1024];
+
+    static void testPacking() throws IOException {
+        // make a copy of the test specimen to local directory
+        File testFile = new File("tools_java.jar");
+        // Add a large number of small files to the golden jar
+        generateLargeJar(testFile, Utils.locateJar("golden.jar"));
+
+        List<String> cmdsList = new ArrayList<>();
+
+        // Repack file to get the Java-based result
+        cmdsList.add(Utils.getPack200Cmd());
+        cmdsList.add("--repack");
+        cmdsList.add(testFile.getName());
+        Utils.runExec(cmdsList);
+        cmdsList.clear();
+
+        // Pack file with pack200 and unpack in with unpack200
+        File packedFile = new File("tools.pack.gz");
+        cmdsList.add(Utils.getPack200Cmd());
+        cmdsList.add(packedFile.getName());
+        cmdsList.add(testFile.getName());
+        Utils.runExec(cmdsList);
+        cmdsList.clear();
+
+        File unpackedFile = new File("tools_native.jar");
+        cmdsList.add(Utils.getUnpack200Cmd());
+        cmdsList.add(packedFile.getName());
+        cmdsList.add(unpackedFile.getName());
+        Utils.runExec(cmdsList);
+
+        // Compare files binary
+        compareTwoFiles(testFile, unpackedFile);
+
+        // Cleaning up generated files
+        testFile.delete();
+        packedFile.delete();
+        unpackedFile.delete();
+    }
+
+    static void compareTwoFiles(File src, File dst) throws IOException {
+        if (!src.exists()) {
+            throw new IOException("File " + src.getName() + " does not exist!");
+        }
+
+        if(!dst.exists()) {
+            throw new IOException("File " + dst.getName() + " does not exist!");
+        }
+
+        BufferedInputStream srcis, dstis;
+        srcis = new BufferedInputStream(new FileInputStream(src));
+        dstis = new BufferedInputStream(new FileInputStream(dst));
+
+        int s = 0, d, pos = 0;
+        while (s != -1) { // Checking of just one result for EOF is enough
+            s = srcis.read();
+            d = dstis.read();
+
+            if (s != d) {
+                throw new IOException("Files are differ starting at position: "
+                + Integer.toHexString(pos));
+            }
+
+            pos++;
+        }
+
+        srcis.close();
+        dstis.close();
+    }
+
+    static void generateLargeJar(File result, File source) throws IOException {
+        if (result.exists()) {
+            result.delete();
+        }
+
+        try (JarOutputStream copyTo = new JarOutputStream(new FileOutputStream(result));
+             JarFile srcJar = new JarFile(source)) {
+
+            for (JarEntry je : Collections.list(srcJar.entries())) {
+                copyTo.putNextEntry(je);
+                if (!je.isDirectory()) {
+                    copyStream(srcJar.getInputStream(je), copyTo);
+                }
+                copyTo.closeEntry();
+            }
+
+            int many = Short.MAX_VALUE * 2 + 2;
+
+            for (int i = 0 ; i < many ; i++) {
+                JarEntry e = new JarEntry("F-" + i + ".txt");
+                copyTo.putNextEntry(e);
+            }
+            copyTo.flush();
+            copyTo.close();
+        }
+    }
+
+    static void copyStream(InputStream in, OutputStream out) throws IOException {
+        int bytesRead;
+        while ((bytesRead = in.read(BUFFER))!= -1) {
+            out.write(BUFFER, 0, bytesRead);
+        }
+    }
+}