4206909: want java.util.zip to work for interactive use (Z_SYNC_FLUSH)

Summary: Add sync_flush option into Deflater/DefalterOutputStream
Reviewed-by: martin, alanb

src/share/classes/java/util/zip/Deflater.java

@@ -122,6 +122,33 @@ class Deflater {
      */
     public static final int DEFAULT_STRATEGY = 0;
 
+    /**
+     * Compression flush mode used to achieve best compression result.
+     *
+     * @see Deflater#deflate(byte[], int, int, int)
+     * @since 1.7
+     */
+    public static final int NO_FLUSH = 0;
+
+    /**
+     * Compression flush mode used to flush out all pending output; may
+     * degrade compression for some compression algorithms.
+     *
+     * @see Deflater#deflate(byte[], int, int, int)
+     * @since 1.7
+     */
+    public static final int SYNC_FLUSH = 2;
+
+    /**
+     * Compression flush mode used to flush out all pending output and
+     * reset the deflater. Using this mode too often can seriously degrade
+     * compression.
+     *
+     * @see Deflater#deflate(byte[], int, int, int)
+     * @since 1.7
+     */
+    public static final int FULL_FLUSH = 3;
+
     static {
         /* Zip library is loaded from System.initializeSystemClass */
         initIDs();
@@ -289,35 +316,100 @@ class Deflater {
     }
 
     /**
-     * Fills specified buffer with compressed data. Returns actual number
-     * of bytes of compressed data. A return value of 0 indicates that
-     * needsInput() should be called in order to determine if more input
-     * data is required.
+     * Compresses the input data and fills specified buffer with compressed
+     * data. Returns actual number of bytes of compressed data. A return value
+     * of 0 indicates that {@link needsInput() needsInput} should be called
+     * in order to determine if more input data is required.
+     *
+     * <p>This method uses {@link #NO_FLUSH} as its compression flush mode.
+     * An invocation of this method of the form {@code deflater.deflate(b, off, len)}
+     * yields the same result as the invocation of
+     * {@code deflater.deflate(b, off, len, Deflater.NO_FLUSH)}.
+     *
      * @param b the buffer for the compressed data
      * @param off the start offset of the data
      * @param len the maximum number of bytes of compressed data
-     * @return the actual number of bytes of compressed data
+     * @return the actual number of bytes of compressed data written to the
+     *         output buffer
      */
-    public synchronized int deflate(byte[] b, int off, int len) {
-        if (b == null) {
-            throw new NullPointerException();
-        }
-        if (off < 0 || len < 0 || off > b.length - len) {
-            throw new ArrayIndexOutOfBoundsException();
-        }
-        return deflateBytes(b, off, len);
+    public int deflate(byte[] b, int off, int len) {
+        return deflateBytes(b, off, len, NO_FLUSH);
     }
 
     /**
-     * Fills specified buffer with compressed data. Returns actual number
-     * of bytes of compressed data. A return value of 0 indicates that
-     * needsInput() should be called in order to determine if more input
-     * data is required.
+     * Compresses the input data and fills specified buffer with compressed
+     * data. Returns actual number of bytes of compressed data. A return value
+     * of 0 indicates that {@link needsInput() needsInput} should be called
+     * in order to determine if more input data is required.
+     *
+     * <p>This method uses {@link #NO_FLUSH} as its compression flush mode.
+     * An invocation of this method of the form {@code deflater.deflate(b)}
+     * yields the same result as the invocation of
+     * {@code deflater.deflate(b, 0, b.length, Deflater.NO_FLUSH)}.
+     *
      * @param b the buffer for the compressed data
-     * @return the actual number of bytes of compressed data
+     * @return the actual number of bytes of compressed data written to the
+     *         output buffer
      */
     public int deflate(byte[] b) {
-        return deflate(b, 0, b.length);
+        return deflate(b, 0, b.length, NO_FLUSH);
+    }
+
+    /**
+     * Compresses the input data and fills the specified buffer with compressed
+     * data. Returns actual number of bytes of data compressed.
+     *
+     * <p>Compression flush mode is one of the following three modes:
+     *
+     * <ul>
+     * <li>{@link #NO_FLUSH}: allows the deflater to decide how much data
+     * to accumulate, before producing output, in order to achieve the best
+     * compression (should be used in normal use scenario). A return value
+     * of 0 in this flush mode indicates that {@link #needsInput()} should
+     * be called in order to determine if more input data is required.
+     *
+     * <li>{@link #SYNC_FLUSH}: all pending output in the deflater is flushed,
+     * to the specified output buffer, so that an inflater that works on
+     * compressed data can get all input data available so far (In particular
+     * the {@link #needsInput()} returns {@code true} after this invocation
+     * if enough output space is provided). Flushing with {@link #SYNC_FLUSH}
+     * may degrade compression for some compression algorithms and so it
+     * should be used only when necessary.
+     *
+     * <li>{@link #FULL_FLUSH}: all pending output is flushed out as with
+     * {@link #SYNC_FLUSH}. The compression state is reset so that the inflater
+     * that works on the compressed output data can restart from this point
+     * if previous compressed data has been damaged or if random access is
+     * desired. Using {@link #FULL_FLUSH} too often can seriously degrade
+     * compression.
+     * </ul>
+     *
+     * <p>In the case of {@link #FULL_FLUSH} or {@link #SYNC_FLUSH}, if
+     * the return value is {@code len}, the space available in output
+     * buffer {@code b}, this method should be invoked again with the same
+     * {@code flush} parameter and more output space.
+     *
+     * @param b the buffer for the compressed data
+     * @param off the start offset of the data
+     * @param len the maximum number of bytes of compressed data
+     * @param flush the compression flush mode
+     * @return the actual number of bytes of compressed data written to
+     *         the output buffer
+     *
+     * @throws IllegalArgumentException if the flush mode is invalid
+     * @since 1.7
+     */
+    public synchronized int deflate(byte[] b, int off, int len, int flush) {
+        if (b == null) {
+            throw new NullPointerException();
+        }
+        if (off < 0 || len < 0 || off > b.length - len) {
+            throw new ArrayIndexOutOfBoundsException();
+        }
+        if (flush == NO_FLUSH || flush == SYNC_FLUSH ||
+            flush == FULL_FLUSH)
+            return deflateBytes(b, off, len, flush);
+        throw new IllegalArgumentException();
     }
 
     /**
@@ -420,7 +512,7 @@ class Deflater {
     private native static long init(int level, int strategy, boolean nowrap);
     private native static void setDictionary(long strm, byte[] b, int off,
                                              int len);
-    private native int deflateBytes(byte[] b, int off, int len);
+    private native int deflateBytes(byte[] b, int off, int len, int flush);
     private native static int getAdler(long strm);
     private native static long getBytesRead(long strm);
     private native static long getBytesWritten(long strm);

src/share/classes/java/util/zip/DeflaterOutputStream.java

 /*
- * Copyright 1996-2006 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1996-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -56,15 +56,29 @@ class DeflaterOutputStream extends FilterOutputStream {
 
     private boolean closed = false;
 
+    private final boolean syncFlush;
+
     /**
-     * Creates a new output stream with the specified compressor and
-     * buffer size.
+     * Creates a new output stream with the specified compressor,
+     * buffer size and flush mode.
+
      * @param out the output stream
      * @param def the compressor ("deflater")
      * @param size the output buffer size
-     * @exception IllegalArgumentException if size is <= 0
+     * @param syncFlush
+     *        if {@code true} the {@link flush()} method of this
+     *        instance flushes the compressor with flush mode
+     *        {@link Deflater#SYNC_FLUSH} before flushing the output
+     *        stream, otherwise only flushes the output stream
+     *
+     * @throws IllegalArgumentException if size is <= 0
+     *
+     * @since 1.7
      */
-    public DeflaterOutputStream(OutputStream out, Deflater def, int size) {
+    public DeflaterOutputStream(OutputStream out,
+                                Deflater def,
+                                int size,
+                                boolean syncFlush) {
         super(out);
         if (out == null || def == null) {
             throw new NullPointerException();
@@ -72,27 +86,93 @@ class DeflaterOutputStream extends FilterOutputStream {
             throw new IllegalArgumentException("buffer size <= 0");
         }
         this.def = def;
-        buf = new byte[size];
+        this.buf = new byte[size];
+        this.syncFlush = syncFlush;
     }
 
+
+    /**
+     * Creates a new output stream with the specified compressor and
+     * buffer size.
+     *
+     * <p>The new output stream instance is created as if by invoking
+     * the 4-argument constructor DeflaterOutputStream(out, def, size, false).
+     *
+     * @param out the output stream
+     * @param def the compressor ("deflater")
+     * @param size the output buffer size
+     * @exception IllegalArgumentException if size is <= 0
+     */
+    public DeflaterOutputStream(OutputStream out, Deflater def, int size) {
+        this(out, def, size, false);
+    }
+
+    /**
+     * Creates a new output stream with the specified compressor, flush
+     * mode and a default buffer size.
+     *
+     * @param out the output stream
+     * @param def the compressor ("deflater")
+     * @param syncFlush
+     *        if {@code true} the {@link flush()} method of this
+     *        instance flushes the compressor with flush mode
+     *        {@link Deflater#SYNC_FLUSH} before flushing the output
+     *        stream, otherwise only flushes the output stream
+     *
+     * @since 1.7
+     */
+    public DeflaterOutputStream(OutputStream out,
+                                Deflater def,
+                                boolean syncFlush) {
+        this(out, def, 512, syncFlush);
+    }
+
+
     /**
      * Creates a new output stream with the specified compressor and
      * a default buffer size.
+     *
+     * <p>The new output stream instance is created as if by invoking
+     * the 3-argument constructor DeflaterOutputStream(out, def, false).
+     *
      * @param out the output stream
      * @param def the compressor ("deflater")
      */
     public DeflaterOutputStream(OutputStream out, Deflater def) {
-        this(out, def, 512);
+        this(out, def, 512, false);
     }
 
     boolean usesDefaultDeflater = false;
 
+
+    /**
+     * Creates a new output stream with a default compressor, a default
+     * buffer size and the specified flush mode.
+     *
+     * @param out the output stream
+     * @param syncFlush
+     *        if {@code true} the {@link flush()} method of this
+     *        instance flushes the compressor with flush mode
+     *        {@link Deflater#SYNC_FLUSH} before flushing the output
+     *        stream, otherwise only flushes the output stream
+     *
+     * @since 1.7
+     */
+    public DeflaterOutputStream(OutputStream out, boolean syncFlush) {
+        this(out, new Deflater(), 512, syncFlush);
+        usesDefaultDeflater = true;
+    }
+
     /**
      * Creates a new output stream with a default compressor and buffer size.
+     *
+     * <p>The new output stream instance is created as if by invoking
+     * the 2-argument constructor DeflaterOutputStream(out, false).
+     *
      * @param out the output stream
      */
     public DeflaterOutputStream(OutputStream out) {
-        this(out, new Deflater());
+        this(out, false);
         usesDefaultDeflater = true;
     }
 
@@ -178,4 +258,32 @@ class DeflaterOutputStream extends FilterOutputStream {
             out.write(buf, 0, len);
         }
     }
+
+    /**
+     * Flushes the compressed output stream.
+     *
+     * If {@link DeflaterOutputStream(OutputStream, Deflater, int, boolean)
+     * syncFlush} is {@code true} when this compressed output stream is
+     * constructed this method flushes the underlying {@code compressor}
+     * first with the flush mode {@link Deflater#SYNC_FLUSH} to force
+     * all pending data to be flushed out to the output stream and then
+     * flushes the output stream. Otherwise this method only flushes the
+     * output stream without flushing the {@code compressor}.
+     *
+     * @throws IOException if an I/O error has occurred
+     *
+     * @since 1.7
+     */
+    public void flush() throws IOException {
+        if (syncFlush && !def.finished()) {
+            int len = 0;
+            while ((len = def.deflate(buf, 0, buf.length, Deflater.SYNC_FLUSH)) > 0)
+            {
+                out.write(buf, 0, len);
+                if (len < buf.length)
+                    break;
+            }
+        }
+        out.flush();
+    }
 }

src/share/native/java/util/zip/Deflater.c

@@ -118,7 +118,7 @@ Java_java_util_zip_Deflater_setDictionary(JNIEnv *env, jclass cls, jlong strm,
 
 JNIEXPORT jint JNICALL
 Java_java_util_zip_Deflater_deflateBytes(JNIEnv *env, jobject this,
-                                         jarray b, jint off, jint len)
+                                         jarray b, jint off, jint len, jint flush)
 {
     z_stream *strm = jlong_to_ptr((*env)->GetLongField(env, this, strmID));
 
@@ -197,7 +197,7 @@ Java_java_util_zip_Deflater_deflateBytes(JNIEnv *env, jobject this,
             strm->next_out = (Bytef *) out_buf;
             strm->avail_in = this_len;
             strm->avail_out = len;
-            res = deflate(strm, finish ? Z_FINISH : Z_NO_FLUSH);
+            res = deflate(strm, finish ? Z_FINISH : flush);
 
             if (res == Z_STREAM_END || res == Z_OK) {
                 (*env)->SetByteArrayRegion(env, b, off, len - strm->avail_out, out_buf);

test/java/util/zip/InflateIn_DeflateOut.java

+/*
+ * Copyright 2009 Google, Inc.  All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ */
+
+/**
+ * @test
+ * @bug 4206909
+ * @summary Test basic functionality of DeflaterOutputStream and InflaterInputStream including flush
+ */
+
+import java.io.*;
+import java.util.*;
+import java.util.zip.*;
+
+public class InflateIn_DeflateOut {
+
+    private static class PairedInputStream extends ByteArrayInputStream {
+        private PairedOutputStream out = null;
+        private Random random;
+
+        public PairedInputStream() {
+            // The ByteArrayInputStream needs to start with a buffer, but we
+            // need to set it to have no data
+            super(new byte[1]);
+            count = 0;
+            pos = 0;
+            random = new Random(new Date().getTime());
+        }
+
+        public void setPairedOutputStream(PairedOutputStream out) {
+            this.out = out;
+        }
+
+        private void maybeFlushPair() {
+            if (random.nextInt(100) < 10) {
+                out.flush();
+            }
+        }
+
+        public int read() {
+            maybeFlushPair();
+            return super.read();
+        }
+
+        public int read(byte b[], int off, int len) {
+            maybeFlushPair();
+            return super.read(b, off, len);
+        }
+
+        public void addBytes(byte[] bytes, int len) {
+            int oldavail = count - pos;
+            int newcount = oldavail + len;
+            byte[] newbuf = new byte[newcount];
+            System.arraycopy(buf, pos, newbuf, 0, oldavail);
+            System.arraycopy(bytes, 0, newbuf, oldavail, len);
+            pos = 0;
+            count = newcount;
+            buf = newbuf;
+        }
+    }
+
+    private static class PairedOutputStream extends ByteArrayOutputStream {
+      private PairedInputStream pairedStream = null;
+
+      public PairedOutputStream(PairedInputStream inputPair) {
+        super();
+        this.pairedStream = inputPair;
+      }
+
+      public void flush() {
+        if (count > 0) {
+          pairedStream.addBytes(buf, count);
+          reset();
+        }
+      }
+
+      public void close() {
+        flush();
+      }
+    }
+
+    private static boolean readFully(InputStream in, byte[] buf, int length)
+            throws IOException {
+        int pos = 0;
+        int n;
+        while ((n = in.read(buf, pos, length - pos)) > 0) {
+            pos += n;
+            if (pos == length) return true;
+        }
+        return false;
+    }
+
+    private static boolean readLineIfAvailable(InputStream in, StringBuilder sb)
+            throws IOException {
+        try {
+            while (in.available() > 0) {
+                int i = in.read();
+                if (i < 0) break;
+                char c = (char) (((byte) i) & 0xff);
+                sb.append(c);
+                if (c == '\n') return true;
+            }
+        } catch (EOFException e) {
+          // empty
+        }
+        return false;
+    }
+
+    /** Check that written, closed and read */
+    private static void WriteCloseRead() throws Throwable {
+        Random random = new Random(new Date().getTime());
+
+        PairedInputStream pis = new PairedInputStream();
+        InflaterInputStream iis = new InflaterInputStream(pis);
+
+        PairedOutputStream pos = new PairedOutputStream(pis);
+        pis.setPairedOutputStream(pos);
+        DeflaterOutputStream dos = new DeflaterOutputStream(pos, true);
+
+        byte[] data = new byte[random.nextInt(1024 * 1024)];
+        byte[] buf = new byte[data.length];
+        random.nextBytes(data);
+
+        dos.write(data);
+        dos.close();
+        check(readFully(iis, buf, buf.length));
+        check(Arrays.equals(data, buf));
+    }
+
+    /** Check that written, flushed and read */
+    private static void WriteFlushRead() throws Throwable {
+        Random random = new Random(new Date().getTime());
+
+        PairedInputStream pis = new PairedInputStream();
+        InflaterInputStream iis = new InflaterInputStream(pis);
+
+        PairedOutputStream pos = new PairedOutputStream(pis);
+        pis.setPairedOutputStream(pos);
+        DeflaterOutputStream dos = new DeflaterOutputStream(pos, true);
+
+        // Large writes
+        for (int x = 0; x < 200 ; x++) {
+            // byte[] data = new byte[random.nextInt(1024 * 1024)];
+            byte[] data = new byte[1024];
+            byte[] buf = new byte[data.length];
+            random.nextBytes(data);
+
+            dos.write(data);
+            dos.flush();
+            check(readFully(iis, buf, buf.length));
+            check(Arrays.equals(data, buf));
+        }
+
+        // Small writes
+        for (int x = 0; x < 2000 ; x++) {
+            byte[] data = new byte[random.nextInt(20) + 10];
+            byte[] buf = new byte[data.length];
+            random.nextBytes(data);
+
+            dos.write(data);
+            dos.flush();
+            if (!readFully(iis, buf, buf.length)) {
+                fail("Didn't read full buffer of " + buf.length);
+            }
+            check(Arrays.equals(data, buf));
+        }
+
+        String quit = "QUIT\r\n";
+
+        // Close it out
+        dos.write(quit.getBytes());
+        dos.close();
+
+        StringBuilder sb = new StringBuilder();
+        check(readLineIfAvailable(iis, sb));
+        equal(sb.toString(), quit);
+    }
+
+    /** Validate that we need to use flush at least once on a line
+     * oriented protocol */
+    private static void LineOrientedProtocol() throws Throwable {
+        PairedInputStream pis = new PairedInputStream();
+        InflaterInputStream iis = new InflaterInputStream(pis);
+
+        PairedOutputStream pos = new PairedOutputStream(pis);
+        pis.setPairedOutputStream(pos);
+        DeflaterOutputStream dos = new DeflaterOutputStream(pos, true);
+
+        boolean flushed = false;
+        int count = 0;
+
+        // Do at least a certain number of lines, but too many without a
+        // flush means this test isn't testing anything
+        while ((count < 10 && flushed) || (count < 1000 && !flushed)) {
+            String command = "PING " + count + "\r\n";
+            dos.write(command.getBytes());
+
+            StringBuilder buf = new StringBuilder();
+            if (!readLineIfAvailable(iis, buf)) {
+                flushed = true;
+                dos.flush();
+                check(readLineIfAvailable(iis, buf));
+            }
+            equal(buf.toString(), command);
+            count++;
+        }
+        check(flushed);
+    }
+
+    public static void realMain(String[] args) throws Throwable {
+        WriteCloseRead();
+
+        WriteFlushRead();
+
+        LineOrientedProtocol();
+    }
+
+    //--------------------- Infrastructure ---------------------------
+    static volatile int passed = 0, failed = 0;
+    static void pass() {passed++;}
+    static void fail() {failed++; Thread.dumpStack();}
+    static void fail(String msg) {System.out.println(msg); fail();}
+    static void unexpected(Throwable t) {failed++; t.printStackTrace();}
+    static void check(boolean cond) {if (cond) pass(); else fail();}
+    static void equal(Object x, Object y) {
+        if (x == null ? y == null : x.equals(y)) pass();
+        else fail(x + " not equal to " + y);}
+    public static void main(String[] args) throws Throwable {
+        try {realMain(args);} catch (Throwable t) {unexpected(t);}
+        System.out.println("\nPassed = " + passed + " failed = " + failed);
+        if (failed > 0) throw new AssertionError("Some tests failed");}
+}