6750401: SSL stress test with GF leads to 32 bit max process size in less than...

6750401: SSL stress test with GF leads to 32 bit max process size in less than 5 minutes,with PCKS11 provider
Summary: This is the JSSE portion of the fix. Main part is in PKCS11.
Reviewed-by: valeriep, xuelei

src/share/classes/sun/security/ssl/CipherBox.java

 /*
- * Copyright 1996-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1996-2008 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -486,4 +486,21 @@ final class CipherBox {
 
         return newlen;
     }
+
+    /*
+     * Dispose of any intermediate state in the underlying cipher.
+     * For PKCS11 ciphers, this will release any attached sessions, and
+     * thus make finalization faster.
+     */
+    void dispose() {
+        try {
+            if (cipher != null) {
+                // ignore return value.
+                cipher.doFinal();
+            }
+        } catch (GeneralSecurityException e) {
+            // swallow for now.
+        }
+    }
+
 }

src/share/classes/sun/security/ssl/SSLEngineImpl.java

 /*
- * Copyright 2003-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2008 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -547,6 +547,8 @@ final public class SSLEngineImpl extends SSLEngine {
 
         // ... create decompressor
 
+        CipherBox oldCipher = readCipher;
+
         try {
             readCipher = handshaker.newReadCipher();
             readMAC = handshaker.newReadMAC();
@@ -555,6 +557,16 @@ final public class SSLEngineImpl extends SSLEngine {
             throw (SSLException)new SSLException
                                 ("Algorithm missing:  ").initCause(e);
         }
+
+        /*
+         * Dispose of any intermediate state in the underlying cipher.
+         * For PKCS11 ciphers, this will release any attached sessions,
+         * and thus make finalization faster.
+         *
+         * Since MAC's doFinal() is called for every SSL/TLS packet, it's
+         * not necessary to do the same with MAC's.
+         */
+        oldCipher.dispose();
     }
 
     /*
@@ -572,6 +584,8 @@ final public class SSLEngineImpl extends SSLEngine {
 
         // ... create compressor
 
+        CipherBox oldCipher = writeCipher;
+
         try {
             writeCipher = handshaker.newWriteCipher();
             writeMAC = handshaker.newWriteMAC();
@@ -580,6 +594,9 @@ final public class SSLEngineImpl extends SSLEngine {
             throw (SSLException)new SSLException
                                 ("Algorithm missing:  ").initCause(e);
         }
+
+        // See comment above.
+        oldCipher.dispose();
     }
 
     /*
@@ -1231,6 +1248,9 @@ final public class SSLEngineImpl extends SSLEngine {
             break;
         }
 
+        // See comment in changeReadCiphers()
+        writeCipher.dispose();
+
         connectionState = cs_CLOSED;
     }
 
@@ -1271,6 +1291,10 @@ final public class SSLEngineImpl extends SSLEngine {
 
         closeOutboundInternal();
         inboundDone = true;
+
+        // See comment in changeReadCiphers()
+        readCipher.dispose();
+
         connectionState = cs_CLOSED;
     }
 
@@ -1457,6 +1481,10 @@ final public class SSLEngineImpl extends SSLEngine {
 
         connectionState = cs_CLOSED;
 
+        // See comment in changeReadCiphers()
+        readCipher.dispose();
+        writeCipher.dispose();
+
         if (cause instanceof RuntimeException) {
             throw (RuntimeException)cause;
         } else {

src/share/classes/sun/security/ssl/SSLSocketImpl.java

@@ -1427,6 +1427,10 @@ final public class SSLSocketImpl extends BaseSSLSocketImpl {
                     waitForClose(false);
                 }
 
+                // See comment in changeReadCiphers()
+                readCipher.dispose();
+                writeCipher.dispose();
+
                 // state will be set to cs_CLOSED in the finally block below
 
                 break;
@@ -1633,6 +1637,11 @@ final public class SSLSocketImpl extends BaseSSLSocketImpl {
          * Clean up our side.
          */
         closeSocket();
+
+        // See comment in changeReadCiphers()
+        readCipher.dispose();
+        writeCipher.dispose();
+
         connectionState = (oldState == cs_APP_CLOSED) ? cs_APP_CLOSED
                                                       : cs_CLOSED;
         throw closeReason;
@@ -1763,6 +1772,8 @@ final public class SSLSocketImpl extends BaseSSLSocketImpl {
 
         // ... create decompressor
 
+        CipherBox oldCipher = readCipher;
+
         try {
             readCipher = handshaker.newReadCipher();
             readMAC = handshaker.newReadMAC();
@@ -1771,6 +1782,16 @@ final public class SSLSocketImpl extends BaseSSLSocketImpl {
             throw (SSLException)new SSLException
                                 ("Algorithm missing:  ").initCause(e);
         }
+
+        /*
+         * Dispose of any intermediate state in the underlying cipher.
+         * For PKCS11 ciphers, this will release any attached sessions,
+         * and thus make finalization faster.
+         *
+         * Since MAC's doFinal() is called for every SSL/TLS packet, it's
+         * not necessary to do the same with MAC's.
+         */
+        oldCipher.dispose();
     }
 
     // used by Handshaker
@@ -1783,6 +1804,8 @@ final public class SSLSocketImpl extends BaseSSLSocketImpl {
 
         // ... create compressor
 
+        CipherBox oldCipher = writeCipher;
+
         try {
             writeCipher = handshaker.newWriteCipher();
             writeMAC = handshaker.newWriteMAC();
@@ -1791,6 +1814,9 @@ final public class SSLSocketImpl extends BaseSSLSocketImpl {
             throw (SSLException)new SSLException
                                 ("Algorithm missing:  ").initCause(e);
         }
+
+        // See comment above.
+        oldCipher.dispose();
     }
 
     /*