6745052: SLServerSocket file descriptor leak

Summary: SSLServerSocketImpl.checkEnabledSuites() does not release the temporary socket properly
Reviewed-by: wetmore, weijun

src/share/classes/sun/security/ssl/BaseSSLSocketImpl.java

 /*
- * Copyright 2002-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2002-2008 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -256,10 +256,12 @@ abstract class BaseSSLSocketImpl extends SSLSocket {
                 // ignore
             }
         } finally {
-            // we call close on the underlying socket anyway, but be
-            // doubly sure all resources get released.
-            // note that we don't need to worry about self, the GC
-            // will finalize that separately
+            // We called close on the underlying socket above to
+            // make doubly sure all resources got released.  We
+            // don't finalize self in the case of overlain sockets,
+            // that's a different object which the GC will finalize
+            // separately.
+
             super.finalize();
         }
     }

src/share/classes/sun/security/ssl/SSLServerSocketImpl.java

 /*
- * Copyright 1996-2007 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 1996-2008 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -304,14 +304,18 @@ class SSLServerSocketImpl extends SSLServerSocket
                          enabledCipherSuites, doClientAuth,
                          enableSessionCreation, enabledProtocols);
 
-            ServerHandshaker handshaker = tmp.getServerHandshaker();
+            try {
+                ServerHandshaker handshaker = tmp.getServerHandshaker();
 
-            for (Iterator t = enabledCipherSuites.iterator(); t.hasNext(); ) {
-                CipherSuite suite = (CipherSuite)t.next();
-                if (handshaker.trySetCipherSuite(suite)) {
-                    checkedEnabled = true;
-                    return;
+                for (Iterator t = enabledCipherSuites.iterator(); t.hasNext(); ) {
+                    CipherSuite suite = (CipherSuite)t.next();
+                    if (handshaker.trySetCipherSuite(suite)) {
+                        checkedEnabled = true;
+                        return;
+                    }
                 }
+            } finally {
+                tmp.closeSocket();
             }
 
             //

src/share/classes/sun/security/ssl/SSLSocketImpl.java

@@ -1012,6 +1012,22 @@ final public class SSLSocketImpl extends BaseSSLSocketImpl {
      */
     ServerHandshaker getServerHandshaker() throws SSLException {
         initHandshaker();
+
+         // The connection state would have been set to cs_HANDSHAKE during the
+         // handshaking initializing, however the caller may not have the
+         // the low level connection's established, which is not consistent with
+         // the HANDSHAKE state. As if it is unconnected, we need to reset the
+         // connection state to cs_START.
+         if (!isConnected()) {
+             connectionState = cs_START;
+         }
+
+         // Make sure that we get a ServerHandshaker.
+         // This should never happen.
+         if (!(handshaker instanceof ServerHandshaker)) {
+             throw new SSLProtocolException("unexpected handshaker instance");
+         }
+
         return (ServerHandshaker)handshaker;
     }
 
@@ -1273,7 +1289,8 @@ final public class SSLSocketImpl extends BaseSSLSocketImpl {
         }
     }
 
-    private void closeSocket() throws IOException {
+    protected void closeSocket() throws IOException {
+
         if ((debug != null) && Debug.isOn("ssl")) {
             System.out.println(threadName() + ", called closeSocket()");
         }