8203182: Release session if initialization of SunPKCS11 Signature fails

Summary: Ensure session is properly released in P11Signature class
Reviewed-by: valeriep
Contributed-by: NMartin Balao <mbalao@redhat.com>

src/share/classes/sun/security/pkcs11/P11Signature.java

@@ -257,47 +257,51 @@ final class P11Signature extends SignatureSpi {
             session = token.killSession(session);
             return;
         }
-        // "cancel" operation by finishing it
-        // XXX make sure all this always works correctly
-        if (mode == M_SIGN) {
-            try {
-                if (type == T_UPDATE) {
-                    token.p11.C_SignFinal(session.id(), 0);
-                } else {
-                    byte[] digest;
-                    if (type == T_DIGEST) {
-                        digest = md.digest();
-                    } else { // T_RAW
-                        digest = buffer;
+        try {
+            // "cancel" operation by finishing it
+            // XXX make sure all this always works correctly
+            if (mode == M_SIGN) {
+                try {
+                    if (type == T_UPDATE) {
+                        token.p11.C_SignFinal(session.id(), 0);
+                    } else {
+                        byte[] digest;
+                        if (type == T_DIGEST) {
+                            digest = md.digest();
+                        } else { // T_RAW
+                            digest = buffer;
+                        }
+                        token.p11.C_Sign(session.id(), digest);
                     }
-                    token.p11.C_Sign(session.id(), digest);
-                }
-            } catch (PKCS11Exception e) {
-                throw new ProviderException("cancel failed", e);
-            }
-        } else { // M_VERIFY
-            try {
-                byte[] signature;
-                if (keyAlgorithm.equals("DSA")) {
-                    signature = new byte[40];
-                } else {
-                    signature = new byte[(p11Key.length() + 7) >> 3];
+                } catch (PKCS11Exception e) {
+                    throw new ProviderException("cancel failed", e);
                 }
-                if (type == T_UPDATE) {
-                    token.p11.C_VerifyFinal(session.id(), signature);
-                } else {
-                    byte[] digest;
-                    if (type == T_DIGEST) {
-                        digest = md.digest();
-                    } else { // T_RAW
-                        digest = buffer;
+            } else { // M_VERIFY
+                try {
+                    byte[] signature;
+                    if (keyAlgorithm.equals("DSA")) {
+                        signature = new byte[40];
+                    } else {
+                        signature = new byte[(p11Key.length() + 7) >> 3];
                     }
-                    token.p11.C_Verify(session.id(), digest, signature);
+                    if (type == T_UPDATE) {
+                        token.p11.C_VerifyFinal(session.id(), signature);
+                    } else {
+                        byte[] digest;
+                        if (type == T_DIGEST) {
+                            digest = md.digest();
+                        } else { // T_RAW
+                            digest = buffer;
+                        }
+                        token.p11.C_Verify(session.id(), digest, signature);
+                    }
+                } catch (PKCS11Exception e) {
+                    // will fail since the signature is incorrect
+                    // XXX check error code
                 }
-            } catch (PKCS11Exception e) {
-                // will fail since the signature is incorrect
-                // XXX check error code
             }
+        } finally {
+            session = token.releaseSession(session);
         }
     }
 
@@ -316,6 +320,8 @@ final class P11Signature extends SignatureSpi {
             }
             initialized = true;
         } catch (PKCS11Exception e) {
+            // release session when initialization failed
+            session = token.releaseSession(session);
             throw new ProviderException("Initialization failed", e);
         }
         if (bytesProcessed != 0) {
@@ -476,6 +482,8 @@ final class P11Signature extends SignatureSpi {
                 }
                 bytesProcessed += len;
             } catch (PKCS11Exception e) {
+                initialized = false;
+                session = token.releaseSession(session);
                 throw new ProviderException(e);
             }
             break;
@@ -523,6 +531,8 @@ final class P11Signature extends SignatureSpi {
                 bytesProcessed += len;
                 byteBuffer.position(ofs + len);
             } catch (PKCS11Exception e) {
+                initialized = false;
+                session = token.releaseSession(session);
                 throw new ProviderException("Update failed", e);
             }
             break;