6924489: sun.security.pkcs11.wrapper.PKCS11Exception: CKR_OPERATION_NOT_INITIALIZED

Summary: Reset cipher state to un-initialized wherever necessary.
Reviewed-by: weijun

src/share/classes/sun/security/pkcs11/P11Cipher.java

@@ -395,6 +395,8 @@ final class P11Cipher extends CipherSpi {
             }
         } catch (PKCS11Exception e) {
             throw new ProviderException("Cancel failed", e);
+        } finally {
+            reset();
         }
     }
 
@@ -408,12 +410,18 @@ final class P11Cipher extends CipherSpi {
         if (session == null) {
             session = token.getOpSession();
         }
-        if (encrypt) {
-            token.p11.C_EncryptInit(session.id(),
-                    new CK_MECHANISM(mechanism, iv), p11Key.keyID);
-        } else {
-            token.p11.C_DecryptInit(session.id(),
-                    new CK_MECHANISM(mechanism, iv), p11Key.keyID);
+        try {
+            if (encrypt) {
+                token.p11.C_EncryptInit(session.id(),
+                        new CK_MECHANISM(mechanism, iv), p11Key.keyID);
+            } else {
+                token.p11.C_DecryptInit(session.id(),
+                        new CK_MECHANISM(mechanism, iv), p11Key.keyID);
+            }
+        } catch (PKCS11Exception ex) {
+            // release session when initialization failed
+            session = token.releaseSession(session);
+            throw ex;
         }
         bytesBuffered = 0;
         padBufferLen = 0;
@@ -448,6 +456,16 @@ final class P11Cipher extends CipherSpi {
         return result;
     }
 
+    // reset the states to the pre-initialized values
+    private void reset() {
+        initialized = false;
+        bytesBuffered = 0;
+        padBufferLen = 0;
+        if (session != null) {
+            session = token.releaseSession(session);
+        }
+    }
+
     // see JCE spec
     protected byte[] engineUpdate(byte[] in, int inOfs, int inLen) {
         try {
@@ -566,6 +584,7 @@ final class P11Cipher extends CipherSpi {
                 throw (ShortBufferException)
                         (new ShortBufferException().initCause(e));
             }
+            reset();
             throw new ProviderException("update() failed", e);
         }
     }
@@ -683,6 +702,7 @@ final class P11Cipher extends CipherSpi {
                 throw (ShortBufferException)
                         (new ShortBufferException().initCause(e));
             }
+            reset();
             throw new ProviderException("update() failed", e);
         }
     }
@@ -729,10 +749,7 @@ final class P11Cipher extends CipherSpi {
             handleException(e);
             throw new ProviderException("doFinal() failed", e);
         } finally {
-            initialized = false;
-            bytesBuffered = 0;
-            padBufferLen = 0;
-            session = token.releaseSession(session);
+            reset();
         }
     }
 
@@ -806,9 +823,7 @@ final class P11Cipher extends CipherSpi {
             handleException(e);
             throw new ProviderException("doFinal() failed", e);
         } finally {
-            initialized = false;
-            bytesBuffered = 0;
-            session = token.releaseSession(session);
+            reset();
         }
     }