8210870: Libsunmscapi improved interactions

Reviewed-by: valeriep, mschoene, rhalade

src/windows/classes/sun/security/mscapi/KeyStore.java

@@ -753,6 +753,7 @@ abstract class KeyStore extends KeyStoreSpi {
     /**
      * Generates a certificate chain from the collection of
      * certificates and stores the result into a key entry.
+     * This method is called by native code in libsunmscapi.
      */
     private void generateCertificateChain(String alias,
         Collection<? extends Certificate> certCollection)
@@ -775,13 +776,15 @@ abstract class KeyStore extends KeyStoreSpi {
         catch (Throwable e)
         {
             // Ignore the exception and skip this entry
-            // TODO - throw CertificateException?
+            // If e is thrown, remember to deal with it in
+            // native code.
         }
     }
 
     /**
      * Generates RSA key and certificate chain from the private key handle,
      * collection of certificates and stores the result into key entries.
+     * This method is called by native code in libsunmscapi.
      */
     private void generateRSAKeyAndCertificateChain(String alias,
         long hCryptProv, long hCryptKey, int keyLength,
@@ -807,12 +810,14 @@ abstract class KeyStore extends KeyStoreSpi {
         catch (Throwable e)
         {
             // Ignore the exception and skip this entry
-            // TODO - throw CertificateException?
+            // If e is thrown, remember to deal with it in
+            // native code.
         }
     }
 
     /**
      * Generates certificates from byte data and stores into cert collection.
+     * This method is called by native code in libsunmscapi.
      *
      * @param data Byte data.
      * @param certCollection Collection of certificates.
@@ -836,12 +841,14 @@ abstract class KeyStore extends KeyStoreSpi {
         catch (CertificateException e)
         {
             // Ignore the exception and skip this certificate
-            // TODO - throw CertificateException?
+            // If e is thrown, remember to deal with it in
+            // native code.
         }
         catch (Throwable te)
         {
             // Ignore the exception and skip this certificate
-            // TODO - throw CertificateException?
+            // If e is thrown, remember to deal with it in
+            // native code.
         }
     }
 

src/windows/native/sun/security/mscapi/security.cpp

@@ -425,6 +425,15 @@ JNIEXPORT void JNICALL Java_sun_security_mscapi_KeyStore_loadKeysOrCertificateCh
                     // Create ArrayList to store certs in each chain
                     jobject jArrayList =
                         env->NewObject(clazzArrayList, mNewArrayList);
+                    if (jArrayList == NULL) {
+                        __leave;
+                    }
+
+                    // Cleanup the previous allocated name
+                    if (pszNameString) {
+                        delete [] pszNameString;
+                        pszNameString = NULL;
+                    }
 
                     for (unsigned int j=0; j < rgpChain->cElement; j++)
                     {
@@ -463,6 +472,9 @@ JNIEXPORT void JNICALL Java_sun_security_mscapi_KeyStore_loadKeysOrCertificateCh
 
                         // Allocate and populate byte array
                         jbyteArray byteArray = env->NewByteArray(cbCertEncoded);
+                        if (byteArray == NULL) {
+                            __leave;
+                        }
                         env->SetByteArrayRegion(byteArray, 0, cbCertEncoded,
                             (jbyte*) pbCertEncoded);
 
@@ -471,12 +483,21 @@ JNIEXPORT void JNICALL Java_sun_security_mscapi_KeyStore_loadKeysOrCertificateCh
                         env->CallVoidMethod(obj, mGenCert, byteArray, jArrayList);
                     }
 
+                    // Usually pszNameString should be non-NULL. It's either
+                    // the friendly name or an element from the subject name
+                    // or SAN.
+                    if (pszNameString)
+                    {
                         if (bHasNoPrivateKey)
                         {
                             // Generate certificate chain and store into cert chain
                             // collection
+                            jstring name = env->NewStringUTF(pszNameString);
+                            if (name == NULL) {
+                                __leave;
+                            }
                             env->CallVoidMethod(obj, mGenCertChain,
-                            env->NewStringUTF(pszNameString),
+                                name,
                                 jArrayList);
                         }
                         else
@@ -491,13 +512,18 @@ JNIEXPORT void JNICALL Java_sun_security_mscapi_KeyStore_loadKeysOrCertificateCh
                             {
                                 // Generate RSA certificate chain and store into cert
                                 // chain collection
+                                jstring name = env->NewStringUTF(pszNameString);
+                                if (name == NULL) {
+                                    __leave;
+                                }
                                 env->CallVoidMethod(obj, mGenRSAKeyAndCertChain,
-                                    env->NewStringUTF(pszNameString),
+                                        name,
                                         (jlong) hCryptProv, (jlong) hUserKey,
                                         dwPublicKeyLength, jArrayList);
                             }
                         }
                     }
+                }
 
                 // Free cert chain
                 if (pCertChainContext)
@@ -641,6 +667,9 @@ JNIEXPORT jbyteArray JNICALL Java_sun_security_mscapi_RSASignature_signHash
 
         // Create new byte array
         jbyteArray temp = env->NewByteArray(dwBufLen);
+        if (temp == NULL) {
+            __leave;
+        }
 
         // Copy data from native buffer
         env->SetByteArrayRegion(temp, 0, dwBufLen, pSignedHashBuffer);
@@ -964,6 +993,9 @@ JNIEXPORT void JNICALL Java_sun_security_mscapi_KeyStore_storeCertificate
         }
 
         jCertAliasChars = env->GetStringChars(jCertAliasName, NULL);
+        if (jCertAliasChars == NULL) {
+            __leave;
+        }
         memcpy(pszCertAliasName, jCertAliasChars, size * sizeof(WCHAR));
         pszCertAliasName[size] = 0; // append the string terminator
 
@@ -1600,7 +1632,9 @@ JNIEXPORT jbyteArray JNICALL Java_sun_security_mscapi_RSACipher_encryptDecrypt
         }
 
         // Create new byte array
-        result = env->NewByteArray(dwBufLen);
+        if ((result = env->NewByteArray(dwBufLen)) == NULL) {
+            __leave;
+        }
 
         // Copy data from native buffer to Java buffer
         env->SetByteArrayRegion(result, 0, dwBufLen, (jbyte*) pData);
@@ -1651,7 +1685,9 @@ JNIEXPORT jbyteArray JNICALL Java_sun_security_mscapi_RSAPublicKey_getPublicKeyB
         }
 
         // Create new byte array
-        blob = env->NewByteArray(dwBlobLen);
+        if ((blob = env->NewByteArray(dwBlobLen)) == NULL) {
+            __leave;
+        }
 
         // Copy data from native buffer to Java buffer
         env->SetByteArrayRegion(blob, 0, dwBlobLen, (jbyte*) pbKeyBlob);
@@ -1680,6 +1716,13 @@ JNIEXPORT jbyteArray JNICALL Java_sun_security_mscapi_RSAPublicKey_getExponent
     __try {
 
         jsize length = env->GetArrayLength(jKeyBlob);
+        jsize headerLength = sizeof(PUBLICKEYSTRUC) + sizeof(RSAPUBKEY);
+
+        if (length < headerLength) {
+            ThrowExceptionWithMessage(env, KEY_EXCEPTION, "Invalid BLOB");
+            __leave;
+        }
+
         if ((keyBlob = env->GetByteArrayElements(jKeyBlob, 0)) == NULL) {
             __leave;
         }
@@ -1706,7 +1749,9 @@ JNIEXPORT jbyteArray JNICALL Java_sun_security_mscapi_RSAPublicKey_getExponent
             exponentBytes[i] = ((BYTE*) &pRsaPubKey->pubexp)[j];
         }
 
-        exponent = env->NewByteArray(len);
+        if ((exponent = env->NewByteArray(len)) == NULL) {
+            __leave;
+        }
         env->SetByteArrayRegion(exponent, 0, len, exponentBytes);
     }
     __finally
@@ -1736,6 +1781,13 @@ JNIEXPORT jbyteArray JNICALL Java_sun_security_mscapi_RSAPublicKey_getModulus
     __try {
 
         jsize length = env->GetArrayLength(jKeyBlob);
+        jsize headerLength = sizeof(PUBLICKEYSTRUC) + sizeof(RSAPUBKEY);
+
+        if (length < headerLength) {
+            ThrowExceptionWithMessage(env, KEY_EXCEPTION, "Invalid BLOB");
+            __leave;
+        }
+
         if ((keyBlob = env->GetByteArrayElements(jKeyBlob, 0)) == NULL) {
             __leave;
         }
@@ -1752,19 +1804,25 @@ JNIEXPORT jbyteArray JNICALL Java_sun_security_mscapi_RSAPublicKey_getModulus
             (RSAPUBKEY *) (keyBlob + sizeof(PUBLICKEYSTRUC));
 
         int len = pRsaPubKey->bitlen / 8;
+        if (len < 0 || len > length - headerLength) {
+            ThrowExceptionWithMessage(env, KEY_EXCEPTION, "Invalid key length");
+            __leave;
+        }
+
         modulusBytes = new (env) jbyte[len];
         if (modulusBytes == NULL) {
             __leave;
         }
-        BYTE * pbModulus =
-            (BYTE *) (keyBlob + sizeof(PUBLICKEYSTRUC) + sizeof(RSAPUBKEY));
+        BYTE * pbModulus = (BYTE *) (keyBlob + headerLength);
 
         // convert from little-endian while copying from blob
         for (int i = 0, j = len - 1; i < len; i++, j--) {
             modulusBytes[i] = pbModulus[j];
         }
 
-        modulus = env->NewByteArray(len);
+        if ((modulus = env->NewByteArray(len)) == NULL) {
+            __leave;
+        }
         env->SetByteArrayRegion(modulus, 0, len, modulusBytes);
     }
     __finally
@@ -1972,7 +2030,9 @@ jbyteArray generateKeyBlob(
             }
         }
 
-        jBlob = env->NewByteArray(jBlobLength);
+        if ((jBlob = env->NewByteArray(jBlobLength)) == NULL) {
+            __leave;
+        }
         env->SetByteArrayRegion(jBlob, 0, jBlobLength, jBlobBytes);
 
     }