8165463: Native implementation of sunmscapi should use operator new (nothrow) for allocations

Reviewed-by: clanger, jnimeh, vinnie

src/windows/native/sun/security/mscapi/security.cpp

@@ -35,6 +35,7 @@
 #include <BaseTsd.h>
 #include <wincrypt.h>
 #include <stdio.h>
+#include <memory>
 
 
 #define OID_EKU_ANY         "2.5.29.37.0"
@@ -47,14 +48,27 @@
 #define KEYSTORE_EXCEPTION  "java/security/KeyStoreException"
 #define PROVIDER_EXCEPTION  "java/security/ProviderException"
 #define SIGNATURE_EXCEPTION "java/security/SignatureException"
+#define OUT_OF_MEMORY_ERROR "java/lang/OutOfMemoryError"
 
 extern "C" {
 
+/*
+ * Throws an arbitrary Java exception with the given message.
+ */
+void ThrowExceptionWithMessage(JNIEnv *env, const char *exceptionName,
+                               const char *szMessage)
+{
+    jclass exceptionClazz = env->FindClass(exceptionName);
+    if (exceptionClazz != NULL) {
+        env->ThrowNew(exceptionClazz, szMessage);
+    }
+}
+
 /*
  * Throws an arbitrary Java exception.
  * The exception message is a Windows system error message.
  */
-void ThrowException(JNIEnv *env, char *exceptionName, DWORD dwError)
+void ThrowException(JNIEnv *env, const char *exceptionName, DWORD dwError)
 {
     char szMessage[1024];
     szMessage[0] = '\0';
@@ -65,12 +79,22 @@ void ThrowException(JNIEnv *env, char *exceptionName, DWORD dwError)
         strcpy(szMessage, "Unknown error");
     }
 
-    jclass exceptionClazz = env->FindClass(exceptionName);
-    if (exceptionClazz != NULL) {
-        env->ThrowNew(exceptionClazz, szMessage);
-    }
+    ThrowExceptionWithMessage(env, exceptionName, szMessage);
 }
 
+/*
+ * Overloaded 'operator new[]' variant, which will raise Java's
+ * OutOfMemoryError in the case of a failure.
+ */
+static void* operator new[](std::size_t size, JNIEnv *env)
+{
+    void* buf = ::operator new[](size, std::nothrow);
+    if (buf == NULL) {
+        ThrowExceptionWithMessage(env, OUT_OF_MEMORY_ERROR,
+                "Native memory allocation failed");
+    }
+    return buf;
+}
 
 /*
  * Maps the name of a hash algorithm to an algorithm identifier.
@@ -205,7 +229,10 @@ JNIEXPORT jbyteArray JNICALL Java_sun_security_mscapi_PRNG_generateSeed
 
         } else if (length > 0) {
 
-            pbData = new BYTE[length];
+            pbData = new (env) BYTE[length];
+            if (pbData == NULL) {
+                __leave;
+            }
 
             if (::CryptGenRandom(
                 hCryptProv,
@@ -435,7 +462,11 @@ JNIEXPORT void JNICALL Java_sun_security_mscapi_KeyStore_loadKeysOrCertificateCh
                                 NULL, 0)) > 1) {
 
                                 // Found friendly name
-                                pszNameString = new char[cchNameString];
+                                pszNameString = new (env) char[cchNameString];
+                                if (pszNameString == NULL) {
+                                    __leave;
+                                }
+
                                 CertGetNameString(pc,
                                     CERT_NAME_FRIENDLY_DISPLAY_TYPE, 0, NULL,
                                     pszNameString, cchNameString);
@@ -572,7 +603,10 @@ JNIEXPORT jbyteArray JNICALL Java_sun_security_mscapi_RSASignature_signHash
         }
 
         // Copy hash from Java to native buffer
-        pHashBuffer = new jbyte[jHashSize];
+        pHashBuffer = new (env) jbyte[jHashSize];
+        if (pHashBuffer == NULL) {
+            __leave;
+        }
         env->GetByteArrayRegion(jHash, 0, jHashSize, pHashBuffer);
 
         // Set hash value in the hash object
@@ -610,7 +644,10 @@ JNIEXPORT jbyteArray JNICALL Java_sun_security_mscapi_RSASignature_signHash
             __leave;
         }
 
-        pSignedHashBuffer = new jbyte[dwBufLen];
+        pSignedHashBuffer = new (env) jbyte[dwBufLen];
+        if (pSignedHashBuffer == NULL) {
+            __leave;
+        }
         if (::CryptSignHash(hHash, dwKeySpec, NULL, dwFlags, (BYTE*)pSignedHashBuffer, &dwBufLen) == FALSE)
         {
             ThrowException(env, SIGNATURE_EXCEPTION, GetLastError());
@@ -698,9 +735,16 @@ JNIEXPORT jboolean JNICALL Java_sun_security_mscapi_RSASignature_verifySignedHas
         }
 
         // Copy hash and signedHash from Java to native buffer
-        pHashBuffer = new jbyte[jHashSize];
+        pHashBuffer = new (env) jbyte[jHashSize];
+        if (pHashBuffer == NULL) {
+            __leave;
+        }
         env->GetByteArrayRegion(jHash, 0, jHashSize, pHashBuffer);
-        pSignedHashBuffer = new jbyte[jSignedHashSize];
+
+        pSignedHashBuffer = new (env) jbyte[jSignedHashSize];
+        if (pSignedHashBuffer == NULL) {
+            __leave;
+        }
         env->GetByteArrayRegion(jSignedHash, 0, jSignedHashSize,
             pSignedHashBuffer);
 
@@ -913,7 +957,10 @@ JNIEXPORT void JNICALL Java_sun_security_mscapi_KeyStore_storeCertificate
         }
 
         // Copy encoding from Java to native buffer
-        pbCertEncoding = new jbyte[jCertEncodingSize];
+        pbCertEncoding = new (env) jbyte[jCertEncodingSize];
+        if (pbCertEncoding == NULL) {
+            __leave;
+        }
         env->GetByteArrayRegion(jCertEncoding, 0, jCertEncodingSize, pbCertEncoding);
 
         // Create a certificate context from the encoded cert
@@ -926,7 +973,10 @@ JNIEXPORT void JNICALL Java_sun_security_mscapi_KeyStore_storeCertificate
 
         // Set the certificate's friendly name
         int size = env->GetStringLength(jCertAliasName);
-        pszCertAliasName = new WCHAR[size + 1];
+        pszCertAliasName = new (env) WCHAR[size + 1];
+        if (pszCertAliasName == NULL) {
+            __leave;
+        }
 
         jCertAliasChars = env->GetStringChars(jCertAliasName, NULL);
         memcpy(pszCertAliasName, jCertAliasChars, size * sizeof(WCHAR));
@@ -964,7 +1014,10 @@ JNIEXPORT void JNICALL Java_sun_security_mscapi_KeyStore_storeCertificate
                 __leave;
             }
 
-            pszContainerName = new char[dwDataLen];
+            pszContainerName = new (env) char[dwDataLen];
+            if (pszContainerName == NULL) {
+                __leave;
+            }
 
             if (! ::CryptGetProvParam(
                 (HCRYPTPROV) hCryptProv,
@@ -978,7 +1031,10 @@ JNIEXPORT void JNICALL Java_sun_security_mscapi_KeyStore_storeCertificate
             }
 
             // Convert to a wide char string
-            pwszContainerName = new WCHAR[dwDataLen];
+            pwszContainerName = new (env) WCHAR[dwDataLen];
+            if (pwszContainerName == NULL) {
+                __leave;
+            }
 
             if (mbstowcs(pwszContainerName, pszContainerName, dwDataLen) == 0) {
                 ThrowException(env, KEYSTORE_EXCEPTION, GetLastError());
@@ -1001,7 +1057,10 @@ JNIEXPORT void JNICALL Java_sun_security_mscapi_KeyStore_storeCertificate
                 __leave;
             }
 
-            pszProviderName = new char[dwDataLen];
+            pszProviderName = new (env) char[dwDataLen];
+            if (pszProviderName == NULL) {
+                __leave;
+            }
 
             if (! ::CryptGetProvParam(
                 (HCRYPTPROV) hCryptProv,
@@ -1015,7 +1074,10 @@ JNIEXPORT void JNICALL Java_sun_security_mscapi_KeyStore_storeCertificate
             }
 
             // Convert to a wide char string
-            pwszProviderName = new WCHAR[dwDataLen];
+            pwszProviderName = new (env) WCHAR[dwDataLen];
+            if (pwszProviderName == NULL) {
+                __leave;
+            }
 
             if (mbstowcs(pwszProviderName, pszProviderName, dwDataLen) == 0) {
                 ThrowException(env, KEYSTORE_EXCEPTION, GetLastError());
@@ -1155,7 +1217,10 @@ JNIEXPORT void JNICALL Java_sun_security_mscapi_KeyStore_removeCertificate
         }
 
         // Copy encoding from Java to native buffer
-        pbCertEncoding = new jbyte[jCertEncodingSize];
+        pbCertEncoding = new (env) jbyte[jCertEncodingSize];
+        if (pbCertEncoding == NULL) {
+            __leave;
+        }
         env->GetByteArrayRegion(jCertEncoding, 0, jCertEncodingSize, pbCertEncoding);
 
         // Create a certificate context from the encoded cert
@@ -1178,7 +1243,10 @@ JNIEXPORT void JNICALL Java_sun_security_mscapi_KeyStore_removeCertificate
         if ((cchNameString = ::CertGetNameString(pTBDCertContext,
                 CERT_NAME_FRIENDLY_DISPLAY_TYPE, 0, NULL, NULL, 0)) > 1) {
 
-            pszNameString = new char[cchNameString];
+            pszNameString = new (env) char[cchNameString];
+            if (pszNameString == NULL) {
+                __leave;
+            }
 
             ::CertGetNameString(pTBDCertContext,
                 CERT_NAME_FRIENDLY_DISPLAY_TYPE, 0, NULL, pszNameString,
@@ -1328,7 +1396,10 @@ JNIEXPORT jlong JNICALL Java_sun_security_mscapi_RSACipher_findCertificateUsingA
                 continue; // not found
             }
 
-            pszNameString = new char[cchNameString];
+            pszNameString = new (env) char[cchNameString];
+            if (pszNameString == NULL) {
+                __leave;
+            }
 
             if (::CertGetNameString(pCertContext,
                 CERT_NAME_FRIENDLY_DISPLAY_TYPE, 0, NULL, pszNameString,
@@ -1504,7 +1575,10 @@ JNIEXPORT jbyteArray JNICALL Java_sun_security_mscapi_RSACipher_encryptDecrypt
     __try
     {
         // Copy data from Java buffer to native buffer
-        pData = new jbyte[dwBufLen];
+        pData = new (env) jbyte[dwBufLen];
+        if (pData == NULL) {
+            __leave;
+        }
         env->GetByteArrayRegion(jData, 0, dwBufLen, pData);
 
         if (doEncrypt == JNI_TRUE) {
@@ -1578,7 +1652,10 @@ JNIEXPORT jbyteArray JNICALL Java_sun_security_mscapi_RSAPublicKey_getPublicKeyB
             __leave;
         }
 
-        pbKeyBlob = new BYTE[dwBlobLen];
+        pbKeyBlob = new (env) BYTE[dwBlobLen];
+        if (pbKeyBlob == NULL) {
+            __leave;
+        }
 
         // Generate key blob
         if (! ::CryptExportKey((HCRYPTKEY) hCryptKey, 0, PUBLICKEYBLOB, 0,
@@ -1632,8 +1709,12 @@ JNIEXPORT jbyteArray JNICALL Java_sun_security_mscapi_RSAPublicKey_getExponent
 
         RSAPUBKEY* pRsaPubKey =
             (RSAPUBKEY *) (keyBlob + sizeof(PUBLICKEYSTRUC));
+
         int len = sizeof(pRsaPubKey->pubexp);
-        exponentBytes = new jbyte[len];
+        exponentBytes = new (env) jbyte[len];
+        if (exponentBytes == NULL) {
+            __leave;
+        }
 
         // convert from little-endian while copying from blob
         for (int i = 0, j = len - 1; i < len; i++, j--) {
@@ -1684,9 +1765,12 @@ JNIEXPORT jbyteArray JNICALL Java_sun_security_mscapi_RSAPublicKey_getModulus
 
         RSAPUBKEY* pRsaPubKey =
             (RSAPUBKEY *) (keyBlob + sizeof(PUBLICKEYSTRUC));
-        int len = pRsaPubKey->bitlen / 8;
 
-        modulusBytes = new jbyte[len];
+        int len = pRsaPubKey->bitlen / 8;
+        modulusBytes = new (env) jbyte[len];
+        if (modulusBytes == NULL) {
+            __leave;
+        }
         BYTE * pbModulus =
             (BYTE *) (keyBlob + sizeof(PUBLICKEYSTRUC) + sizeof(RSAPUBKEY));
 
@@ -1807,12 +1891,16 @@ jbyteArray generateKeyBlob(
                         (jKeyBitLength / 8);
     }
 
-    jbyte* jBlobBytes = new jbyte[jBlobLength];
+    jbyte* jBlobBytes = NULL;
     jbyte* jBlobElement;
     jbyteArray jBlob = NULL;
     jsize  jElementLength;
 
     __try {
+        jBlobBytes = new (env) jbyte[jBlobLength];
+        if (jBlobBytes == NULL) {
+            __leave;
+        }
 
         BLOBHEADER *pBlobHeader = (BLOBHEADER *) jBlobBytes;
         if (bGeneratePrivateKeyBlob) {