8153438: Avoid repeated "Please insert a smart card" popup windows

Reviewed-by: vinnie, valeriep

src/windows/native/sun/security/mscapi/security.cpp

@@ -352,22 +352,35 @@ JNIEXPORT void JNICALL Java_sun_security_mscapi_KeyStore_loadKeysOrCertificateCh
             BOOL bHasNoPrivateKey = FALSE;
             DWORD dwPublicKeyLength = 0;
 
-            if (::CryptAcquireCertificatePrivateKey(pCertContext, NULL, NULL,
-                                                    &hCryptProv, &dwKeySpec, &bCallerFreeProv) == FALSE)
+            // First, probe it silently
+            if (::CryptAcquireCertificatePrivateKey(pCertContext, CRYPT_ACQUIRE_SILENT_FLAG, NULL,
+                    &hCryptProv, &dwKeySpec, &bCallerFreeProv) == FALSE
+                && GetLastError() != NTE_SILENT_CONTEXT)
             {
                 bHasNoPrivateKey = TRUE;
+            }
+            else
+            {
+                if (bCallerFreeProv == TRUE) {
+                    ::CryptReleaseContext(hCryptProv, NULL);
+                    bCallerFreeProv = FALSE;
+                }
 
-            } else {
+                // Second, acquire the key normally (not silently)
+                if (::CryptAcquireCertificatePrivateKey(pCertContext, 0, NULL,
+                        &hCryptProv, &dwKeySpec, &bCallerFreeProv) == FALSE)
+                {
+                    bHasNoPrivateKey = TRUE;
+                }
+                else
+                {
                     // Private key is available
-
                     BOOL bGetUserKey = ::CryptGetUserKey(hCryptProv, dwKeySpec, &hUserKey);
 
                     // Skip certificate if cannot find private key
-                if (bGetUserKey == FALSE)
-                {
+                    if (bGetUserKey == FALSE) {
                         if (bCallerFreeProv)
                             ::CryptReleaseContext(hCryptProv, NULL);
-
                         continue;
                     }
 
@@ -375,7 +388,6 @@ JNIEXPORT void JNICALL Java_sun_security_mscapi_KeyStore_loadKeysOrCertificateCh
                     DWORD dwCipherMode = CRYPT_MODE_ECB;
                     ::CryptSetKeyParam(hUserKey, KP_MODE, (BYTE*)&dwCipherMode, NULL);
 
-
                     // If the private key is present in smart card, we may not be able to
                     // determine the key length by using the private key handle. However,
                     // since public/private key pairs must have the same length, we could
@@ -383,7 +395,7 @@ JNIEXPORT void JNICALL Java_sun_security_mscapi_KeyStore_loadKeysOrCertificateCh
                     // in the certificate.
                     dwPublicKeyLength = ::CertGetPublicKeyLength(X509_ASN_ENCODING | PKCS_7_ASN_ENCODING,
                             &(pCertContext->pCertInfo->SubjectPublicKeyInfo));
-
+                }
             }
             PCCERT_CHAIN_CONTEXT pCertChainContext = NULL;
 
@@ -392,8 +404,7 @@ JNIEXPORT void JNICALL Java_sun_security_mscapi_KeyStore_loadKeysOrCertificateCh
             //
             if (GetCertificateChain(OID_EKU_ANY, pCertContext, &pCertChainContext))
             {
-
-                for (unsigned int i=0; i < pCertChainContext->cChain; i++)
+                for (DWORD i = 0; i < pCertChainContext->cChain; i++)
                 {
                     // Found cert chain
                     PCERT_SIMPLE_CHAIN rgpChain =
@@ -443,6 +454,7 @@ JNIEXPORT void JNICALL Java_sun_security_mscapi_KeyStore_loadKeysOrCertificateCh
                         // cert collection
                         env->CallVoidMethod(obj, mGenCert, byteArray, jArrayList);
                     }
+
                     if (bHasNoPrivateKey)
                     {
                         // Generate certificate chain and store into cert chain
@@ -1361,43 +1373,57 @@ JNIEXPORT jlong JNICALL Java_sun_security_mscapi_RSACipher_getKeyFromCert
     HCRYPTPROV hCryptProv = NULL;
     HCRYPTKEY hKey = NULL;
     DWORD dwKeySpec;
+    BOOL bCallerFreeProv = FALSE;
+    BOOL bRes;
 
     __try
     {
         if (usePrivateKey == JNI_TRUE) {
             // Locate the key container for the certificate's private key
-            if (!(::CryptAcquireCertificatePrivateKey(
-                (PCCERT_CONTEXT) pCertContext, 0, NULL, &hCryptProv,
-                &dwKeySpec, NULL))) {
 
+            // First, probe it silently
+            bRes = ::CryptAcquireCertificatePrivateKey(
+                    (PCCERT_CONTEXT) pCertContext, CRYPT_ACQUIRE_SILENT_FLAG,
+                    NULL, &hCryptProv, &dwKeySpec, &bCallerFreeProv);
+
+            if (bRes == FALSE && GetLastError() != NTE_SILENT_CONTEXT)
+            {
+                ThrowException(env, KEYSTORE_EXCEPTION, GetLastError());
+                __leave;
+            }
+
+            if (bCallerFreeProv == TRUE) {
+                ::CryptReleaseContext(hCryptProv, NULL);
+                bCallerFreeProv = FALSE;
+            }
+
+            // Now, do it normally (not silently)
+            if (::CryptAcquireCertificatePrivateKey(
+                    (PCCERT_CONTEXT) pCertContext, 0, NULL, &hCryptProv,
+                    &dwKeySpec, &bCallerFreeProv) == FALSE)
+            {
                 ThrowException(env, KEYSTORE_EXCEPTION, GetLastError());
                 __leave;
             }
 
             // Get a handle to the private key
-            if (!(::CryptGetUserKey(hCryptProv, dwKeySpec, &hKey))) {
+            if (::CryptGetUserKey(hCryptProv, dwKeySpec, &hKey) == FALSE) {
                 ThrowException(env, KEY_EXCEPTION, GetLastError());
                 __leave;
             }
-
-        } else { // use public key
+        }
+        else // use public key
+        {
+            bCallerFreeProv = TRUE;
 
             //  Acquire a CSP context.
-            if(::CryptAcquireContext(
-               &hCryptProv,
-               "J2SE",
-               NULL,
-               PROV_RSA_FULL,
-               0) == FALSE)
+            if (::CryptAcquireContext(&hCryptProv, "J2SE", NULL,
+                    PROV_RSA_FULL, 0) == FALSE)
             {
                 // If CSP context hasn't been created, create one.
                 //
-                if (::CryptAcquireContext(
-                    &hCryptProv,
-                    "J2SE",
-                    NULL,
-                    PROV_RSA_FULL,
-                    CRYPT_NEWKEYSET) == FALSE)
+                if (::CryptAcquireContext(&hCryptProv, "J2SE", NULL,
+                        PROV_RSA_FULL, CRYPT_NEWKEYSET) == FALSE)
                 {
                     ThrowException(env, KEYSTORE_EXCEPTION, GetLastError());
                     __leave;
@@ -1405,10 +1431,10 @@ JNIEXPORT jlong JNICALL Java_sun_security_mscapi_RSACipher_getKeyFromCert
             }
 
             // Import the certificate's public key into the key container
-            if (!(::CryptImportPublicKeyInfo(hCryptProv, X509_ASN_ENCODING,
+            if (::CryptImportPublicKeyInfo(hCryptProv, X509_ASN_ENCODING,
                     &(((PCCERT_CONTEXT) pCertContext)->pCertInfo->SubjectPublicKeyInfo),
-                &hKey))) {
-
+                    &hKey) == FALSE)
+            {
                 ThrowException(env, KEY_EXCEPTION, GetLastError());
                 __leave;
             }
@@ -1419,7 +1445,7 @@ JNIEXPORT jlong JNICALL Java_sun_security_mscapi_RSACipher_getKeyFromCert
         //--------------------------------------------------------------------
         // Clean up.
 
-        if (hCryptProv)
+        if (bCallerFreeProv == TRUE && hCryptProv != NULL)
             ::CryptReleaseContext(hCryptProv, 0);
     }