8023546: sun/security/mscapi/ShortRSAKey1024.sh fails intermittently

Reviewed-by: vinnie

src/windows/classes/sun/security/mscapi/KeyStore.java

@@ -41,12 +41,7 @@ import java.security.cert.Certificate;
 import java.security.cert.CertificateException;
 import java.security.cert.CertificateFactory;
 import java.security.interfaces.RSAPrivateCrtKey;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Date;
-import java.util.Enumeration;
-import java.util.Iterator;
-import java.util.UUID;
+import java.util.*;
 
 import sun.security.action.GetPropertyAction;
 
@@ -142,7 +137,7 @@ abstract class KeyStore extends KeyStoreSpi {
                 key.getPrimeExponentQ().toByteArray(),
                 key.getCrtCoefficient().toByteArray());
 
-            privateKey = storePrivateKey(keyBlob,
+            privateKey = storePrivateKey(Objects.requireNonNull(keyBlob),
                 "{" + UUID.randomUUID().toString() + "}", keyBitLength);
         }
 

src/windows/native/sun/security/mscapi/security.cpp

@@ -1659,29 +1659,36 @@ JNIEXPORT jbyteArray JNICALL Java_sun_security_mscapi_RSAPublicKey_getModulus
 int convertToLittleEndian(JNIEnv *env, jbyteArray source, jbyte* destination,
     int destinationLength) {
 
-    int count = 0;
     int sourceLength = env->GetArrayLength(source);
 
-    if (sourceLength < destinationLength) {
-        return -1;
-    }
-
     jbyte* sourceBytes = env->GetByteArrayElements(source, 0);
     if (sourceBytes == NULL) {
         return -1;
     }
 
+    int copyLen = sourceLength;
+    if (sourceLength > destinationLength) {
+        // source might include an extra sign byte
+        if (sourceLength == destinationLength + 1 && sourceBytes[0] == 0) {
+            copyLen--;
+        } else {
+            return -1;
+        }
+    }
+
     // Copy bytes from the end of the source array to the beginning of the
     // destination array (until the destination array is full).
     // This ensures that the sign byte from the source array will be excluded.
-    for (int i = 0; i < destinationLength; i++) {
-        destination[i] = sourceBytes[sourceLength - i - 1];
-        count++;
+    for (int i = 0; i < copyLen; i++) {
+        destination[i] = sourceBytes[sourceLength - 1 - i];
     }
-    if (sourceBytes)
-        env->ReleaseByteArrayElements(source, sourceBytes, JNI_ABORT);
+    if (copyLen < destinationLength) {
+        memset(destination + copyLen, 0, destinationLength - copyLen);
+    }
+
+    env->ReleaseByteArrayElements(source, sourceBytes, JNI_ABORT);
 
-    return count;
+    return destinationLength;
 }
 
 /*

test/sun/security/mscapi/SmallPrimeExponentP.java

+/*
+ * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import sun.security.tools.keytool.CertAndKeyGen;
+import sun.security.x509.X500Name;
+
+import java.security.KeyStore;
+import java.security.SecureRandom;
+import java.security.cert.X509Certificate;
+import java.security.interfaces.RSAPrivateCrtKey;
+import java.util.HashSet;
+import java.util.Set;
+
+/*
+ * @test
+ * @bug 8023546
+ * @summary sun/security/mscapi/ShortRSAKey1024.sh fails intermittently
+ */
+public class SmallPrimeExponentP {
+
+    public static void main(String argv[]) throws Exception {
+
+        String osName = System.getProperty("os.name");
+        if (!osName.startsWith("Windows")) {
+            System.out.println("Not windows");
+            return;
+        }
+        KeyStore ks = KeyStore.getInstance("Windows-MY");
+        ks.load(null, null);
+        CertAndKeyGen ckg = new CertAndKeyGen("RSA", "SHA1withRSA");
+        ckg.setRandom(new SecureRandom());
+        boolean see63 = false, see65 = false;
+        while (!see63 || !see65) {
+            ckg.generate(1024);
+            RSAPrivateCrtKey k = (RSAPrivateCrtKey) ckg.getPrivateKey();
+            int len = k.getPrimeExponentP().toByteArray().length;
+            if (len == 63 || len == 65) {
+                if (len == 63) {
+                    if (see63) continue;
+                    else see63 = true;
+                }
+                if (len == 65) {
+                    if (see65) continue;
+                    else see65 = true;
+                }
+                System.err.print(len);
+                ks.setKeyEntry("anything", k, null, new X509Certificate[]{
+                        ckg.getSelfCertificate(new X500Name("CN=Me"), 1000)
+                });
+            }
+            System.err.print('.');
+        }
+        ks.store(null, null);
+    }
+}