6888925: SunMSCAPI's Cipher can't use RSA public keys obtained from other sources.

Reviewed-by: mullan

src/windows/classes/sun/security/mscapi/RSACipher.java

 /*
- * Copyright (c) 2005, 2009, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,6 +25,7 @@
 
 package sun.security.mscapi;
 
+import java.math.BigInteger;
 import java.security.*;
 import java.security.Key;
 import java.security.interfaces.*;
@@ -33,6 +34,8 @@ import java.security.spec.*;
 import javax.crypto.*;
 import javax.crypto.spec.*;
 
+import sun.security.rsa.RSAKeyFactory;
+
 /**
  * RSA cipher implementation using the Microsoft Crypto API.
  * Supports RSA en/decryption and signing/verifying using PKCS#1 v1.5 padding.
@@ -189,8 +192,38 @@ public final class RSACipher extends CipherSpi {
         default:
             throw new InvalidKeyException("Unknown mode: " + opmode);
         }
+
         if (!(key instanceof sun.security.mscapi.Key)) {
-            throw new InvalidKeyException("Unsupported key type: " + key);
+            if (key instanceof java.security.interfaces.RSAPublicKey) {
+                java.security.interfaces.RSAPublicKey rsaKey =
+                    (java.security.interfaces.RSAPublicKey) key;
+
+                // Convert key to MSCAPI format
+
+                BigInteger modulus = rsaKey.getModulus();
+                BigInteger exponent =  rsaKey.getPublicExponent();
+
+                // Check against the local and global values to make sure
+                // the sizes are ok.  Round up to the nearest byte.
+                RSAKeyFactory.checkKeyLengths(((modulus.bitLength() + 7) & ~7),
+                    exponent, -1, RSAKeyPairGenerator.KEY_SIZE_MAX);
+
+                byte[] modulusBytes = modulus.toByteArray();
+                byte[] exponentBytes = exponent.toByteArray();
+
+                // Adjust key length due to sign bit
+                int keyBitLength = (modulusBytes[0] == 0)
+                    ? (modulusBytes.length - 1) * 8
+                    : modulusBytes.length * 8;
+
+                byte[] keyBlob = RSASignature.generatePublicKeyBlob(
+                    keyBitLength, modulusBytes, exponentBytes);
+
+                key = RSASignature.importPublicKey(keyBlob, keyBitLength);
+
+            } else {
+                throw new InvalidKeyException("Unsupported key type: " + key);
+            }
         }
 
         if (key instanceof PublicKey) {
@@ -358,6 +391,10 @@ public final class RSACipher extends CipherSpi {
 
         if (key instanceof sun.security.mscapi.Key) {
             return ((sun.security.mscapi.Key) key).bitLength();
+
+        } else if (key instanceof RSAKey) {
+            return ((RSAKey) key).getModulus().bitLength();
+
         } else {
             throw new InvalidKeyException("Unsupported key type: " + key);
         }

src/windows/classes/sun/security/mscapi/RSAPublicKey.java

 /*
- * Copyright (c) 2005, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -91,7 +91,7 @@ class RSAPublicKey extends Key implements java.security.interfaces.RSAPublicKey
         if (exponent == null) {
             publicKeyBlob = getPublicKeyBlob(hCryptKey);
 
-            exponent = new BigInteger(getExponent(publicKeyBlob));
+            exponent = new BigInteger(1, getExponent(publicKeyBlob));
         }
 
         return exponent;
@@ -104,7 +104,7 @@ class RSAPublicKey extends Key implements java.security.interfaces.RSAPublicKey
 
         if (modulus == null) {
             publicKeyBlob = getPublicKeyBlob(hCryptKey);
-            modulus = new BigInteger(getModulus(publicKeyBlob));
+            modulus = new BigInteger(1, getModulus(publicKeyBlob));
         }
 
         return modulus;

src/windows/classes/sun/security/mscapi/RSASignature.java

@@ -379,11 +379,13 @@ abstract class RSASignature extends java.security.SignatureSpi
     /**
      * Generates a public-key BLOB from a key's components.
      */
-    private native byte[] generatePublicKeyBlob(
+    // used by RSACipher
+    static native byte[] generatePublicKeyBlob(
         int keyBitLength, byte[] modulus, byte[] publicExponent);
 
     /**
      * Imports a public-key BLOB.
      */
-    private native RSAPublicKey importPublicKey(byte[] keyBlob, int keySize);
+    // used by RSACipher
+    static native RSAPublicKey importPublicKey(byte[] keyBlob, int keySize);
 }

test/sun/security/mscapi/PublicKeyInterop.java

+/*
+ * Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @see PublicKeyInterop.sh
+ */
+
+import java.security.*;
+import java.util.*;
+import javax.crypto.*;
+
+import sun.misc.HexDumpEncoder;
+
+/*
+ * Confirm interoperability of RSA public keys between SunMSCAPI and SunJCE
+ * security providers.
+ */
+public class PublicKeyInterop {
+
+    public static void main(String[] arg) throws Exception {
+        PrivateKey privKey = null;
+        Certificate cert = null;
+        KeyStore ks = KeyStore.getInstance("Windows-MY");
+        ks.load(null, null);
+        System.out.println("Loaded keystore: Windows-MY");
+
+        PublicKey myPuKey =
+            (PublicKey) ks.getCertificate("6888925").getPublicKey();
+        System.out.println("Public key is a " + myPuKey.getClass().getName());
+        PrivateKey myPrKey = (PrivateKey) ks.getKey("6888925", null);
+        System.out.println("Private key is a " + myPrKey.getClass().getName());
+        System.out.println();
+
+        byte[] plain = new byte[] {0x01, 0x02, 0x03, 0x04, 0x05};
+        HexDumpEncoder hde = new HexDumpEncoder();
+        System.out.println("Plaintext:\n" + hde.encode(plain) + "\n");
+
+        Cipher rsa = Cipher.getInstance("RSA/ECB/PKCS1Padding");
+        rsa.init(Cipher.ENCRYPT_MODE, myPuKey);
+        byte[] encrypted = rsa.doFinal(plain);
+        System.out.println("Encrypted plaintext using RSA Cipher from " +
+            rsa.getProvider().getName() + " JCE provider\n");
+        System.out.println(hde.encode(encrypted) + "\n");
+
+        Cipher rsa2 = Cipher.getInstance("RSA/ECB/PKCS1Padding", "SunMSCAPI");
+        rsa2.init(Cipher.ENCRYPT_MODE, myPuKey);
+        byte[] encrypted2 = rsa2.doFinal(plain);
+        System.out.println("Encrypted plaintext using RSA Cipher from " +
+            rsa2.getProvider().getName() + " JCE provider\n");
+        System.out.println(hde.encode(encrypted2) + "\n");
+
+        Cipher rsa3 = Cipher.getInstance("RSA/ECB/PKCS1Padding", "SunMSCAPI");
+        rsa3.init(Cipher.DECRYPT_MODE, myPrKey);
+        byte[] decrypted = rsa3.doFinal(encrypted);
+        System.out.println("Decrypted first ciphertext using RSA Cipher from " +
+            rsa3.getProvider().getName() + " JCE provider\n");
+        System.out.println(hde.encode(decrypted) + "\n");
+        if (! Arrays.equals(plain, decrypted)) {
+            throw new Exception("First decrypted ciphertext does not match " +
+                "original plaintext");
+        }
+
+        decrypted = rsa3.doFinal(encrypted2);
+        System.out.println("Decrypted second ciphertext using RSA Cipher from "
+            + rsa3.getProvider().getName() + " JCE provider\n");
+        System.out.println(hde.encode(decrypted) + "\n");
+        if (! Arrays.equals(plain, decrypted)) {
+            throw new Exception("Second decrypted ciphertext does not match " +
+                "original plaintext");
+        }
+    }
+}

test/sun/security/mscapi/PublicKeyInterop.sh

+#!/bin/sh
+
+#
+# Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+
+# @test
+# @bug 6888925
+# @run shell PublicKeyInterop.sh
+# @summary SunMSCAPI's Cipher can't use RSA public keys obtained from other
+#          sources.
+#
+
+# set a few environment variables so that the shell-script can run stand-alone
+# in the source directory
+if [ "${TESTSRC}" = "" ] ; then
+   TESTSRC="."
+fi
+
+if [ "${TESTCLASSES}" = "" ] ; then
+   TESTCLASSES="."
+fi
+
+if [ "${TESTJAVA}" = "" ] ; then
+   echo "TESTJAVA not set.  Test cannot execute."
+   echo "FAILED!!!"
+   exit 1
+fi
+
+OS=`uname -s`
+case "$OS" in
+    Windows* | CYGWIN* )
+
+        echo "Creating a temporary RSA keypair in the Windows-My store..."
+        ${TESTJAVA}/bin/keytool \
+	    -genkeypair \
+	    -storetype Windows-My \
+	    -keyalg RSA \
+	    -alias 6888925 \
+	    -dname "cn=6888925,c=US" \
+	    -noprompt
+
+        echo
+	echo "Running the test..."
+        ${TESTJAVA}/bin/javac -d . ${TESTSRC}\\PublicKeyInterop.java
+        ${TESTJAVA}/bin/java PublicKeyInterop
+
+        rc=$?
+
+        echo
+        echo "Removing the temporary RSA keypair from the Windows-My store..."
+        ${TESTJAVA}/bin/keytool \
+	    -delete \
+	    -storetype Windows-My \
+	    -alias 6888925
+
+	echo done.
+        exit $rc
+        ;;
+
+    * )
+        echo "This test is not intended for '$OS' - passing test"
+        exit 0
+        ;;
+esac