6497740: Limit the size of RSA public keys

Reviewed-by: andreas, valeriep, vinnie

src/share/classes/sun/security/pkcs11/P11KeyPairGenerator.java

 /*
- * Copyright 2003-2006 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2008 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -38,6 +38,8 @@ import static sun.security.pkcs11.TemplateManager.*;
 import sun.security.pkcs11.wrapper.*;
 import static sun.security.pkcs11.wrapper.PKCS11Constants.*;
 
+import sun.security.rsa.RSAKeyFactory;
+
 /**
  * KeyPairGenerator implementation class. This class currently supports
  * RSA, DSA, DH, and EC.
@@ -66,7 +68,7 @@ final class P11KeyPairGenerator extends KeyPairGeneratorSpi {
     private AlgorithmParameterSpec params;
 
     // for RSA, selected or default value of public exponent, always valid
-    private BigInteger rsaPublicExponent;
+    private BigInteger rsaPublicExponent = RSAKeyGenParameterSpec.F4;
 
     // SecureRandom instance, if specified in init
     private SecureRandom random;
@@ -88,19 +90,19 @@ final class P11KeyPairGenerator extends KeyPairGeneratorSpi {
     public void initialize(int keySize, SecureRandom random) {
         token.ensureValid();
         try {
-            checkKeySize(keySize);
+            checkKeySize(keySize, null);
         } catch (InvalidAlgorithmParameterException e) {
             throw new InvalidParameterException(e.getMessage());
         }
         this.keySize = keySize;
         this.params = null;
         this.random = random;
-        this.rsaPublicExponent = RSAKeyGenParameterSpec.F4;
         if (algorithm.equals("EC")) {
             params = P11ECKeyFactory.getECParameterSpec(keySize);
             if (params == null) {
-                throw new InvalidParameterException
-                ("No EC parameters available for key size " + keySize + " bits");
+                throw new InvalidParameterException(
+                    "No EC parameters available for key size "
+                    + keySize + " bits");
             }
         }
     }
@@ -115,8 +117,10 @@ final class P11KeyPairGenerator extends KeyPairGeneratorSpi {
                         ("DHParameterSpec required for Diffie-Hellman");
             }
             DHParameterSpec dhParams = (DHParameterSpec)params;
-            this.keySize = dhParams.getP().bitLength();
-            this.params = params;
+            int tmpKeySize = dhParams.getP().bitLength();
+            checkKeySize(tmpKeySize, dhParams);
+            this.keySize = tmpKeySize;
+            this.params = dhParams;
             // XXX sanity check params
         } else if (algorithm.equals("RSA")) {
             if (params instanceof RSAKeyGenParameterSpec == false) {
@@ -124,7 +128,9 @@ final class P11KeyPairGenerator extends KeyPairGeneratorSpi {
                         ("RSAKeyGenParameterSpec required for RSA");
             }
             RSAKeyGenParameterSpec rsaParams = (RSAKeyGenParameterSpec)params;
-            this.keySize = rsaParams.getKeysize();
+            int tmpKeySize = rsaParams.getKeysize();
+            checkKeySize(tmpKeySize, rsaParams);
+            this.keySize = tmpKeySize;
             this.params = null;
             this.rsaPublicExponent = rsaParams.getPublicExponent();
             // XXX sanity check params
@@ -134,13 +140,16 @@ final class P11KeyPairGenerator extends KeyPairGeneratorSpi {
                         ("DSAParameterSpec required for DSA");
             }
             DSAParameterSpec dsaParams = (DSAParameterSpec)params;
-            this.keySize = dsaParams.getP().bitLength();
-            this.params = params;
+            int tmpKeySize = dsaParams.getP().bitLength();
+            checkKeySize(tmpKeySize, dsaParams);
+            this.keySize = tmpKeySize;
+            this.params = dsaParams;
             // XXX sanity check params
         } else if (algorithm.equals("EC")) {
             ECParameterSpec ecParams;
             if (params instanceof ECParameterSpec) {
-                ecParams = P11ECKeyFactory.getECParameterSpec((ECParameterSpec)params);
+                ecParams = P11ECKeyFactory.getECParameterSpec(
+                    (ECParameterSpec)params);
                 if (ecParams == null) {
                     throw new InvalidAlgorithmParameterException
                         ("Unsupported curve: " + params);
@@ -156,16 +165,17 @@ final class P11KeyPairGenerator extends KeyPairGeneratorSpi {
                 throw new InvalidAlgorithmParameterException
                     ("ECParameterSpec or ECGenParameterSpec required for EC");
             }
-            this.keySize = ecParams.getCurve().getField().getFieldSize();
+            int tmpKeySize = ecParams.getCurve().getField().getFieldSize();
+            checkKeySize(tmpKeySize, ecParams);
+            this.keySize = tmpKeySize;
             this.params = ecParams;
         } else {
             throw new ProviderException("Unknown algorithm: " + algorithm);
         }
         this.random = random;
-        checkKeySize(keySize);
     }
 
-    private void checkKeySize(int keySize)
+    private void checkKeySize(int keySize, AlgorithmParameterSpec params)
             throws InvalidAlgorithmParameterException {
         if (algorithm.equals("EC")) {
             if (keySize < 112) {
@@ -178,13 +188,28 @@ final class P11KeyPairGenerator extends KeyPairGeneratorSpi {
                     ("Key size must be at most 2048 bit");
             }
             return;
+        } else if (algorithm.equals("RSA")) {
+            BigInteger tmpExponent = rsaPublicExponent;
+            if (params != null) {
+                // Already tested for instanceof RSAKeyGenParameterSpec above
+                tmpExponent =
+                    ((RSAKeyGenParameterSpec)params).getPublicExponent();
+            }
+            try {
+                // This provider supports 64K or less.
+                RSAKeyFactory.checkKeyLengths(keySize, tmpExponent,
+                    512, 64 * 1024);
+            } catch (InvalidKeyException e) {
+                throw new InvalidAlgorithmParameterException(e.getMessage());
+            }
+            return;
         }
+
         if (keySize < 512) {
             throw new InvalidAlgorithmParameterException
                 ("Key size must be at least 512 bit");
         }
-        if (algorithm.equals("RSA") ||
-                (algorithm.equals("DH") && (params != null))) {
+        if (algorithm.equals("DH") && (params != null)) {
             // sanity check, nobody really wants keys this large
             if (keySize > 64 * 1024) {
                 throw new InvalidAlgorithmParameterException

src/share/classes/sun/security/pkcs11/P11KeyStore.java

@@ -80,6 +80,8 @@ import static sun.security.pkcs11.P11Util.*;
 import sun.security.pkcs11.wrapper.*;
 import static sun.security.pkcs11.wrapper.PKCS11Constants.*;
 
+import sun.security.rsa.RSAKeyFactory;
+
 final class P11KeyStore extends KeyStoreSpi {
 
     private static final CK_ATTRIBUTE ATTR_CLASS_CERT =
@@ -1328,6 +1330,15 @@ final class P11KeyStore extends KeyStoreSpi {
             BigInteger modulus = attrs[0].getBigInteger();
             keyLength = modulus.bitLength();
 
+            // This check will combine our "don't care" values here
+            // with the system-wide min/max values.
+            try {
+                RSAKeyFactory.checkKeyLengths(keyLength, null,
+                    -1, Integer.MAX_VALUE);
+            } catch (InvalidKeyException e) {
+                throw new KeyStoreException(e.getMessage());
+            }
+
             return P11Key.privateKey(session,
                                 oHandle,
                                 keyType,

src/share/classes/sun/security/pkcs11/P11RSAKeyFactory.java

 /*
- * Copyright 2003 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2008 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -35,6 +35,8 @@ import static sun.security.pkcs11.TemplateManager.*;
 import sun.security.pkcs11.wrapper.*;
 import static sun.security.pkcs11.wrapper.PKCS11Constants.*;
 
+import sun.security.rsa.RSAKeyFactory;
+
 /**
  * RSA KeyFactory implemenation.
  *
@@ -131,6 +133,9 @@ final class P11RSAKeyFactory extends P11KeyFactory {
         } catch (PKCS11Exception e) {
             throw new InvalidKeySpecException
                 ("Could not create RSA public key", e);
+        } catch (InvalidKeyException e) {
+            throw new InvalidKeySpecException
+                ("Could not create RSA public key", e);
         }
     }
 
@@ -175,11 +180,15 @@ final class P11RSAKeyFactory extends P11KeyFactory {
         } catch (PKCS11Exception e) {
             throw new InvalidKeySpecException
                 ("Could not create RSA private key", e);
+        } catch (InvalidKeyException e) {
+            throw new InvalidKeySpecException
+                ("Could not create RSA private key", e);
         }
     }
 
     private PublicKey generatePublic(BigInteger n, BigInteger e)
-            throws PKCS11Exception {
+            throws PKCS11Exception, InvalidKeyException {
+        RSAKeyFactory.checkKeyLengths(n.bitLength(), e, -1, 64 * 1024);
         CK_ATTRIBUTE[] attributes = new CK_ATTRIBUTE[] {
             new CK_ATTRIBUTE(CKA_CLASS, CKO_PUBLIC_KEY),
             new CK_ATTRIBUTE(CKA_KEY_TYPE, CKK_RSA),
@@ -200,7 +209,8 @@ final class P11RSAKeyFactory extends P11KeyFactory {
     }
 
     private PrivateKey generatePrivate(BigInteger n, BigInteger d)
-            throws PKCS11Exception {
+            throws PKCS11Exception, InvalidKeyException {
+        RSAKeyFactory.checkKeyLengths(n.bitLength(), null, -1, 64 * 1024);
         CK_ATTRIBUTE[] attributes = new CK_ATTRIBUTE[] {
             new CK_ATTRIBUTE(CKA_CLASS, CKO_PRIVATE_KEY),
             new CK_ATTRIBUTE(CKA_KEY_TYPE, CKK_RSA),
@@ -222,7 +232,9 @@ final class P11RSAKeyFactory extends P11KeyFactory {
 
     private PrivateKey generatePrivate(BigInteger n, BigInteger e,
             BigInteger d, BigInteger p, BigInteger q, BigInteger pe,
-            BigInteger qe, BigInteger coeff) throws PKCS11Exception {
+            BigInteger qe, BigInteger coeff) throws PKCS11Exception,
+            InvalidKeyException {
+        RSAKeyFactory.checkKeyLengths(n.bitLength(), e, -1, 64 * 1024);
         CK_ATTRIBUTE[] attributes = new CK_ATTRIBUTE[] {
             new CK_ATTRIBUTE(CKA_CLASS, CKO_PRIVATE_KEY),
             new CK_ATTRIBUTE(CKA_KEY_TYPE, CKK_RSA),

src/share/classes/sun/security/pkcs11/SunPKCS11.java

@@ -120,11 +120,13 @@ public final class SunPKCS11 extends AuthProvider {
     }
 
     /**
-     * @deprecated use new SunPKCS11(String) or new SunPKCS11(InputStream) instead
+     * @deprecated use new SunPKCS11(String) or new SunPKCS11(InputStream)
+     *         instead
      */
     @Deprecated
     public SunPKCS11(String configName, InputStream configStream) {
-        super("SunPKCS11-" + Config.getConfig(configName, configStream).getName(),
+        super("SunPKCS11-" +
+            Config.getConfig(configName, configStream).getName(),
             1.7d, Config.getConfig(configName, configStream).getDescription());
         this.configName = configName;
         this.config = Config.removeConfig(configName);
@@ -153,7 +155,8 @@ public final class SunPKCS11 extends AuthProvider {
         //
         // If we are in Secmod mode and configured to use either the
         // nssKeyStore or the nssTrustAnchors module, we automatically
-        // switch to using the NSS trust attributes for trusted certs (KeyStore).
+        // switch to using the NSS trust attributes for trusted certs
+        // (KeyStore).
         //
 
         if (useSecmod) {
@@ -168,33 +171,40 @@ public final class SunPKCS11 extends AuthProvider {
                 if (secmod.isInitialized()) {
                     if (nssSecmodDirectory != null) {
                         String s = secmod.getConfigDir();
-                        if ((s != null) && (s.equals(nssSecmodDirectory) == false)) {
+                        if ((s != null) &&
+                                (s.equals(nssSecmodDirectory) == false)) {
                             throw new ProviderException("Secmod directory "
                                 + nssSecmodDirectory
-                                + " invalid, NSS already initialized with " + s);
+                                + " invalid, NSS already initialized with "
+                                + s);
                         }
                     }
                     if (nssLibraryDirectory != null) {
                         String s = secmod.getLibDir();
-                        if ((s != null) && (s.equals(nssLibraryDirectory) == false)) {
+                        if ((s != null) &&
+                                (s.equals(nssLibraryDirectory) == false)) {
                             throw new ProviderException("NSS library directory "
                                 + nssLibraryDirectory
-                                + " invalid, NSS already initialized with " + s);
+                                + " invalid, NSS already initialized with "
+                                + s);
                         }
                     }
                 } else {
                     if (nssDbMode != DbMode.NO_DB) {
                         if (nssSecmodDirectory == null) {
-                            throw new ProviderException("Secmod not initialized and "
-                                + "nssSecmodDirectory not specified");
+                            throw new ProviderException(
+                                "Secmod not initialized and "
+                                 + "nssSecmodDirectory not specified");
                         }
                     } else {
                         if (nssSecmodDirectory != null) {
-                            throw new ProviderException
-                            ("nssSecmodDirectory must not be specified in noDb mode");
+                            throw new ProviderException(
+                                "nssSecmodDirectory must not be "
+                                + "specified in noDb mode");
                         }
                     }
-                    secmod.initialize(nssDbMode, nssSecmodDirectory, nssLibraryDirectory);
+                    secmod.initialize(nssDbMode, nssSecmodDirectory,
+                        nssLibraryDirectory);
                 }
             } catch (IOException e) {
                 // XXX which exception to throw
@@ -211,7 +221,8 @@ public final class SunPKCS11 extends AuthProvider {
                 if (nssModule != null) {
                     moduleName = "fips";
                 } else {
-                    moduleName = (nssDbMode == DbMode.NO_DB) ? "crypto" : "keystore";
+                    moduleName = (nssDbMode == DbMode.NO_DB) ?
+                        "crypto" : "keystore";
                 }
             }
             if (moduleName.equals("fips")) {
@@ -253,10 +264,12 @@ public final class SunPKCS11 extends AuthProvider {
                         + ": only " + k + " external NSS modules available");
                 }
             } else {
-                throw new ProviderException("Unknown NSS module: " + moduleName);
+                throw new ProviderException(
+                    "Unknown NSS module: " + moduleName);
             }
             if (nssModule == null) {
-                throw new ProviderException("NSS module not available: " + moduleName);
+                throw new ProviderException(
+                    "NSS module not available: " + moduleName);
             }
             if (nssModule.hasInitializedProvider()) {
                 throw new ProviderException("Secmod module already configured");
@@ -296,8 +309,9 @@ public final class SunPKCS11 extends AuthProvider {
             initArgs.flags = CKF_OS_LOCKING_OK;
             PKCS11 tmpPKCS11;
             try {
-                tmpPKCS11 = PKCS11.getInstance
-                    (library, functionList, initArgs, config.getOmitInitialize());
+                tmpPKCS11 = PKCS11.getInstance(
+                    library, functionList, initArgs,
+                    config.getOmitInitialize());
             } catch (PKCS11Exception e) {
                 if (debug != null) {
                     debug.println("Multi-threaded initialization failed: " + e);
@@ -312,8 +326,8 @@ public final class SunPKCS11 extends AuthProvider {
                 } else {
                     initArgs.flags = 0;
                 }
-                tmpPKCS11 = PKCS11.getInstance
-                    (library, functionList, initArgs, config.getOmitInitialize());
+                tmpPKCS11 = PKCS11.getInstance(library,
+                    functionList, initArgs, config.getOmitInitialize());
             }
             p11 = tmpPKCS11;
 
@@ -336,8 +350,10 @@ public final class SunPKCS11 extends AuthProvider {
                     System.out.println("Slots with tokens: " + toString(slots));
                 }
                 if (slotID < 0) {
-                    if ((slotListIndex < 0) || (slotListIndex >= slots.length)) {
-                        throw new ProviderException("slotListIndex is " + slotListIndex
+                    if ((slotListIndex < 0)
+                            || (slotListIndex >= slots.length)) {
+                        throw new ProviderException("slotListIndex is "
+                            + slotListIndex
                             + " but token only has " + slots.length + " slots");
                     }
                     slotID = slots[slotListIndex];
@@ -575,12 +591,15 @@ public final class SunPKCS11 extends AuthProvider {
         d(KF, "DH",             P11DHKeyFactory,        s("DiffieHellman"),
                 m(CKM_DH_PKCS_KEY_PAIR_GEN, CKM_DH_PKCS_DERIVE));
         d(KF, "EC",             P11DHKeyFactory,
-                m(CKM_EC_KEY_PAIR_GEN, CKM_ECDH1_DERIVE, CKM_ECDSA, CKM_ECDSA_SHA1));
+                m(CKM_EC_KEY_PAIR_GEN, CKM_ECDH1_DERIVE,
+                    CKM_ECDSA, CKM_ECDSA_SHA1));
 
         // AlgorithmParameters for EC.
         // Only needed until we have an EC implementation in the SUN provider.
-        d(AGP, "EC",            "sun.security.ec.ECParameters", s("1.2.840.10045.2.1"),
-                m(CKM_EC_KEY_PAIR_GEN, CKM_ECDH1_DERIVE, CKM_ECDSA, CKM_ECDSA_SHA1));
+        d(AGP, "EC",            "sun.security.ec.ECParameters",
+                                                s("1.2.840.10045.2.1"),
+                m(CKM_EC_KEY_PAIR_GEN, CKM_ECDH1_DERIVE,
+                    CKM_ECDSA, CKM_ECDSA_SHA1));
 
         d(KA, "DH",             P11KeyAgreement,        s("DiffieHellman"),
                 m(CKM_DH_PKCS_DERIVE));
@@ -654,12 +673,16 @@ public final class SunPKCS11 extends AuthProvider {
         d(SIG, "SHA512withRSA", P11Signature,
                 m(CKM_SHA512_RSA_PKCS, CKM_RSA_PKCS, CKM_RSA_X_509));
 
-        d(KG, "SunTlsRsaPremasterSecret", "sun.security.pkcs11.P11TlsRsaPremasterSecretGenerator",
+        d(KG, "SunTlsRsaPremasterSecret",
+                    "sun.security.pkcs11.P11TlsRsaPremasterSecretGenerator",
                 m(CKM_SSL3_PRE_MASTER_KEY_GEN, CKM_TLS_PRE_MASTER_KEY_GEN));
-        d(KG, "SunTlsMasterSecret", "sun.security.pkcs11.P11TlsMasterSecretGenerator",
+        d(KG, "SunTlsMasterSecret",
+                    "sun.security.pkcs11.P11TlsMasterSecretGenerator",
                 m(CKM_SSL3_MASTER_KEY_DERIVE, CKM_TLS_MASTER_KEY_DERIVE,
-                    CKM_SSL3_MASTER_KEY_DERIVE_DH, CKM_TLS_MASTER_KEY_DERIVE_DH));
-        d(KG, "SunTlsKeyMaterial", "sun.security.pkcs11.P11TlsKeyMaterialGenerator",
+                    CKM_SSL3_MASTER_KEY_DERIVE_DH,
+                    CKM_TLS_MASTER_KEY_DERIVE_DH));
+        d(KG, "SunTlsKeyMaterial",
+                    "sun.security.pkcs11.P11TlsKeyMaterialGenerator",
                 m(CKM_SSL3_KEY_AND_MAC_DERIVE, CKM_TLS_KEY_AND_MAC_DERIVE));
         d(KG, "SunTlsPrf", "sun.security.pkcs11.P11TlsPrfGenerator",
                 m(CKM_TLS_PRF, CKM_NSS_TLS_PRF_GENERAL));
@@ -773,6 +796,13 @@ public final class SunPKCS11 extends AuthProvider {
             System.out.println(token.tokenInfo);
         }
         long[] supportedMechanisms = p11.C_GetMechanismList(slotID);
+
+        // Create a map from the various Descriptors to the "most
+        // preferred" mechanism that was defined during the
+        // static initialization.  For example, DES/CBC/PKCS5Padding
+        // could be mapped to CKM_DES_CBC_PAD or CKM_DES_CBC.  Prefer
+        // the earliest entry.  When asked for "DES/CBC/PKCS5Padding", we
+        // return a CKM_DES_CBC_PAD.
         final Map<Descriptor,Integer> supportedAlgs =
                                         new HashMap<Descriptor,Integer>();
         for (int i = 0; i < supportedMechanisms.length; i++) {
@@ -807,6 +837,9 @@ public final class SunPKCS11 extends AuthProvider {
                     supportedAlgs.put(d, integerMech);
                     continue;
                 }
+                // See if there is something "more preferred"
+                // than what we currently have in the supportedAlgs
+                // map.
                 int intOldMech = oldMech.intValue();
                 for (int j = 0; j < d.mechanisms.length; j++) {
                     int nextMech = d.mechanisms[j];

src/share/classes/sun/security/rsa/RSAKeyFactory.java

 /*
- * Copyright 2003-2006 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2008 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -31,6 +31,8 @@ import java.security.*;
 import java.security.interfaces.*;
 import java.security.spec.*;
 
+import sun.security.action.GetPropertyAction;
+
 /**
  * KeyFactory for RSA keys. Keys must be instances of PublicKey or PrivateKey
  * and getAlgorithm() must return "RSA". For such keys, it supports conversion
@@ -68,6 +70,24 @@ public final class RSAKeyFactory extends KeyFactorySpi {
     private final static Class<?> x509KeySpecClass  = X509EncodedKeySpec.class;
     private final static Class<?> pkcs8KeySpecClass = PKCS8EncodedKeySpec.class;
 
+    public final static int MIN_MODLEN = 512;
+    public final static int MAX_MODLEN = 16384;
+
+    /*
+     * If the modulus length is above this value, restrict the size of
+     * the exponent to something that can be reasonably computed.  We
+     * could simply hardcode the exp len to something like 64 bits, but
+     * this approach allows flexibility in case impls would like to use
+     * larger module and exponent values.
+     */
+    public final static int MAX_MODLEN_RESTRICT_EXP = 3072;
+    public final static int MAX_RESTRICTED_EXPLEN = 64;
+
+    private static final boolean restrictExpLen =
+        "true".equalsIgnoreCase(AccessController.doPrivileged(
+            new GetPropertyAction(
+                "sun.security.rsa.restrictRSAExponent", "true")));
+
     // instance used for static translateKey();
     private final static RSAKeyFactory INSTANCE = new RSAKeyFactory();
 
@@ -76,74 +96,79 @@ public final class RSAKeyFactory extends KeyFactorySpi {
     }
 
     /**
-     * Static method to convert Key into a useable instance of
-     * RSAPublicKey or RSAPrivate(Crt)Key. Check the key and convert it
-     * to a SunRsaSign key if necessary. If the key is not an RSA key
-     * or cannot be used, throw an InvalidKeyException.
-     *
-     * The difference between this method and engineTranslateKey() is that
-     * we do not convert keys of other providers that are already an
-     * instance of RSAPublicKey or RSAPrivate(Crt)Key.
+     * Static method to convert Key into an instance of RSAPublicKeyImpl
+     * or RSAPrivate(Crt)KeyImpl. If the key is not an RSA key or cannot be
+     * used, throw an InvalidKeyException.
      *
      * Used by RSASignature and RSACipher.
      */
     public static RSAKey toRSAKey(Key key) throws InvalidKeyException {
-        if (key instanceof RSAKey) {
-            RSAKey rsaKey = (RSAKey)key;
-            checkKey(rsaKey);
-            return rsaKey;
+        if ((key instanceof RSAPrivateKeyImpl) ||
+            (key instanceof RSAPrivateCrtKeyImpl) ||
+            (key instanceof RSAPublicKeyImpl)) {
+            return (RSAKey)key;
         } else {
             return (RSAKey)INSTANCE.engineTranslateKey(key);
         }
     }
 
-    /**
-     * Check that the given RSA key is valid.
+    /*
+     * Single test entry point for all of the mechanisms in the SunRsaSign
+     * provider (RSA*KeyImpls).  All of the tests are the same.
+     *
+     * For compatibility, we round up to the nearest byte here:
+     * some Key impls might pass in a value within a byte of the
+     * real value.
      */
-    private static void checkKey(RSAKey key) throws InvalidKeyException {
-        // check for subinterfaces, omit additional checks for our keys
-        if (key instanceof RSAPublicKey) {
-            if (key instanceof RSAPublicKeyImpl) {
-                return;
-            }
-        } else if (key instanceof RSAPrivateKey) {
-            if ((key instanceof RSAPrivateCrtKeyImpl)
-                    || (key instanceof RSAPrivateKeyImpl)) {
-                return;
-            }
-        } else {
-            throw new InvalidKeyException("Neither a public nor a private key");
-        }
-        // RSAKey does not extend Key, so we need to do a cast
-        String keyAlg = ((Key)key).getAlgorithm();
-        if (keyAlg.equals("RSA") == false) {
-            throw new InvalidKeyException("Not an RSA key: " + keyAlg);
-        }
-        BigInteger modulus;
-        // some providers implement RSAKey for keys where the values are
-        // not accessible (although they should). Detect those here
-        // for a more graceful failure.
-        try {
-            modulus = key.getModulus();
-            if (modulus == null) {
-                throw new InvalidKeyException("Modulus is missing");
-            }
-        } catch (RuntimeException e) {
-            throw new InvalidKeyException(e);
-        }
-        checkKeyLength(modulus);
+    static void checkRSAProviderKeyLengths(int modulusLen, BigInteger exponent)
+            throws InvalidKeyException {
+        checkKeyLengths(((modulusLen + 7) & ~7), exponent,
+            RSAKeyFactory.MIN_MODLEN, Integer.MAX_VALUE);
     }
 
     /**
-     * Check the length of the modulus of an RSA key. We only support keys
-     * at least 505 bits long.
+     * Check the length of an RSA key modulus/exponent to make sure it
+     * is not too short or long.  Some impls have their own min and
+     * max key sizes that may or may not match with a system defined value.
+     *
+     * @param modulusLen the bit length of the RSA modulus.
+     * @param exponent the RSA exponent
+     * @param minModulusLen if > 0, check to see if modulusLen is at
+     *        least this long, otherwise unused.
+     * @param maxModulusLen caller will allow this max number of bits.
+     *        Allow the smaller of the system-defined maximum and this param.
+     *
+     * @throws InvalidKeyException if any of the values are unacceptable.
      */
-    static void checkKeyLength(BigInteger modulus) throws InvalidKeyException {
-        if (modulus.bitLength() < 505) {
-            // some providers may generate slightly shorter keys
-            // accept them if the encoding is at least 64 bytes long
-            throw new InvalidKeyException
-                ("RSA keys must be at least 512 bits long");
+     public static void checkKeyLengths(int modulusLen, BigInteger exponent,
+            int minModulusLen, int maxModulusLen) throws InvalidKeyException {
+
+        if ((minModulusLen > 0) && (modulusLen < (minModulusLen))) {
+            throw new InvalidKeyException( "RSA keys must be at least " +
+                minModulusLen + " bits long");
+        }
+
+        // Even though our policy file may allow this, we don't want
+        // either value (mod/exp) to be too big.
+
+        int maxLen = Math.min(maxModulusLen, MAX_MODLEN);
+
+        // If a RSAPrivateKey/RSAPublicKey, make sure the
+        // modulus len isn't too big.
+        if (modulusLen > maxLen) {
+            throw new InvalidKeyException(
+                "RSA keys must be no longer than " + maxLen + " bits");
+        }
+
+        // If a RSAPublicKey, make sure the exponent isn't too big.
+        if (restrictExpLen && (exponent != null) &&
+                (modulusLen > MAX_MODLEN_RESTRICT_EXP) &&
+                (exponent.bitLength() > MAX_RESTRICTED_EXPLEN)) {
+            throw new InvalidKeyException(
+                "RSA exponents can be no longer than " +
+                MAX_RESTRICTED_EXPLEN + " bits " +
+                " if modulus is greater than " +
+                MAX_MODLEN_RESTRICT_EXP + " bits");
         }
     }
 

src/share/classes/sun/security/rsa/RSAKeyPairGenerator.java

 /*
- * Copyright 2003-2004 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2008 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -47,7 +47,7 @@ public final class RSAKeyPairGenerator extends KeyPairGeneratorSpi {
     // public exponent to use
     private BigInteger publicExponent;
 
-    // size of the key to generate, >= 512
+    // size of the key to generate, >= RSAKeyFactory.MIN_MODLEN
     private int keySize;
 
     // PRNG to use
@@ -60,15 +60,16 @@ public final class RSAKeyPairGenerator extends KeyPairGeneratorSpi {
 
     // initialize the generator. See JCA doc
     public void initialize(int keySize, SecureRandom random) {
-        if (keySize < 512) {
-            throw new InvalidParameterException
-                ("Key size must be at least 512 bits");
-        }
-        if (keySize > 64 * 1024) {
-            // do not allow unreasonably large key sizes, probably user error
-            throw new InvalidParameterException
-                ("Key size must be 65536 bits or less");
+
+        // do not allow unreasonably small or large key sizes,
+        // probably user error
+        try {
+            RSAKeyFactory.checkKeyLengths(keySize, RSAKeyGenParameterSpec.F4,
+                512, 64 * 1024);
+        } catch (InvalidKeyException e) {
+            throw new InvalidParameterException(e.getMessage());
         }
+
         this.keySize = keySize;
         this.random = random;
         this.publicExponent = RSAKeyGenParameterSpec.F4;
@@ -77,35 +78,41 @@ public final class RSAKeyPairGenerator extends KeyPairGeneratorSpi {
     // second initialize method. See JCA doc.
     public void initialize(AlgorithmParameterSpec params, SecureRandom random)
             throws InvalidAlgorithmParameterException {
+
         if (params instanceof RSAKeyGenParameterSpec == false) {
             throw new InvalidAlgorithmParameterException
                 ("Params must be instance of RSAKeyGenParameterSpec");
         }
+
         RSAKeyGenParameterSpec rsaSpec = (RSAKeyGenParameterSpec)params;
-        keySize = rsaSpec.getKeysize();
-        publicExponent = rsaSpec.getPublicExponent();
-        this.random = random;
-        if (keySize < 512) {
-            throw new InvalidAlgorithmParameterException
-                ("Key size must be at least 512 bits");
-        }
-        if (keySize > 64 * 1024) {
-            // do not allow unreasonably large key sizes, probably user error
-            throw new InvalidAlgorithmParameterException
-                ("Key size must be 65536 bits or less");
-        }
-        if (publicExponent == null) {
-            publicExponent = RSAKeyGenParameterSpec.F4;
+        int tmpKeySize = rsaSpec.getKeysize();
+        BigInteger tmpPublicExponent = rsaSpec.getPublicExponent();
+
+        if (tmpPublicExponent == null) {
+            tmpPublicExponent = RSAKeyGenParameterSpec.F4;
         } else {
-            if (publicExponent.compareTo(RSAKeyGenParameterSpec.F0) < 0) {
+            if (tmpPublicExponent.compareTo(RSAKeyGenParameterSpec.F0) < 0) {
                 throw new InvalidAlgorithmParameterException
                         ("Public exponent must be 3 or larger");
             }
-            if (publicExponent.bitLength() > keySize) {
+            if (tmpPublicExponent.bitLength() > tmpKeySize) {
                 throw new InvalidAlgorithmParameterException
                         ("Public exponent must be smaller than key size");
             }
         }
+
+        // do not allow unreasonably large key sizes, probably user error
+        try {
+            RSAKeyFactory.checkKeyLengths(tmpKeySize, tmpPublicExponent,
+                512, 64 * 1024);
+        } catch (InvalidKeyException e) {
+            throw new InvalidAlgorithmParameterException(
+                "Invalid key sizes", e);
+        }
+
+        this.keySize = tmpKeySize;
+        this.publicExponent = tmpPublicExponent;
+        this.random = random;
     }
 
     // generate the keypair. See JCA doc

src/share/classes/sun/security/rsa/RSAPrivateCrtKeyImpl.java

 /*
- * Copyright 2003-2005 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2008 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -89,7 +89,7 @@ public final class RSAPrivateCrtKeyImpl
      */
     RSAPrivateCrtKeyImpl(byte[] encoded) throws InvalidKeyException {
         decode(encoded);
-        RSAKeyFactory.checkKeyLength(n);
+        RSAKeyFactory.checkRSAProviderKeyLengths(n.bitLength(), e);
     }
 
     /**
@@ -107,7 +107,8 @@ public final class RSAPrivateCrtKeyImpl
         this.pe = pe;
         this.qe = qe;
         this.coeff = coeff;
-        RSAKeyFactory.checkKeyLength(n);
+        RSAKeyFactory.checkRSAProviderKeyLengths(n.bitLength(), e);
+
         // generate the encoding
         algid = rsaId;
         try {

src/share/classes/sun/security/rsa/RSAPrivateKeyImpl.java

 /*
- * Copyright 2003 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2008 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -61,7 +61,7 @@ public final class RSAPrivateKeyImpl extends PKCS8Key implements RSAPrivateKey {
     RSAPrivateKeyImpl(BigInteger n, BigInteger d) throws InvalidKeyException {
         this.n = n;
         this.d = d;
-        RSAKeyFactory.checkKeyLength(n);
+        RSAKeyFactory.checkRSAProviderKeyLengths(n.bitLength(), null);
         // generate the encoding
         algid = RSAPrivateCrtKeyImpl.rsaId;
         try {

src/share/classes/sun/security/rsa/RSAPublicKeyImpl.java

 /*
- * Copyright 2003-2005 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2008 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -56,10 +56,11 @@ public final class RSAPublicKeyImpl extends X509Key implements RSAPublicKey {
      * Construct a key from its components. Used by the
      * RSAKeyFactory and the RSAKeyPairGenerator.
      */
-    public RSAPublicKeyImpl(BigInteger n, BigInteger e) throws InvalidKeyException {
+    public RSAPublicKeyImpl(BigInteger n, BigInteger e)
+            throws InvalidKeyException {
         this.n = n;
         this.e = e;
-        RSAKeyFactory.checkKeyLength(n);
+        RSAKeyFactory.checkRSAProviderKeyLengths(n.bitLength(), e);
         // generate the encoding
         algid = RSAPrivateCrtKeyImpl.rsaId;
         try {
@@ -80,7 +81,7 @@ public final class RSAPublicKeyImpl extends X509Key implements RSAPublicKey {
      */
     public RSAPublicKeyImpl(byte[] encoded) throws InvalidKeyException {
         decode(encoded);
-        RSAKeyFactory.checkKeyLength(n);
+        RSAKeyFactory.checkRSAProviderKeyLengths(n.bitLength(), e);
     }
 
     // see JCA doc

src/windows/classes/sun/security/mscapi/RSAKeyPairGenerator.java

 /*
- * Copyright 2005 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2005-2008 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -31,6 +31,7 @@ import java.security.spec.AlgorithmParameterSpec;
 import java.security.spec.RSAKeyGenParameterSpec;
 
 import sun.security.jca.JCAUtil;
+import sun.security.rsa.RSAKeyFactory;
 
 /**
  * RSA keypair generator.
@@ -43,8 +44,8 @@ import sun.security.jca.JCAUtil;
 public final class RSAKeyPairGenerator extends KeyPairGeneratorSpi {
 
     // Supported by Microsoft Base, Strong and Enhanced Cryptographic Providers
-    private static final int KEY_SIZE_MIN = 512; // disallow MSCAPI min. of 384
-    private static final int KEY_SIZE_MAX = 16384;
+    static final int KEY_SIZE_MIN = 512; // disallow MSCAPI min. of 384
+    static final int KEY_SIZE_MAX = 16384;
     private static final int KEY_SIZE_DEFAULT = 1024;
 
     // size of the key to generate, KEY_SIZE_MIN <= keySize <= KEY_SIZE_MAX
@@ -59,7 +60,14 @@ public final class RSAKeyPairGenerator extends KeyPairGeneratorSpi {
     // random is always ignored
     public void initialize(int keySize, SecureRandom random) {
 
-        checkKeySize(keySize);
+        try {
+            RSAKeyFactory.checkKeyLengths(keySize, null,
+                KEY_SIZE_MIN, KEY_SIZE_MAX);
+        } catch (InvalidKeyException e) {
+            throw new InvalidParameterException(e.getMessage());
+        }
+
+        this.keySize = keySize;
     }
 
     // second initialize method. See JCA doc
@@ -67,21 +75,31 @@ public final class RSAKeyPairGenerator extends KeyPairGeneratorSpi {
     public void initialize(AlgorithmParameterSpec params, SecureRandom random)
             throws InvalidAlgorithmParameterException {
 
+        int tmpSize;
         if (params == null) {
-            checkKeySize(KEY_SIZE_DEFAULT);
-
+            tmpSize = KEY_SIZE_DEFAULT;
         } else if (params instanceof RSAKeyGenParameterSpec) {
 
             if (((RSAKeyGenParameterSpec) params).getPublicExponent() != null) {
                 throw new InvalidAlgorithmParameterException
                     ("Exponent parameter is not supported");
             }
-            checkKeySize(((RSAKeyGenParameterSpec) params).getKeysize());
+            tmpSize = ((RSAKeyGenParameterSpec) params).getKeysize();
 
         } else {
             throw new InvalidAlgorithmParameterException
                 ("Params must be an instance of RSAKeyGenParameterSpec");
         }
+
+        try {
+            RSAKeyFactory.checkKeyLengths(tmpSize, null,
+                KEY_SIZE_MIN, KEY_SIZE_MAX);
+        } catch (InvalidKeyException e) {
+            throw new InvalidAlgorithmParameterException(
+                "Invalid Key sizes", e);
+        }
+
+        this.keySize = tmpSize;
     }
 
     // generate the keypair. See JCA doc
@@ -95,18 +113,6 @@ public final class RSAKeyPairGenerator extends KeyPairGeneratorSpi {
         return new KeyPair(keys.getPublic(), keys.getPrivate());
     }
 
-    private void checkKeySize(int keySize) throws InvalidParameterException {
-        if (keySize < KEY_SIZE_MIN) {
-            throw new InvalidParameterException
-                ("Key size must be at least " + KEY_SIZE_MIN + " bits");
-        }
-        if (keySize > KEY_SIZE_MAX) {
-            throw new InvalidParameterException
-                ("Key size must be " + KEY_SIZE_MAX + " bits or less");
-        }
-        this.keySize = keySize;
-    }
-
     private static native RSAKeyPair generateRSAKeyPair(int keySize,
         String keyContainerName);
 }

src/windows/classes/sun/security/mscapi/RSASignature.java

 /*
- * Copyright 2005 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2005-2008 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -38,6 +38,9 @@ import java.security.SecureRandom;
 import java.security.Signature;
 import java.security.SignatureSpi;
 import java.security.SignatureException;
+import java.math.BigInteger;
+
+import sun.security.rsa.RSAKeyFactory;
 
 /**
  * RSA signature implementation. Supports RSA signing using PKCS#1 v1.5 padding.
@@ -124,7 +127,16 @@ abstract class RSASignature extends java.security.SignatureSpi
 
             // convert key to MSCAPI format
 
-            byte[] modulusBytes = rsaKey.getModulus().toByteArray();
+            BigInteger modulus = rsaKey.getModulus();
+            BigInteger exponent =  rsaKey.getPublicExponent();
+
+            // Check against the local and global values to make sure
+            // the sizes are ok.  Round up to the nearest byte.
+            RSAKeyFactory.checkKeyLengths(((modulus.bitLength() + 7) & ~7),
+                exponent, -1, RSAKeyPairGenerator.KEY_SIZE_MAX);
+
+            byte[] modulusBytes = modulus.toByteArray();
+            byte[] exponentBytes = exponent.toByteArray();
 
             // Adjust key length due to sign bit
             int keyBitLength = (modulusBytes[0] == 0)
@@ -132,8 +144,7 @@ abstract class RSASignature extends java.security.SignatureSpi
                 : modulusBytes.length * 8;
 
             byte[] keyBlob = generatePublicKeyBlob(
-                keyBitLength, modulusBytes,
-                rsaKey.getPublicExponent().toByteArray());
+                keyBitLength, modulusBytes, exponentBytes);
 
             publicKey = importPublicKey(keyBlob, keyBitLength);
 
@@ -166,12 +177,11 @@ abstract class RSASignature extends java.security.SignatureSpi
         }
         privateKey = (sun.security.mscapi.RSAPrivateKey) key;
 
-        // Determine byte length from bit length
-        int keySize = (privateKey.bitLength() + 7) >> 3;
-
-        if (keySize < 64)
-            throw new InvalidKeyException(
-                "RSA keys must be at least 512 bits long");
+        // Check against the local and global values to make sure
+        // the sizes are ok.  Round up to nearest byte.
+        RSAKeyFactory.checkKeyLengths(((privateKey.bitLength() + 7) & ~7),
+            null, RSAKeyPairGenerator.KEY_SIZE_MIN,
+            RSAKeyPairGenerator.KEY_SIZE_MAX);
 
         if (needsReset) {
             messageDigest.reset();