6695485: SignedObject constructor throws ProviderException if it's called...

6695485: SignedObject constructor throws ProviderException if it's called using provider "SunPKCS11-Solaris"
Summary: Added checking for RSA key lengths in initSign and initVerify
Reviewed-by: vinnie

src/share/classes/sun/security/pkcs11/P11Signature.java

 /*
- * Copyright 2003-2006 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2010 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -30,8 +30,7 @@ import java.math.BigInteger;
 import java.nio.ByteBuffer;
 
 import java.security.*;
-import java.security.interfaces.ECPublicKey;
-
+import java.security.interfaces.*;
 import sun.nio.ch.DirectBuffer;
 
 import sun.security.util.*;
@@ -88,7 +87,7 @@ final class P11Signature extends SignatureSpi {
     // mechanism id
     private final long mechanism;
 
-    // digest algorithm OID, if we do RSA padding ourselves
+    // digest algorithm OID, if we encode RSA signature ourselves
     private final ObjectIdentifier digestOID;
 
     // type, one of T_* below
@@ -103,7 +102,7 @@ final class P11Signature extends SignatureSpi {
     // associated session, if any
     private Session session;
 
-    // mode, on of M_* below
+    // mode, one of M_* below
     private int mode;
 
     // flag indicating whether an operation is initialized
@@ -137,6 +136,9 @@ final class P11Signature extends SignatureSpi {
         this.token = token;
         this.algorithm = algorithm;
         this.mechanism = mechanism;
+        byte[] buffer = null;
+        ObjectIdentifier digestOID = null;
+        MessageDigest md = null;
         switch ((int)mechanism) {
         case (int)CKM_MD2_RSA_PKCS:
         case (int)CKM_MD5_RSA_PKCS:
@@ -146,34 +148,25 @@ final class P11Signature extends SignatureSpi {
         case (int)CKM_SHA512_RSA_PKCS:
             keyAlgorithm = "RSA";
             type = T_UPDATE;
-            digestOID = null;
             buffer = new byte[1];
-            md = null;
             break;
         case (int)CKM_DSA_SHA1:
             keyAlgorithm = "DSA";
             type = T_UPDATE;
-            digestOID = null;
             buffer = new byte[1];
-            md = null;
             break;
         case (int)CKM_ECDSA_SHA1:
             keyAlgorithm = "EC";
             type = T_UPDATE;
-            digestOID = null;
             buffer = new byte[1];
-            md = null;
             break;
         case (int)CKM_DSA:
             keyAlgorithm = "DSA";
-            digestOID = null;
             if (algorithm.equals("DSA")) {
                 type = T_DIGEST;
                 md = MessageDigest.getInstance("SHA-1");
-                buffer = null;
             } else if (algorithm.equals("RawDSA")) {
                 type = T_RAW;
-                md = null;
                 buffer = new byte[20];
             } else {
                 throw new ProviderException(algorithm);
@@ -181,10 +174,8 @@ final class P11Signature extends SignatureSpi {
             break;
         case (int)CKM_ECDSA:
             keyAlgorithm = "EC";
-            digestOID = null;
             if (algorithm.equals("NONEwithECDSA")) {
                 type = T_RAW;
-                md = null;
                 buffer = new byte[RAW_ECDSA_MAX];
             } else {
                 String digestAlg;
@@ -201,14 +192,12 @@ final class P11Signature extends SignatureSpi {
                 }
                 type = T_DIGEST;
                 md = MessageDigest.getInstance(digestAlg);
-                buffer = null;
             }
             break;
         case (int)CKM_RSA_PKCS:
         case (int)CKM_RSA_X_509:
             keyAlgorithm = "RSA";
             type = T_DIGEST;
-            buffer = null;
             if (algorithm.equals("MD5withRSA")) {
                 md = MessageDigest.getInstance("MD5");
                 digestOID = AlgorithmId.MD5_oid;
@@ -234,6 +223,9 @@ final class P11Signature extends SignatureSpi {
         default:
             throw new ProviderException("Unknown mechanism: " + mechanism);
         }
+        this.buffer = buffer;
+        this.digestOID = digestOID;
+        this.md = md;
         session = token.getOpSession();
     }
 
@@ -326,9 +318,52 @@ final class P11Signature extends SignatureSpi {
         }
     }
 
+    private void checkRSAKeyLength(int len) throws InvalidKeyException {
+        RSAPadding padding;
+        try {
+            padding = RSAPadding.getInstance
+                (RSAPadding.PAD_BLOCKTYPE_1, (len + 7) >> 3);
+        } catch (InvalidAlgorithmParameterException iape) {
+            throw new InvalidKeyException(iape.getMessage());
+        }
+        int maxDataSize = padding.getMaxDataSize();
+        int encodedLength;
+        if (algorithm.equals("MD5withRSA") ||
+            algorithm.equals("MD2withRSA")) {
+            encodedLength = 34;
+        } else if (algorithm.equals("SHA1withRSA")) {
+            encodedLength = 35;
+        } else if (algorithm.equals("SHA256withRSA")) {
+            encodedLength = 51;
+        } else if (algorithm.equals("SHA384withRSA")) {
+            encodedLength = 67;
+        } else if (algorithm.equals("SHA512withRSA")) {
+            encodedLength = 83;
+        } else {
+            throw new ProviderException("Unknown signature algo: " + algorithm);
+        }
+        if (encodedLength > maxDataSize) {
+            throw new InvalidKeyException
+                ("Key is too short for this signature algorithm");
+        }
+    }
+
     // see JCA spec
     protected void engineInitVerify(PublicKey publicKey)
             throws InvalidKeyException {
+        if (publicKey == null) {
+            throw new InvalidKeyException("Key must not be null");
+        }
+        // Need to check RSA key length whenever a new key is set
+        if (keyAlgorithm.equals("RSA") && publicKey != p11Key) {
+            int keyLen;
+            if (publicKey instanceof P11Key) {
+                keyLen = ((P11Key) publicKey).keyLength();
+            } else {
+                keyLen = ((RSAKey) publicKey).getModulus().bitLength();
+            }
+            checkRSAKeyLength(keyLen);
+        }
         cancelOperation();
         mode = M_VERIFY;
         p11Key = P11KeyFactory.convertKey(token, publicKey, keyAlgorithm);
@@ -338,6 +373,19 @@ final class P11Signature extends SignatureSpi {
     // see JCA spec
     protected void engineInitSign(PrivateKey privateKey)
             throws InvalidKeyException {
+        if (privateKey == null) {
+            throw new InvalidKeyException("Key must not be null");
+        }
+        // Need to check RSA key length whenever a new key is set
+        if (keyAlgorithm.equals("RSA") && privateKey != p11Key) {
+            int keyLen;
+            if (privateKey instanceof P11Key) {
+                keyLen = ((P11Key) privateKey).keyLength;
+            } else {
+                keyLen = ((RSAKey) privateKey).getModulus().bitLength();
+            }
+            checkRSAKeyLength(keyLen);
+        }
         cancelOperation();
         mode = M_SIGN;
         p11Key = P11KeyFactory.convertKey(token, privateKey, keyAlgorithm);

test/sun/security/pkcs11/Signature/TestRSAKeyLength.java

+/*
+ * Copyright 2010 Sun Microsystems, Inc.  All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ */
+
+/**
+ * @test %W% %E%
+ * @bug 6695485
+ * @summary Make sure initSign/initVerify() check RSA key lengths
+ * @author Yu-Ching Valerie Peng
+ * @library ..
+ */
+
+import java.security.*;
+
+public class TestRSAKeyLength extends PKCS11Test {
+    public static void main(String[] args) throws Exception {
+        main(new TestRSAKeyLength());
+    }
+    public void main(Provider p) throws Exception {
+        boolean isValidKeyLength[] = { true, true, false, false };
+        String algos[] = { "SHA1withRSA", "SHA256withRSA",
+                           "SHA384withRSA", "SHA512withRSA" };
+        KeyPairGenerator kpg = KeyPairGenerator.getInstance("RSA", p);
+        kpg.initialize(512);
+        KeyPair kp = kpg.generateKeyPair();
+        PrivateKey privKey = kp.getPrivate();
+        PublicKey pubKey = kp.getPublic();
+
+        for (int i = 0; i < algos.length; i++) {
+            Signature sig = Signature.getInstance(algos[i], p);
+            System.out.println("Testing RSA signature " + algos[i]);
+            try {
+                sig.initSign(privKey);
+                if (!isValidKeyLength[i]) {
+                    throw new Exception("initSign: Expected IKE not thrown!");
+                }
+            } catch (InvalidKeyException ike) {
+                if (isValidKeyLength[i]) {
+                    throw new Exception("initSign: Unexpected " + ike);
+                }
+            }
+            try {
+                sig.initVerify(pubKey);
+                if (!isValidKeyLength[i]) {
+                    throw new RuntimeException("initVerify: Expected IKE not thrown!");
+                }
+                new SignedObject("Test string for getSignature test.", privKey, sig);
+            } catch (InvalidKeyException ike) {
+                if (isValidKeyLength[i]) {
+                    throw new Exception("initSign: Unexpected " + ike);
+                }
+            }
+        }
+    }
+}