8081297: SSL Problem with Tomcat

Reviewed-by: xuelei, jnimeh, ahgross

src/share/classes/com/sun/crypto/provider/TlsRsaPremasterSecretGenerator.java

@@ -74,11 +74,14 @@ public final class TlsRsaPremasterSecretGenerator extends KeyGeneratorSpi {
                 "TlsRsaPremasterSecretGenerator must be initialized");
         }
 
-        if (random == null) {
-            random = new SecureRandom();
+        byte[] b = spec.getEncodedSecret();
+        if (b == null) {
+            if (random == null) {
+                random = new SecureRandom();
+            }
+            b = new byte[48];
+            random.nextBytes(b);
         }
-        byte[] b = new byte[48];
-        random.nextBytes(b);
         b[0] = (byte)spec.getMajorVersion();
         b[1] = (byte)spec.getMinorVersion();
 

src/share/classes/sun/security/internal/spec/TlsRsaPremasterSecretParameterSpec.java

@@ -43,6 +43,8 @@ import java.security.PrivilegedAction;
 public class TlsRsaPremasterSecretParameterSpec
         implements AlgorithmParameterSpec {
 
+    private final byte[] encodedSecret;
+
     /*
      * The TLS spec says that the version in the RSA premaster secret must
      * be the maximum version supported by the client (i.e. the version it
@@ -89,6 +91,33 @@ public class TlsRsaPremasterSecretParameterSpec
 
         this.clientVersion = checkVersion(clientVersion);
         this.serverVersion = checkVersion(serverVersion);
+        this.encodedSecret = null;
+    }
+
+    /**
+     * Constructs a new TlsRsaPremasterSecretParameterSpec.
+     *
+     * @param clientVersion the version of the TLS protocol by which the
+     *        client wishes to communicate during this session
+     * @param serverVersion the negotiated version of the TLS protocol which
+     *        contains the lower of that suggested by the client in the client
+     *        hello and the highest supported by the server.
+     * @param encodedSecret the encoded secret key
+     *
+     * @throws IllegalArgumentException if clientVersion or serverVersion are
+     *   negative or larger than (2^16 - 1) or if encodedSecret is not
+     *   exactly 48 bytes
+     */
+    public TlsRsaPremasterSecretParameterSpec(
+            int clientVersion, int serverVersion, byte[] encodedSecret) {
+
+        this.clientVersion = checkVersion(clientVersion);
+        this.serverVersion = checkVersion(serverVersion);
+        if (encodedSecret == null || encodedSecret.length != 48) {
+            throw new IllegalArgumentException(
+                        "Encoded secret is not exactly 48 bytes");
+        }
+        this.encodedSecret = encodedSecret.clone();
     }
 
     /**
@@ -147,4 +176,13 @@ public class TlsRsaPremasterSecretParameterSpec
         }
         return version;
     }
+
+    /**
+     * Returns the encoded secret.
+     *
+     * @return the encoded secret, may be null if no encoded secret.
+     */
+    public byte[] getEncodedSecret() {
+        return encodedSecret == null ? null : encodedSecret.clone();
+    }
 }

src/share/classes/sun/security/ssl/RSAClientKeyExchange.java

@@ -111,14 +111,41 @@ final class RSAClientKeyExchange extends HandshakeMessage {
             }
         }
 
+        boolean needFailover = false;
+        byte[] encoded = null;
         try {
             Cipher cipher = JsseJce.getCipher(JsseJce.CIPHER_RSA_PKCS1);
-            cipher.init(Cipher.UNWRAP_MODE, privateKey,
-                    new TlsRsaPremasterSecretParameterSpec(
-                            maxVersion.v, currentVersion.v),
-                    generator);
-            preMaster = (SecretKey)cipher.unwrap(encrypted,
-                                "TlsRsaPremasterSecret", Cipher.SECRET_KEY);
+            needFailover = !KeyUtil.isOracleJCEProvider(
+                                        cipher.getProvider().getName());
+            if (needFailover) {
+                cipher.init(Cipher.DECRYPT_MODE, privateKey);
+                encoded = cipher.doFinal(encrypted);
+                encoded = KeyUtil.checkTlsPreMasterSecretKey(
+                                maxVersion.v, currentVersion.v,
+                                generator, encoded, false);
+                preMaster = generatePreMasterSecret(
+                                maxVersion.v, currentVersion.v,
+                                encoded, generator);
+            } else {
+                cipher.init(Cipher.UNWRAP_MODE, privateKey,
+                        new TlsRsaPremasterSecretParameterSpec(
+                                maxVersion.v, currentVersion.v),
+                        generator);
+                preMaster = (SecretKey)cipher.unwrap(encrypted,
+                        "TlsRsaPremasterSecret", Cipher.SECRET_KEY);
+            }
+        } catch (BadPaddingException bpe) {
+            if (needFailover) {
+                encoded = KeyUtil.checkTlsPreMasterSecretKey(
+                                maxVersion.v, currentVersion.v,
+                                generator, null, false);
+                preMaster = generatePreMasterSecret(
+                                maxVersion.v, currentVersion.v,
+                                encoded, generator);
+            } else {
+                //  Otherwise, unlikely to happen
+                throw new RuntimeException("Unexpected exception", bpe);
+            }
         } catch (InvalidKeyException ibk) {
             // the message is too big to process with RSA
             throw new SSLProtocolException(
@@ -133,6 +160,35 @@ final class RSAClientKeyExchange extends HandshakeMessage {
         }
     }
 
+    // generate a premaster secret with the specified version number
+    @SuppressWarnings("deprecation")
+    private static SecretKey generatePreMasterSecret(
+            int clientVersion, int serverVersion,
+            byte[] encodedSecret, SecureRandom generator) {
+
+        if (debug != null && Debug.isOn("handshake")) {
+            System.out.println("Generating a premaster secret");
+        }
+
+        try {
+            String s = ((clientVersion >= ProtocolVersion.TLS12.v) ?
+                "SunTls12RsaPremasterSecret" : "SunTlsRsaPremasterSecret");
+            KeyGenerator kg = JsseJce.getKeyGenerator(s);
+            kg.init(new TlsRsaPremasterSecretParameterSpec(
+                    clientVersion, serverVersion, encodedSecret),
+                    generator);
+            return kg.generateKey();
+        } catch (InvalidAlgorithmParameterException |
+                NoSuchAlgorithmException iae) {
+            // unlikely to happen, otherwise, must be a provider exception
+            if (debug != null && Debug.isOn("handshake")) {
+                System.out.println("RSA premaster secret generation error:");
+                iae.printStackTrace(System.out);
+            }
+            throw new RuntimeException("Could not generate premaster secret", iae);
+        }
+    }
+
     @Override
     int messageType() {
         return ht_client_key_exchange;

src/share/classes/sun/security/util/KeyUtil.java

@@ -144,8 +144,6 @@ public final class KeyUtil {
 
     /**
      * Returns whether the specified provider is Oracle provider or not.
-     * <P>
-     * Note that this method is only apply to SunJCE and SunPKCS11 at present.
      *
      * @param  providerName
      *         the provider name
@@ -153,8 +151,11 @@ public final class KeyUtil {
      *         {@code providerName} is Oracle provider
      */
     public static final boolean isOracleJCEProvider(String providerName) {
-        return providerName != null && (providerName.equals("SunJCE") ||
-                                        providerName.startsWith("SunPKCS11"));
+        return providerName != null &&
+                (providerName.equals("SunJCE") ||
+                    providerName.equals("SunMSCAPI") ||
+                    providerName.equals("OracleUcrypto") ||
+                    providerName.startsWith("SunPKCS11"));
     }
 
     /**