Replace SSL_PKEY_RSA_ENC, SSL_PKEY_RSA_SIGN

The original intent of SSL_PKEY_RSA_SIGN and SSL_PKEY_RSA_ENC was to
support two different keys for RSA signing and decrypt. However this
was never implemented and we only ever set one key and the other was
always NULL. Replace with single SSL_PKEY_RSA type.
Reviewed-by: NMatt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/2587)

ssl/ssl_cert.c

@@ -63,7 +63,7 @@ CERT *ssl_cert_new(void)
         return NULL;
     }
 
-    ret->key = &(ret->pkeys[SSL_PKEY_RSA_ENC]);
+    ret->key = &(ret->pkeys[SSL_PKEY_RSA]);
     ret->references = 1;
     ret->sec_cb = ssl_security_default_callback;
     ret->sec_level = OPENSSL_TLS_SECURITY_LEVEL;

ssl/ssl_ciph.c

@@ -1909,7 +1909,7 @@ int ssl_cipher_get_cert_index(const SSL_CIPHER *c)
     else if (alg_a & SSL_aDSS)
         return SSL_PKEY_DSA_SIGN;
     else if (alg_a & SSL_aRSA)
-        return SSL_PKEY_RSA_ENC;
+        return SSL_PKEY_RSA;
     else if (alg_a & SSL_aGOST12)
         return SSL_PKEY_GOST_EC;
     else if (alg_a & SSL_aGOST01)

ssl/ssl_lib.c

@@ -2740,8 +2740,8 @@ void ssl_set_masks(SSL *s)
     dh_tmp = 0;
 #endif
 
-    rsa_enc = pvalid[SSL_PKEY_RSA_ENC] & CERT_PKEY_VALID;
-    rsa_sign = pvalid[SSL_PKEY_RSA_SIGN] & CERT_PKEY_SIGN;
+    rsa_enc = pvalid[SSL_PKEY_RSA] & CERT_PKEY_VALID;
+    rsa_sign = pvalid[SSL_PKEY_RSA] & CERT_PKEY_SIGN;
     dsa_sign = pvalid[SSL_PKEY_DSA_SIGN] & CERT_PKEY_SIGN;
 #ifndef OPENSSL_NO_EC
     have_ecc_cert = pvalid[SSL_PKEY_ECC] & CERT_PKEY_VALID;
@@ -2855,8 +2855,6 @@ static int ssl_get_server_cert_index(const SSL *s)
     }
 
     idx = ssl_cipher_get_cert_index(s->s3->tmp.new_cipher);
-    if (idx == SSL_PKEY_RSA_ENC && !s->cert->pkeys[SSL_PKEY_RSA_ENC].x509)
-        idx = SSL_PKEY_RSA_SIGN;
     if (idx == SSL_PKEY_GOST_EC) {
         if (s->cert->pkeys[SSL_PKEY_GOST12_512].x509)
             idx = SSL_PKEY_GOST12_512;
@@ -2902,15 +2900,12 @@ EVP_PKEY *ssl_get_sign_pkey(SSL *s, const SSL_CIPHER *cipher,
     alg_a = cipher->algorithm_auth;
     c = s->cert;
 
-    if ((alg_a & SSL_aDSS) && (c->pkeys[SSL_PKEY_DSA_SIGN].privatekey != NULL))
+    if (alg_a & SSL_aDSS && c->pkeys[SSL_PKEY_DSA_SIGN].privatekey != NULL)
         idx = SSL_PKEY_DSA_SIGN;
-    else if (alg_a & SSL_aRSA) {
-        if (c->pkeys[SSL_PKEY_RSA_SIGN].privatekey != NULL)
-            idx = SSL_PKEY_RSA_SIGN;
-        else if (c->pkeys[SSL_PKEY_RSA_ENC].privatekey != NULL)
-            idx = SSL_PKEY_RSA_ENC;
-    } else if ((alg_a & SSL_aECDSA) &&
-               (c->pkeys[SSL_PKEY_ECC].privatekey != NULL))
+    else if (alg_a & SSL_aRSA && c->pkeys[SSL_PKEY_RSA].privatekey != NULL)
+            idx = SSL_PKEY_RSA;
+    else if (alg_a & SSL_aECDSA &&
+             c->pkeys[SSL_PKEY_ECC].privatekey != NULL)
         idx = SSL_PKEY_ECC;
     if (idx == -1) {
         SSLerr(SSL_F_SSL_GET_SIGN_PKEY, ERR_R_INTERNAL_ERROR);

ssl/ssl_locl.h

@@ -398,14 +398,13 @@
 # define SSL_USE_ETM(s) (s->s3->flags & TLS1_FLAGS_ENCRYPT_THEN_MAC)
 
 /* Mostly for SSLv3 */
-# define SSL_PKEY_RSA_ENC        0
-# define SSL_PKEY_RSA_SIGN       1
-# define SSL_PKEY_DSA_SIGN       2
-# define SSL_PKEY_ECC            3
-# define SSL_PKEY_GOST01         4
-# define SSL_PKEY_GOST12_256     5
-# define SSL_PKEY_GOST12_512     6
-# define SSL_PKEY_NUM            7
+# define SSL_PKEY_RSA            0
+# define SSL_PKEY_DSA_SIGN       1
+# define SSL_PKEY_ECC            2
+# define SSL_PKEY_GOST01         3
+# define SSL_PKEY_GOST12_256     4
+# define SSL_PKEY_GOST12_512     5
+# define SSL_PKEY_NUM            6
 /*
  * Pseudo-constant. GOST cipher suites can use different certs for 1
  * SSL_CIPHER. So let's see which one we have in fact.
@@ -413,10 +412,10 @@
 # define SSL_PKEY_GOST_EC SSL_PKEY_NUM+1
 
 /*
- * TODO(TLS1.3) for now use RSA_SIGN keys for PSS
+ * TODO(TLS1.3) for now use SSL_PKEY_RSA keys for PSS
  */
 
-#define SSL_PKEY_RSA_PSS_SIGN SSL_PKEY_RSA_SIGN
+#define SSL_PKEY_RSA_PSS_SIGN SSL_PKEY_RSA
 
 /*-
  * SSL_kRSA <- RSA_ENC

ssl/statem/statem_lib.c

@@ -1071,7 +1071,7 @@ int ssl_cert_type(const X509 *x, const EVP_PKEY *pk)
     default:
         return -1;
     case EVP_PKEY_RSA:
-        return SSL_PKEY_RSA_ENC;
+        return SSL_PKEY_RSA;
     case EVP_PKEY_DSA:
         return SSL_PKEY_DSA_SIGN;
 #ifndef OPENSSL_NO_EC

ssl/statem/statem_srvr.c

@@ -2465,7 +2465,7 @@ static int tls_process_cke_rsa(SSL *s, PACKET *pkt, int *al)
     unsigned char *rsa_decrypt = NULL;
     int ret = 0;
 
-    rsa = EVP_PKEY_get0_RSA(s->cert->pkeys[SSL_PKEY_RSA_ENC].privatekey);
+    rsa = EVP_PKEY_get0_RSA(s->cert->pkeys[SSL_PKEY_RSA].privatekey);
     if (rsa == NULL) {
         *al = SSL_AD_HANDSHAKE_FAILURE;
         SSLerr(SSL_F_TLS_PROCESS_CKE_RSA, SSL_R_MISSING_RSA_CERTIFICATE);

ssl/t1_lib.c

@@ -730,16 +730,16 @@ static const SIGALG_LOOKUP sigalg_lookup_tbl[] = {
      NID_sha512, SSL_MD_SHA512_IDX, EVP_PKEY_RSA_PSS, SSL_PKEY_RSA_PSS_SIGN,
      NID_undef, NID_undef},
     {"rsa_pkcs1_sha256", TLSEXT_SIGALG_rsa_pkcs1_sha256,
-     NID_sha256, SSL_MD_SHA256_IDX, EVP_PKEY_RSA, SSL_PKEY_RSA_SIGN,
+     NID_sha256, SSL_MD_SHA256_IDX, EVP_PKEY_RSA, SSL_PKEY_RSA,
      NID_sha256WithRSAEncryption, NID_undef},
     {"rsa_pkcs1_sha384", TLSEXT_SIGALG_rsa_pkcs1_sha384,
-     NID_sha384, SSL_MD_SHA384_IDX, EVP_PKEY_RSA, SSL_PKEY_RSA_SIGN,
+     NID_sha384, SSL_MD_SHA384_IDX, EVP_PKEY_RSA, SSL_PKEY_RSA,
      NID_sha384WithRSAEncryption, NID_undef},
     {"rsa_pkcs1_sha512", TLSEXT_SIGALG_rsa_pkcs1_sha512,
-     NID_sha512, SSL_MD_SHA512_IDX, EVP_PKEY_RSA, SSL_PKEY_RSA_SIGN,
+     NID_sha512, SSL_MD_SHA512_IDX, EVP_PKEY_RSA, SSL_PKEY_RSA,
      NID_sha512WithRSAEncryption, NID_undef},
     {"rsa_pkcs1_sha1", TLSEXT_SIGALG_rsa_pkcs1_sha1,
-     NID_sha1, SSL_MD_SHA1_IDX, EVP_PKEY_RSA, SSL_PKEY_RSA_SIGN,
+     NID_sha1, SSL_MD_SHA1_IDX, EVP_PKEY_RSA, SSL_PKEY_RSA,
      NID_sha1WithRSAEncryption, NID_undef},
 #ifndef OPENSSL_NO_DSA
     {NULL, TLSEXT_SIGALG_dsa_sha256,
@@ -1022,10 +1022,9 @@ void ssl_set_default_md(SSL *s)
 #endif
 #ifndef OPENSSL_NO_RSA
     if (SSL_USE_SIGALGS(s))
-        pmd[SSL_PKEY_RSA_SIGN] = ssl_md(SSL_MD_SHA1_IDX);
+        pmd[SSL_PKEY_RSA] = ssl_md(SSL_MD_SHA1_IDX);
     else
-        pmd[SSL_PKEY_RSA_SIGN] = ssl_md(SSL_MD_MD5_SHA1_IDX);
-    pmd[SSL_PKEY_RSA_ENC] = pmd[SSL_PKEY_RSA_SIGN];
+        pmd[SSL_PKEY_RSA] = ssl_md(SSL_MD_MD5_SHA1_IDX);
 #endif
 #ifndef OPENSSL_NO_EC
     pmd[SSL_PKEY_ECC] = ssl_md(SSL_MD_SHA1_IDX);
@@ -1358,13 +1357,13 @@ static int tls12_get_pkey_idx(int sig_nid)
     switch (sig_nid) {
 #ifndef OPENSSL_NO_RSA
     case EVP_PKEY_RSA:
-        return SSL_PKEY_RSA_SIGN;
+        return SSL_PKEY_RSA;
     /*
      * For now return RSA key for PSS. When we support PSS only keys
      * this will need to be updated.
      */
     case EVP_PKEY_RSA_PSS:
-        return SSL_PKEY_RSA_SIGN;
+        return SSL_PKEY_RSA;
 #endif
 #ifndef OPENSSL_NO_DSA
     case EVP_PKEY_DSA:
@@ -1605,10 +1604,6 @@ int tls1_process_sigalgs(SSL *s)
             md = ssl_md(sigptr->hash_idx);
             pmd[idx] = md;
             pvalid[idx] = CERT_PKEY_EXPLICIT_SIGN;
-            if (idx == SSL_PKEY_RSA_SIGN) {
-                pvalid[SSL_PKEY_RSA_ENC] = CERT_PKEY_EXPLICIT_SIGN;
-                pmd[SSL_PKEY_RSA_ENC] = md;
-            }
         }
     }
     /*
@@ -1626,9 +1621,8 @@ int tls1_process_sigalgs(SSL *s)
             pmd[SSL_PKEY_DSA_SIGN] = EVP_sha1();
 #endif
 #ifndef OPENSSL_NO_RSA
-        if (pmd[SSL_PKEY_RSA_SIGN] == NULL) {
-            pmd[SSL_PKEY_RSA_SIGN] = EVP_sha1();
-            pmd[SSL_PKEY_RSA_ENC] = EVP_sha1();
+        if (pmd[SSL_PKEY_RSA] == NULL) {
+            pmd[SSL_PKEY_RSA] = EVP_sha1();
         }
 #endif
 #ifndef OPENSSL_NO_EC
@@ -1945,8 +1939,7 @@ int tls1_check_chain(SSL *s, X509 *x, EVP_PKEY *pk, STACK_OF(X509) *chain,
         /* If no sigalgs extension use defaults from RFC5246 */
         else {
             switch (idx) {
-            case SSL_PKEY_RSA_ENC:
-            case SSL_PKEY_RSA_SIGN:
+            case SSL_PKEY_RSA:
                 rsign = EVP_PKEY_RSA;
                 default_nid = NID_sha1WithRSAEncryption;
                 break;
@@ -2133,8 +2126,7 @@ int tls1_check_chain(SSL *s, X509 *x, EVP_PKEY *pk, STACK_OF(X509) *chain,
 /* Set validity of certificates in an SSL structure */
 void tls1_set_cert_validity(SSL *s)
 {
-    tls1_check_chain(s, NULL, NULL, NULL, SSL_PKEY_RSA_ENC);
-    tls1_check_chain(s, NULL, NULL, NULL, SSL_PKEY_RSA_SIGN);
+    tls1_check_chain(s, NULL, NULL, NULL, SSL_PKEY_RSA);
     tls1_check_chain(s, NULL, NULL, NULL, SSL_PKEY_DSA_SIGN);
     tls1_check_chain(s, NULL, NULL, NULL, SSL_PKEY_ECC);
     tls1_check_chain(s, NULL, NULL, NULL, SSL_PKEY_GOST01);
@@ -2299,14 +2291,8 @@ int tls_choose_sigalg(SSL *s)
                 continue;
             idx = lu->sig_idx;
             c = &s->cert->pkeys[idx];
-            if (c->x509 == NULL || c->privatekey == NULL) {
-                if (idx != SSL_PKEY_RSA_SIGN)
-                    continue;
-                idx = SSL_PKEY_RSA_ENC;
-                c = s->cert->pkeys + idx;
-                if (c->x509 == NULL || c->privatekey == NULL)
+            if (c->x509 == NULL || c->privatekey == NULL)
                     continue;
-            }
             if (lu->sig == EVP_PKEY_EC) {
 #ifndef OPENSSL_NO_EC
                 if (curve == -1) {