Allow RSA certificates to be used for RSA-PSS

Allo RSA certificate to be used for RSA-PSS signatures: this needs
to be explicit because RSA and RSA-PSS certificates are now distinct
types.
Reviewed-by: NBen Kaduk <kaduk@mit.edu>
(Merged from https://github.com/openssl/openssl/pull/4368)

ssl/t1_lib.c

@@ -2268,18 +2268,23 @@ int ssl_security_cert_chain(SSL *s, STACK_OF(X509) *sk, X509 *x, int vfy)
 
 /*
  * For TLS 1.2 servers check if we have a certificate which can be used
- * with the signature algorithm "lu".
+ * with the signature algorithm "lu" and return index of certificate.
  */
 
-static int tls12_check_cert_sigalg(const SSL *s, const SIGALG_LOOKUP *lu)
+static int tls12_get_cert_sigalg_idx(const SSL *s, const SIGALG_LOOKUP *lu)
 {
-    const SSL_CERT_LOOKUP *clu = ssl_cert_lookup_by_idx(lu->sig_idx);
+    int sig_idx = lu->sig_idx;
+    const SSL_CERT_LOOKUP *clu = ssl_cert_lookup_by_idx(sig_idx);
 
     /* If not recognised or not supported by cipher mask it is not suitable */
     if (clu == NULL || !(clu->amask & s->s3->tmp.new_cipher->algorithm_auth))
-        return 0;
+        return -1;
+
+    /* If PSS and we have no PSS cert use RSA */
+    if (sig_idx == SSL_PKEY_RSA_PSS_SIGN && !ssl_has_cert(s, sig_idx))
+        sig_idx = SSL_PKEY_RSA;
 
-    return s->s3->tmp.valid_flags[lu->sig_idx] & CERT_PKEY_VALID ? 1 : 0;
+    return s->s3->tmp.valid_flags[sig_idx] & CERT_PKEY_VALID ? sig_idx : -1;
 }
 
 /*
@@ -2296,6 +2301,7 @@ static int tls12_check_cert_sigalg(const SSL *s, const SIGALG_LOOKUP *lu)
 int tls_choose_sigalg(SSL *s, int *al)
 {
     const SIGALG_LOOKUP *lu = NULL;
+    int sig_idx = -1;
 
     s->s3->tmp.cert = NULL;
     s->s3->tmp.sigalg = NULL;
@@ -2318,8 +2324,12 @@ int tls_choose_sigalg(SSL *s, int *al)
                 continue;
             if (!tls1_lookup_md(lu, NULL))
                 continue;
-            if (!ssl_has_cert(s, lu->sig_idx))
+            if (!ssl_has_cert(s, lu->sig_idx)) {
+                if (lu->sig_idx != SSL_PKEY_RSA_PSS_SIGN
+                        || !ssl_has_cert(s, SSL_PKEY_RSA))
                     continue;
+                    sig_idx = SSL_PKEY_RSA;
+            }
             if (lu->sig == EVP_PKEY_EC) {
 #ifndef OPENSSL_NO_EC
                 if (curve == -1) {
@@ -2376,10 +2386,18 @@ int tls_choose_sigalg(SSL *s, int *al)
                     lu = s->cert->shared_sigalgs[i];
 
                     if (s->server) {
-                        if (!tls12_check_cert_sigalg(s, lu))
-                            continue;
-                    } else if (lu->sig_idx != s->cert->key - s->cert->pkeys) {
+                        if ((sig_idx = tls12_get_cert_sigalg_idx(s, lu)) == -1)
                             continue;
+                    } else {
+                        int cc_idx = s->cert->key - s->cert->pkeys;
+
+                        sig_idx = lu->sig_idx;
+                        if (cc_idx != sig_idx) {
+                            if (sig_idx != SSL_PKEY_RSA_PSS_SIGN
+                                || cc_idx != SSL_PKEY_RSA)
+                                continue;
+                            sig_idx = SSL_PKEY_RSA;
+                        }
                     }
 #ifndef OPENSSL_NO_EC
                     if (curve == -1 || lu->curve == curve)
@@ -2432,7 +2450,9 @@ int tls_choose_sigalg(SSL *s, int *al)
             }
         }
     }
-    s->s3->tmp.cert = &s->cert->pkeys[lu->sig_idx];
+    if (sig_idx == -1)
+        sig_idx = lu->sig_idx;
+    s->s3->tmp.cert = &s->cert->pkeys[sig_idx];
     s->cert->key = s->s3->tmp.cert;
     s->s3->tmp.sigalg = lu;
     return 1;