Honour mandatory digest on private key in has_usable_cert()

If the private key says it can only support one specific digest, then
don't ask it to perform a different one.

Fixes: #7348
Reviewed-by: NNicola Tuveri <nic.tuv@gmail.com>
Reviewed-by: NRichard Levitte <levitte@openssl.org>

(cherry picked from commit 2d263a4a73f852005b16359873475d48755999ad)
Reviewed-by: NMatt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/7609)

ssl/t1_lib.c

@@ -2537,7 +2537,8 @@ static int tls12_get_cert_sigalg_idx(const SSL *s, const SIGALG_LOOKUP *lu)
 static int has_usable_cert(SSL *s, const SIGALG_LOOKUP *sig, int idx)
 {
     const SIGALG_LOOKUP *lu;
-    int mdnid, pknid;
+    int mdnid, pknid, default_mdnid;
+    int mandatory_md = 0;
     size_t i;
 
     /* TLS 1.2 callers can override lu->sig_idx, but not TLS 1.3 callers. */
@@ -2545,12 +2546,26 @@ static int has_usable_cert(SSL *s, const SIGALG_LOOKUP *sig, int idx)
         idx = sig->sig_idx;
     if (!ssl_has_cert(s, idx))
         return 0;
+    /* If the EVP_PKEY reports a mandatory digest, allow nothing else. */
+    ERR_set_mark();
+    switch (EVP_PKEY_get_default_digest_nid(s->cert->pkeys[idx].privatekey,
+                                            &default_mdnid)) {
+    case 2:
+        mandatory_md = 1;
+        break;
+    case 1:
+        break;
+    default: /* If it didn't report a mandatory NID, for whatever reasons,
+              * just clear the error and allow all hashes to be used. */
+        ERR_pop_to_mark();
+    }
     if (s->s3->tmp.peer_cert_sigalgs != NULL) {
         for (i = 0; i < s->s3->tmp.peer_cert_sigalgslen; i++) {
             lu = tls1_lookup_sigalg(s->s3->tmp.peer_cert_sigalgs[i]);
             if (lu == NULL
                 || !X509_get_signature_info(s->cert->pkeys[idx].x509, &mdnid,
-                                            &pknid, NULL, NULL))
+                                            &pknid, NULL, NULL)
+                || (mandatory_md && mdnid != default_mdnid))
                 continue;
             /*
              * TODO this does not differentiate between the
@@ -2563,7 +2578,7 @@ static int has_usable_cert(SSL *s, const SIGALG_LOOKUP *sig, int idx)
         }
         return 0;
     }
-    return 1;
+    return !mandatory_md || sig->hash == default_mdnid;
 }
 
 /*