Use certificate tables instead of ssl_cert_type

Reviewed-by: NRich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/3858)

ssl/ssl_locl.h

@@ -2136,7 +2136,6 @@ __owur int ssl_undefined_const_function(const SSL *s);
 __owur int ssl_get_server_cert_serverinfo(SSL *s,
                                           const unsigned char **serverinfo,
                                           size_t *serverinfo_length);
-__owur int ssl_cert_type(const X509 *x, const EVP_PKEY *pkey);
 void ssl_set_masks(SSL *s);
 __owur STACK_OF(SSL_CIPHER) *ssl_get_ciphers_by_id(SSL *s);
 __owur int ssl_verify_alarm_type(long type);

ssl/ssl_rsa.c

@@ -127,9 +127,9 @@ int SSL_use_RSAPrivateKey(SSL *ssl, RSA *rsa)
 
 static int ssl_set_pkey(CERT *c, EVP_PKEY *pkey)
 {
-    int i;
-    i = ssl_cert_type(NULL, pkey);
-    if (i < 0) {
+    size_t i;
+
+    if (ssl_cert_lookup_by_pkey(pkey, &i) == NULL) {
         SSLerr(SSL_F_SSL_SET_PKEY, SSL_R_UNKNOWN_CERTIFICATE_TYPE);
         return (0);
     }
@@ -167,8 +167,8 @@ static int ssl_set_pkey(CERT *c, EVP_PKEY *pkey)
     EVP_PKEY_free(c->pkeys[i].privatekey);
     EVP_PKEY_up_ref(pkey);
     c->pkeys[i].privatekey = pkey;
-    c->key = &(c->pkeys[i]);
-    return (1);
+    c->key = &c->pkeys[i];
+    return 1;
 }
 
 #ifndef OPENSSL_NO_RSA
@@ -316,7 +316,7 @@ int SSL_CTX_use_certificate(SSL_CTX *ctx, X509 *x)
 static int ssl_set_cert(CERT *c, X509 *x)
 {
     EVP_PKEY *pkey;
-    int i;
+    size_t i;
 
     pkey = X509_get0_pubkey(x);
     if (pkey == NULL) {
@@ -324,8 +324,7 @@ static int ssl_set_cert(CERT *c, X509 *x)
         return (0);
     }
 
-    i = ssl_cert_type(x, pkey);
-    if (i < 0) {
+    if (ssl_cert_lookup_by_pkey(pkey, &i) == NULL) {
         SSLerr(SSL_F_SSL_SET_CERT, SSL_R_UNKNOWN_CERTIFICATE_TYPE);
         return 0;
     }

ssl/statem/statem_lib.c

@@ -1238,35 +1238,6 @@ int tls_get_message_body(SSL *s, size_t *len)
     return 1;
 }
 
-int ssl_cert_type(const X509 *x, const EVP_PKEY *pk)
-{
-    if (pk == NULL && (pk = X509_get0_pubkey(x)) == NULL)
-        return -1;
-
-    switch (EVP_PKEY_id(pk)) {
-    default:
-        return -1;
-    case EVP_PKEY_RSA:
-        return SSL_PKEY_RSA;
-    case EVP_PKEY_DSA:
-        return SSL_PKEY_DSA_SIGN;
-#ifndef OPENSSL_NO_EC
-    case EVP_PKEY_EC:
-        return SSL_PKEY_ECC;
-    case EVP_PKEY_ED25519:
-        return SSL_PKEY_ED25519;
-#endif
-#ifndef OPENSSL_NO_GOST
-    case NID_id_GostR3410_2001:
-        return SSL_PKEY_GOST01;
-    case NID_id_GostR3410_2012_256:
-        return SSL_PKEY_GOST12_256;
-    case NID_id_GostR3410_2012_512:
-        return SSL_PKEY_GOST12_512;
-#endif
-    }
-}
-
 int ssl_verify_alarm_type(long type)
 {
     int al;

ssl/t1_lib.c

@@ -875,9 +875,12 @@ static const SIGALG_LOOKUP *tls1_get_legacy_sigalg(const SSL *s, int idx)
 /* Set peer sigalg based key type */
 int tls1_set_peer_legacy_sigalg(SSL *s, const EVP_PKEY *pkey)
 {
-    int idx = ssl_cert_type(NULL, pkey);
+    size_t idx;
+    const SIGALG_LOOKUP *lu;
 
-    const SIGALG_LOOKUP *lu = tls1_get_legacy_sigalg(s, idx);
+    if (ssl_cert_lookup_by_pkey(pkey, &idx) == NULL)
+        return 0;
+    lu = tls1_get_legacy_sigalg(s, idx);
     if (lu == NULL)
         return 0;
     s->s3->tmp.peer_sigalg = lu;
@@ -1923,11 +1926,14 @@ int tls1_check_chain(SSL *s, X509 *x, EVP_PKEY *pk, STACK_OF(X509) *chain,
         if (!x || !pk)
             goto end;
     } else {
+        size_t certidx;
+
         if (!x || !pk)
             return 0;
-        idx = ssl_cert_type(x, pk);
-        if (idx == -1)
+
+        if (ssl_cert_lookup_by_pkey(pk, &certidx) == NULL)
             return 0;
+        idx = certidx;
         pvalid = s->s3->tmp.valid_flags + idx;
 
         if (c->cert_flags & SSL_CERT_FLAGS_CHECK_TLS_STRICT)