Store peer signature type.

Store peer signature type in s->s3->tmp.peer_sigtype and check it
to see if the peer used PSS.
Reviewed-by: NRichard Levitte <levitte@openssl.org>
Reviewed-by: NMatt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/2301)

ssl/ssl_locl.h

@@ -1271,6 +1271,8 @@ typedef struct ssl3_state_st {
         size_t peer_sigalgslen;
         /* Digest peer uses for signing */
         const EVP_MD *peer_md;
+        /* Signature type: public key type or EVP_PKEY_RSA_PSS for PSS */
+        int peer_sigtype;
         /* Array of digests used for signing */
         const EVP_MD *md[SSL_PKEY_NUM];
         /*
@@ -1741,10 +1743,7 @@ typedef enum tlsext_index_en {
 /* An invalid index into the TLSv1.3 PSK identities */
 #define TLSEXT_PSK_BAD_IDENTITY                                 -1
 
-#define SIGID_IS_PSS(sigid) ((sigid) == TLSEXT_SIGALG_rsa_pss_sha256 \
-                             || (sigid) == TLSEXT_SIGALG_rsa_pss_sha384 \
-                             || (sigid) == TLSEXT_SIGALG_rsa_pss_sha512)
-
+#define SSL_USE_PSS(s) (s->s3->tmp.peer_sigtype == EVP_PKEY_RSA_PSS)
 
 /* A dummy signature value not valid for TLSv1.2 signature algs */
 #define TLSEXT_signature_rsa_pss                                0x0101
@@ -2254,8 +2253,7 @@ __owur int tls12_copy_sigalgs(SSL *s, WPACKET *pkt,
 __owur int tls1_save_sigalgs(SSL *s, PACKET *pkt);
 __owur int tls1_process_sigalgs(SSL *s);
 __owur size_t tls12_get_psigalgs(SSL *s, int sent, const uint16_t **psigs);
-__owur int tls12_check_peer_sigalg(const EVP_MD **pmd, SSL *s, unsigned int sig,
-                                   EVP_PKEY *pkey);
+__owur int tls12_check_peer_sigalg(SSL *s, unsigned int sig, EVP_PKEY *pkey);
 void ssl_set_client_disabled(SSL *s);
 __owur int ssl_cipher_disabled(SSL *s, const SSL_CIPHER *c, int op);
 

ssl/statem/statem_clnt.c

@@ -1902,7 +1902,7 @@ static int tls_process_ske_ecdhe(SSL *s, PACKET *pkt, EVP_PKEY **pkey, int *al)
 
 MSG_PROCESS_RETURN tls_process_key_exchange(SSL *s, PACKET *pkt)
 {
-    int al = -1, ispss = 0;
+    int al = -1;
     long alg_k;
     EVP_PKEY *pkey = NULL;
     EVP_MD_CTX *md_ctx = NULL;
@@ -1967,7 +1967,7 @@ MSG_PROCESS_RETURN tls_process_key_exchange(SSL *s, PACKET *pkt)
                 SSLerr(SSL_F_TLS_PROCESS_KEY_EXCHANGE, SSL_R_LENGTH_TOO_SHORT);
                 goto err;
             }
-            rv = tls12_check_peer_sigalg(&md, s, sigalg, pkey);
+            rv = tls12_check_peer_sigalg(s, sigalg, pkey);
             if (rv == -1) {
                 al = SSL_AD_INTERNAL_ERROR;
                 goto err;
@@ -1975,7 +1975,7 @@ MSG_PROCESS_RETURN tls_process_key_exchange(SSL *s, PACKET *pkt)
                 al = SSL_AD_DECODE_ERROR;
                 goto err;
             }
-            ispss = SIGID_IS_PSS(sigalg);
+            md = s->s3->tmp.peer_md;
 #ifdef SSL_DEBUG
             fprintf(stderr, "USING TLSv1.2 HASH %s\n", EVP_MD_name(md));
 #endif
@@ -2021,7 +2021,7 @@ MSG_PROCESS_RETURN tls_process_key_exchange(SSL *s, PACKET *pkt)
             SSLerr(SSL_F_TLS_PROCESS_KEY_EXCHANGE, ERR_R_EVP_LIB);
             goto err;
         }
-        if (ispss) {
+        if (SSL_USE_PSS(s)) {
             if (EVP_PKEY_CTX_set_rsa_padding(pctx, RSA_PKCS1_PSS_PADDING) <= 0
                        /* -1 here means set saltlen to the digest len */
                     || EVP_PKEY_CTX_set_rsa_pss_saltlen(pctx,

ssl/statem/statem_lib.c

@@ -286,7 +286,7 @@ MSG_PROCESS_RETURN tls_process_cert_verify(SSL *s, PACKET *pkt)
     unsigned char *gost_data = NULL;
 #endif
     int al = SSL_AD_INTERNAL_ERROR, ret = MSG_PROCESS_ERROR;
-    int type = 0, j, pktype, ispss = 0;
+    int type = 0, j, pktype;
     unsigned int len;
     X509 *peer;
     const EVP_MD *md = NULL;
@@ -333,14 +333,14 @@ MSG_PROCESS_RETURN tls_process_cert_verify(SSL *s, PACKET *pkt)
                 al = SSL_AD_DECODE_ERROR;
                 goto f_err;
             }
-            rv = tls12_check_peer_sigalg(&md, s, sigalg, pkey);
+            rv = tls12_check_peer_sigalg(s, sigalg, pkey);
             if (rv == -1) {
                 goto f_err;
             } else if (rv == 0) {
                 al = SSL_AD_DECODE_ERROR;
                 goto f_err;
             }
-            ispss = SIGID_IS_PSS(sigalg);
+            md = s->s3->tmp.peer_md;
 #ifdef SSL_DEBUG
             fprintf(stderr, "USING TLSv1.2 HASH %s\n", EVP_MD_name(md));
 #endif
@@ -402,7 +402,7 @@ MSG_PROCESS_RETURN tls_process_cert_verify(SSL *s, PACKET *pkt)
     }
 #endif
 
-    if (ispss) {
+    if (SSL_USE_PSS(s)) {
         if (EVP_PKEY_CTX_set_rsa_padding(pctx, RSA_PKCS1_PSS_PADDING) <= 0
             || EVP_PKEY_CTX_set_rsa_pss_saltlen(pctx,
                                                 RSA_PSS_SALTLEN_DIGEST) <= 0) {

ssl/t1_lib.c

@@ -809,22 +809,22 @@ size_t tls12_get_psigalgs(SSL *s, int sent, const uint16_t **psigs)
  * algorithms and if so set relevant digest and signature scheme in
  * s.
  */
-int tls12_check_peer_sigalg(const EVP_MD **pmd, SSL *s, unsigned int sig,
-                            EVP_PKEY *pkey)
+int tls12_check_peer_sigalg(SSL *s, unsigned int sig, EVP_PKEY *pkey)
 {
     const uint16_t *sent_sigs;
+    const EVP_MD *md = NULL;
     char sigalgstr[2];
     size_t sent_sigslen, i;
     int pkeyid = EVP_PKEY_id(pkey);
-    int peer_pkeyid;
+    int peer_sigtype;
     /* Should never happen */
     if (pkeyid == -1)
         return -1;
     /* Check key type is consistent with signature */
-    peer_pkeyid = tls_sigalg_get_sig(sig);
+    peer_sigtype = tls_sigalg_get_sig(sig);
     /* RSA keys can be used for RSA-PSS */
-    if (pkeyid != peer_pkeyid
-        && (peer_pkeyid != EVP_PKEY_RSA_PSS || pkeyid != EVP_PKEY_RSA)) {
+    if (pkeyid != peer_sigtype
+        && (peer_sigtype != EVP_PKEY_RSA_PSS || pkeyid != EVP_PKEY_RSA)) {
         SSLerr(SSL_F_TLS12_CHECK_PEER_SIGALG, SSL_R_WRONG_SIGNATURE_TYPE);
         return 0;
     }
@@ -874,8 +874,8 @@ int tls12_check_peer_sigalg(const EVP_MD **pmd, SSL *s, unsigned int sig,
         SSLerr(SSL_F_TLS12_CHECK_PEER_SIGALG, SSL_R_WRONG_SIGNATURE_TYPE);
         return 0;
     }
-    *pmd = tls12_get_hash(tls_sigalg_get_hash(sig));
-    if (*pmd == NULL) {
+    md = tls12_get_hash(tls_sigalg_get_hash(sig));
+    if (md == NULL) {
         SSLerr(SSL_F_TLS12_CHECK_PEER_SIGALG, SSL_R_UNKNOWN_DIGEST);
         return 0;
     }
@@ -886,7 +886,7 @@ int tls12_check_peer_sigalg(const EVP_MD **pmd, SSL *s, unsigned int sig,
     sigalgstr[0] = (sig >> 8) & 0xff;
     sigalgstr[1] = sig & 0xff;
     if (!ssl_security(s, SSL_SECOP_SIGALG_CHECK,
-                      EVP_MD_size(*pmd) * 4, EVP_MD_type(*pmd),
+                      EVP_MD_size(md) * 4, EVP_MD_type(md),
                       (void *)sigalgstr)) {
         SSLerr(SSL_F_TLS12_CHECK_PEER_SIGALG, SSL_R_WRONG_SIGNATURE_TYPE);
         return 0;
@@ -894,7 +894,8 @@ int tls12_check_peer_sigalg(const EVP_MD **pmd, SSL *s, unsigned int sig,
     /*
      * Store the digest used so applications can retrieve it if they wish.
      */
-    s->s3->tmp.peer_md = *pmd;
+    s->s3->tmp.peer_md = md;
+    s->s3->tmp.peer_sigtype = peer_sigtype;
     return 1;
 }