Reject compressed point format with TLS 1.3

Reviewed-by: NRich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/2739)

include/openssl/ssl.h

@@ -2508,6 +2508,7 @@ int ERR_load_SSL_strings(void);
 # define SSL_R_GOT_A_FIN_BEFORE_A_CCS                     154
 # define SSL_R_HTTPS_PROXY_REQUEST                        155
 # define SSL_R_HTTP_REQUEST                               156
+# define SSL_R_ILLEGAL_POINT_COMPRESSION                  162
 # define SSL_R_ILLEGAL_SUITEB_DIGEST                      380
 # define SSL_R_INAPPROPRIATE_FALLBACK                     373
 # define SSL_R_INCONSISTENT_COMPRESSION                   340

ssl/ssl_err.c

@@ -570,6 +570,8 @@ static ERR_STRING_DATA SSL_str_reasons[] = {
     {ERR_REASON(SSL_R_GOT_A_FIN_BEFORE_A_CCS), "got a fin before a ccs"},
     {ERR_REASON(SSL_R_HTTPS_PROXY_REQUEST), "https proxy request"},
     {ERR_REASON(SSL_R_HTTP_REQUEST), "http request"},
+    {ERR_REASON(SSL_R_ILLEGAL_POINT_COMPRESSION),
+     "illegal point compression"},
     {ERR_REASON(SSL_R_ILLEGAL_SUITEB_DIGEST), "illegal Suite B digest"},
     {ERR_REASON(SSL_R_INAPPROPRIATE_FALLBACK), "inappropriate fallback"},
     {ERR_REASON(SSL_R_INCONSISTENT_COMPRESSION), "inconsistent compression"},

ssl/t1_lib.c

@@ -916,8 +916,12 @@ int tls12_check_peer_sigalg(SSL *s, uint16_t sig, EVP_PKEY *pkey)
         int curve = EC_GROUP_get_curve_name(EC_KEY_get0_group(ec));
 
         if (SSL_IS_TLS13(s)) {
+            if (EC_KEY_get_conv_form(ec) != POINT_CONVERSION_UNCOMPRESSED) {
+                SSLerr(SSL_F_TLS12_CHECK_PEER_SIGALG,
+                       SSL_R_ILLEGAL_POINT_COMPRESSION);
+                return 0;
+            }
             /* For TLS 1.3 check curve matches signature algorithm */
-
             if (lu->curve != NID_undef && curve != lu->curve) {
                 SSLerr(SSL_F_TLS12_CHECK_PEER_SIGALG, SSL_R_WRONG_CURVE);
                 return 0;
@@ -2237,7 +2241,7 @@ int tls_choose_sigalg(SSL *s, int *al)
     if (SSL_IS_TLS13(s)) {
         size_t i;
 #ifndef OPENSSL_NO_EC
-        int curve = -1;
+        int curve = -1, skip_ec = 0;
 #endif
 
         /* Look for a certificate matching shared sigaglgs */
@@ -2258,8 +2262,11 @@ int tls_choose_sigalg(SSL *s, int *al)
                     EC_KEY *ec = EVP_PKEY_get0_EC_KEY(s->cert->pkeys[idx].privatekey);
 
                     curve = EC_GROUP_get_curve_name(EC_KEY_get0_group(ec));
+                    if (EC_KEY_get_conv_form(ec)
+                        != POINT_CONVERSION_UNCOMPRESSED)
+                        skip_ec = 1;
                 }
-                if (lu->curve != NID_undef && curve != lu->curve)
+                if (skip_ec || (lu->curve != NID_undef && curve != lu->curve))
                     continue;
 #else
                 continue;