Move decisions about whether to accept reneg into the state machine

If a server receives an unexpected ClientHello then we may or may not
accept it. Make sure all such decisions are made in the state machine
and not in the record layer. This also removes a disparity between the
TLS and the DTLS code. The TLS code was making this decision in the
record layer, while the DTLS code was making it later.

Finally it also solves a problem where a warning alert was being sent
during tls_setup_handshake() and the function was returning a failure
return code. This is problematic because it can be called from a
transition function - which we only allow fatal errors to occur in.
Reviewed-by: NRich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/5190)

ssl/record/rec_layer_s3.c

@@ -1491,27 +1491,6 @@ int ssl3_read_bytes(SSL *s, int type, int *recvd_type, unsigned char *buf,
      * (Possibly rr is 'empty' now, i.e. rr->length may be 0.)
      */
 
-    /*
-     * If we are a server and get a client hello when renegotiation isn't
-     * allowed send back a no renegotiation alert and carry on. WARNING:
-     * experimental code, needs reviewing (steve)
-     */
-    if (s->server &&
-        SSL_is_init_finished(s) &&
-        (s->version > SSL3_VERSION) &&
-        !SSL_IS_TLS13(s) &&
-        (SSL3_RECORD_get_type(rr) == SSL3_RT_HANDSHAKE) &&
-        (s->rlayer.handshake_fragment_len >= 4) &&
-        (s->rlayer.handshake_fragment[0] == SSL3_MT_CLIENT_HELLO) &&
-        (s->session != NULL) && (s->session->cipher != NULL) &&
-        ((!s->s3->send_connection_binding &&
-          !(s->options & SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION)) ||
-         (s->options & SSL_OP_NO_RENEGOTIATION))) {
-        SSL3_RECORD_set_length(rr, 0);
-        SSL3_RECORD_set_read(rr);
-        ssl3_send_alert(s, SSL3_AL_WARNING, SSL_AD_NO_RENEGOTIATION);
-        goto start;
-    }
     if (SSL3_RECORD_get_type(rr) == SSL3_RT_ALERT) {
         unsigned int alert_level, alert_descr;
         unsigned char *alert_bytes = SSL3_RECORD_get_data(rr)

ssl/statem/statem_lib.c

@@ -123,20 +123,6 @@ int tls_setup_handshake(SSL *s)
             /* N.B. s->session_ctx == s->ctx here */
             CRYPTO_atomic_add(&s->session_ctx->stats.sess_accept, 1, &i,
                               s->session_ctx->lock);
-        } else if ((s->options & SSL_OP_NO_RENEGOTIATION)) {
-            /* Renegotiation is disabled */
-            ssl3_send_alert(s, SSL3_AL_WARNING, SSL_AD_NO_RENEGOTIATION);
-            return 0;
-        } else if (!s->s3->send_connection_binding &&
-                   !(s->options &
-                     SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION)) {
-            /*
-             * Server attempting to renegotiate with client that doesn't
-             * support secure renegotiation.
-             */
-            SSLfatal(s, SSL_AD_HANDSHAKE_FAILURE, SSL_F_TLS_SETUP_HANDSHAKE,
-                     SSL_R_UNSAFE_LEGACY_RENEGOTIATION_DISABLED);
-            return 0;
         } else {
             /* N.B. s->ctx may not equal s->session_ctx */
             CRYPTO_atomic_add(&s->ctx->stats.sess_accept_renegotiate, 1, &i,

ssl/statem/statem_srvr.c

@@ -13,6 +13,7 @@
 #include "../ssl_locl.h"
 #include "statem_locl.h"
 #include "internal/constant_time_locl.h"
+#include "internal/cryptlib.h"
 #include <openssl/buffer.h>
 #include <openssl/rand.h>
 #include <openssl/objects.h>
@@ -514,10 +515,15 @@ WRITE_TRAN ossl_statem_server_write_transition(SSL *s)
 
     case TLS_ST_SR_CLNT_HELLO:
         if (SSL_IS_DTLS(s) && !s->d1->cookie_verified
-            && (SSL_get_options(s) & SSL_OP_COOKIE_EXCHANGE))
+            && (SSL_get_options(s) & SSL_OP_COOKIE_EXCHANGE)) {
             st->hand_state = DTLS_ST_SW_HELLO_VERIFY_REQUEST;
-        else
+        } else if (s->renegotiate == 0 && !SSL_IS_FIRST_HANDSHAKE(s)) {
+            /* We must have rejected the renegotiation */
+            st->hand_state = TLS_ST_OK;
+            return WRITE_TRAN_CONTINUE;
+        } else {
             st->hand_state = TLS_ST_SW_SRVR_HELLO;
+        }
         return WRITE_TRAN_CONTINUE;
 
     case DTLS_ST_SW_HELLO_VERIFY_REQUEST:
@@ -1254,24 +1260,33 @@ MSG_PROCESS_RETURN tls_process_client_hello(SSL *s, PACKET *pkt)
     /* |cookie| will only be initialized for DTLS. */
     PACKET session_id, compression, extensions, cookie;
     static const unsigned char null_compression = 0;
-    CLIENTHELLO_MSG *clienthello;
+    CLIENTHELLO_MSG *clienthello = NULL;
 
-    clienthello = OPENSSL_zalloc(sizeof(*clienthello));
-    if (clienthello == NULL) {
-        SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_F_TLS_PROCESS_CLIENT_HELLO,
-                 ERR_R_INTERNAL_ERROR);
-        goto err;
-    }
     /* Check if this is actually an unexpected renegotiation ClientHello */
     if (s->renegotiate == 0 && !SSL_IS_FIRST_HANDSHAKE(s)) {
-        if ((s->options & SSL_OP_NO_RENEGOTIATION)) {
-            ssl3_send_alert(s, SSL3_AL_WARNING, SSL_AD_NO_RENEGOTIATION);
+        if (!ossl_assert(!SSL_IS_TLS13(s))) {
+            SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_F_TLS_PROCESS_CLIENT_HELLO,
+                     ERR_R_INTERNAL_ERROR);
             goto err;
         }
+        if ((s->options & SSL_OP_NO_RENEGOTIATION) != 0
+                || (!s->s3->send_connection_binding
+                    && (s->options
+                        & SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION) == 0)) {
+            ssl3_send_alert(s, SSL3_AL_WARNING, SSL_AD_NO_RENEGOTIATION);
+            return MSG_PROCESS_FINISHED_READING;
+        }
         s->renegotiate = 1;
         s->new_session = 1;
     }
 
+    clienthello = OPENSSL_zalloc(sizeof(*clienthello));
+    if (clienthello == NULL) {
+        SSLfatal(s, SSL_AD_INTERNAL_ERROR, SSL_F_TLS_PROCESS_CLIENT_HELLO,
+                 ERR_R_INTERNAL_ERROR);
+        goto err;
+    }
+
     /*
      * First, parse the raw ClientHello data into the CLIENTHELLO_MSG structure.
      */