Don't attempt to send fragments > max_send_fragment in DTLS

We were allocating the write buffer based on the size of max_send_fragment,
but ignoring it when writing data. We should fragment handshake messages
if they exceed max_send_fragment and reject application data writes that
are too large.
Reviewed-by: NRichard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/3286)

include/openssl/ssl.h

@@ -2607,6 +2607,7 @@ int ERR_load_SSL_strings(void);
 # define SSL_R_ENCRYPTED_LENGTH_TOO_LONG                  150
 # define SSL_R_ERROR_IN_RECEIVED_CIPHER_LIST              151
 # define SSL_R_ERROR_SETTING_TLSA_BASE_DOMAIN             204
+# define SSL_R_EXCEEDS_MAX_FRAGMENT_SIZE                  194
 # define SSL_R_EXCESSIVE_MESSAGE_SIZE                     152
 # define SSL_R_EXTRA_DATA_IN_MESSAGE                      153
 # define SSL_R_EXT_LENGTH_MISMATCH                        163

ssl/record/rec_layer_d1.c

@@ -882,6 +882,11 @@ int do_dtls1_write(SSL *s, int type, const unsigned char *buf,
     if (len == 0 && !create_empty_fragment)
         return 0;
 
+    if (len > s->max_send_fragment) {
+        SSLerr(SSL_F_DO_DTLS1_WRITE, SSL_R_EXCEEDS_MAX_FRAGMENT_SIZE);
+        return 0;
+    }
+
     sess = s->session;
 
     if ((sess == NULL) ||

ssl/ssl_err.c

@@ -590,6 +590,8 @@ static ERR_STRING_DATA SSL_str_reasons[] = {
      "error in received cipher list"},
     {ERR_REASON(SSL_R_ERROR_SETTING_TLSA_BASE_DOMAIN),
      "error setting tlsa base domain"},
+    {ERR_REASON(SSL_R_EXCEEDS_MAX_FRAGMENT_SIZE),
+     "exceeds max fragment size"},
     {ERR_REASON(SSL_R_EXCESSIVE_MESSAGE_SIZE), "excessive message size"},
     {ERR_REASON(SSL_R_EXTRA_DATA_IN_MESSAGE), "extra data in message"},
     {ERR_REASON(SSL_R_EXT_LENGTH_MISMATCH), "ext length mismatch"},

ssl/statem/statem_dtls.c

@@ -214,6 +214,9 @@ int dtls1_do_write(SSL *s, int type)
         else
             len = s->init_num;
 
+        if (len > s->max_send_fragment)
+            len = s->max_send_fragment;
+
         /*
          * XDTLS: this function is too long.  split out the CCS part
          */