Limit padded record to max plaintext

Reviewed-by: NRich Salz <rsalz@openssl.org>
Reviewed-by: NMatt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/3374)

ssl/record/rec_layer_s3.c

@@ -860,7 +860,7 @@ int do_ssl3_write(SSL *s, int type, const unsigned char *buf,
         }
 
         if (SSL_TREAT_AS_TLS13(s) && s->enc_write_ctx != NULL) {
-            size_t padding = 0;
+            size_t rlen;
 
             if (!WPACKET_put_bytes_u8(thispkt, type)) {
                 SSLerr(SSL_F_DO_SSL3_WRITE, ERR_R_INTERNAL_ERROR);
@@ -869,22 +869,21 @@ int do_ssl3_write(SSL *s, int type, const unsigned char *buf,
             SSL3_RECORD_add_length(thiswr, 1);
 
             /* Add TLS1.3 padding */
+            rlen = SSL3_RECORD_get_length(thiswr);
+            if (rlen < SSL3_RT_MAX_PLAIN_LENGTH) {
+                size_t padding = 0;
+                size_t max_padding = SSL3_RT_MAX_PLAIN_LENGTH - rlen;
                 if (s->record_padding_cb != NULL) {
-                size_t rlen = SSL3_RECORD_get_length(thiswr);
-
                     padding = s->record_padding_cb(s, type, rlen, s->record_padding_arg);
-                /* do not allow the record to exceed max plaintext length */
-                if (padding > (SSL3_RT_MAX_PLAIN_LENGTH - rlen))
-                    padding = SSL3_RT_MAX_PLAIN_LENGTH - rlen;
                 } else if (s->block_padding > 0) {
                     size_t mask = s->block_padding - 1;
                     size_t remainder;
 
                     /* optimize for power of 2 */
                     if ((s->block_padding & mask) == 0)
-                    remainder = SSL3_RECORD_get_length(thiswr) & mask;
+                        remainder = rlen & mask;
                     else
-                    remainder = SSL3_RECORD_get_length(thiswr) % s->block_padding;
+                        remainder = rlen % s->block_padding;
                     /* don't want to add a block of padding if we don't have to */
                     if (remainder == 0)
                         padding = 0;
@@ -892,6 +891,9 @@ int do_ssl3_write(SSL *s, int type, const unsigned char *buf,
                         padding = s->block_padding - remainder;
                 }
                 if (padding > 0) {
+                    /* do not allow the record to exceed max plaintext length */
+                    if (padding > max_padding)
+                        padding = max_padding;
                     if (!WPACKET_memset(thispkt, 0, padding)) {
                         SSLerr(SSL_F_DO_SSL3_WRITE, ERR_R_INTERNAL_ERROR);
                         goto err;
@@ -899,6 +901,7 @@ int do_ssl3_write(SSL *s, int type, const unsigned char *buf,
                     SSL3_RECORD_add_length(thiswr, padding);
                 }
             }
+        }
 
         /*
          * we should still have the output to thiswr->data and the input from