Fix DTLS failure when used in a build which has SCTP enabled

The value of BIO_CTRL_DGRAM_SET_PEEK_MODE was clashing with the value for
BIO_CTRL_DGRAM_SCTP_SET_IN_HANDSHAKE. In an SCTP enabled build
BIO_CTRL_DGRAM_SCTP_SET_IN_HANDSHAKE was used unconditionally with
the reasoning that it would be ignored if SCTP wasn't in use. Unfortunately
due to this clash, this wasn't the case. The BIO ended up going into peek
mode and was continually reading the same data over and over - throwing it
away as a replay.

Fixes #3723
Reviewed-by: NRich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/3724)

crypto/bio/bss_dgram.c

@@ -787,6 +787,15 @@ static long dgram_ctrl(BIO *b, int cmd, long num, void *ptr)
     case BIO_CTRL_DGRAM_GET_MTU_OVERHEAD:
         ret = dgram_get_mtu_overhead(data);
         break;
+
+    /*
+     * BIO_CTRL_DGRAM_SCTP_SET_IN_HANDSHAKE is used here for compatibility
+     * reasons. When BIO_CTRL_DGRAM_SET_PEEK_MODE was first defined its value
+     * was incorrectly clashing with BIO_CTRL_DGRAM_SCTP_SET_IN_HANDSHAKE. The
+     * value has been updated to a non-clashing value. However to preserve
+     * binary compatiblity we now respond to both the old value and the new one
+     */
+    case BIO_CTRL_DGRAM_SCTP_SET_IN_HANDSHAKE:
     case BIO_CTRL_DGRAM_SET_PEEK_MODE:
         data->peekmode = (unsigned int)num;
         break;

include/openssl/bio.h

@@ -126,11 +126,10 @@ extern "C" {
 
 # define BIO_CTRL_DGRAM_GET_MTU_OVERHEAD   49
 
-# define BIO_CTRL_DGRAM_SET_PEEK_MODE      50
-
+/* Deliberately outside of OPENSSL_NO_SCTP - used in bss_dgram.c */
+#  define BIO_CTRL_DGRAM_SCTP_SET_IN_HANDSHAKE    50
 # ifndef OPENSSL_NO_SCTP
 /* SCTP stuff */
-#  define BIO_CTRL_DGRAM_SCTP_SET_IN_HANDSHAKE    50
 #  define BIO_CTRL_DGRAM_SCTP_ADD_AUTH_KEY                51
 #  define BIO_CTRL_DGRAM_SCTP_NEXT_AUTH_KEY               52
 #  define BIO_CTRL_DGRAM_SCTP_AUTH_CCS_RCVD               53
@@ -143,6 +142,8 @@ extern "C" {
 #  define BIO_CTRL_DGRAM_SCTP_SAVE_SHUTDOWN               70
 # endif
 
+# define BIO_CTRL_DGRAM_SET_PEEK_MODE      71
+
 /* modifiers */
 # define BIO_FP_READ             0x02
 # define BIO_FP_WRITE            0x04

ssl/statem/statem.c

@@ -300,10 +300,10 @@ static int state_machine(SSL *s, int server)
             return -1;
     }
 #ifndef OPENSSL_NO_SCTP
-    if (SSL_IS_DTLS(s)) {
+    if (SSL_IS_DTLS(s) && BIO_dgram_is_sctp(SSL_get_wbio(s))) {
         /*
          * Notify SCTP BIO socket to enter handshake mode and prevent stream
-         * identifier other than 0. Will be ignored if no SCTP is used.
+         * identifier other than 0.
          */
         BIO_ctrl(SSL_get_wbio(s), BIO_CTRL_DGRAM_SCTP_SET_IN_HANDSHAKE,
                  st->in_handshake, NULL);
@@ -421,10 +421,10 @@ static int state_machine(SSL *s, int server)
     st->in_handshake--;
 
 #ifndef OPENSSL_NO_SCTP
-    if (SSL_IS_DTLS(s)) {
+    if (SSL_IS_DTLS(s) && BIO_dgram_is_sctp(SSL_get_wbio(s))) {
         /*
          * Notify SCTP BIO socket to leave handshake mode and allow stream
-         * identifier other than 0. Will be ignored if no SCTP is used.
+         * identifier other than 0.
          */
         BIO_ctrl(SSL_get_wbio(s), BIO_CTRL_DGRAM_SCTP_SET_IN_HANDSHAKE,
                  st->in_handshake, NULL);