PR: 2602

Submitted by: Robin Seggelmann <seggelmann@fh-muenster.de>
Reviewed by: steve

Fix DTLS bug which prevents manual MTU setting

ssl/d1_both.c

@@ -158,7 +158,6 @@ static unsigned char bitmask_end_values[]   = {0xff, 0x01, 0x03, 0x07, 0x0f, 0x1
 /* XDTLS:  figure out the right values */
 static unsigned int g_probable_mtu[] = {1500 - 28, 512 - 28, 256 - 28};
 
-static unsigned int dtls1_min_mtu(void);
 static unsigned int dtls1_guess_mtu(unsigned int curr_mtu);
 static void dtls1_fix_message_header(SSL *s, unsigned long frag_off, 
 	unsigned long frag_len);
@@ -264,11 +263,10 @@ int dtls1_do_write(SSL *s, int type)
 			return ret;
 		mtu = s->d1->mtu - (DTLS1_HM_HEADER_LENGTH + DTLS1_RT_HEADER_LENGTH);
 		}
-
-	OPENSSL_assert(mtu > 0);  /* should have something reasonable now */
-
 #endif
 
+	OPENSSL_assert(s->d1->mtu >= dtls1_min_mtu());  /* should have something reasonable now */
+
 	if ( s->init_off == 0  && type == SSL3_RT_HANDSHAKE)
 		OPENSSL_assert(s->init_num == 
 			(int)s->d1->w_msg_hdr.msg_len + DTLS1_HM_HEADER_LENGTH);
@@ -1367,7 +1365,7 @@ dtls1_write_message_header(SSL *s, unsigned char *p)
 	return p;
 	}
 
-static unsigned int 
+unsigned int 
 dtls1_min_mtu(void)
 	{
 	return (g_probable_mtu[(sizeof(g_probable_mtu) / 

ssl/d1_lib.c

@@ -204,7 +204,8 @@ void dtls1_clear(SSL *s)
     pqueue buffered_messages;
 	pqueue sent_messages;
 	pqueue buffered_app_data;
-	
+	unsigned int mtu;
+
 	if (s->d1)
 		{
 		unprocessed_rcds = s->d1->unprocessed_rcds.q;
@@ -212,6 +213,7 @@ void dtls1_clear(SSL *s)
 		buffered_messages = s->d1->buffered_messages;
 		sent_messages = s->d1->sent_messages;
 		buffered_app_data = s->d1->buffered_app_data.q;
+		mtu = s->d1->mtu;
 
 		dtls1_clear_queues(s);
 
@@ -222,6 +224,11 @@ void dtls1_clear(SSL *s)
 			s->d1->cookie_len = sizeof(s->d1->cookie);
 			}
 
+		if (SSL_get_options(s) & SSL_OP_NO_QUERY_MTU)
+			{
+			s->d1->mtu = mtu;
+			}
+
 		s->d1->unprocessed_rcds.q = unprocessed_rcds;
 		s->d1->processed_rcds.q = processed_rcds;
 		s->d1->buffered_messages = buffered_messages;

ssl/ssl_lib.c

@@ -1075,6 +1075,9 @@ long SSL_ctrl(SSL *s,int cmd,long larg,void *parg)
 		s->max_cert_list=larg;
 		return(l);
 	case SSL_CTRL_SET_MTU:
+		if (larg < dtls1_min_mtu())
+			return 0;
+
 		if (SSL_version(s) == DTLS1_VERSION ||
 		    SSL_version(s) == DTLS1_BAD_VER)
 			{

ssl/ssl_locl.h

@@ -966,6 +966,7 @@ void dtls1_stop_timer(SSL *s);
 int dtls1_is_timer_expired(SSL *s);
 void dtls1_double_timeout(SSL *s);
 int dtls1_send_newsession_ticket(SSL *s);
+unsigned int dtls1_min_mtu(void);
 
 /* some client-only functions */
 int ssl3_client_hello(SSL *s);