tcp: Overflow bug in Vegas

From: Lachlan Andrew <lachlan.andrew@gmail.com>

There is an overflow bug in net/ipv4/tcp_vegas.c for large BDPs
(e.g. 400Mbit/s, 400ms).  The multiplication (old_wnd *
vegas->baseRTT) << V_PARAM_SHIFT overflows a u32.

[ Fix tcp_veno.c too, it has similar calculations. -DaveM ]
Signed-off-by: NDavid S. Miller <davem@davemloft.net>

net/ipv4/tcp_vegas.c

@@ -229,7 +229,8 @@ static void tcp_vegas_cong_avoid(struct sock *sk, u32 ack, u32 in_flight)
 			 */
 			tcp_reno_cong_avoid(sk, ack, in_flight);
 		} else {
-			u32 rtt, target_cwnd, diff;
+			u32 rtt, diff;
+			u64 target_cwnd;
 
 			/* We have enough RTT samples, so, using the Vegas
 			 * algorithm, we determine if we should increase or
@@ -252,8 +253,9 @@ static void tcp_vegas_cong_avoid(struct sock *sk, u32 ack, u32 in_flight)
 			 * We keep it as a fixed point number with
 			 * V_PARAM_SHIFT bits to the right of the binary point.
 			 */
-			target_cwnd = ((old_wnd * vegas->baseRTT)
-				       << V_PARAM_SHIFT) / rtt;
+			target_cwnd = ((u64)old_wnd * vegas->baseRTT);
+			target_cwnd <<= V_PARAM_SHIFT;
+			do_div(target_cwnd, rtt);
 
 			/* Calculate the difference between the window we had,
 			 * and the window we would like to have. This quantity
@@ -279,7 +281,7 @@ static void tcp_vegas_cong_avoid(struct sock *sk, u32 ack, u32 in_flight)
 				 * utilization.
 				 */
 				tp->snd_cwnd = min(tp->snd_cwnd,
-						   (target_cwnd >>
+						   ((u32)target_cwnd >>
 						    V_PARAM_SHIFT)+1);
 
 			} else if (tp->snd_cwnd <= tp->snd_ssthresh) {

net/ipv4/tcp_veno.c

@@ -133,7 +133,8 @@ static void tcp_veno_cong_avoid(struct sock *sk, u32 ack, u32 in_flight)
 		 */
 		tcp_reno_cong_avoid(sk, ack, in_flight);
 	} else {
-		u32 rtt, target_cwnd;
+		u64 target_cwnd;
+		u32 rtt;
 
 		/* We have enough rtt samples, so, using the Veno
 		 * algorithm, we determine the state of the network.
@@ -141,8 +142,9 @@ static void tcp_veno_cong_avoid(struct sock *sk, u32 ack, u32 in_flight)
 
 		rtt = veno->minrtt;
 
-		target_cwnd = ((tp->snd_cwnd * veno->basertt)
-			       << V_PARAM_SHIFT) / rtt;
+		target_cwnd = (tp->snd_cwnd * veno->basertt);
+		target_cwnd <<= V_PARAM_SHIFT;
+		do_div(target_cwnd, rtt);
 
 		veno->diff = (tp->snd_cwnd << V_PARAM_SHIFT) - target_cwnd;